From b3c0c65f907573f92a493da57b29a99a8c3932b1 Mon Sep 17 00:00:00 2001 From: stansmith Date: Thu, 6 Sep 2007 18:17:48 +0000 Subject: [PATCH] [DAT/DAPL 2.0] Initial checkin; not enable as part of the standard build yet. git-svn-id: svn://openib.tc.cornell.edu/gen1@783 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/dapl2/AUTHORS | 17 + trunk/ulp/dapl2/COPYING | 36 + trunk/ulp/dapl2/LICENSE.txt | 235 ++ trunk/ulp/dapl2/LICENSE2.txt | 30 + trunk/ulp/dapl2/LICENSE3.txt | 340 +++ trunk/ulp/dapl2/README | 419 +++ .../ulp/dapl2/dapl/common/dapl_adapter_util.h | 298 +++ trunk/ulp/dapl2/dapl/common/dapl_cno_util.c | 336 +++ trunk/ulp/dapl2/dapl/common/dapl_cno_util.h | 66 + trunk/ulp/dapl2/dapl/common/dapl_cookie.c | 368 +++ trunk/ulp/dapl2/dapl/common/dapl_cookie.h | 72 + trunk/ulp/dapl2/dapl/common/dapl_cr_accept.c | 251 ++ .../ulp/dapl2/dapl/common/dapl_cr_callback.c | 592 +++++ trunk/ulp/dapl2/dapl/common/dapl_cr_handoff.c | 67 + trunk/ulp/dapl2/dapl/common/dapl_cr_query.c | 104 + trunk/ulp/dapl2/dapl/common/dapl_cr_reject.c | 144 + trunk/ulp/dapl2/dapl/common/dapl_cr_util.c | 110 + trunk/ulp/dapl2/dapl/common/dapl_cr_util.h | 58 + trunk/ulp/dapl2/dapl/common/dapl_csp.c | 108 + trunk/ulp/dapl2/dapl/common/dapl_debug.c | 109 + trunk/ulp/dapl2/dapl/common/dapl_ep_connect.c | 423 +++ trunk/ulp/dapl2/dapl/common/dapl_ep_create.c | 343 +++ .../dapl/common/dapl_ep_create_with_srq.c | 369 +++ .../dapl2/dapl/common/dapl_ep_disconnect.c | 190 ++ .../dapl2/dapl/common/dapl_ep_dup_connect.c | 130 + trunk/ulp/dapl2/dapl/common/dapl_ep_free.c | 205 ++ .../dapl2/dapl/common/dapl_ep_get_status.c | 120 + trunk/ulp/dapl2/dapl/common/dapl_ep_modify.c | 792 ++++++ .../dapl/common/dapl_ep_post_rdma_read.c | 106 + .../common/dapl_ep_post_rdma_read_to_rmr.c | 92 + .../dapl/common/dapl_ep_post_rdma_write.c | 105 + .../ulp/dapl2/dapl/common/dapl_ep_post_recv.c | 134 + .../ulp/dapl2/dapl/common/dapl_ep_post_send.c | 104 + .../common/dapl_ep_post_send_invalidate.c | 94 + trunk/ulp/dapl2/dapl/common/dapl_ep_query.c | 128 + .../dapl2/dapl/common/dapl_ep_recv_query.c | 99 + trunk/ulp/dapl2/dapl/common/dapl_ep_reset.c | 107 + .../dapl2/dapl/common/dapl_ep_set_watermark.c | 100 + trunk/ulp/dapl2/dapl/common/dapl_ep_util.c | 676 +++++ trunk/ulp/dapl2/dapl/common/dapl_ep_util.h | 87 + .../dapl/common/dapl_evd_connection_callb.c | 236 ++ .../common/dapl_evd_cq_async_error_callb.c | 96 + .../ulp/dapl2/dapl/common/dapl_evd_dequeue.c | 146 ++ .../dapl2/dapl/common/dapl_evd_dto_callb.c | 168 ++ trunk/ulp/dapl2/dapl/common/dapl_evd_free.c | 141 + .../ulp/dapl2/dapl/common/dapl_evd_post_se.c | 104 + .../common/dapl_evd_qp_async_error_callb.c | 146 ++ trunk/ulp/dapl2/dapl/common/dapl_evd_resize.c | 144 + .../common/dapl_evd_un_async_error_callb.c | 98 + trunk/ulp/dapl2/dapl/common/dapl_evd_util.c | 1423 ++++++++++ trunk/ulp/dapl2/dapl/common/dapl_evd_util.h | 154 ++ .../dapl/common/dapl_get_consumer_context.c | 102 + .../dapl2/dapl/common/dapl_get_handle_type.c | 89 + trunk/ulp/dapl2/dapl/common/dapl_hash.c | 538 ++++ trunk/ulp/dapl2/dapl/common/dapl_hash.h | 109 + trunk/ulp/dapl2/dapl/common/dapl_hca_util.c | 180 ++ trunk/ulp/dapl2/dapl/common/dapl_hca_util.h | 60 + trunk/ulp/dapl2/dapl/common/dapl_ia_close.c | 97 + trunk/ulp/dapl2/dapl/common/dapl_ia_ha.c | 79 + trunk/ulp/dapl2/dapl/common/dapl_ia_open.c | 510 ++++ trunk/ulp/dapl2/dapl/common/dapl_ia_query.c | 238 ++ trunk/ulp/dapl2/dapl/common/dapl_ia_util.c | 1248 +++++++++ trunk/ulp/dapl2/dapl/common/dapl_ia_util.h | 158 ++ trunk/ulp/dapl2/dapl/common/dapl_init.h | 57 + trunk/ulp/dapl2/dapl/common/dapl_llist.c | 384 +++ trunk/ulp/dapl2/dapl/common/dapl_lmr_free.c | 140 + trunk/ulp/dapl2/dapl/common/dapl_lmr_query.c | 90 + .../dapl/common/dapl_lmr_sync_rdma_read.c | 85 + .../dapl/common/dapl_lmr_sync_rdma_write.c | 85 + trunk/ulp/dapl2/dapl/common/dapl_lmr_util.c | 98 + trunk/ulp/dapl2/dapl/common/dapl_lmr_util.h | 63 + trunk/ulp/dapl2/dapl/common/dapl_mr_util.c | 115 + trunk/ulp/dapl2/dapl/common/dapl_mr_util.h | 96 + .../ulp/dapl2/dapl/common/dapl_name_service.c | 286 ++ .../ulp/dapl2/dapl/common/dapl_name_service.h | 61 + trunk/ulp/dapl2/dapl/common/dapl_provider.c | 494 ++++ trunk/ulp/dapl2/dapl/common/dapl_provider.h | 108 + trunk/ulp/dapl2/dapl/common/dapl_psp_create.c | 213 ++ .../dapl2/dapl/common/dapl_psp_create_any.c | 220 ++ trunk/ulp/dapl2/dapl/common/dapl_psp_free.c | 154 ++ trunk/ulp/dapl2/dapl/common/dapl_psp_query.c | 104 + trunk/ulp/dapl2/dapl/common/dapl_pz_create.c | 105 + trunk/ulp/dapl2/dapl/common/dapl_pz_free.c | 89 + trunk/ulp/dapl2/dapl/common/dapl_pz_query.c | 91 + trunk/ulp/dapl2/dapl/common/dapl_pz_util.c | 116 + trunk/ulp/dapl2/dapl/common/dapl_pz_util.h | 51 + .../dapl2/dapl/common/dapl_ring_buffer_util.c | 356 +++ .../dapl2/dapl/common/dapl_ring_buffer_util.h | 78 + trunk/ulp/dapl2/dapl/common/dapl_rmr_bind.c | 349 +++ trunk/ulp/dapl2/dapl/common/dapl_rmr_create.c | 143 + trunk/ulp/dapl2/dapl/common/dapl_rmr_free.c | 96 + trunk/ulp/dapl2/dapl/common/dapl_rmr_query.c | 96 + trunk/ulp/dapl2/dapl/common/dapl_rmr_util.c | 90 + trunk/ulp/dapl2/dapl/common/dapl_rmr_util.h | 86 + trunk/ulp/dapl2/dapl/common/dapl_rsp_create.c | 218 ++ trunk/ulp/dapl2/dapl/common/dapl_rsp_free.c | 162 ++ trunk/ulp/dapl2/dapl/common/dapl_rsp_query.c | 103 + .../dapl/common/dapl_set_consumer_context.c | 90 + trunk/ulp/dapl2/dapl/common/dapl_sp_util.c | 314 +++ trunk/ulp/dapl2/dapl/common/dapl_sp_util.h | 63 + trunk/ulp/dapl2/dapl/common/dapl_srq_create.c | 159 ++ trunk/ulp/dapl2/dapl/common/dapl_srq_free.c | 133 + .../dapl2/dapl/common/dapl_srq_post_recv.c | 130 + trunk/ulp/dapl2/dapl/common/dapl_srq_query.c | 101 + trunk/ulp/dapl2/dapl/common/dapl_srq_resize.c | 112 + trunk/ulp/dapl2/dapl/common/dapl_srq_set_lw.c | 101 + trunk/ulp/dapl2/dapl/common/dapl_srq_util.c | 149 ++ trunk/ulp/dapl2/dapl/common/dapl_srq_util.h | 54 + trunk/ulp/dapl2/dapl/common/dapl_timer_util.c | 337 +++ trunk/ulp/dapl2/dapl/common/dapl_timer_util.h | 47 + trunk/ulp/dapl2/dapl/dirs | 1 + trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cm.c | 2320 +++++++++++++++++ trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cq.c | 489 ++++ trunk/ulp/dapl2/dapl/ibal/dapl_ibal_dto.h | 556 ++++ .../dapl2/dapl/ibal/dapl_ibal_extensions.c | 332 +++ trunk/ulp/dapl2/dapl/ibal/dapl_ibal_kmod.h | 91 + trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.c | 392 +++ trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.h | 52 + .../dapl2/dapl/ibal/dapl_ibal_name_service.h | 70 + trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c | 692 +++++ trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.c | 2257 ++++++++++++++++ trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.h | 497 ++++ trunk/ulp/dapl2/dapl/include/dapl.h | 1267 +++++++++ trunk/ulp/dapl2/dapl/include/dapl_debug.h | 145 ++ .../ulp/dapl2/dapl/include/dapl_ipoib_names.h | 262 ++ trunk/ulp/dapl2/dapl/include/dapl_vendor.h | 116 + trunk/ulp/dapl2/dapl/udapl/Makefile.linux | 444 ++++ trunk/ulp/dapl2/dapl/udapl/SOURCES | 59 + trunk/ulp/dapl2/dapl/udapl/dapl_cno_create.c | 105 + trunk/ulp/dapl2/dapl/udapl/dapl_cno_free.c | 90 + .../dapl2/dapl/udapl/dapl_cno_modify_agent.c | 83 + trunk/ulp/dapl2/dapl/udapl/dapl_cno_query.c | 96 + trunk/ulp/dapl2/dapl/udapl/dapl_cno_wait.c | 133 + .../dapl/udapl/dapl_evd_clear_unwaitable.c | 82 + trunk/ulp/dapl2/dapl/udapl/dapl_evd_create.c | 194 ++ trunk/ulp/dapl2/dapl/udapl/dapl_evd_disable.c | 80 + trunk/ulp/dapl2/dapl/udapl/dapl_evd_enable.c | 94 + .../dapl2/dapl/udapl/dapl_evd_modify_cno.c | 117 + trunk/ulp/dapl2/dapl/udapl/dapl_evd_query.c | 114 + .../dapl/udapl/dapl_evd_set_unwaitable.c | 104 + trunk/ulp/dapl2/dapl/udapl/dapl_evd_wait.c | 281 ++ trunk/ulp/dapl2/dapl/udapl/dapl_init.c | 316 +++ trunk/ulp/dapl2/dapl/udapl/dapl_lmr_create.c | 539 ++++ trunk/ulp/dapl2/dapl/udapl/libdaplcma.map | 7 + trunk/ulp/dapl2/dapl/udapl/libdaplscm.map | 6 + trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.c | 635 +++++ trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.h | 542 ++++ trunk/ulp/dapl2/dapl/udapl/makefile | 7 + trunk/ulp/dapl2/dapl/udapl/udapl.rc | 48 + trunk/ulp/dapl2/dapl/udapl/udapl_exports.src | 14 + trunk/ulp/dapl2/dapl/udapl/udapl_sources.c | 100 + trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.c | 268 ++ trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.h | 536 ++++ trunk/ulp/dapl2/dat/common/dat_api.c | 1301 +++++++++ trunk/ulp/dapl2/dat/common/dat_dictionary.c | 472 ++++ trunk/ulp/dapl2/dat/common/dat_dictionary.h | 112 + trunk/ulp/dapl2/dat/common/dat_dr.c | 386 +++ trunk/ulp/dapl2/dat/common/dat_dr.h | 103 + trunk/ulp/dapl2/dat/common/dat_init.c | 161 ++ trunk/ulp/dapl2/dat/common/dat_init.h | 106 + trunk/ulp/dapl2/dat/common/dat_sr.c | 492 ++++ trunk/ulp/dapl2/dat/common/dat_sr.h | 104 + trunk/ulp/dapl2/dat/common/dat_strerror.c | 630 +++++ trunk/ulp/dapl2/dat/dirs | 1 + trunk/ulp/dapl2/dat/include/dat/dat.h | 1377 ++++++++++ trunk/ulp/dapl2/dat/include/dat/dat_error.h | 380 +++ .../dapl2/dat/include/dat/dat_ib_extensions.h | 310 +++ .../dapl2/dat/include/dat/dat_iw_extensions.h | 167 ++ .../dat/include/dat/dat_platform_specific.h | 270 ++ .../dapl2/dat/include/dat/dat_redirection.h | 858 ++++++ .../ulp/dapl2/dat/include/dat/dat_registry.h | 132 + .../dat/include/dat/dat_vendor_specific.h | 81 + trunk/ulp/dapl2/dat/include/dat/kdat.h | 487 ++++ trunk/ulp/dapl2/dat/include/dat/kdat_config.h | 83 + .../dapl2/dat/include/dat/kdat_redirection.h | 330 +++ .../dat/include/dat/kdat_vendor_specific.h | 83 + trunk/ulp/dapl2/dat/include/dat/udat.h | 504 ++++ trunk/ulp/dapl2/dat/include/dat/udat_config.h | 82 + .../dapl2/dat/include/dat/udat_redirection.h | 356 +++ .../dat/include/dat/udat_vendor_specific.h | 83 + trunk/ulp/dapl2/dat/udat/Makefile.linux | 256 ++ trunk/ulp/dapl2/dat/udat/SOURCES | 32 + .../dat/udat/linux/dat-registry-1.1.spec | 106 + trunk/ulp/dapl2/dat/udat/linux/dat_osd.c | 183 ++ trunk/ulp/dapl2/dat/udat/linux/dat_osd.h | 404 +++ trunk/ulp/dapl2/dat/udat/makefile | 7 + trunk/ulp/dapl2/dat/udat/udat.c | 473 ++++ trunk/ulp/dapl2/dat/udat/udat.rc | 48 + trunk/ulp/dapl2/dat/udat/udat_api.c | 316 +++ trunk/ulp/dapl2/dat/udat/udat_exports.src | 53 + trunk/ulp/dapl2/dat/udat/udat_sources.c | 11 + trunk/ulp/dapl2/dat/udat/udat_sr_parser.c | 1525 +++++++++++ trunk/ulp/dapl2/dat/udat/udat_sr_parser.h | 63 + trunk/ulp/dapl2/dat/udat/windows/dat_osd.c | 205 ++ trunk/ulp/dapl2/dat/udat/windows/dat_osd.h | 416 +++ trunk/ulp/dapl2/dat/udat/windows/dat_osd_sr.h | 43 + trunk/ulp/dapl2/dirs | 4 + trunk/ulp/dapl2/doc/dapl_coding_style.txt | 264 ++ trunk/ulp/dapl2/doc/dapl_end_point_design.txt | 1129 ++++++++ trunk/ulp/dapl2/doc/dapl_environ.txt | 42 + trunk/ulp/dapl2/doc/dapl_event_design.txt | 875 +++++++ .../ulp/dapl2/doc/dapl_ibm_api_variations.txt | 34 + .../doc/dapl_memory_management_design.txt | 173 ++ trunk/ulp/dapl2/doc/dapl_patch.txt | 83 + trunk/ulp/dapl2/doc/dapl_registry_design.txt | 631 +++++ .../dapl2/doc/dapl_shared_memory_design.txt | 876 +++++++ .../doc/dapl_vendor_specific_changes.txt | 394 +++ trunk/ulp/dapl2/doc/dat_environ.txt | 45 + trunk/ulp/dapl2/man/dapltest.1 | 390 +++ trunk/ulp/dapl2/man/dtest.1 | 78 + trunk/ulp/dapl2/test/dapltest/README | 293 +++ trunk/ulp/dapl2/test/dapltest/SOURCES | 32 + .../dapl2/test/dapltest/cmd/dapl_fft_cmd.c | 354 +++ .../ulp/dapl2/test/dapltest/cmd/dapl_getopt.c | 181 ++ .../dapl2/test/dapltest/cmd/dapl_limit_cmd.c | 235 ++ trunk/ulp/dapl2/test/dapltest/cmd/dapl_main.c | 142 + .../dapl2/test/dapltest/cmd/dapl_netaddr.c | 147 ++ .../ulp/dapl2/test/dapltest/cmd/dapl_params.c | 305 +++ .../test/dapltest/cmd/dapl_performance_cmd.c | 320 +++ .../dapl2/test/dapltest/cmd/dapl_qos_util.c | 65 + .../dapl2/test/dapltest/cmd/dapl_quit_cmd.c | 142 + .../dapl2/test/dapltest/cmd/dapl_server_cmd.c | 123 + .../dapl2/test/dapltest/cmd/dapl_test_data.c | 58 + .../test/dapltest/cmd/dapl_transaction_cmd.c | 500 ++++ .../dapl2/test/dapltest/common/dapl_endian.c | 100 + .../dapl2/test/dapltest/common/dapl_global.c | 36 + .../common/dapl_performance_cmd_util.c | 64 + .../test/dapltest/common/dapl_quit_cmd_util.c | 40 + .../common/dapl_transaction_cmd_util.c | 65 + trunk/ulp/dapl2/test/dapltest/dapltest.rc | 47 + trunk/ulp/dapl2/test/dapltest/dt_cmd.c | 14 + trunk/ulp/dapl2/test/dapltest/dt_common.c | 5 + trunk/ulp/dapl2/test/dapltest/dt_mdep.c | 3 + trunk/ulp/dapl2/test/dapltest/dt_test.c | 32 + trunk/ulp/dapl2/test/dapltest/dt_udapl.c | 3 + .../dapl2/test/dapltest/include/dapl_bpool.h | 62 + .../test/dapltest/include/dapl_client_info.h | 47 + .../dapl2/test/dapltest/include/dapl_common.h | 53 + .../test/dapltest/include/dapl_execute.h | 40 + .../test/dapltest/include/dapl_fft_cmd.h | 72 + .../test/dapltest/include/dapl_fft_util.h | 97 + .../dapl2/test/dapltest/include/dapl_getopt.h | 52 + .../dapl2/test/dapltest/include/dapl_global.h | 39 + .../test/dapltest/include/dapl_limit_cmd.h | 69 + .../dapl2/test/dapltest/include/dapl_mdep.h | 40 + .../test/dapltest/include/dapl_memlist.h | 57 + .../dapl2/test/dapltest/include/dapl_params.h | 75 + .../dapltest/include/dapl_performance_cmd.h | 69 + .../dapltest/include/dapl_performance_stats.h | 65 + .../dapltest/include/dapl_performance_test.h | 91 + .../dapl2/test/dapltest/include/dapl_proto.h | 669 +++++ .../test/dapltest/include/dapl_quit_cmd.h | 44 + .../test/dapltest/include/dapl_server_cmd.h | 44 + .../test/dapltest/include/dapl_server_info.h | 58 + .../dapl2/test/dapltest/include/dapl_tdep.h | 81 + .../test/dapltest/include/dapl_tdep_print.h | 68 + .../test/dapltest/include/dapl_test_data.h | 106 + .../dapltest/include/dapl_transaction_cmd.h | 69 + .../dapltest/include/dapl_transaction_stats.h | 50 + .../dapltest/include/dapl_transaction_test.h | 101 + .../test/dapltest/include/dapl_version.h | 41 + trunk/ulp/dapl2/test/dapltest/makefile | 7 + .../dapltest/mdep/linux/dapl_mdep_kernel.c | 381 +++ .../dapltest/mdep/linux/dapl_mdep_kernel.h | 180 ++ .../test/dapltest/mdep/linux/dapl_mdep_user.c | 603 +++++ .../test/dapltest/mdep/linux/dapl_mdep_user.h | 188 ++ .../dapltest/mdep/solaris/dapl_mdep_user.c | 466 ++++ .../dapltest/mdep/solaris/dapl_mdep_user.h | 151 ++ .../dapltest/mdep/windows/dapl_mdep_user.c | 436 ++++ .../dapltest/mdep/windows/dapl_mdep_user.h | 185 ++ .../ulp/dapl2/test/dapltest/test/dapl_bpool.c | 391 +++ .../dapl2/test/dapltest/test/dapl_client.c | 641 +++++ .../test/dapltest/test/dapl_client_info.c | 57 + .../ulp/dapl2/test/dapltest/test/dapl_cnxn.c | 77 + .../dapl2/test/dapltest/test/dapl_execute.c | 102 + .../test/dapltest/test/dapl_fft_connmgt.c | 126 + .../test/dapltest/test/dapl_fft_dataxfer.c | 156 ++ .../dapltest/test/dapl_fft_dataxfer_client.c | 147 ++ .../test/dapltest/test/dapl_fft_endpoint.c | 296 +++ .../test/dapltest/test/dapl_fft_hwconn.c | 229 ++ .../dapl2/test/dapltest/test/dapl_fft_mem.c | 253 ++ .../dapl2/test/dapltest/test/dapl_fft_pz.c | 272 ++ .../test/dapltest/test/dapl_fft_queryinfo.c | 664 +++++ .../dapl2/test/dapltest/test/dapl_fft_test.c | 92 + .../dapl2/test/dapltest/test/dapl_fft_util.c | 382 +++ .../ulp/dapl2/test/dapltest/test/dapl_limit.c | 1554 +++++++++++ .../dapl2/test/dapltest/test/dapl_memlist.c | 140 + .../dapltest/test/dapl_performance_client.c | 520 ++++ .../dapltest/test/dapl_performance_server.c | 412 +++ .../dapltest/test/dapl_performance_stats.c | 399 +++ .../dapltest/test/dapl_performance_util.c | 678 +++++ .../dapl2/test/dapltest/test/dapl_quit_util.c | 44 + .../dapl2/test/dapltest/test/dapl_server.c | 885 +++++++ .../test/dapltest/test/dapl_server_info.c | 51 + .../dapl2/test/dapltest/test/dapl_test_data.c | 58 + .../dapl2/test/dapltest/test/dapl_test_util.c | 720 +++++ .../dapl2/test/dapltest/test/dapl_thread.c | 134 + .../dapltest/test/dapl_transaction_stats.c | 176 ++ .../dapltest/test/dapl_transaction_test.c | 2010 ++++++++++++++ .../dapltest/test/dapl_transaction_util.c | 800 ++++++ .../ulp/dapl2/test/dapltest/test/dapl_util.c | 247 ++ .../dapl2/test/dapltest/udapl/udapl_tdep.c | 132 + trunk/ulp/dapl2/test/dirs | 1 + trunk/ulp/dapl2/test/dtest/GETOPT.C | 148 ++ trunk/ulp/dapl2/test/dtest/GETOPT.H | 78 + trunk/ulp/dapl2/test/dtest/README | 22 + trunk/ulp/dapl2/test/dtest/SOURCES | 28 + trunk/ulp/dapl2/test/dtest/dtest.c | 1904 ++++++++++++++ trunk/ulp/dapl2/test/dtest/dtest.rc | 47 + trunk/ulp/dapl2/test/dtest/makefile | 7 + 310 files changed, 77785 insertions(+) create mode 100644 trunk/ulp/dapl2/AUTHORS create mode 100644 trunk/ulp/dapl2/COPYING create mode 100644 trunk/ulp/dapl2/LICENSE.txt create mode 100644 trunk/ulp/dapl2/LICENSE2.txt create mode 100644 trunk/ulp/dapl2/LICENSE3.txt create mode 100644 trunk/ulp/dapl2/README create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_adapter_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cno_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cno_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cookie.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cookie.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_accept.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_callback.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_handoff.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_reject.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_cr_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_csp.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_debug.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_connect.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_create.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_create_with_srq.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_disconnect.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_dup_connect.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_get_status.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_modify.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read_to_rmr.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_write.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_post_recv.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_post_send.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_post_send_invalidate.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_recv_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_reset.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_set_watermark.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ep_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_connection_callb.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_cq_async_error_callb.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_dequeue.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_dto_callb.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_post_se.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_qp_async_error_callb.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_resize.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_un_async_error_callb.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_evd_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_get_consumer_context.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_get_handle_type.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_hash.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_hash.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_hca_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_hca_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ia_close.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ia_ha.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ia_open.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ia_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ia_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ia_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_init.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_llist.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_lmr_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_lmr_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_read.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_write.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_lmr_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_lmr_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_mr_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_mr_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_name_service.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_name_service.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_provider.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_provider.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_psp_create.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_psp_create_any.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_psp_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_psp_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_pz_create.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_pz_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_pz_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_pz_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_pz_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rmr_bind.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rmr_create.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rmr_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rmr_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rmr_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rmr_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rsp_create.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rsp_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_rsp_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_set_consumer_context.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_sp_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_sp_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_create.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_free.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_post_recv.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_query.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_resize.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_set_lw.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_srq_util.h create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_timer_util.c create mode 100644 trunk/ulp/dapl2/dapl/common/dapl_timer_util.h create mode 100644 trunk/ulp/dapl2/dapl/dirs create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cm.c create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cq.c create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_dto.h create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_extensions.c create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_kmod.h create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.c create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.h create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_name_service.h create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.c create mode 100644 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.h create mode 100644 trunk/ulp/dapl2/dapl/include/dapl.h create mode 100644 trunk/ulp/dapl2/dapl/include/dapl_debug.h create mode 100644 trunk/ulp/dapl2/dapl/include/dapl_ipoib_names.h create mode 100644 trunk/ulp/dapl2/dapl/include/dapl_vendor.h create mode 100644 trunk/ulp/dapl2/dapl/udapl/Makefile.linux create mode 100644 trunk/ulp/dapl2/dapl/udapl/SOURCES create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_cno_create.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_cno_free.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_cno_modify_agent.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_cno_query.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_cno_wait.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_clear_unwaitable.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_create.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_disable.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_enable.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_modify_cno.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_query.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_set_unwaitable.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_evd_wait.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_init.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/dapl_lmr_create.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/libdaplcma.map create mode 100644 trunk/ulp/dapl2/dapl/udapl/libdaplscm.map create mode 100644 trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.h create mode 100644 trunk/ulp/dapl2/dapl/udapl/makefile create mode 100644 trunk/ulp/dapl2/dapl/udapl/udapl.rc create mode 100644 trunk/ulp/dapl2/dapl/udapl/udapl_exports.src create mode 100644 trunk/ulp/dapl2/dapl/udapl/udapl_sources.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.c create mode 100644 trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.h create mode 100644 trunk/ulp/dapl2/dat/common/dat_api.c create mode 100644 trunk/ulp/dapl2/dat/common/dat_dictionary.c create mode 100644 trunk/ulp/dapl2/dat/common/dat_dictionary.h create mode 100644 trunk/ulp/dapl2/dat/common/dat_dr.c create mode 100644 trunk/ulp/dapl2/dat/common/dat_dr.h create mode 100644 trunk/ulp/dapl2/dat/common/dat_init.c create mode 100644 trunk/ulp/dapl2/dat/common/dat_init.h create mode 100644 trunk/ulp/dapl2/dat/common/dat_sr.c create mode 100644 trunk/ulp/dapl2/dat/common/dat_sr.h create mode 100644 trunk/ulp/dapl2/dat/common/dat_strerror.c create mode 100644 trunk/ulp/dapl2/dat/dirs create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_error.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_ib_extensions.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_iw_extensions.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_platform_specific.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_redirection.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_registry.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/dat_vendor_specific.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/kdat.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/kdat_config.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/kdat_redirection.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/kdat_vendor_specific.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/udat.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/udat_config.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/udat_redirection.h create mode 100644 trunk/ulp/dapl2/dat/include/dat/udat_vendor_specific.h create mode 100644 trunk/ulp/dapl2/dat/udat/Makefile.linux create mode 100644 trunk/ulp/dapl2/dat/udat/SOURCES create mode 100644 trunk/ulp/dapl2/dat/udat/linux/dat-registry-1.1.spec create mode 100644 trunk/ulp/dapl2/dat/udat/linux/dat_osd.c create mode 100644 trunk/ulp/dapl2/dat/udat/linux/dat_osd.h create mode 100644 trunk/ulp/dapl2/dat/udat/makefile create mode 100644 trunk/ulp/dapl2/dat/udat/udat.c create mode 100644 trunk/ulp/dapl2/dat/udat/udat.rc create mode 100644 trunk/ulp/dapl2/dat/udat/udat_api.c create mode 100644 trunk/ulp/dapl2/dat/udat/udat_exports.src create mode 100644 trunk/ulp/dapl2/dat/udat/udat_sources.c create mode 100644 trunk/ulp/dapl2/dat/udat/udat_sr_parser.c create mode 100644 trunk/ulp/dapl2/dat/udat/udat_sr_parser.h create mode 100644 trunk/ulp/dapl2/dat/udat/windows/dat_osd.c create mode 100644 trunk/ulp/dapl2/dat/udat/windows/dat_osd.h create mode 100644 trunk/ulp/dapl2/dat/udat/windows/dat_osd_sr.h create mode 100644 trunk/ulp/dapl2/dirs create mode 100644 trunk/ulp/dapl2/doc/dapl_coding_style.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_end_point_design.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_environ.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_event_design.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_ibm_api_variations.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_memory_management_design.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_patch.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_registry_design.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_shared_memory_design.txt create mode 100644 trunk/ulp/dapl2/doc/dapl_vendor_specific_changes.txt create mode 100644 trunk/ulp/dapl2/doc/dat_environ.txt create mode 100644 trunk/ulp/dapl2/man/dapltest.1 create mode 100644 trunk/ulp/dapl2/man/dtest.1 create mode 100644 trunk/ulp/dapl2/test/dapltest/README create mode 100644 trunk/ulp/dapl2/test/dapltest/SOURCES create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_fft_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_getopt.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_limit_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_main.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_netaddr.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_params.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_performance_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_qos_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_quit_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_server_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_test_data.c create mode 100644 trunk/ulp/dapl2/test/dapltest/cmd/dapl_transaction_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/common/dapl_endian.c create mode 100644 trunk/ulp/dapl2/test/dapltest/common/dapl_global.c create mode 100644 trunk/ulp/dapl2/test/dapltest/common/dapl_performance_cmd_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/common/dapl_quit_cmd_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/common/dapl_transaction_cmd_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/dapltest.rc create mode 100644 trunk/ulp/dapl2/test/dapltest/dt_cmd.c create mode 100644 trunk/ulp/dapl2/test/dapltest/dt_common.c create mode 100644 trunk/ulp/dapl2/test/dapltest/dt_mdep.c create mode 100644 trunk/ulp/dapl2/test/dapltest/dt_test.c create mode 100644 trunk/ulp/dapl2/test/dapltest/dt_udapl.c create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_bpool.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_client_info.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_common.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_execute.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_fft_cmd.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_fft_util.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_getopt.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_global.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_limit_cmd.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_mdep.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_memlist.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_params.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_performance_cmd.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_performance_stats.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_performance_test.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_proto.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_quit_cmd.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_server_cmd.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_server_info.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_tdep.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_tdep_print.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_test_data.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_cmd.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_stats.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_test.h create mode 100644 trunk/ulp/dapl2/test/dapltest/include/dapl_version.h create mode 100644 trunk/ulp/dapl2/test/dapltest/makefile create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.c create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.h create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.c create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.h create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.c create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.h create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.c create mode 100644 trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.h create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_bpool.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_client.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_client_info.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_cnxn.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_execute.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_connmgt.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer_client.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_endpoint.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_hwconn.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_mem.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_pz.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_queryinfo.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_test.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_fft_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_limit.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_memlist.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_performance_client.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_performance_server.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_performance_stats.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_performance_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_quit_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_server.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_server_info.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_test_data.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_test_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_thread.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_stats.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_test.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/test/dapl_util.c create mode 100644 trunk/ulp/dapl2/test/dapltest/udapl/udapl_tdep.c create mode 100644 trunk/ulp/dapl2/test/dirs create mode 100644 trunk/ulp/dapl2/test/dtest/GETOPT.C create mode 100644 trunk/ulp/dapl2/test/dtest/GETOPT.H create mode 100644 trunk/ulp/dapl2/test/dtest/README create mode 100644 trunk/ulp/dapl2/test/dtest/SOURCES create mode 100644 trunk/ulp/dapl2/test/dtest/dtest.c create mode 100644 trunk/ulp/dapl2/test/dtest/dtest.rc create mode 100644 trunk/ulp/dapl2/test/dtest/makefile diff --git a/trunk/ulp/dapl2/AUTHORS b/trunk/ulp/dapl2/AUTHORS new file mode 100644 index 00000000..8f6ba2c3 --- /dev/null +++ b/trunk/ulp/dapl2/AUTHORS @@ -0,0 +1,17 @@ +This is an incomplete list of people who have contributed to the +DAPL project: + + Caitlin Bestler + Philip Christopher + Jay Danielsen + Arlin Davis + Tom Duffy + Matthew Finlay + Jimmy Hill + James Lentini + Gil Rubin + Steve Sears + Randy Smith + Stan Smith + Anthony Topper + Steve Wise diff --git a/trunk/ulp/dapl2/COPYING b/trunk/ulp/dapl2/COPYING new file mode 100644 index 00000000..a1e8bacd --- /dev/null +++ b/trunk/ulp/dapl2/COPYING @@ -0,0 +1,36 @@ +# +# Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. +# Copyright (c) 2005 Voltaire Inc. All rights reserved. +# Copyright (c) 2005 Intel Corporation. All rights reserved. +# Copyright (c) 2004-2005, Mellanox Technologies, Inc. All rights reserved. +# Copyright (c) 2003 Topspin Corporation. All rights reserved. +# Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. +# +# This Software is licensed under one of the following licenses: +# +# 1) under the terms of the "Common Public License 1.0" a copy of which is +# in the file LICENSE.txt in the root directory. The license is also +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/cpl.php. +# +# 2) under the terms of the "The BSD License" a copy of which is in the file +# LICENSE2.txt in the root directory. The license is also available from +# the Open Source Initiative, see +# http://www.opensource.org/licenses/bsd-license.php. +# +# 3) under the terms of the "GNU General Public License (GPL) Version 2" a +# copy of which is in the file LICENSE3.txt in the root directory. The +# license is also available from the Open Source Initiative, see +# http://www.opensource.org/licenses/gpl-license.php. +# +# Licensee has the right to choose one of the above licenses. +# +# Redistributions of source code must retain the above copyright +# notice and one of the license notices. +# +# Redistributions in binary form must reproduce both the above copyright +# notice, one of the license notices in the documentation +# and/or other materials provided with the distribution. +# + + diff --git a/trunk/ulp/dapl2/LICENSE.txt b/trunk/ulp/dapl2/LICENSE.txt new file mode 100644 index 00000000..85720312 --- /dev/null +++ b/trunk/ulp/dapl2/LICENSE.txt @@ -0,0 +1,235 @@ +Common Public License 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON +PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF +THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's + behalf. Contributions do not include additions to the Program + which: (i) are separate modules of software distributed in + conjunction with the Program under their own license agreement, and + (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents " mean patent claims licensable by a Contributor +which are necessarily infringed by the use or sale of its Contribution +alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare derivative works of, publicly + display, publicly perform, distribute and sublicense the + Contribution of such Contributor, if any, and such derivative + works, in source code and object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in source code and object code form. This patent license + shall apply to the combination of the Contribution and the Program + if, at the time the Contribution is added by the Contributor, such + addition of the Contribution causes such combination to be covered + by the Licensed Patents. The patent license shall not apply to any + other combinations which include the Contribution. No hardware per + se is licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other + entity. Each Contributor disclaims any liability to Recipient for + claims brought by any other entity based on infringement of + intellectual property rights or otherwise. As a condition to + exercising the rights and licenses granted hereunder, each + Recipient hereby assumes sole responsibility to secure any other + intellectual property rights needed, if any. For example, if a + third party patent license is required to allow Recipient to + distribute the Program, it is Recipient's responsibility to acquire + that license before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form +under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and implied + warranties or conditions of merchantability and fitness for a + particular purpose; + + ii) effectively excludes on behalf of all Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement + are offered by that Contributor alone and not by any other party; + and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a reasonable + manner on or through a medium customarily used for software + exchange. + +When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of the + Program. + +Contributors may not remove or alter any copyright notices contained +within the Program. + +Each Contributor must identify itself as the originator of its +Contribution, if any, in a manner that reasonably allows subsequent +Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, the +Contributor who includes the Program in a commercial product offering +should do so in a manner which does not create potential liability for +other Contributors. Therefore, if a Contributor includes the Program in +a commercial product offering, such Contributor ("Commercial +Contributor") hereby agrees to defend and indemnify every other +Contributor ("Indemnified Contributor") against any losses, damages and +costs (collectively "Losses") arising from claims, lawsuits and other +legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the +Program in a commercial product offering. The obligations in this +section do not apply to any claims or Losses relating to any actual or +alleged intellectual property infringement. In order to qualify, an +Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial +Contributor to control, and cooperate with the Commercial Contributor +in, the defense and any related settlement negotiations. The Indemnified +Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those +performance claims and warranties, and if a court requires any other +Contributor to pay any damages as a result, the Commercial Contributor +must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS +PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY +WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR +FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible +for determining the appropriateness of using and distributing the +Program and assumes all risks associated with its exercise of rights +under this Agreement, including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss +of data, programs or equipment, and unavailability or interruption of +operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR +ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING +WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR +DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED +HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against a Contributor with +respect to a patent applicable to software (including a cross-claim or +counterclaim in a lawsuit), then any patent licenses granted by that +Contributor to such Recipient under this Agreement shall terminate as +of the date such litigation is filed. In addition, if Recipient +institutes patent litigation against any entity (including a cross- +claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights +granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails +to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of time +after becoming aware of such noncompliance. If all Recipient's rights +under this Agreement terminate, Recipient agrees to cease use and +distribution of the Program as soon as reasonably practicable. However, +Recipient's obligations under this Agreement and any licenses granted +by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. IBM is the initial +Agreement Steward. IBM may assign the responsibility to serve as the +Agreement Steward to a suitable separate entity. Each new version of +the Agreement will be given a distinguishing version number. The +Program (including Contributions) may always be distributed subject to +the version of the Agreement under which it was received. In addition, +after a new version of the Agreement is published, Contributor may +elect to distribute the Program (including its Contributions) under the +new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual +property of any Contributor under this Agreement, whether expressly, by +implication, estoppel or otherwise. All rights in the Program not +expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to +this Agreement will bring a legal action under this Agreement more than +one year after the cause of action arose. Each party waives its rights +to a jury trial in any resulting litigation. + diff --git a/trunk/ulp/dapl2/LICENSE2.txt b/trunk/ulp/dapl2/LICENSE2.txt new file mode 100644 index 00000000..0f84e3bb --- /dev/null +++ b/trunk/ulp/dapl2/LICENSE2.txt @@ -0,0 +1,30 @@ +Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + Neither the name of the Network Appliance, Inc. nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/trunk/ulp/dapl2/LICENSE3.txt b/trunk/ulp/dapl2/LICENSE3.txt new file mode 100644 index 00000000..d826fce6 --- /dev/null +++ b/trunk/ulp/dapl2/LICENSE3.txt @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + +To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + +We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + +Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + +Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and +modification follow. + +GNU GENERAL PUBLIC LICENSE +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + +a) You must cause the modified files to carry prominent notices +stating that you changed the files and the date of any change. + +b) You must cause any work that you distribute or publish, that in +whole or in part contains or is derived from the Program or any +part thereof, to be licensed as a whole at no charge to all third +parties under the terms of this License. + +c) If the modified program normally reads commands interactively +when run, you must cause it, when started running for such +interactive use in the most ordinary way, to print or display an +announcement including an appropriate copyright notice and a +notice that there is no warranty (or else, saying that you provide +a warranty) and that users may redistribute the program under +these conditions, and telling the user how to view a copy of this +License. (Exception: if the Program itself is interactive but +does not normally print such an announcement, your work based on +the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + +3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + +a) Accompany it with the complete corresponding machine-readable +source code, which must be distributed under the terms of Sections +1 and 2 above on a medium customarily used for software interchange; or, + +b) Accompany it with a written offer, valid for at least three +years, to give any third party, for a charge no more than your +cost of physically performing source distribution, a complete +machine-readable copy of the corresponding source code, to be +distributed under the terms of Sections 1 and 2 above on a medium +customarily used for software interchange; or, + +c) Accompany it with the information you received as to the offer +to distribute corresponding source code. (This alternative is +allowed only for noncommercial distribution and only if you +received the program in object code or executable form with such +an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + +5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + +9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + +10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + +Copyright (C) + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + +Gnomovision version 69, Copyright (C) year name of author +Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. +This is free software, and you are welcome to redistribute it +under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + +Yoyodyne, Inc., hereby disclaims all copyright interest in the program +`Gnomovision' (which makes passes at compilers) written by James Hacker. + +, 1 April 1989 +Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/trunk/ulp/dapl2/README b/trunk/ulp/dapl2/README new file mode 100644 index 00000000..8a865249 --- /dev/null +++ b/trunk/ulp/dapl2/README @@ -0,0 +1,419 @@ + +========== +1.0 BUILD: +========== + +The default build includes a non-debug version of libdat and libdapl-cma uDAPL provider. It will also builds test suites dtest and dapltest and provides manpages for each. This version requires libibverbs and librdmacm installation, IPoIB installation, and IPoIB configuration with an IP address. + +Building : +---------- +./autogen.sh +./configure +make + +Building debug version: +---------------------- +./autogen.sh +./configure --enable-debug +make + +Build example with OFED prefix (x86_64) +----------------------------------------- +./autogen.sh +./configure --prefix /usr/local/ofed --libdir /usr/local/ofed/lib64 LDFLAGS=-L/usr/local/ofed/lib64 CPPFLAGS="-I/usr/local/ofed/include" +make + +Installing: +---------- +make install + +NOTE: to link these libraries you must either use libtool and +specify the full pathname of the library, or use the `-LLIBDIR' +flag during linking and do at least one of the following: + - add LIBDIR to the `LD_LIBRARY_PATH' environment variable + during execution + - add LIBDIR to the `LD_RUN_PATH' environment variable + during linking + - use the `-Wl,--rpath -Wl,LIBDIR' linker flag + - have your system administrator add LIBDIR to `/etc/ld.so.conf' + +See any operating system documentation about shared libraries for +more information, such as the ld(1) and ld.so(8) manual pages. + +=================== +2.0 CONFIGURATION: +=================== + +sample /etc/dat.conf + +# +# DAT 1.2 configuration file, sample OFED +# +# Each entry should have the following fields: +# +# \ +# +# +# For openib-cma provider you can specify as either: +# network address, network hostname, or netdev name and 0 for port +# +# This example shows netdev name, enabling administrator to use same copy across cluster +# +OpenIB-cma u1.2 nonthreadsafe default /usr/local/ofed/lib64/libdapl-cma.so mv_dapl.1.2 "ib0 0" "" + +============================= +3.0 Bugs/Known issues +============================= + + + +============================= +4.0 SAMPLE uDAPL APPLICATION: +============================= + +There are 2 sample programs, with manpages, provided with this package. + +(dapl/test/dtest/) + +NAME + dtest - simple uDAPL send/receive and RDMA test + +SYNOPSIS + dtest [-P provider] [-b buf size] [-B burst count][-v] [-c] [-p] [-d] [-s] + + dtest [-P provider] [-b buf size] [-B burst count][-v] [-c] [-p] [-d] [-h HOSTNAME] + +DESCRIPTION + dtest is a simple test used to exercise and verify the uDAPL interfaces. At least two instantia- + tions of the test must be run. One acts as the server and the other the client. The server side of + the test, once invoked listens for connection requests, until timing out or killed. Upon receipt + of a cd connection request, the connection is established, the server and client sides exchange + information necessary to perform RDMA writes and reads. + +OPTIONS + -P=PROVIDER + use PROVIDER to specify uDAPL interface using /etc/dat.conf (default OpenIB-cma) + + -b=BUFFER_SIZE + use buffer size BUFFER_SIZE for RDMA(default 64) + + -B=BURST_COUNT + use busrt count BURST_COUNT for interations (default 10) + + -v, verbose output(default off) + + -c, use consumer notification events (default off) + + -p, use polling (default wait for event) + + -d, delay in seconds before close (default off) + + -s, run as server (default - run as server) + + -h=HOSTNAME + use HOSTNAME to specify server hostname or IP address (default - none) + +EXAMPLES + dtest -P OpenIB-cma -v -s + Starts a server process with debug verbosity using provider OpenIB-cma. + + dtest -P OpenIB-cma -h server1-ib0 + + Starts a client process, using OpenIB-cma provider to connect to hostname server1-ib0. + +SEE ALSO + dapltest(1) + +AUTHORS + Arlin Davis + + +BUGS + +/dapl/test/dapltest/ + +NAME + dapltest - test for the Direct Access Programming Library (DAPL) + +DESCRIPTION + Dapltest is a set of tests developed to exercise, characterize, and verify the DAPL interfaces + during development and porting. At least two instantiations of the test must be run. One acts as + the server, fielding requests and spawning server-side test threads as needed. Other client invo- + cations connect to the server and issue test requests. The server side of the test, once invoked, + listens continuously for client connection requests, until quit or killed. Upon receipt of a con- + nection request, the connection is established, the server and client sides swap version numbers + to verify that they are able to communicate, and the client sends the test request to the server. + If the version numbers match, and the test request is well-formed, the server spawns the threads + needed to run the test before awaiting further connections. + +USAGE + dapltest [ -f script_file_name ] [ -T S|Q|T|P|L ] [ -D device_name ] [ -d ] [ -R HT|LL|EC|PM|BE ] + + With no arguments, dapltest runs as a server using default values, and loops accepting requests + from clients. + + The -f option allows all arguments to be placed in a file, to ease test automation. + + The following arguments are common to all tests: + + [ -T S|Q|T|P|L ] + Test function to be performed: + + S - server loop + + Q - quit, client requests that server wait for any outstanding tests to complete, then + clean up and exit + + T - transaction test, transfers data between client and server + + P - performance test, times DTO operations + + L - limit test, exhausts various resources, runs in client w/o server interaction + Default: S + + [ -D device_name ] + Specifies the interface adapter name as documented in the /etc/dat.conf static configura- + tion file. This name corresponds to the provider library to open. Default: none + + [ -d ] Enables extra debug verbosity, primarily tracing of the various DAPL operations as they + progress. Repeating this parameter increases debug spew. Errors encountered result in the + test spewing some explanatory text and stopping; this flag provides more detail about what + lead up to the error. Default: zero + + [ -R BE ] + Indicate the quality of service (QoS) desired. Choices are: + + HT - high throughput + + LL - low latency + + EC - economy (neither HT nor LL) + + PM - premium + + BE - best effort Default: BE + + Usage - Quit test client + + dapltest [Common_Args] [ -s server_name ] + + Quit testing (-T Q) connects to the server to ask it to clean up and + exit (after it waits for any outstanding test runs to complete). + In addition to being more polite than simply killing the server, + this test exercises the DAPL object teardown code paths. + There is only one argument other than those supported by all tests: + + -s server_name Specifies the name of the server interface. + No default. + + Usage - Transaction test client + + dapltest [Common_Args] [ -s server_name ] + [ -t threads ] [ -w endpoints ] [ -i iterations ] [ -Q ] + [ -V ] [ -P ] OPclient OPserver [ op3, + + Transaction testing (-T T) transfers a variable amount of data between +: + client and server. The data transfer can be described as a sequence of + individual operations; that entire sequence is transferred ’iterations’ + times by each thread over all of its endpoint(s). + + The following parameters determine the behavior of the transaction test: + + -s server_name Specifies the name or IP address of the server interface. + No default. + + [ -t threads ] Specify the number of threads to be used. + Default: 1 + + [ -w endpoints ] Specify the number of connected endpoints per thread. + Default: 1 + + [ -i iterations ] Specify the number of times the entire sequence + of data transfers will be made over each endpoint. + Default: 1000 + + [ -Q ] Funnel completion events into a CNO. + Default: use EVDs + + [ -V ] Validate the data being transferred. + Default: ignore the data + + [ -P ] Turn on DTO completion polling + Default: off + + OP1 OP2 [ OP3, ... ] + A single transaction (OPx) consists of: + + server|client Indicates who initiates the + data transfer. + + SR|RR|RW Indicates the type of transfer: + SR send/recv + RR RDMA read + RW RDMA write + Defaults: none + + [ seg_size [ num_segs ] ] +: + + Indicates the amount and format + of the data to be transferred. + Default: 4096 1 + (i.e., 1 4KB buffer) + + [ -f ] For SR transfers only, indicates + that a client’s send transfer + completion should be reaped when + the next recv completion is reaped. + Sends and receives must be paired + (one client, one server, and in that + order) for this option to be used. + + Restrictions: + + Due to the flow control algorithm used by the transaction test, there + must be at least one SR OP for both the client and the server. + + Requesting data validation (-V) causes the test to automatically append + three OPs to those specified. These additional operations provide + synchronization points during each iteration, at which all user-specified + transaction buffers are checked. These three appended operations satisfy + the "one SR in each direction" requirement. + + The transaction OP list is printed out if -d is supplied. + + Usage - Performance test client + + dapltest [Common_Args] -s server_name [ -m p|b ] + [ -i iterations ] [ -p pipeline ] OP + + Performance testing (-T P) times the transfer of an operation. + The operation is posted ’iterations’ times. + + The following parameters determine the behavior of the transaction test: + + -s server_name Specifies the name or IP address of the server interface. + No default. + + -m b|p Used to choose either blocking (b) or polling (p) + Default: blocking (b) + [ -i iterations ] Specify the number of times the entire sequence + of data transfers will be made over each endpoint. + Default: 1000 + + [ -p pipeline ] Specify the pipline length, valid arguments are in + the range [0,MAX_SEND_DTOS]. If a value greater than + MAX_SEND_DTOS is requested the value will be + adjusted down to MAX_SEND_DTOS. + Default: MAX_SEND_DTOS + + OP Specifies the operation as follow: + + RR|RW Indicates the type of transfer: + RR RDMA read + RW RDMA write + Defaults: none + + [ seg_size [ num_segs ] ] + Indicates the amount and format + of the data to be transferred. + Default: 4096 1 + (i.e., 1 4KB buffer) + + Usage - Limit test client + + Limit testing (-T L) neither requires nor connects to any server + instance. The client runs one or more tests which attempt to + exhaust various resources to determine DAPL limits and exercise + DAPL error paths. If no arguments are given, all tests are run. + + Limit testing creates the sequence of DAT objects needed to + move data back and forth, attempting to find the limits supported + for the DAPL object requested. For example, if the LMR creation + limit is being examined, the test will create a set of + {IA, PZ, CNO, EVD, EP} before trying to run dat_lmr_create() to + failure using that set of DAPL objects. The ’width’ parameter + can be used to control how many of these parallel DAPL object + sets are created before beating upon the requested constructor. + Use of -m limits the number of dat_*_create() calls that will + be attempted, which can be helpful if the DAPL in use supports + essentailly unlimited numbers of some objects. + The limit test arguments are: + + [ -m maximum ] Specify the maximum number of dapl_*_create() + attempts. + Default: run to object creation failure + + [ -w width ] Specify the number of DAPL object sets to + create while initializing. + Default: 1 + + [ limit_ia ] Attempt to exhaust dat_ia_open() + + [ limit_pz ] Attempt to exhaust dat_pz_create() + + [ limit_cno ] Attempt to exhaust dat_cno_create() + + [ limit_evd ] Attempt to exhaust dat_evd_create() + + [ limit_ep ] Attempt to exhaust dat_ep_create() + + [ limit_rsp ] Attempt to exhaust dat_rsp_create() + + [ limit_psp ] Attempt to exhaust dat_psp_create() + + [ limit_lmr ] Attempt to exhaust dat_lmr_create(4KB) + + [ limit_rpost ] Attempt to exhaust dat_ep_post_recv(4KB) + + [ limit_size_lmr ] Probe maximum size dat_lmr_create() + + Default: run all tests + +EXAMPLES + dapltest -T S -d -D OpenIB-cma + + Starts a server process with debug verbosity. + + dapltest -T T -d -s host1-ib0 -D OpenIB-cma -i 100 client SR 4096 2 server SR 4096 2 + + Runs a transaction test, with both sides + sending one buffer with two 4KB segments, + one hundred times. + + dapltest -T P -d -s host1-ib0 -D OpenIB-cma -i 100 SR 4096 2 + + Runs a performance test, with the client + sending one buffer with two 4KB segments, + one hundred times. + + dapltest -T Q -s host1-ib0 -D OpenIB-cma + + Asks the server to clean up and exit. + + dapltest -T L -D OpenIB-cma -d -w 16 -m 1000 + + Runs all of the limit tests, setting up + 16 complete sets of DAPL objects, and + creating at most a thousand instances + when trying to exhaust resources. + + dapltest -T T -V -d -t 2 -w 4 -i 55555 -s linux3 -D OpenIB-cma client RW 4096 1 server RW 2048 4 + client SR 1024 4 server SR 4096 2 client SR 1024 3 -f server SR 2048 1 -f + + Runs a more complicated transaction test, + with two thread using four EPs each, + sending a more complicated buffer pattern + for a larger number of iterations, + validating the data received. + + BUGS (and To Do List) + + Use of CNOs (-Q) is not yet supported. + + Further limit tests could be added. + + + + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_adapter_util.h b/trunk/ulp/dapl2/dapl/common/dapl_adapter_util.h new file mode 100644 index 00000000..a962a882 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_adapter_util.h @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_adapter_util.h + * + * PURPOSE: Utility defs & routines for the adapter data structure + * + * $Id: dapl_adapter_util.h 1317 2005-04-25 17:29:42Z jlentini $ + * + **********************************************************************/ + +#ifndef _DAPL_ADAPTER_UTIL_H_ +#define _DAPL_ADAPTER_UTIL_H_ + + +typedef enum async_handler_type +{ + DAPL_ASYNC_UNAFILIATED, + DAPL_ASYNC_CQ_ERROR, + DAPL_ASYNC_CQ_COMPLETION, + DAPL_ASYNC_QP_ERROR +} DAPL_ASYNC_HANDLER_TYPE; + + +int dapls_ib_init (void); + +int dapls_ib_release (void); + +DAT_RETURN dapls_ib_enum_hcas ( + IN const char *vendor, + OUT DAPL_HCA_NAME **hca_names, + OUT DAT_COUNT *total_hca_count); + +DAT_RETURN dapls_ib_get_instance_data( + IN DAPL_HCA_NAME hca_name, + OUT char *instance); + +DAT_RETURN dapls_ib_open_hca ( + IN char *namestr, + IN DAPL_HCA *hca_ptr); + +DAT_RETURN dapls_ib_close_hca ( + IN DAPL_HCA *hca_ptr); + +DAT_RETURN dapls_ib_qp_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr, + IN DAPL_EP *ep_ctx_ptr); + +DAT_RETURN dapls_ib_qp_free ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr); + +DAT_RETURN dapls_ib_qp_modify ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr, + IN DAT_EP_ATTR *ep_attr); + +DAT_RETURN dapls_ib_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + IN DAT_CONN_QUAL remote_conn_qual, + IN DAT_COUNT private_data_size, + IN DAT_PVOID private_data); + +DAT_RETURN dapls_ib_disconnect ( + IN DAPL_EP *ep_ptr, + IN DAT_CLOSE_FLAGS close_flags); + +DAT_RETURN dapls_ib_setup_conn_listener ( + IN DAPL_IA *ia_ptr, + IN DAT_UINT64 ServiceID, + IN DAPL_SP *sp_ptr); + +DAT_RETURN dapls_ib_remove_conn_listener ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_ptr); + +DAT_RETURN dapls_ib_accept_connection ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data); + +DAT_RETURN dapls_ib_reject_connection ( + IN dp_ib_cm_handle_t cm_handle, + IN int reject_reason); + +DAT_RETURN dapls_ib_setup_async_callback ( + IN DAPL_IA *ia_ptr, + IN DAPL_ASYNC_HANDLER_TYPE handler_type, + IN DAPL_EVD *evd_ptr, + IN ib_async_handler_t callback, + IN void *context); + +DAT_RETURN dapls_ib_cq_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT *cqlen); + +DAT_RETURN dapls_ib_cq_free ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr); + +DAT_RETURN dapls_set_cq_notify ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr); + +DAT_RETURN dapls_ib_cq_resize ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT *cqlen); + +DAT_RETURN dapls_ib_pd_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_PZ *pz); + +DAT_RETURN dapls_ib_pd_free ( + IN DAPL_PZ *pz); + +DAT_RETURN dapls_ib_mr_register ( + IN DAPL_IA *ia_ptr, + IN DAPL_LMR *lmr, + IN DAT_PVOID virt_addr, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type); + +#if defined(__KDAPL__) +DAT_RETURN dapls_ib_mr_register_physical ( + IN DAPL_IA *ia_ptr, + INOUT DAPL_LMR *lmr, + IN DAT_PADDR phys_addr, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS privileges); +#endif /* __KDAPL__ */ + +DAT_RETURN dapls_ib_mr_deregister ( + IN DAPL_LMR *lmr); + +DAT_RETURN dapls_ib_mr_register_shared ( + IN DAPL_IA *ia_ptr, + IN DAPL_LMR *lmr, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type); + +DAT_RETURN dapls_ib_mw_alloc ( + IN DAPL_RMR *rmr); + +DAT_RETURN dapls_ib_mw_free ( + IN DAPL_RMR *rmr); + +DAT_RETURN dapls_ib_mw_bind ( + IN DAPL_RMR *rmr, + IN DAPL_LMR *lmr, + IN DAPL_EP *ep, + IN DAPL_COOKIE *cookie, + IN DAT_VADDR virtual_address, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN DAT_BOOLEAN is_signaled); + +DAT_RETURN dapls_ib_mw_unbind ( + IN DAPL_RMR *rmr, + IN DAPL_EP *ep, + IN DAPL_COOKIE *cookie, + IN DAT_BOOLEAN is_signaled); + +DAT_RETURN dapls_ib_query_hca ( + IN DAPL_HCA *hca_ptr, + OUT DAT_IA_ATTR *ia_attr, + OUT DAT_EP_ATTR *ep_attr, + OUT DAT_SOCK_ADDR6 *ip_addr); + +DAT_RETURN dapls_ib_completion_poll ( + IN DAPL_HCA *hca_ptr, + IN DAPL_EVD *evd_ptr, + IN ib_work_completion_t *cqe_ptr); + +DAT_RETURN dapls_ib_completion_notify ( + IN ib_hca_handle_t hca_handle, + IN DAPL_EVD *evd_ptr, + IN ib_notification_type_t type); + +DAT_DTO_COMPLETION_STATUS dapls_ib_get_dto_status ( + IN ib_work_completion_t *cqe_ptr); + +void dapls_ib_reinit_ep ( + IN DAPL_EP *ep_ptr); + +void dapls_ib_disconnect_clean ( + IN DAPL_EP *ep_ptr, + IN DAT_BOOLEAN passive, + IN const ib_cm_events_t ib_cm_event); + +DAT_RETURN dapls_ib_get_async_event ( + IN ib_error_record_t *cause_ptr, + OUT DAT_EVENT_NUMBER *async_event); + +DAT_EVENT_NUMBER dapls_ib_get_dat_event ( + IN const ib_cm_events_t ib_cm_event, + IN DAT_BOOLEAN active); + +ib_cm_events_t dapls_ib_get_cm_event ( + IN DAT_EVENT_NUMBER dat_event_num); + +DAT_RETURN dapls_ib_cm_remote_addr ( + IN DAT_HANDLE dat_handle, + OUT DAT_SOCK_ADDR6 *remote_ia_address); + +int dapls_ib_private_data_size ( + IN DAPL_PRIVATE *prd_ptr, + IN DAPL_PDATA_OP conn_op); + +void +dapls_query_provider_specific_attr( + IN DAT_PROVIDER_ATTR *provider_attr ); + +#ifdef CQ_WAIT_OBJECT +DAT_RETURN +dapls_ib_wait_object_create ( + IN DAPL_EVD *evd_ptr, + IN ib_wait_obj_handle_t *p_cq_wait_obj_handle); + +DAT_RETURN +dapls_ib_wait_object_destroy ( + IN ib_wait_obj_handle_t cq_wait_obj_handle); + +DAT_RETURN +dapls_ib_wait_object_wakeup ( + IN ib_wait_obj_handle_t cq_wait_obj_handle); + +DAT_RETURN +dapls_ib_wait_object_wait ( + IN ib_wait_obj_handle_t cq_wait_obj_handle, + IN u_int32_t timeout); +#endif + +#ifdef DAT_EXTENSIONS +void +dapls_cqe_to_event_extension( + IN DAPL_EP *ep_ptr, + IN DAPL_COOKIE *cookie, + IN ib_work_completion_t *cqe_ptr, + IN DAT_EVENT *event_ptr); +#endif + +/* + * Values for provider DAT_NAMED_ATTR + */ +#define IB_QP_STATE 1 /* QP state change request */ + + +#ifdef IBAPI +#include "dapl_ibapi_dto.h" +#elif VAPI +#include "dapl_vapi_dto.h" +#elif __OPENIB__ +#include "dapl_openib_dto.h" +#elif DUMMY +#include "dapl_dummy_dto.h" +#elif OPENIB +#include "dapl_ib_dto.h" +#else +#include "dapl_ibal_dto.h" +#endif + + +#endif /* _DAPL_ADAPTER_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cno_util.c b/trunk/ulp/dapl2/dapl/common/dapl_cno_util.c new file mode 100644 index 00000000..a4c3146a --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cno_util.c @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cno_util.c + * + * PURPOSE: Manage CNO Info structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ia_util.h" +#include "dapl_cno_util.h" +#include "dapl_adapter_util.h" + + + +/* + * dapl_cno_alloc + * + * alloc and initialize an EVD struct + * + * Input: + * ia + * + * Returns: + * cno_ptr, or null on failure. + */ +#if defined(__KDAPL__) +DAPL_CNO * +dapl_cno_alloc ( + IN DAPL_IA *ia_ptr, + IN const DAT_UPCALL_OBJECT *upcall) +#else +DAPL_CNO * +dapl_cno_alloc ( + IN DAPL_IA *ia_ptr, + IN DAT_OS_WAIT_PROXY_AGENT wait_agent) +#endif /* defined(__KDAPL__) */ +{ + DAPL_CNO *cno_ptr; + + cno_ptr = (DAPL_CNO *) dapl_os_alloc (sizeof (DAPL_CNO)); + if (!cno_ptr) + { + return NULL; + } + + /* zero the structure */ + dapl_os_memzero (cno_ptr, sizeof (DAPL_CNO)); + + /* + * Initialize the header. + */ + cno_ptr->header.provider = ia_ptr->header.provider; + cno_ptr->header.magic = DAPL_MAGIC_CNO; +#if !defined(__KDAPL__) + cno_ptr->header.handle_type = DAT_HANDLE_TYPE_CNO; +#endif /* defined(__KDAPL__) */ + cno_ptr->header.owner_ia = ia_ptr; + cno_ptr->header.user_context.as_64 = 0; + cno_ptr->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&cno_ptr->header.ia_list_entry); + dapl_os_lock_init (&cno_ptr->header.lock); + + /* + * Initialize the body + */ + cno_ptr->cno_waiters = 0; + dapl_os_atomic_set (&cno_ptr->cno_ref_count, 0); + cno_ptr->cno_state = DAPL_CNO_STATE_UNTRIGGERED; + cno_ptr->cno_evd_triggered = NULL; +#if defined(__KDAPL__) + cno_ptr->cno_upcall = *upcall; +#else + cno_ptr->cno_wait_agent = wait_agent; +#endif /* defined(__KDAPL__) */ + dapl_os_wait_object_init (&cno_ptr->cno_wait_object); + + return cno_ptr; +} + +/* + * dapl_cno_dealloc + * + * Free the passed in CNO structure. + * + * Input: + * cno_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_cno_dealloc ( + IN DAPL_CNO *cno_ptr) +{ + dapl_os_assert (cno_ptr->header.magic == DAPL_MAGIC_CNO); + dapl_os_assert (dapl_os_atomic_read (&cno_ptr->cno_ref_count) == 0); + + /* + * deinitialize the header + */ + cno_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ + + dapl_os_wait_object_destroy (&cno_ptr->cno_wait_object); + dapl_os_free (cno_ptr, sizeof (DAPL_CNO)); +} + + +/* + * dapl_internal_cno_trigger + * + * DAPL Internal routine to trigger the specified CNO. + * Called by the callback of some EVD associated with the CNO. + * + * + * + * Input: + * cno_ptr + * evd_ptr EVD triggering + * + * Output: + * None + * + * Returns: + * None + */ +void +dapl_internal_cno_trigger ( + IN DAPL_CNO *cno_ptr, + IN DAPL_EVD *evd_ptr) +{ + DAT_RETURN dat_status; +#if defined(__KDAPL__) + DAT_EVENT event; +#endif /* defined(__KDAPL__) */ + + dat_status = DAT_SUCCESS; + + dapl_os_assert (cno_ptr->header.magic == DAPL_MAGIC_CNO); + /* The spec allows NULL EVDs. kDAPL doesn't have CNOs, they + * are strictly used behind the scenes + */ + dapl_os_assert (evd_ptr == NULL || evd_ptr->header.magic == DAPL_MAGIC_EVD); + + dapl_os_lock (&cno_ptr->header.lock); + + /* Maybe I should just return, but this really shouldn't happen. */ + dapl_os_assert (cno_ptr->cno_state != DAPL_CNO_STATE_DEAD); + + if (cno_ptr->cno_state == DAPL_CNO_STATE_UNTRIGGERED) + { +#if !defined(__KDAPL__) + DAT_OS_WAIT_PROXY_AGENT agent; + + /* Squirrel away wait agent, and delete link. */ + agent = cno_ptr->cno_wait_agent; +#endif /* !defined(__KDAPL__) */ + + /* Separate assignments for windows compiler. */ +#ifndef _WIN32 +#if defined(__KDAPL__) + cno_ptr->cno_upcall = DAT_UPCALL_NULL; +#else + cno_ptr->cno_wait_agent = DAT_OS_WAIT_PROXY_AGENT_NULL; +#endif /* defined(__KDAPL__) */ +#else + cno_ptr->cno_wait_agent.instance_data = NULL; + cno_ptr->cno_wait_agent.proxy_agent_func = NULL; +#endif + + cno_ptr->cno_evd_triggered = evd_ptr; + + /* + * Must set to triggerred and let waiter untrigger to handle + * timeout of waiter. + */ + cno_ptr->cno_state = DAPL_CNO_STATE_TRIGGERED; + if (cno_ptr->cno_waiters > 0) + { + dapl_os_wait_object_wakeup (&cno_ptr->cno_wait_object); + } + + dapl_os_unlock (&cno_ptr->header.lock); + + /* Trigger the OS proxy wait agent, if one exists. */ +#if defined(__KDAPL__) + dat_status = dapl_evd_dequeue ( (DAT_EVD_HANDLE)evd_ptr, + &event ); + while (dat_status == DAT_SUCCESS ) + { + if (cno_ptr->cno_upcall.upcall_func != (DAT_UPCALL_FUNC) NULL) + { + cno_ptr->cno_upcall.upcall_func ( + cno_ptr->cno_upcall.instance_data, + &event, + DAT_FALSE); + } + dat_status = dapl_evd_dequeue ( (DAT_EVD_HANDLE)evd_ptr, + &event ); + } +#else + if (agent.proxy_agent_func != (DAT_AGENT_FUNC) NULL) + { + agent.proxy_agent_func ( + agent.instance_data, (DAT_EVD_HANDLE) evd_ptr ); + } +#endif /* defined(__KDAPL__) */ + } + else + { + dapl_os_unlock (&cno_ptr->header.lock); +#if defined(__KDAPL__) + dat_status = dapl_evd_dequeue ( (DAT_EVD_HANDLE)evd_ptr, + &event ); + while (dat_status == DAT_SUCCESS ) + { + if (cno_ptr->cno_upcall.upcall_func != (DAT_UPCALL_FUNC) NULL) + { + cno_ptr->cno_upcall.upcall_func ( + cno_ptr->cno_upcall.instance_data, + &event, + DAT_FALSE); + } + dat_status = dapl_evd_dequeue ( (DAT_EVD_HANDLE)evd_ptr, + &event ); + } +#endif /* defined(__KDAPL__) */ + } + + return; +} + +/* + * dapl_cno_fd_create + * + * DAPL Requirements Version 2.0, 6.3.2.x + * + * creates a CNO instance. Upon creation, there are no + * Event Dispatchers feeding it. os_fd is a File Descriptor in Unix, + * i.e. struct pollfd or an equivalent object in other OSes that is + * always associates with the created CNO. Consumer can multiplex event + * waiting using UNIX poll or select functions. Upon creation, the CNO + * is not associated with any EVDs, has no waiters and has the os_fd + * associated with it. + * + * Input: + * ia_handle + * + * Output: + * file descripter + * cno_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + * DAT_LENGTH_ERROR + * DAT_PROTECTION_VIOLATION + * DAT_PRIVILEGES_VIOLATION + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_cno_fd_create ( + IN DAT_IA_HANDLE ia_handle, /* ia_handle */ + OUT DAT_FD *fd, /* file_descriptor */ + OUT DAT_CNO_HANDLE *cno_handle) /* cno_handle */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * dapl_cno_fd_create + * + * DAPL Requirements Version 2.0, 6.3.2.x + * + * Returns the latest EVD that triggered the CNO. + * + * Input: + * cno_handle + * + * Output: + * evd_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_cno_trigger( + IN DAT_CNO_HANDLE cno_handle, + OUT DAT_EVD_HANDLE *evd_handle) +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cno_util.h b/trunk/ulp/dapl2/dapl/common/dapl_cno_util.h new file mode 100644 index 00000000..7cd204cb --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cno_util.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_cno_util.h + * + * PURPOSE: Utility defs & routines for the cno data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_CNO_UTIL_H_ +#define _DAPL_CNO_UTIL_H_ + +#include "dapl.h" + +#if defined(__KDAPL__) +DAPL_CNO * +dapl_cno_alloc ( + IN DAPL_IA *ia_ptr, + IN const DAT_UPCALL_OBJECT *upcall) ; + +#else +DAPL_CNO * +dapl_cno_alloc ( + IN DAPL_IA *ia_ptr, + IN DAT_OS_WAIT_PROXY_AGENT wait_agent) ; + +#endif /* defined(__KDAPL__) */ + +void +dapl_cno_dealloc ( + IN DAPL_CNO *cno_ptr) ; + +void +dapl_internal_cno_trigger( + IN DAPL_CNO *cno_ptr, + IN DAPL_EVD *evd_ptr); + +#endif diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cookie.c b/trunk/ulp/dapl2/dapl/common/dapl_cookie.c new file mode 100644 index 00000000..78191876 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cookie.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cookie.c + * + * PURPOSE: Manage CQE cookie structures + * + * The DAPL spec requires that all a cookies passed to a posting operation + * be returned in the operation's corresponding completion. + * + * Implementing this feature is complicated by the user's ability to + * suppress event generation for specific operations. When these operations + * complete successfully, the provider does not have an easy way to + * deallocate resources devoted to storing context data for these operations. + * + * To support this feature, a pool of memory is allocated up front large + * enough to hold cookie data for the maximum number of operations possible + * on an endpoint. + * + * Two pieces of information are maintained to manage cookie allocation: + * + * head index : index of next unallocated cookie + * tail index : index of last unallocated cookie + * + * Each cookie store its index in this memory pool. + * + * When an event is received, the index stored in the event's cookie will be + * used to update the tail. This will implicitly deallocate all of the cookies + * "between" the old tail and the new tail. + * + * The implementation relies on the following assumptions: + * + * - there can be only 1 thread in dat_ep_post_send(), dat_ep_post_rdma_write(), + * dat_ep_post_rdma_read(), or dat_rmr_bind() at a time, therefore + * dapls_cb_get() does not need to be thread safe when manipulating + * request data structures. + * + * - there can be only 1 thread in dat_ep_post_recv(), therefore + * dapls_cb_get() does not need to be thread safe when manipulating + * receive data structures. + * + * - there can be only 1 thread generating completions for a given EP's request + * opeartions, therefore dapls_cb_put() does not need to be thread safe when + * manipulating request data structures. + * + * - there can be only 1 thread generating completions for a given EP's receive + * opeartions therefore dapls_cb_put() does not need to be thread safe when + * manipulating receive data structures. + * + * - completions are delivered in order + * + * $Id:$ + **********************************************************************/ + +#include "dapl_cookie.h" +#include "dapl_ring_buffer_util.h" + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +DAT_RETURN +dapls_cb_get ( + DAPL_COOKIE_BUFFER *buffer, + DAPL_COOKIE **cookie_ptr ); + +DAT_RETURN +dapls_cb_put ( + DAPL_COOKIE_BUFFER *buffer, + DAPL_COOKIE *cookie ); + + +/********************************************************************* + * * + * Function Definitions * + * * + *********************************************************************/ + + +/* + * dapls_cb_create + * + * Given a DAPL_COOKIE_BUFFER, allocate and initialize memory for + * the data structure. + * + * Input: + * buffer pointer to DAPL_COOKIE_BUFFER + * ep endpoint to associate with cookies + * size number of elements to allocate & manage + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_cb_create ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_EP *ep, + IN DAT_COUNT size ) +{ + DAT_COUNT i; + + /* + * allocate one additional entry so that the tail + * can always point at an empty location + */ + size++; + + buffer->pool = dapl_os_alloc (size * sizeof (DAPL_COOKIE)); + if ( NULL != buffer->pool ) + { + buffer->pool_size = size; + dapl_os_atomic_set(&buffer->head, 0); + dapl_os_atomic_set(&buffer->tail, 0); + + for ( i = 0; i < size; i++ ) + { + buffer->pool[i].index = i; + buffer->pool[i].ep = ep; + } + + return (DAT_SUCCESS); + } + else + { + return (DAT_INSUFFICIENT_RESOURCES); + } +} + + +/* + * dapls_cb_free + * + * Free the data structure + * + * Input: + * buffer pointer to DAPL_COOKIE_BUFFER + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapls_cb_free ( + IN DAPL_COOKIE_BUFFER *buffer ) +{ + if ( NULL != buffer->pool ) + { + dapl_os_free (buffer->pool, buffer->pool_size * sizeof (DAPL_COOKIE)); + } +} + + +/* + * dapls_cb_get + * + * Remove an entry from the buffer + * + * Input: + * buffer pointer to DAPL_COOKIE_BUFFER + * + * Output: + * cookie_ptr pointer to pointer to cookie + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_cb_get ( + IN DAPL_COOKIE_BUFFER *buffer, + OUT DAPL_COOKIE **cookie_ptr ) +{ + DAT_RETURN dat_status; + DAT_COUNT new_head; + + dapl_os_assert(NULL != cookie_ptr); + + new_head = (dapl_os_atomic_read (&buffer->head) + 1) % buffer->pool_size; + + if ( new_head == dapl_os_atomic_read (&buffer->tail) ) + { + dat_status = DAT_INSUFFICIENT_RESOURCES; + goto bail; + } + else + { + dapl_os_atomic_set (&buffer->head, new_head); + + *cookie_ptr = &buffer->pool[dapl_os_atomic_read (&buffer->head)]; + dat_status = DAT_SUCCESS; + } +bail: + return dat_status; +} + + +/* + * dapls_cb_put + * + * Add entry(s) to the buffer + * + * Input: + * buffer pointer to DAPL_COOKIE_BUFFER + * cookie pointer to cookie + * + * Output: + * entry entry removed from the ring buffer + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_EMPTY + * + */ +DAT_RETURN +dapls_cb_put ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_COOKIE *cookie ) +{ + dapl_os_atomic_set (&buffer->tail, cookie->index); + + return DAT_SUCCESS; +} + + +/* + * dapls_rmr_cookie_alloc + * + * Allocate an RMR Bind cookie + * + * Input: + * buffer pointer to DAPL_COOKIE_BUFFER + * rmr rmr to associate with the cookie + * user_cookie user's cookie data + * + * Output: + * cookie_ptr pointer to pointer to allocated cookie + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_EMPTY + * + */ +DAT_RETURN +dapls_rmr_cookie_alloc ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_RMR *rmr, + IN DAT_RMR_COOKIE user_cookie, + OUT DAPL_COOKIE **cookie_ptr ) +{ + DAPL_COOKIE *cookie; + DAT_RETURN dat_status; + + if ( DAT_SUCCESS != dapls_cb_get (buffer, &cookie) ) + { + *cookie_ptr = NULL; + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_status = DAT_SUCCESS; + cookie->type = DAPL_COOKIE_TYPE_RMR; + cookie->val.rmr.rmr = rmr; + cookie->val.rmr.cookie = user_cookie; + + *cookie_ptr = cookie; + + bail: + return dat_status; +} + + +/* + * dapls_dto_cookie_alloc + * + * Allocate a DTO cookie + * + * Input: + * buffer pointer to DAPL_COOKIE_BUFFER + * type DTO type + * user_cookie user's cookie data + * + * Output: + * cookie_ptr pointer to pointer to allocated cookie + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_EMPTY + * + */ +DAT_RETURN +dapls_dto_cookie_alloc ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_DTO_TYPE type, + IN DAT_DTO_COOKIE user_cookie, + OUT DAPL_COOKIE **cookie_ptr ) +{ + DAPL_COOKIE *cookie; + + if ( DAT_SUCCESS != dapls_cb_get (buffer, &cookie) ) + { + *cookie_ptr = NULL; + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + } + + cookie->type = DAPL_COOKIE_TYPE_DTO; + cookie->val.dto.type = type; + cookie->val.dto.cookie = user_cookie; + cookie->val.dto.size = 0; + + *cookie_ptr = cookie; + return DAT_SUCCESS; +} + +void +dapls_cookie_dealloc ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_COOKIE *cookie) +{ + dapls_cb_put (buffer, cookie); +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cookie.h b/trunk/ulp/dapl2/dapl/common/dapl_cookie.h new file mode 100644 index 00000000..7aa4457f --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cookie.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_cookie.h + * + * PURPOSE: Utility defs & routines for the cookie data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_COOKIE_H +#define _DAPL_COOKIE_H_ + +#include "dapl.h" + +extern DAT_RETURN +dapls_cb_create ( + DAPL_COOKIE_BUFFER *buffer, + DAPL_EP *ep, + DAT_COUNT size ); + +extern void +dapls_cb_free ( + DAPL_COOKIE_BUFFER *buffer ); + +extern DAT_RETURN +dapls_rmr_cookie_alloc ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_RMR *rmr, + IN DAT_RMR_COOKIE user_cookie, + OUT DAPL_COOKIE **cookie_ptr ); + +extern DAT_RETURN +dapls_dto_cookie_alloc ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_DTO_TYPE type, + IN DAT_DTO_COOKIE user_cookie, + OUT DAPL_COOKIE **cookie_ptr ); + +extern void +dapls_cookie_dealloc ( + IN DAPL_COOKIE_BUFFER *buffer, + IN DAPL_COOKIE *cookie ); + +#endif /* _DAPL_COOKIE_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_accept.c b/trunk/ulp/dapl2/dapl/common/dapl_cr_accept.c new file mode 100644 index 00000000..3db6f753 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_accept.c @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cr_accept.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" + +/* + * dapl_cr_accept + * + * DAPL Requirements Version xxx, 6.4.2.1 + * + * Establish a connection between active remote side requesting Endpoint + * and passic side local Endpoint. + * + * Input: + * cr_handle + * ep_handle + * private_data_size + * private_data + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_ATTRIBUTE + */ +DAT_RETURN DAT_API +dapl_cr_accept ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data ) +{ + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + DAPL_CR *cr_ptr; + DAT_EP_STATE entry_ep_state; + DAT_EP_HANDLE entry_ep_handle; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_cr_accept (CR %p EP %p, PDsz %d)\n", + cr_handle, + ep_handle, + private_data_size); + + if ( DAPL_BAD_HANDLE (cr_handle, DAPL_MAGIC_CR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CR); + goto bail; + } + + cr_ptr = (DAPL_CR *) cr_handle; + + /* + * Return an error if we have an ep_handle and the CR already has an + * EP, indicating this is an RSP connection or PSP_PROVIDER_FLAG was + * specified. + */ + if ( ep_handle != NULL && + ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) || + cr_ptr->param.local_ep_handle != NULL ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + /* Make sure we have an EP handle in one place or another */ + if ( ep_handle == NULL && cr_ptr->param.local_ep_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + if ( (0 != private_data_size) && (NULL == private_data) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4); + goto bail; + } + + /* + * ep_handle is NULL if the user specified DAT_PSP_PROVIDER_FLAG + * OR this is an RSP connection; retrieve it from the cr. + */ + if ( ep_handle == NULL ) + { + ep_handle = cr_ptr->param.local_ep_handle; + if ( (((DAPL_EP *) ep_handle)->param.ep_state != DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING) + && (((DAPL_EP *)ep_handle)->param.ep_state != DAT_EP_STATE_PASSIVE_CONNECTION_PENDING) ) + { + return DAT_INVALID_STATE; + } + } else + { + /* ensure this EP isn't connected or in use*/ + if ( ((DAPL_EP *) ep_handle)->param.ep_state != DAT_EP_STATE_UNCONNECTED ) + { + return DAT_INVALID_STATE; + } + } + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify the attributes of the EP handle before we connect it. Test + * all of the handles to make sure they are currently valid. + * Specifically: + * pz_handle required + * recv_evd_handle optional, but must be valid + * request_evd_handle optional, but must be valid + * connect_evd_handle required + * We do all verification and state change under lock, at which + * point the EP state should protect us from most races. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + if ( ep_ptr->param.pz_handle == NULL + || DAPL_BAD_HANDLE (ep_ptr->param.pz_handle, DAPL_MAGIC_PZ) + /* test connect handle */ + || ep_ptr->param.connect_evd_handle == NULL + || DAPL_BAD_HANDLE (ep_ptr->param.connect_evd_handle, DAPL_MAGIC_EVD) + || ! (((DAPL_EVD *)ep_ptr->param.connect_evd_handle)->evd_flags & DAT_EVD_CONNECTION_FLAG) + /* test optional completion handles */ + || (ep_ptr->param.recv_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (ep_ptr->param.recv_evd_handle, DAPL_MAGIC_EVD))) + + || (ep_ptr->param.request_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (ep_ptr->param.request_evd_handle, DAPL_MAGIC_EVD)))) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + + /* The qp must be attached by this point! */ + if ( ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED ) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + + entry_ep_state = ep_ptr->param.ep_state; + entry_ep_handle = cr_ptr->param.local_ep_handle; + ep_ptr->param.ep_state = DAT_EP_STATE_COMPLETION_PENDING; + ep_ptr->cm_handle = cr_ptr->ib_cm_handle; + ep_ptr->cr_ptr = cr_ptr; + ep_ptr->param.remote_ia_address_ptr = cr_ptr->param.remote_ia_address_ptr; + cr_ptr->param.local_ep_handle = ep_handle; + + dapl_os_unlock ( &ep_ptr->header.lock ); + + dat_status = dapls_ib_accept_connection ( cr_handle, + ep_handle, + private_data_size, + private_data ); + + /* + * If the provider failed, unwind the damage so we are back at + * the initial state. + */ + if ( dat_status != DAT_SUCCESS) + { + if ( DAT_GET_TYPE(dat_status) == DAT_INVALID_ADDRESS ) + { + /* The remote connection request has disappeared; timeout, + * system error, app termination, perhaps other reasons. + */ + dat_status = dapls_evd_post_connection_event ( + ep_ptr->param.connect_evd_handle, + DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR, + (DAT_HANDLE) ep_ptr, + 0, + 0 ); + + cr_ptr->header.magic = DAPL_MAGIC_CR_DESTROYED; + } + else + { + ep_ptr->param.ep_state = entry_ep_state; + cr_ptr->param.local_ep_handle = entry_ep_handle; + ep_ptr->cr_ptr = NULL; + ep_ptr->param.remote_ia_address_ptr = NULL; + } + + /* + * After restoring values above, we now check if we need + * to translate the error + */ + if ( DAT_GET_TYPE(dat_status) == DAT_LENGTH_ERROR ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + } + + } + else + { + /* Make this CR invalid. We need to hang on to it until + * the connection terminates, but it's destroyed from + * the app point of view. + */ + cr_ptr->header.magic = DAPL_MAGIC_CR_DESTROYED; + } + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_callback.c b/trunk/ulp/dapl2/dapl/common/dapl_cr_callback.c new file mode 100644 index 00000000..0359ddb8 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_callback.c @@ -0,0 +1,592 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapls_cr_callback.c + * + * PURPOSE: implements passive side connection callbacks + * + * Description: Accepts asynchronous callbacks from the Communications Manager + * for EVDs that have been specified as the connection_evd. + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" +#include "dapl_ia_util.h" +#include "dapl_sp_util.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" + + +/* + * Prototypes + */ +DAT_RETURN dapli_connection_request ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN DAPL_SP *sp_ptr, + IN DAPL_PRIVATE *prd_ptr, + IN DAPL_EVD *evd_ptr); + +DAPL_EP * dapli_get_sp_ep ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN DAPL_SP *sp_ptr, + IN DAT_EVENT_NUMBER dat_event_num); + + +/* + * dapls_cr_callback + * + * The callback function registered with verbs for passive side of + * connection requests. The interface is specified by cm_api.h + * + * + * Input: + * ib_cm_handle, Handle to CM + * ib_cm_event Specific CM event + * instant_data Private data with DAT ADDRESS header + * context SP pointer + * + * Output: + * None + * + */ +void +dapls_cr_callback ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN const ib_cm_events_t ib_cm_event, + IN const void *private_data_ptr, /* event data */ + IN const void *context) +{ + DAPL_EP *ep_ptr; + DAPL_EVD *evd_ptr; + DAPL_SP *sp_ptr; + DAPL_PRIVATE *prd_ptr; + DAT_EVENT_NUMBER dat_event_num; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> dapls_cr_callback! context %p event %d cm_hdl %lx\n", + context, ib_cm_event); + + /* + * Passive side of the connection, context is a SP and + * we need to look up the EP. + */ + sp_ptr = (DAPL_SP *) context; + /* + * The context pointer could have been cleaned up in a racing + * CM callback, check to see if we should just exit here + */ + if (sp_ptr->header.magic == DAPL_MAGIC_INVALID) + { + return; + } + dapl_os_assert ( sp_ptr->header.magic == DAPL_MAGIC_PSP || + sp_ptr->header.magic == DAPL_MAGIC_RSP ); + + /* Obtain the event number from the provider layer */ + dat_event_num = dapls_ib_get_dat_event (ib_cm_event, DAT_FALSE); + + /* + * CONNECT_REQUEST events create an event on the PSP + * EVD, which will trigger connection processing. The + * sequence is: + * CONNECT_REQUEST Event to SP + * CONNECTED Event to EP + * DISCONNECT Event to EP + * + * Obtain the EP if required and set an event up on the correct + * EVD. + */ + if (dat_event_num == DAT_CONNECTION_REQUEST_EVENT) + { + ep_ptr = NULL; + evd_ptr = sp_ptr->evd_handle; + } + else + { + /* see if there is an EP connected with this CM handle */ + ep_ptr = dapli_get_sp_ep ( ib_cm_handle, + sp_ptr, + dat_event_num ); + + /* if we lost a race with the CM just exit. */ + if (ep_ptr == NULL) + { + return; + } + + evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle; + /* if something has happened to our EVD, bail. */ + if (evd_ptr == NULL) + { + return; + } + } + + prd_ptr = (DAPL_PRIVATE *)private_data_ptr; + + dat_status = DAT_INTERNAL_ERROR; /* init to ERR */ + + switch (dat_event_num) + { + case DAT_CONNECTION_REQUEST_EVENT: + { + /* + * Requests arriving on a disabled SP are immediatly rejected + */ + + dapl_os_lock (&sp_ptr->header.lock); + if ( sp_ptr->listening == DAT_FALSE ) + { + dapl_os_unlock (&sp_ptr->header.lock); + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "---> dapls_cr_callback: conn event on down SP\n"); + (void)dapls_ib_reject_connection (ib_cm_handle, + DAT_CONNECTION_EVENT_UNREACHABLE ); + + return; + } + + if ( sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP ) + { + /* + * RSP connections only allow a single connection. Close + * it down NOW so we reject any further connections. + */ + sp_ptr->listening = DAT_FALSE; + } + dapl_os_unlock (&sp_ptr->header.lock); + + /* + * Only occurs on the passive side of a connection + * dapli_connection_request will post the connection + * event if appropriate. + */ + dat_status = dapli_connection_request ( ib_cm_handle, + sp_ptr, + prd_ptr, + evd_ptr ); + /* Set evd_ptr = NULL so we don't generate an event below */ + evd_ptr = NULL; + + break; + } + case DAT_CONNECTION_EVENT_ESTABLISHED: + { + /* This is just a notification the connection is now + * established, there isn't any private data to deal with. + * + * Update the EP state and cache a copy of the cm handle, + * then let the user know we are ready to go. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + if (ep_ptr->header.magic != DAPL_MAGIC_EP || + ep_ptr->param.ep_state != DAT_EP_STATE_COMPLETION_PENDING) + { + /* If someone pulled the plug on the EP or connection, + * just exit + */ + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_SUCCESS; + /* Set evd_ptr = NULL so we don't generate an event below */ + evd_ptr = NULL; + + break; + } + + ep_ptr->param.ep_state = DAT_EP_STATE_CONNECTED; + ep_ptr->cm_handle = ib_cm_handle; + dapl_os_unlock ( &ep_ptr->header.lock ); + + break; + } + case DAT_CONNECTION_EVENT_DISCONNECTED: + { + /* + * EP is now fully disconnected; initiate any post processing + * to reset the underlying QP and get the EP ready for + * another connection + */ + dapl_os_lock ( &ep_ptr->header.lock ); + if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) + { + /* The disconnect has already occurred, we are now + * cleaned up and ready to exit + */ + dapl_os_unlock ( &ep_ptr->header.lock ); + return; + } + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + dapls_ib_disconnect_clean (ep_ptr, DAT_FALSE, ib_cm_event); + dapl_os_unlock ( &ep_ptr->header.lock ); + + break; + } + case DAT_CONNECTION_EVENT_NON_PEER_REJECTED: + case DAT_CONNECTION_EVENT_PEER_REJECTED: + case DAT_CONNECTION_EVENT_UNREACHABLE: + { + /* + * After posting an accept the requesting node has + * stopped talking. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + ep_ptr->cm_handle = IB_INVALID_HANDLE; + dapls_ib_disconnect_clean (ep_ptr, DAT_FALSE, ib_cm_event); + dapl_os_unlock ( &ep_ptr->header.lock ); + + break; + } + case DAT_CONNECTION_EVENT_BROKEN: + { + dapl_os_lock ( &ep_ptr->header.lock ); + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + dapls_ib_disconnect_clean (ep_ptr, DAT_FALSE, ib_cm_event); + dapl_os_unlock ( &ep_ptr->header.lock ); + + break; + } + default: + { + evd_ptr = NULL; + dapl_os_assert (0); /* shouldn't happen */ + break; + } + } + + if (evd_ptr != NULL ) + { + dat_status = dapls_evd_post_connection_event ( + evd_ptr, + dat_event_num, + (DAT_HANDLE) ep_ptr, + 0, + NULL); + } + + if (dat_status != DAT_SUCCESS) + { + /* The event post failed; take appropriate action. */ + (void)dapls_ib_reject_connection ( ib_cm_handle, + DAT_CONNECTION_EVENT_BROKEN); + + return; + } +} + + +/* + * dapli_connection_request + * + * Process a connection request on the Passive side of a connection. + * Create a CR record and link it on to the SP so we can update it + * and free it later. Create an EP if specified by the PSP flags. + * + * Input: + * ib_cm_handle, + * sp_ptr + * event_ptr + * prd_ptr + * + * Output: + * None + * + * Returns + * DAT_INSUFFICIENT_RESOURCES + * DAT_SUCCESS + * + */ +DAT_RETURN +dapli_connection_request ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN DAPL_SP *sp_ptr, + IN DAPL_PRIVATE *prd_ptr, + IN DAPL_EVD *evd_ptr) +{ + DAT_RETURN dat_status; + + DAPL_CR *cr_ptr; + DAPL_EP *ep_ptr; + DAPL_IA *ia_ptr; + DAT_SP_HANDLE sp_handle; + + cr_ptr = dapls_cr_alloc (sp_ptr->header.owner_ia); + if ( cr_ptr == NULL ) + { + /* Invoking function will call dapls_ib_cm_reject() */ + return DAT_INSUFFICIENT_RESOURCES; + } + + /* + * Set up the CR + */ + cr_ptr->sp_ptr = sp_ptr; /* maintain sp_ptr in case of reject */ + cr_ptr->param.remote_port_qual = 0; + cr_ptr->ib_cm_handle = ib_cm_handle; + +#ifdef IBHOSTS_NAMING + /* + * Special case: pull the remote HCA address from the private data + * prefix. This is a spec violation as it introduces a protocol, but + * some implementations may find it necessary for a time. + */ + cr_ptr->remote_ia_address = prd_ptr->hca_address; +#endif /* IBHOSTS_NAMING */ + cr_ptr->param.remote_ia_address_ptr = (DAT_IA_ADDRESS_PTR)&cr_ptr->remote_ia_address; + /* + * Copy the remote address and private data out of the private_data + * payload and put them in a local structure + */ + + /* Private data size will be determined by the provider layer */ + cr_ptr->param.private_data = cr_ptr->private_data; + if (prd_ptr == NULL) + { + cr_ptr->param.private_data_size = 0; + } + else + { + cr_ptr->param.private_data_size = + dapls_ib_private_data_size (prd_ptr, DAPL_PDATA_CONN_REQ); + } + if (cr_ptr->param.private_data_size > 0) + { + dapl_os_memcpy (cr_ptr->private_data, + prd_ptr->private_data, + DAPL_MIN(cr_ptr->param.private_data_size, + DAPL_MAX_PRIVATE_DATA_SIZE)); + } + + /* EP will be NULL unless RSP service point */ + ep_ptr = (DAPL_EP *) sp_ptr->ep_handle; + + if ( sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG ) + { + /* + * Never true for RSP connections + * + * Create an EP for the user. If we can't allocate an + * EP we are out of resources and need to tell the + * requestor that we cant help them. + */ + ia_ptr = sp_ptr->header.owner_ia; + ep_ptr = dapl_ep_alloc (ia_ptr, NULL); + if ( ep_ptr == NULL ) + { + dapls_cr_free (cr_ptr); + /* Invoking function will call dapls_ib_cm_reject() */ + return DAT_INSUFFICIENT_RESOURCES; + } + ep_ptr->param.ia_handle = ia_ptr; + ep_ptr->param.local_ia_address_ptr= (DAT_IA_ADDRESS_PTR) + &ia_ptr->hca_ptr->hca_address; + + /* Link the EP onto the IA */ + dapl_ia_link_ep (ia_ptr, ep_ptr); + } + + cr_ptr->param.local_ep_handle = ep_ptr; + + if ( ep_ptr != NULL ) + { + /* Assign valid EP fields: RSP and PSP_PROVIDER_FLAG only */ + if ( sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG ) + { + ep_ptr->param.ep_state = + DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING; + } + else + { + /* RSP */ + dapl_os_assert (sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP); + ep_ptr->param.ep_state = + DAT_EP_STATE_PASSIVE_CONNECTION_PENDING; + } + ep_ptr->cm_handle = ib_cm_handle; + } + + /* link the CR onto the SP so we can pick it up later */ + dapl_sp_link_cr (sp_ptr, cr_ptr); + + /* Post the event. */ + /* assign sp_ptr to union to avoid typecast errors from some compilers */ + sp_handle.psp_handle = (DAT_PSP_HANDLE)sp_ptr; + + dat_status = dapls_evd_post_cr_arrival_event ( + evd_ptr, + DAT_CONNECTION_REQUEST_EVENT, + sp_handle, + (DAT_IA_ADDRESS_PTR)&sp_ptr->header.owner_ia->hca_ptr->hca_address, + sp_ptr->conn_qual, + (DAT_CR_HANDLE)cr_ptr); + + if (dat_status != DAT_SUCCESS) + { + dapls_cr_free (cr_ptr); + (void)dapls_ib_reject_connection (ib_cm_handle, + DAT_CONNECTION_EVENT_BROKEN); + + /* Take the CR off the list, we can't use it */ + dapl_os_lock (&sp_ptr->header.lock); + dapl_sp_remove_cr (sp_ptr, cr_ptr); + dapl_os_unlock (&sp_ptr->header.lock); + return DAT_INSUFFICIENT_RESOURCES; + } + + + return DAT_SUCCESS; +} + + +/* + * dapli_get_sp_ep + * + * Passive side of a connection is now fully established. Clean + * up resources and obtain the EP pointer associated with a CR in + * the SP + * + * Input: + * ib_cm_handle, + * sp_ptr + * connection_event + * + * Output: + * none + * + * Returns + * ep_ptr + * + */ +DAPL_EP * +dapli_get_sp_ep ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN DAPL_SP *sp_ptr, + IN DAT_EVENT_NUMBER dat_event_num) +{ + DAPL_CR *cr_ptr; + DAPL_EP *ep_ptr; + + /* + * acquire the lock, we may be racing with other threads here + */ + dapl_os_lock (&sp_ptr->header.lock); + + /* Verify under lock that the SP is still valid */ + if (sp_ptr->header.magic == DAPL_MAGIC_INVALID) + { + dapl_os_unlock (&sp_ptr->header.lock); + return NULL; + } + /* + * There are potentially multiple connections in progress. Need to + * go through the list and find the one we are interested + * in. There is no guarantee of order. dapl_sp_search_cr + * leaves the CR on the SP queue. + */ + cr_ptr = dapl_sp_search_cr (sp_ptr, ib_cm_handle); + if ( cr_ptr == NULL ) + { + dapl_os_unlock (&sp_ptr->header.lock); + return NULL; + } + + ep_ptr = (DAPL_EP *)cr_ptr->param.local_ep_handle; + + + /* Quick check to ensure our EP is still valid */ + if ( (DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP )) ) + { + ep_ptr = NULL; + } + + /* The CR record is discarded in all except for the CONNECTED case, + * as it will have no further relevance. + */ + if (dat_event_num != DAT_CONNECTION_EVENT_ESTABLISHED) + { + /* Remove the CR from the queue */ + dapl_sp_remove_cr (sp_ptr, cr_ptr); + + if (ep_ptr != NULL) + { + ep_ptr->cr_ptr = NULL; + } + + /* + * If this SP has been removed from service, free it + * up after the last CR is removed + */ + if ( sp_ptr->listening != DAT_TRUE && sp_ptr->cr_list_count == 0 && + sp_ptr->state != DAPL_SP_STATE_FREE ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> dapli_get_sp_ep! disconnect dump sp: %p \n", sp_ptr); + /* Decrement the ref count on the EVD */ + if (sp_ptr->evd_handle) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)sp_ptr->evd_handle)->evd_ref_count); + sp_ptr->evd_handle = NULL; + } + sp_ptr->state = DAPL_SP_STATE_FREE; + dapl_os_unlock (&sp_ptr->header.lock); + (void) dapls_ib_remove_conn_listener ( sp_ptr->header.owner_ia, + sp_ptr ); + dapls_ia_unlink_sp ( (DAPL_IA *)sp_ptr->header.owner_ia, + sp_ptr ); + dapls_sp_free_sp ( sp_ptr ); + dapls_cr_free (cr_ptr); + goto skip_unlock; + } + + dapl_os_unlock (&sp_ptr->header.lock); + /* free memory outside of the lock */ + dapls_cr_free (cr_ptr); + } + else + { + dapl_os_unlock (&sp_ptr->header.lock); + } + +skip_unlock: + return ep_ptr; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_handoff.c b/trunk/ulp/dapl2/dapl/common/dapl_cr_handoff.c new file mode 100644 index 00000000..9852f1bb --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_handoff.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cr_handoff.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_cr_handoff + * + * DAPL Requirements Version xxx, 6.4.2.4 + * + * Hand the connection request to another Sevice pont specified by the + * Connectin Qualifier. + * + * Input: + * cr_handle + * cr_handoff + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_cr_handoff ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_CONN_QUAL cr_handoff ) /* handoff */ +{ + return DAT_ERROR (DAT_NOT_IMPLEMENTED,0); +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_query.c b/trunk/ulp/dapl2/dapl/common/dapl_cr_query.c new file mode 100644 index 00000000..5f96533e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_query.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cr_query.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * dapl_cr_query + * + * DAPL Requirements Version xxx, 6.4.2.1 + * + * Return Connection Request args + * + * Input: + * cr_handle + * cr_param_mask + * + * Output: + * cr_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + */ +DAT_RETURN DAT_API +dapl_cr_query ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_CR_PARAM_MASK cr_param_mask, + OUT DAT_CR_PARAM *cr_param) +{ + DAPL_CR *cr_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_cr_query (%p, %x, %p)\n", + cr_handle, + cr_param_mask, + cr_param); + + dat_status = DAT_SUCCESS; + if ( DAPL_BAD_HANDLE (cr_handle, DAPL_MAGIC_CR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CR); + goto bail; + } + + if (NULL == cr_param) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + cr_ptr = (DAPL_CR *) cr_handle; + + /* obtain the remote IP address */ + if (cr_param_mask & DAT_CR_FIELD_REMOTE_IA_ADDRESS_PTR) + { + dat_status = dapls_ib_cm_remote_addr ((DAT_HANDLE)cr_handle, + &cr_ptr->remote_ia_address ); + } + + /* since the arguments are easily accessible, ignore the mask */ + dapl_os_memcpy (cr_param, &cr_ptr->param, sizeof (DAT_CR_PARAM)); + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_reject.c b/trunk/ulp/dapl2/dapl/common/dapl_cr_reject.c new file mode 100644 index 00000000..37efe49e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_reject.c @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cr_reject.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_cr_util.h" +#include "dapl_sp_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_cr_reject + * + * DAPL Requirements Version xxx, 6.4.2.2 + * + * Reject a connection request from the active remote side requesting + * an Endpoint. + * + * Input: + * cr_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_cr_reject ( + IN DAT_CR_HANDLE cr_handle,/* cr_handle */ + IN DAT_COUNT pdata_size, /* private_data_size */ + IN const DAT_PVOID pdata ) /* private_data */ +{ + DAPL_CR *cr_ptr; + DAPL_EP *ep_ptr; + DAT_EP_STATE entry_ep_state; + DAT_EP_HANDLE entry_ep_handle; + DAPL_SP *sp_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_cr_reject (%p)\n", cr_handle); + + if ( DAPL_BAD_HANDLE (cr_handle, DAPL_MAGIC_CR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CR); + goto bail; + } + + cr_ptr = (DAPL_CR *)cr_handle; + + /* + * Clean up provider created EP if there is one: only if + * DAT_PSP_PROVIDER_FLAG was set on the PSP + */ + ep_ptr = (DAPL_EP *)cr_ptr->param.local_ep_handle; + entry_ep_handle = cr_ptr->param.local_ep_handle; + entry_ep_state = 0; + if ( ep_ptr != NULL ) + { + entry_ep_state = ep_ptr->param.ep_state; + ep_ptr->param.ep_state = DAT_EP_STATE_UNCONNECTED; + cr_ptr->param.local_ep_handle = NULL; + } + + dat_status = dapls_ib_reject_connection ( cr_ptr->ib_cm_handle, + IB_CM_REJ_REASON_CONSUMER_REJ ); + + if ( dat_status != DAT_SUCCESS) + { + if ( ep_ptr != NULL ) + { + /* Revert our state to the beginning */ + ep_ptr->param.ep_state = entry_ep_state; + cr_ptr->param.local_ep_handle = entry_ep_handle; + cr_ptr->param.local_ep_handle = (DAT_EP_HANDLE)ep_ptr; + } + } + else + { + /* + * If this EP has been allocated by the provider, clean it up; + * see DAT 1.1 spec, page 100, lines 3-4 (section 6.4.3.1.1.1). + * RSP and user-provided EPs are in the control of the user. + */ + sp_ptr = cr_ptr->sp_ptr; + if ( ep_ptr != NULL && + sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG ) + { + (void)dapl_ep_free (ep_ptr); + } + + /* Remove the CR from the queue, then free it */ + dapl_os_lock (&sp_ptr->header.lock); + dapl_sp_remove_cr ( sp_ptr, cr_ptr ); + dapl_os_unlock (&sp_ptr->header.lock); + + dapls_cr_free ( cr_ptr ); + } + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_util.c b/trunk/ulp/dapl2/dapl/common/dapl_cr_util.c new file mode 100644 index 00000000..03a046a6 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_util.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cr_util.c + * + * PURPOSE: Manage CR (Connection Request) structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_cr_util.h" + +/* + * dapls_cr_create + * + * Create a CR. Part of the passive side of a connection + * + * Input: + * ia_ptr + * + * Returns: + * DAPL_CR + * + */ + +DAPL_CR * +dapls_cr_alloc ( + DAPL_IA *ia_ptr ) +{ + DAPL_CR *cr_ptr; + + /* Allocate EP */ + cr_ptr = (DAPL_CR *)dapl_os_alloc (sizeof (DAPL_CR)); + if ( cr_ptr == NULL ) + { + return (NULL); + } + + /* zero the structure */ + dapl_os_memzero (cr_ptr, sizeof (DAPL_CR)); + + /* + * initialize the header + */ + cr_ptr->header.provider = ia_ptr->header.provider; + cr_ptr->header.magic = DAPL_MAGIC_CR; + cr_ptr->header.handle_type = DAT_HANDLE_TYPE_CR; + cr_ptr->header.owner_ia = ia_ptr; + cr_ptr->header.user_context.as_64 = 0; + cr_ptr->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&cr_ptr->header.ia_list_entry); + dapl_os_lock_init (&cr_ptr->header.lock); + + return (cr_ptr); +} + + +/* + * dapls_cr_free + * + * Free the passed in EP structure. + * + * Input: + * entry point pointer + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapls_cr_free ( + IN DAPL_CR *cr_ptr ) +{ + dapl_os_assert (cr_ptr->header.magic == DAPL_MAGIC_CR || + cr_ptr->header.magic == DAPL_MAGIC_CR_DESTROYED ); + + cr_ptr->header.magic = DAPL_MAGIC_INVALID;/* reset magic to prevent reuse */ + dapl_os_free (cr_ptr, sizeof (DAPL_CR)); +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_cr_util.h b/trunk/ulp/dapl2/dapl/common/dapl_cr_util.h new file mode 100644 index 00000000..b2b2f606 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_cr_util.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_cr_util.h + * + * PURPOSE: Utility defs & routines for the CR data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_CR_UTIL_H_ +#define _DAPL_CR_UTIL_H_ + +#include "dapl.h" + +DAPL_CR * +dapls_cr_alloc ( + DAPL_IA *ia_ptr ); + +void +dapls_cr_free ( + IN DAPL_CR *cr_ptr ); + +void +dapls_cr_callback ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN const ib_cm_events_t ib_cm_event, + IN const void *instant_data_p, + IN const void *context); + +#endif /* _DAPL_CR_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_csp.c b/trunk/ulp/dapl2/dapl/common/dapl_csp.c new file mode 100644 index 00000000..77d51b36 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_csp.c @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_csp.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 2.0 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_sp_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_psp_create, dapl_csp_query, dapl_csp_free + * + * uDAPL: User Direct Access Program Library Version 2.0, 6.4.4.2 + * + * The Common Service Point is transport-independent analog of the Public + * Service Point. It allows the Consumer to listen on socket-equivalent for + * requests for connections arriving on a specified IP port instead of + * transport-dependent Connection Qualifier. An IA Address follows the + * platform conventions and provides among others the IP port to listen on. + * An IP port of the Common Service Point advertisement is supported by + * existing Ethernet infrastructure or DAT Name Service. + * + * Input: + * ia_handle + * comm_id + * address + * evd_handle + * csp_handle + * + * Output: + * csp_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_CONN_QUAL_IN_USE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_csp_create ( + IN DAT_IA_HANDLE ia_handle, /* ia_handle */ + IN DAT_COMM *comm, /* communicator */ + IN DAT_IA_ADDRESS_PTR addr, /* address */ + IN DAT_EVD_HANDLE evd_handle, /* evd_handle */ + OUT DAT_CSP_HANDLE *csp_handle ) /* csp_handle */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +DAT_RETURN DAT_API +dapl_csp_query ( + IN DAT_CSP_HANDLE csp_handle, /* csp_handle */ + IN DAT_CSP_PARAM_MASK param_mask, /* csp_param_mask */ + OUT DAT_CSP_PARAM *param ) /* csp_param */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +DAT_RETURN DAT_API +dapl_csp_free ( + IN DAT_CSP_HANDLE csp_handle ) /* csp_handle */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_debug.c b/trunk/ulp/dapl2/dapl/common/dapl_debug.c new file mode 100644 index 00000000..73f30ccd --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_debug.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_debug.h" +#include "dapl.h" +#if !defined(__KDAPL__) +#include +#include +#endif /* __KDAPL__ */ + +#ifdef DAPL_DBG +DAPL_DBG_TYPE g_dapl_dbg_type; /* initialized in dapl_init.c */ +DAPL_DBG_DEST g_dapl_dbg_dest; /* initialized in dapl_init.c */ + +void dapl_internal_dbg_log ( DAPL_DBG_TYPE type, const char *fmt, ...) +{ + va_list args; + + if ( type & g_dapl_dbg_type ) + { + va_start (args, fmt); + + if ( DAPL_DBG_DEST_STDOUT & g_dapl_dbg_dest ) + { + dapl_os_vprintf (fmt, args); + } + + if ( DAPL_DBG_DEST_SYSLOG & g_dapl_dbg_dest ) + { + dapl_os_syslog(fmt, args); + } + va_end (args); + } +} + +#if defined(DAPL_COUNTERS) +int dapl_dbg_counters[DCNT_NUM_COUNTERS] = { 0 }; + +/* + * The order of this list must match exactly with the #defines + * in dapl_debug.h + */ +char *dapl_dbg_counter_names[] = { + "dapl_ep_create", + "dapl_ep_free", + "dapl_ep_connect", + "dapl_ep_disconnect", + "dapl_ep_post_send", + "dapl_ep_post_recv", + "dapl_ep_post_rdma_write", + "dapl_ep_post_rdma_read", + "dapl_evd_create", + "dapl_evd_free", + "dapl_evd_wait", + "dapl_evd_blocked", + "dapl_evd_completion_notify", + "dapl_evd_dto_callback", + "dapl_evd_connection_callback", + "dapl_evd_dequeue", + "dapl_evd_poll", + "dapl_evd_found", + "dapl_evd_not_found", + "dapls_timer_set", + "dapls_timer_cancel", +}; + +void dapl_dump_cntr( int cntr ) +{ + int i; + + for ( i = 0; i < DCNT_NUM_COUNTERS; i++ ) + { + if (( cntr == i ) || ( cntr == DCNT_ALL_COUNTERS )) + { + dapl_dbg_log ( DAPL_DBG_TYPE_CNTR, + "DAPL Counter: %s = %lu \n", + dapl_dbg_counter_names[i], + dapl_dbg_counters[i] ); + } + } +} + +#endif /* DAPL_COUNTERS */ +#endif + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_connect.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_connect.c new file mode 100644 index 00000000..d71295c9 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_connect.c @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_connect.c + * + * PURPOSE: Endpoint management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_timer_util.h" + +/* + * dapl_ep_connect + * + * Request a connection be established between the local Endpoint + * and a remote Endpoint. This operation is used by the active/client + * side of a connection + * + * Input: + * ep_handle + * remote_ia_address + * remote_conn_qual + * timeout + * private_data_size + * privaet_data + * qos + * connect_flags + * + * Output: + * None + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOUCRES + * DAT_INVALID_PARAMETER + * DAT_MODLE_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + IN DAT_CONN_QUAL remote_conn_qual, + IN DAT_TIMEOUT timeout, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data, + IN DAT_QOS qos, + IN DAT_CONNECT_FLAGS connect_flags ) +{ + DAPL_EP *ep_ptr; + DAPL_EP alloc_ep; + DAT_RETURN dat_status; + DAT_COUNT req_hdr_size; + DAT_UINT32 max_req_pdata_size; + void *private_data_ptr; + + dapl_dbg_log (DAPL_DBG_TYPE_API | DAPL_DBG_TYPE_CM, + "dapl_ep_connect (%p, {%u.%u.%u.%u}, %X, %d, %d, %p, %x, %x)\n", + ep_handle, + remote_ia_address->sa_data[2], + remote_ia_address->sa_data[3], + remote_ia_address->sa_data[4], + remote_ia_address->sa_data[5], + remote_conn_qual, + timeout, + private_data_size, + private_data, + qos, + connect_flags); + DAPL_CNTR (DCNT_EP_CONNECT); + + dat_status = DAT_SUCCESS; + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify parameter & state. The connection handle must be good + * at this point. + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + if ( DAPL_BAD_HANDLE (ep_ptr->param.connect_evd_handle, DAPL_MAGIC_EVD) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_CONN); + goto bail; + } + + /* Can't do a connection in 0 time, reject outright */ + if ( timeout == 0 ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; + } + + /* + * If the endpoint needs a QP, associated the QP with it. + * This needs to be done carefully, in order to: + * * Avoid allocating under a lock. + * * Not step on data structures being altered by + * routines with which we are racing. + * So we: + * * Confirm that a new QP is needed and is not forbidden by the + * current state. + * * Allocate it into a separate EP. + * * Take the EP lock. + * * Reconfirm that the EP is in a state where it needs a QP. + * * Assign the QP and release the lock. + */ + if ( ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED ) + { + if ( ep_ptr->param.pz_handle == NULL + || DAPL_BAD_HANDLE (ep_ptr->param.pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_EP_NOTREADY); + goto bail; + } + alloc_ep = *ep_ptr; + + dat_status = dapls_ib_qp_alloc ( ep_ptr->header.owner_ia, + &alloc_ep, + ep_ptr ); + if ( dat_status != DAT_SUCCESS ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_lock ( &ep_ptr->header.lock ); + /* + * PZ shouldn't have changed since we're only racing with + * dapl_cr_accept() + */ + if ( ep_ptr->qp_state != DAPL_QP_STATE_UNATTACHED ) + { + /* Bail, cleaning up. */ + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = dapls_ib_qp_free ( ep_ptr->header.owner_ia, + &alloc_ep ); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ep_connect: ib_qp_free failed with %x\n", + dat_status); + } + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + ep_ptr->qp_handle = alloc_ep.qp_handle; + ep_ptr->qpn = alloc_ep.qpn; + ep_ptr->qp_state = alloc_ep.qp_state; + + dapl_os_unlock ( &ep_ptr->header.lock ); + } + + /* + * We do state checks and transitions under lock. + * The only code we're racing against is dapl_cr_accept. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + + /* + * Verify the attributes of the EP handle before we connect it. Test + * all of the handles to make sure they are currently valid. + * Specifically: + * pz_handle required + * recv_evd_handle optional, but must be valid + * request_evd_handle optional, but must be valid + * connect_evd_handle required + */ + if ( ep_ptr->param.pz_handle == NULL + || DAPL_BAD_HANDLE (ep_ptr->param.pz_handle, DAPL_MAGIC_PZ) + /* test connect handle */ + || ep_ptr->param.connect_evd_handle == NULL + || DAPL_BAD_HANDLE (ep_ptr->param.connect_evd_handle, DAPL_MAGIC_EVD) + || ! (((DAPL_EVD *)ep_ptr->param.connect_evd_handle)->evd_flags & DAT_EVD_CONNECTION_FLAG) + /* test optional completion handles */ + || (ep_ptr->param.recv_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (ep_ptr->param.recv_evd_handle, DAPL_MAGIC_EVD))) + || (ep_ptr->param.request_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (ep_ptr->param.request_evd_handle, DAPL_MAGIC_EVD)))) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_EP_NOTREADY); + goto bail; + } + + /* Check both the EP state and the QP state: if we don't have a QP + * we need to attach one now. + */ + if ( ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED ) + { + dat_status = dapls_ib_qp_alloc ( ep_ptr->header.owner_ia, + ep_ptr, ep_ptr ); + + if ( dat_status != DAT_SUCCESS) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_TEP); + goto bail; + } + } + + if ( ep_ptr->param.ep_state != DAT_EP_STATE_UNCONNECTED ) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + if ( qos != DAT_QOS_BEST_EFFORT || + connect_flags != DAT_CONNECT_DEFAULT_FLAG ) + { + /* + * At this point we only support one QOS level + */ + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_MODEL_NOT_SUPPORTED, 0); + goto bail; + } + + /* + * Verify the private data size doesn't exceed the max + * req_hdr_size will evaluate to 0 unless IBHOSTS_NAMING is enabled. + */ + req_hdr_size = (sizeof (DAPL_PRIVATE) - DAPL_MAX_PRIVATE_DATA_SIZE); + + max_req_pdata_size = dapls_ib_private_data_size (NULL, DAPL_PDATA_CONN_REQ); + + if (private_data_size + req_hdr_size > (DAT_COUNT)max_req_pdata_size) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG5); + goto bail; + } + + /* transition the state before requesting a connection to avoid + * race conditions + */ + ep_ptr->param.ep_state = DAT_EP_STATE_ACTIVE_CONNECTION_PENDING; + + /* + * At this point we're committed, and done with the endpoint + * except for the connect, so we can drop the lock. + */ + dapl_os_unlock ( &ep_ptr->header.lock ); + +#ifdef IBHOSTS_NAMING + /* + * Special case: put the remote HCA address into the private data + * prefix. This is a spec violation as it introduces a protocol, but + * some implementations may find it necessary for a time. + * Copy the private data into the EP area so the data is contiguous. + * If the provider needs to pad the buffer with NULLs, it happens at + * the provider layer. + */ + dapl_os_memcpy (&ep_ptr->hca_address, + &ep_ptr->header.owner_ia->hca_ptr->hca_address, + sizeof (DAT_SOCK_ADDR)); + dapl_os_memcpy (ep_ptr->private.private_data, private_data, private_data_size); + private_data_ptr = (void *)&ep_ptr->private.private_data; +#else + private_data_ptr = private_data; +#endif /* IBHOSTS_NAMING */ + + /* Copy the connection qualifiers */ + dapl_os_memcpy ( ep_ptr->param.remote_ia_address_ptr, + remote_ia_address, + sizeof ( DAT_SOCK_ADDR) ); + ep_ptr->param.remote_port_qual = remote_conn_qual; + + dat_status = dapls_ib_connect ( ep_handle, + remote_ia_address, + remote_conn_qual, + private_data_size + req_hdr_size, + private_data_ptr ); + + if ( dat_status != DAT_SUCCESS ) + { + ep_ptr->param.ep_state = DAT_EP_STATE_UNCONNECTED; + + /* + * Some implementations provide us with an error code that the + * remote destination is unreachable, but DAT doesn't have a + * synchronous error code to communicate this. So the provider + * layer generates an INTERNAL_ERROR with a subtype; when + * this happens, return SUCCESS and generate the event + */ + if (dat_status == DAT_ERROR (DAT_INTERNAL_ERROR, 1) ) + { + dapls_evd_post_connection_event ( + (DAPL_EVD *)ep_ptr->param.connect_evd_handle, + DAT_CONNECTION_EVENT_UNREACHABLE, + (DAT_HANDLE) ep_ptr, + 0, + 0); + dat_status = DAT_SUCCESS; + } + } + else + { + /* + * Acquire the lock and recheck the state of the EP; this + * thread could have been descheduled after issuing the connect + * request and the EP is now connected. Set up a timer if + * necessary. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + if (ep_ptr->param.ep_state == DAT_EP_STATE_ACTIVE_CONNECTION_PENDING && + timeout != DAT_TIMEOUT_INFINITE ) + { + ep_ptr->cxn_timer = + (DAPL_OS_TIMER *)dapl_os_alloc (sizeof (DAPL_OS_TIMER)); + + dapls_timer_set ( ep_ptr->cxn_timer, + dapls_ep_timeout, + ep_ptr, + timeout ); + } + dapl_os_unlock ( &ep_ptr->header.lock ); + } + +bail: + dapl_dbg_log (DAPL_DBG_TYPE_RTN | DAPL_DBG_TYPE_CM, + "dapl_ep_connect () returns 0x%x\n", + dat_status); + + return dat_status; +} + +/* + * dapl_ep_common_connect + * + * DAPL Requirements Version 2.0, 6.6.x + * + * Requests that a connection be established + * between the local Endpoint and a remote Endpoint specified by the + * remote_ia_address. This operation is used by the active/client side + * Consumer of the Connection establishment model. + * + * EP must be properly configured for this operation. The EP Communicator + * must be specified. As part of the successful completion of this operation, + * the local Endpoint is bound to a local IA Address if it had these assigned + * before. + * + * The local IP Address, port and protocol are passed to the remote side of + * the requested connection and is available to the remote Consumer in the + * Connection Request of the DAT_CONNECTION_REQUEST_EVENT. + * + * The Consumer-provided private_data is passed to the remote side and is + * provided to the remote Consumer in the Connection Request. Consumers + * can encapsulate any local Endpoint attributes that remote Consumers + * need to know as part of an upper-level protocol. + * + * Input: + * ep_handle + * remote_ia_address + * timeout + * private_data_size + * private_date pointer + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_common_connect ( + IN DAT_EP_HANDLE ep, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR remote_addr, /* remote_ia_address */ + IN DAT_TIMEOUT timeout, /* timeout */ + IN DAT_COUNT pdata_size, /* private_data_size */ + IN const DAT_PVOID pdata ) /* private_data */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_create.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_create.c new file mode 100644 index 00000000..ccb506df --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_create.c @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_create.c + * + * PURPOSE: Endpoint management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" + + +/* + * dapl_ep_create + * + * Create an instance of an Endpoint that is provided to the + * consumer at ep_handle. + * + * Input: + * ia_handle + * pz_handle + * recv_evd_handle (recv DTOs) + * request_evd_handle (xmit DTOs) + * connect_evd_handle + * ep_attrs + * + * Output: + * ep_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_ATTRIBUTE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_EVD_HANDLE recv_evd_handle, + IN DAT_EVD_HANDLE request_evd_handle, + IN DAT_EVD_HANDLE connect_evd_handle, + IN const DAT_EP_ATTR *ep_attr, + OUT DAT_EP_HANDLE *ep_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_EP *ep_ptr; + DAT_EP_ATTR ep_attr_limit; + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_create (%p, %p, %p, %p, %p, %p, %p)\n", + ia_handle, + pz_handle, + recv_evd_handle, + request_evd_handle, + connect_evd_handle, + ep_attr, + ep_handle); + DAPL_CNTR(DCNT_EP_CREATE); + + ia_ptr = (DAPL_IA *)ia_handle; + dat_status = DAT_SUCCESS; + + /* + * Verify parameters + */ + if ( DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + + /* + * Verify non-required parameters. + * N.B. Assumption: any parameter that can be + * modified by dat_ep_modify() is not strictly + * required when the EP is created + */ + if ( pz_handle != DAT_HANDLE_NULL && + DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + goto bail; + } + + /* If connect handle is !NULL verify handle is good */ + if (connect_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (connect_evd_handle, DAPL_MAGIC_EVD) || + ! (((DAPL_EVD *)connect_evd_handle)->evd_flags & DAT_EVD_CONNECTION_FLAG)) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_CONN); + goto bail; + } + + /* If recv_evd is !NULL, verify handle is good and flags are valid */ + if (recv_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (recv_evd_handle, DAPL_MAGIC_EVD) || + ! (((DAPL_EVD *)recv_evd_handle)->evd_flags & DAT_EVD_DTO_FLAG) ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_RECV); + goto bail; + } + + /* If req_evd is !NULL, verify handle is good and flags are valid */ + if (request_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (request_evd_handle, DAPL_MAGIC_EVD) || + ! (((DAPL_EVD *)request_evd_handle)->evd_flags & DAT_EVD_DTO_FLAG) ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + goto bail; + } + + if ( ep_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG7); + goto bail; + } + if (DAPL_BAD_PTR(ep_attr)) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG6); + goto bail; + } + + /* + * Qualify EP Attributes are legal and make sense. Note that if one + * or both of the DTO handles are NULL, then the corresponding + * max_*_dtos must 0 as the user will not be able to post dto ops on + * the respective queue. + */ + if (ep_attr != NULL && + (ep_attr->service_type != DAT_SERVICE_TYPE_RC || + (recv_evd_handle == DAT_HANDLE_NULL && ep_attr->max_recv_dtos != 0) || + (recv_evd_handle != DAT_HANDLE_NULL && ep_attr->max_recv_dtos == 0) || + (request_evd_handle == DAT_HANDLE_NULL && ep_attr->max_request_dtos != 0) || + (request_evd_handle != DAT_HANDLE_NULL && ep_attr->max_request_dtos == 0) || + (recv_evd_handle != DAT_HANDLE_NULL && ep_attr->max_recv_iov == 0) || + ep_attr->max_request_iov == 0 || + (DAT_SUCCESS != dapl_ep_check_recv_completion_flags ( + ep_attr->recv_completion_flags)) )) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG6); + goto bail; + } + + /* Verify the attributes against the transport */ + if (ep_attr != NULL) + { + dapl_os_memzero (&ep_attr_limit, sizeof (DAT_EP_ATTR)); + dat_status = dapls_ib_query_hca (ia_ptr->hca_ptr, + NULL, &ep_attr_limit, NULL); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + if ( ep_attr->max_mtu_size > ep_attr_limit.max_mtu_size || + ep_attr->max_rdma_size > ep_attr_limit.max_rdma_size || + ep_attr->max_recv_dtos > ep_attr_limit.max_recv_dtos || + ep_attr->max_request_dtos > ep_attr_limit.max_request_dtos || + ep_attr->max_recv_iov > ep_attr_limit.max_recv_iov || + ep_attr->max_request_iov > ep_attr_limit.max_request_iov || + ep_attr->max_rdma_read_in > ep_attr_limit.max_rdma_read_in || + ep_attr->max_rdma_read_out> ep_attr_limit.max_rdma_read_out ) + + { + dat_status = DAT_INVALID_PARAMETER | DAT_INVALID_ARG6; + goto bail; + } + } + + /* + * Verify the completion flags for the EVD and the EP + */ + /* + * XXX FIXME + * XXX Need to make assign the EVD to the right completion type + * XXX depending on the EP attributes. Fail if the types don't + * XXX match, they are mutually exclusive. + */ + evd_ptr = (DAPL_EVD *)recv_evd_handle; + if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) + { + if (ep_attr != NULL && + ep_attr->recv_completion_flags == DAT_COMPLETION_DEFAULT_FLAG) + { + evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; + } + else + { + evd_ptr->completion_type = ep_attr->recv_completion_flags; + } + } + + evd_ptr = (DAPL_EVD *)request_evd_handle; + if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) + { + if (ep_attr != NULL && + ep_attr->recv_completion_flags == DAT_COMPLETION_DEFAULT_FLAG) + { + evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; + } + else + { + evd_ptr->completion_type = ep_attr->recv_completion_flags; + } + } + + + /* Allocate EP */ + ep_ptr = dapl_ep_alloc ( ia_ptr, ep_attr ); + if ( ep_ptr == NULL ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + /* + * Fill in the EP + */ + ep_ptr->param.ia_handle = ia_handle; + ep_ptr->param.ep_state = DAT_EP_STATE_UNCONNECTED; + ep_ptr->param.local_ia_address_ptr= + (DAT_IA_ADDRESS_PTR)&ia_ptr->hca_ptr->hca_address; + /* Set the remote address pointer to the end of the EP struct */ + ep_ptr->param.remote_ia_address_ptr = (DAT_IA_ADDRESS_PTR) (ep_ptr + 1); + + ep_ptr->param.pz_handle = pz_handle; + ep_ptr->param.recv_evd_handle = recv_evd_handle; + ep_ptr->param.request_evd_handle = request_evd_handle; + ep_ptr->param.connect_evd_handle = connect_evd_handle; + + /* + * Make sure we handle the NULL DTO EVDs + */ + if ( recv_evd_handle == DAT_HANDLE_NULL && ep_attr == NULL ) + { + ep_ptr->param.ep_attr.max_recv_dtos = 0; + } + + if ( request_evd_handle == DAT_HANDLE_NULL && ep_attr == NULL ) + { + ep_ptr->param.ep_attr.max_request_dtos = 0; + } + + /* + * If the user has specified a PZ handle we allocate a QP for + * this EP; else we defer until it is assigned via ep_modify(). + * As much as possible we try to keep QP creation out of the + * connect path to avoid resource errors in strange places. + */ + if (pz_handle != DAT_HANDLE_NULL ) + { + /* Take a reference on the PZ handle */ + dapl_os_atomic_inc (& ((DAPL_PZ *)pz_handle)->pz_ref_count); + + /* + * Get a QP from the IB provider + */ + dat_status = dapls_ib_qp_alloc ( ia_ptr, ep_ptr, ep_ptr ); + + if ( dat_status != DAT_SUCCESS) + { + dapl_os_atomic_dec (& ((DAPL_PZ *)pz_handle)->pz_ref_count); + dapl_ep_dealloc ( ep_ptr ); + goto bail; + } + } + else + { + ep_ptr->qp_state = DAPL_QP_STATE_UNATTACHED; + } + + /* + * Update ref counts. See the spec where the endpoint marks + * a data object as 'in use' + * pz_handle: dat_pz_free, uDAPL Document, 6.6.1.2 + * evd_handles: + * + * N.B. This should really be done by a util routine. + */ + if (connect_evd_handle != DAT_HANDLE_NULL) + { + dapl_os_atomic_inc (& ((DAPL_EVD *)connect_evd_handle)->evd_ref_count); + } + /* Optional handles */ + if (recv_evd_handle != DAT_HANDLE_NULL) + { + dapl_os_atomic_inc (& ((DAPL_EVD *)recv_evd_handle)->evd_ref_count); + } + if (request_evd_handle != DAT_HANDLE_NULL) + { + dapl_os_atomic_inc (& ((DAPL_EVD *)request_evd_handle)->evd_ref_count); + } + + /* Link it onto the IA */ + dapl_ia_link_ep (ia_ptr, ep_ptr); + + *ep_handle = ep_ptr; + +bail: + return dat_status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_create_with_srq.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_create_with_srq.c new file mode 100644 index 00000000..72fa5835 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_create_with_srq.c @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_create_with_srq.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the kDAPL 1.2 API, Chapter 6, section 6.5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" + + +/* + * dapl_ep_create_with_srq + * + * uDAPL Version 1.2, 6.6.5 + * + * Create an instance of an Endpoint that is provided to the + * consumer at ep_handle. + * + * Input: + * ia_handle + * pz_handle + * recv_evd_handle (recv DTOs) + * request_evd_handle (xmit DTOs) + * connect_evd_handle + * srq_handle + * ep_attrs + * + * Output: + * ep_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * DAT_INVALID_ATTRIBUTE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_create_with_srq ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_EVD_HANDLE recv_evd_handle, + IN DAT_EVD_HANDLE request_evd_handle, + IN DAT_EVD_HANDLE connect_evd_handle, + IN DAT_SRQ_HANDLE srq_handle, + IN const DAT_EP_ATTR *ep_attr, + OUT DAT_EP_HANDLE *ep_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_EP *ep_ptr; + DAT_EP_ATTR ep_attr_limit; + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_create_with_srq (%p, %p, %p, %p, %p, %p, %p %p)\n", + ia_handle, + pz_handle, + recv_evd_handle, + request_evd_handle, + connect_evd_handle, + srq_handle, + ep_attr, + ep_handle); + + ia_ptr = (DAPL_IA *)ia_handle; + + /* + * Verify parameters + */ + if ( DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + + /* + * Verify non-required parameters. + * N.B. Assumption: any parameter that can be + * modified by dat_ep_modify() is not strictly + * required when the EP is created + */ + if ( pz_handle != NULL && + DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + goto bail; + } + + /* If connect handle is !NULL verify handle is good */ + if (connect_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (connect_evd_handle, DAPL_MAGIC_EVD) || + ! (((DAPL_EVD *)connect_evd_handle)->evd_flags & DAT_EVD_CONNECTION_FLAG)) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_CONN); + goto bail; + } + /* If recv_evd is !NULL, verify handle is good and flags are valid */ + if (recv_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (recv_evd_handle, DAPL_MAGIC_EVD) || + ! (((DAPL_EVD *)recv_evd_handle)->evd_flags & DAT_EVD_DTO_FLAG) ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_RECV); + goto bail; + } + + /* If req_evd is !NULL, verify handle is good and flags are valid */ + if (request_evd_handle != DAT_HANDLE_NULL && + (DAPL_BAD_HANDLE (request_evd_handle, DAPL_MAGIC_EVD) || + ! (((DAPL_EVD *)request_evd_handle)->evd_flags & DAT_EVD_DTO_FLAG) ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + goto bail; + } + + /* + * Verify the SRQ handle. It is an error to invoke this call with + * a NULL handle + */ + if (DAPL_BAD_HANDLE (srq_handle, DAPL_MAGIC_SRQ)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + goto bail; + } + + if ( ep_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG7); + goto bail; + } + + if ( DAPL_BAD_PTR(ep_attr) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG6); + goto bail; + } + + /* + * Qualify EP Attributes are legal and make sense. Note that if one + * or both of the DTO handles are NULL, then the corresponding + * max_*_dtos must 0 as the user will not be able to post dto ops on + * the respective queue. + */ + if (ep_attr != NULL && + (ep_attr->service_type != DAT_SERVICE_TYPE_RC || + (recv_evd_handle == DAT_HANDLE_NULL && ep_attr->max_recv_dtos != 0) || + (recv_evd_handle != DAT_HANDLE_NULL && ep_attr->max_recv_dtos == 0) || + (request_evd_handle == DAT_HANDLE_NULL && ep_attr->max_request_dtos != 0) || + (request_evd_handle != DAT_HANDLE_NULL && ep_attr->max_request_dtos == 0) || + ep_attr->max_recv_iov == 0 || + ep_attr->max_request_iov == 0 || + (DAT_SUCCESS != dapl_ep_check_recv_completion_flags ( + ep_attr->recv_completion_flags)) )) + { + dat_status = DAT_INVALID_PARAMETER | DAT_INVALID_ARG6; + goto bail; + } + + /* Verify the attributes against the transport */ + if (ep_attr != NULL) + { + dapl_os_memzero (&ep_attr_limit, sizeof (DAT_EP_ATTR)); + dat_status = dapls_ib_query_hca (ia_ptr->hca_ptr, + NULL, &ep_attr_limit, NULL); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + if ( ep_attr->max_mtu_size > ep_attr_limit.max_mtu_size || + ep_attr->max_rdma_size > ep_attr_limit.max_rdma_size || + ep_attr->max_recv_dtos > ep_attr_limit.max_recv_dtos || + ep_attr->max_request_dtos > ep_attr_limit.max_request_dtos || + ep_attr->max_recv_iov > ep_attr_limit.max_recv_iov || + ep_attr->max_request_iov > ep_attr_limit.max_request_iov || + ep_attr->max_rdma_read_in > ep_attr_limit.max_rdma_read_in || + ep_attr->max_rdma_read_out> ep_attr_limit.max_rdma_read_out ) + + { + dat_status = DAT_INVALID_PARAMETER | DAT_INVALID_ARG6; + goto bail; + } + } + + /* + * Verify the completion flags for the EVD and the EP + */ + /* + * XXX FIXME + * XXX Need to make assign the EVD to the right completion type + * XXX depending on the EP attributes. Fail if the types don't + * XXX match, they are mutually exclusive. + */ + evd_ptr = (DAPL_EVD *)recv_evd_handle; + if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) + { + if (ep_attr != NULL && + ep_attr->recv_completion_flags == DAT_COMPLETION_DEFAULT_FLAG) + { + evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; + } + else + { + evd_ptr->completion_type = ep_attr->recv_completion_flags; + } + } + + evd_ptr = (DAPL_EVD *)request_evd_handle; + if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) + { + if (ep_attr != NULL && + ep_attr->recv_completion_flags == DAT_COMPLETION_DEFAULT_FLAG) + { + evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; + } + else + { + evd_ptr->completion_type = ep_attr->recv_completion_flags; + } + } + + dat_status = DAT_NOT_IMPLEMENTED; + + /* + * XXX The rest of the EP code is useful in this case too, + * XXX but need to complete the SRQ implementation before + * XXX committing resources + */ + *ep_handle = ep_ptr = NULL; + goto bail; +#ifdef notdef + + /* Allocate EP */ + ep_ptr = dapl_ep_alloc ( ia_ptr, ep_attr ); + if ( ep_ptr == NULL ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + /* + * Fill in the EP + */ + ep_ptr->param.ia_handle = ia_handle; + ep_ptr->param.ep_state = DAT_EP_STATE_UNCONNECTED; + ep_ptr->param.local_ia_address_ptr= + (DAT_IA_ADDRESS_PTR)&ia_ptr->hca_ptr->hca_address; + /* Set the remote address pointer to the end of the EP struct */ + ep_ptr->param.remote_ia_address_ptr = (DAT_IA_ADDRESS_PTR) (ep_ptr + 1); + + ep_ptr->param.pz_handle = pz_handle; + ep_ptr->param.recv_evd_handle = recv_evd_handle; + ep_ptr->param.request_evd_handle = request_evd_handle; + ep_ptr->param.connect_evd_handle = connect_evd_handle; + + /* + * Make sure we handle the NULL DTO EVDs + */ + if ( recv_evd_handle == DAT_HANDLE_NULL && ep_attr == NULL ) + { + ep_ptr->param.ep_attr.max_recv_dtos = 0; + } + + if ( request_evd_handle == DAT_HANDLE_NULL && ep_attr == NULL ) + { + ep_ptr->param.ep_attr.max_request_dtos = 0; + } + + /* + * If the user has specified a PZ handle we allocate a QP for + * this EP; else we defer until it is assigned via ep_modify(). + * As much as possible we try to keep QP creation out of the + * connect path to avoid resource errors in strange places. + */ + if (pz_handle != DAT_HANDLE_NULL ) + { + /* Take a reference on the PZ handle */ + dapl_os_atomic_inc (& ((DAPL_PZ *)pz_handle)->pz_ref_count); + + /* + * Get a QP from the IB provider + */ + dat_status = dapls_ib_qp_alloc ( ia_ptr, ep_ptr, ep_ptr ); + + if ( dat_status != DAT_SUCCESS) + { + dapl_os_atomic_dec (& ((DAPL_PZ *)pz_handle)->pz_ref_count); + dapl_ep_dealloc ( ep_ptr ); + goto bail; + } + } + else + { + ep_ptr->qp_state = DAPL_QP_STATE_UNATTACHED; + } + + /* + * Update ref counts. See the spec where the endpoint marks + * a data object as 'in use' + * pz_handle: dat_pz_free, uDAPL Document, 6.6.1.2 + * evd_handles: + * + * N.B. This should really be done by a util routine. + */ + dapl_os_atomic_inc (& ((DAPL_EVD *)connect_evd_handle)->evd_ref_count); + /* Optional handles */ + if (recv_evd_handle != DAT_HANDLE_NULL) + { + dapl_os_atomic_inc (& ((DAPL_EVD *)recv_evd_handle)->evd_ref_count); + } + if (request_evd_handle != DAT_HANDLE_NULL) + { + dapl_os_atomic_inc (& ((DAPL_EVD *)request_evd_handle)->evd_ref_count); + } + + /* Link it onto the IA */ + dapl_ia_link_ep (ia_ptr, ep_ptr); + + *ep_handle = ep_ptr; + +#endif /* notdef */ + +bail: + return dat_status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_disconnect.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_disconnect.c new file mode 100644 index 00000000..8e4c1c28 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_disconnect.c @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_disconnect.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_ep_util.h" +#include "dapl_sp_util.h" +#include "dapl_evd_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_ep_disconnect + * + * DAPL Requirements Version xxx, 6.5.9 + * + * Terminate a connection. + * + * Input: + * ep_handle + * disconnect_flags + * + * Output: + * None + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_ep_disconnect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_CLOSE_FLAGS disconnect_flags) +{ + DAPL_EP *ep_ptr; + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API | DAPL_DBG_TYPE_CM, + "dapl_ep_disconnect (%p, %x)\n", + ep_handle, + disconnect_flags); + DAPL_CNTR(DCNT_EP_DISCONNECT); + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + /* + * Do the verification of parameters and the state change + * atomically. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + + /* Disconnecting a disconnected EP is a no-op. */ + if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED ) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_SUCCESS; + goto bail; + } + + /* Check the EP state to ensure we are queiscent. Note that + * we may get called in UNCONNECTED state in order to remove + * RECV requests from the queue prior to destroying an EP. + * See the states in the spec at 6.5.1 Endpont Lifecycle + */ + if ( ep_ptr->param.ep_state != DAT_EP_STATE_CONNECTED && + ep_ptr->param.ep_state != DAT_EP_STATE_ACTIVE_CONNECTION_PENDING && + ep_ptr->param.ep_state != DAT_EP_STATE_COMPLETION_PENDING && + ep_ptr->param.ep_state != DAT_EP_STATE_DISCONNECT_PENDING ) + { + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING && + disconnect_flags != DAT_CLOSE_ABRUPT_FLAG) + { + /* + * If in state DISCONNECT_PENDING then this must be an + * ABRUPT disconnect + */ + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail; + } + + if ( ep_ptr->param.ep_state == DAT_EP_STATE_ACTIVE_CONNECTION_PENDING || + ep_ptr->param.ep_state == DAT_EP_STATE_COMPLETION_PENDING ) + { + /* + * Beginning or waiting on a connection: abort and reset the + * state + */ + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + + dapl_os_unlock ( &ep_ptr->header.lock ); + /* disconnect and make sure we get no callbacks */ + (void) dapls_ib_disconnect (ep_ptr, DAT_CLOSE_ABRUPT_FLAG); + + /* clean up connection state */ + dapl_sp_remove_ep (ep_ptr); + + evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle; + dapls_evd_post_connection_event (evd_ptr, + DAT_CONNECTION_EVENT_DISCONNECTED, + (DAT_HANDLE) ep_ptr, + 0, + 0); + dat_status = DAT_SUCCESS; + goto bail; + } + + /* + * Transition the EP state to DISCONNECT_PENDING if we are + * CONNECTED. Otherwise we do not get a disconnect event and will be + * stuck in DISCONNECT_PENDING. + * + * If the user specifies a graceful disconnect, the underlying + * provider should complete all DTOs before disconnecting; in IB + * terms, this means setting the QP state to SQD before completing + * the disconnect state transitions. + */ + if ( ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED ) + { + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECT_PENDING; + } + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = dapls_ib_disconnect ( ep_ptr, disconnect_flags ); + +bail: + dapl_dbg_log (DAPL_DBG_TYPE_RTN | DAPL_DBG_TYPE_CM, + "dapl_ep_disconnect (%lx) returns 0x%x\n", + ep_ptr, dat_status); + + return dat_status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_dup_connect.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_dup_connect.c new file mode 100644 index 00000000..5c818102 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_dup_connect.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_dup_connect.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" +#include "dapl_timer_util.h" + +/* + * dapl_ep_dup_connect + * + * DAPL Requirements Version xxx, 6.5.8 + * + * Requst that a connection be established between the local Endpoint + * and a remote Endpoint. The remote Endpoint is identified by the + * dup_ep. + * + * Input: + * ep_handle + * ep_dup_handle + * conn_qual + * timeout + * private_data_size + * private_data + * qos + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_dup_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_HANDLE ep_dup_handle, + IN DAT_TIMEOUT timeout, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data, + IN DAT_QOS qos) +{ + DAPL_EP *ep_dup_ptr; + DAT_RETURN dat_status; + DAT_IA_ADDRESS_PTR remote_ia_address_ptr; + DAT_CONN_QUAL remote_conn_qual; + + ep_dup_ptr = (DAPL_EP *) ep_dup_handle; + + /* + * Verify the dup handle, which must be connected. All other + * parameters will be verified by dapl_ep_connect + */ + if ( DAPL_BAD_HANDLE (ep_dup_handle, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + + /* Can't do a connection in 0 time, reject outright */ + if ( timeout == 0 ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + /* Check both the EP state and the QP state: if we don't have a QP + * there is a problem. Do this under a lock and pull out + * the connection parameters for atomicity. + */ + dapl_os_lock ( &ep_dup_ptr->header.lock ); + if ( ep_dup_ptr->param.ep_state != DAT_EP_STATE_CONNECTED ) + { + dapl_os_unlock ( &ep_dup_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_STATE,dapls_ep_state_subtype (ep_dup_ptr)); + goto bail; + } + remote_ia_address_ptr = ep_dup_ptr->param.remote_ia_address_ptr; + remote_conn_qual = ep_dup_ptr->param.remote_port_qual; + dapl_os_unlock ( &ep_dup_ptr->header.lock ); + + dat_status = dapl_ep_connect ( ep_handle, + remote_ia_address_ptr, + remote_conn_qual, + timeout, + private_data_size, + private_data, + qos, + DAT_CONNECT_DEFAULT_FLAG ); + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c new file mode 100644 index 00000000..1b09d92e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_free.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5.4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" +#include "dapl_ring_buffer_util.h" +#include "dapl_timer_util.h" + +/* + * dapl_ep_free + * + * DAPL Requirements Version xxx, 6.5.3 + * + * Destroy an instance of the Endpoint + * + * Input: + * ep_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + */ +DAT_RETURN DAT_API +dapl_ep_free ( + IN DAT_EP_HANDLE ep_handle) +{ + DAPL_EP *ep_ptr; + DAPL_IA *ia_ptr; + DAT_EP_PARAM *param; + ib_qp_state_t save_qp_state; + DAT_RETURN dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API | DAPL_DBG_TYPE_CM, + "dapl_ep_free (%p)\n", ep_handle); + DAPL_CNTR(DCNT_EP_FREE); + + ep_ptr = (DAPL_EP *) ep_handle; + param = &ep_ptr->param; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + if ( ep_ptr->param.ep_state == DAT_EP_STATE_RESERVED || + ep_ptr->param.ep_state == DAT_EP_STATE_PASSIVE_CONNECTION_PENDING || + ep_ptr->param.ep_state == DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING ) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "--> dapl_ep_free: invalid state: %x, ep %p\n", + ep_ptr->param.ep_state, + ep_ptr); + dat_status = DAT_ERROR (DAT_INVALID_STATE, + dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + ia_ptr = ep_ptr->header.owner_ia; + + /* If we are connected, issue a disconnect. If we are in the + * disconnect_pending state, disconnect with the ABRUPT flag + * set. + */ + + /* + * Invoke ep_disconnect to clean up outstanding connections + */ + (void) dapl_ep_disconnect (ep_ptr, DAT_CLOSE_ABRUPT_FLAG); + dapl_os_assert (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED || + ep_ptr->param.ep_state == DAT_EP_STATE_UNCONNECTED); + + /* + * Do verification of parameters and the state change atomically. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + + if (ep_ptr->cxn_timer != NULL) + { + dapls_timer_cancel ( ep_ptr->cxn_timer ); + dapl_os_free ( ep_ptr->cxn_timer, sizeof ( DAPL_OS_TIMER ) ); + ep_ptr->cxn_timer = NULL; + } + + /* Remove the EP from the IA */ + dapl_ia_unlink_ep ( ia_ptr, ep_ptr ); + + /* + * Update ref counts. Note the user may have used ep_modify + * to set handles to NULL. Set handles to NULL so this routine + * is idempotent. + */ + if ( param->pz_handle != NULL ) + { + dapl_os_atomic_dec (& ((DAPL_PZ *)param->pz_handle)->pz_ref_count); + param->pz_handle = NULL; + } + if ( param->recv_evd_handle != NULL ) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)param->recv_evd_handle)->evd_ref_count); + param->recv_evd_handle = NULL; + } + if ( param->request_evd_handle != NULL ) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)param->request_evd_handle)->evd_ref_count); + param->request_evd_handle = NULL; + } + if ( param->connect_evd_handle != NULL ) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)param->connect_evd_handle)->evd_ref_count); + param->connect_evd_handle = NULL; + } + + /* + * Finish tearing everything down. + */ + dapl_dbg_log (DAPL_DBG_TYPE_EP | DAPL_DBG_TYPE_CM, + "dapl_ep_free: Free EP: %x, ep %p qp_state %x qp_handle %x\n", + ep_ptr->param.ep_state, + ep_ptr, + ep_ptr->qp_state, + ep_ptr->qp_handle); + /* + * Take care of the transport resource. Make a copy of the qp_state + * to prevent race conditions when we exit the lock. + */ + save_qp_state = ep_ptr->qp_state; + ep_ptr->qp_state = DAPL_QP_STATE_UNATTACHED; + dapl_os_unlock (&ep_ptr->header.lock); + + /* Free the QP. If the EP has never been used, the QP is invalid */ + if ( save_qp_state != DAPL_QP_STATE_UNATTACHED ) + { + dat_status = dapls_ib_qp_free (ia_ptr, ep_ptr); + /* This should always succeed, but report to the user if + * there is a problem. The qp_state must be restored so + * they can try it again in the face of EINTR or similar + * where the QP is OK but the call couldn't complete. + */ + if (dat_status != DAT_SUCCESS) + { + ep_ptr->qp_state = save_qp_state; + goto bail; + } + } + + /* Free the resource */ + dapl_ep_dealloc (ep_ptr); + +bail: + return dat_status; + +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_get_status.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_get_status.c new file mode 100644 index 00000000..b3da2063 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_get_status.c @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_get_status.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ring_buffer_util.h" + +/* + * dapl_ep_get_status + * + * DAPL Requirements Version xxx, 6.5.4 + * + * Provide the consumer with a quick snapshot of the Endpoint. + * The snapshot consists of Endpoint state and DTO information. + * + * Input: + * ep_handle + * + * Output: + * ep_state + * in_dto_idle + * out_dto_idle + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_ep_get_status ( + IN DAT_EP_HANDLE ep_handle, + OUT DAT_EP_STATE *ep_state, + OUT DAT_BOOLEAN *in_dto_idle, + OUT DAT_BOOLEAN *out_dto_idle) +{ + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_get_status (%p, %p, %p, %p)\n", + ep_handle, + ep_state, + in_dto_idle, + out_dto_idle); + + ep_ptr = (DAPL_EP *) ep_handle; + dat_status = DAT_SUCCESS; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + + /* + * Gather state info for user + */ + if ( ep_state != NULL ) + { + *ep_state = ep_ptr->param.ep_state; + } + + if ( in_dto_idle != NULL ) + { + *in_dto_idle = (dapl_os_atomic_read (&ep_ptr->recv_count)) ? DAT_FALSE : DAT_TRUE; + } + + if ( out_dto_idle != NULL ) + { + *out_dto_idle = (dapl_os_atomic_read (&ep_ptr->req_count)) ? DAT_FALSE : DAT_TRUE; + } + + bail: + return dat_status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_modify.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_modify.c new file mode 100644 index 00000000..59742e04 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_modify.c @@ -0,0 +1,792 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_modify.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.0 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_cookie.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" + +/* + * Internal prototypes + */ + +static _INLINE_ DAT_RETURN +dapli_ep_modify_validate_parameters ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_PARAM_MASK ep_param_mask, + IN const DAT_EP_PARAM *ep_param, + OUT DAPL_IA **ia_ptr, + OUT DAPL_EP **ep_ptr, + OUT DAT_EP_ATTR *ep_attr_ptr ); + + +/* + * dapl_ep_modify + * + * DAPL Requirements Version xxx, 6.5.6 + * + * Provide the consumer parameters, including attributes and status of + * the Endpoint. + * + * Input: + * ep_handle + * ep_args_mask + * + * Output: + * ep_args + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_ATTRIBUTE + * DAT_INVALID_STATE + */ +DAT_RETURN DAT_API +dapl_ep_modify ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_PARAM_MASK ep_param_mask, + IN const DAT_EP_PARAM *ep_param ) +{ + DAPL_IA *ia; + DAPL_EP *ep1, *ep2; + DAT_EP_ATTR ep_attr1, ep_attr2; + DAPL_EP new_ep, copy_of_old_ep; + DAPL_EP alloc_ep; /* Holder for resources. */ + DAPL_PZ *tmp_pz; + DAPL_EVD *tmp_evd; + DAT_RETURN dat_status; + + /* Flag indicating we've allocated a new one of these. */ + DAT_BOOLEAN qp_allocated = DAT_FALSE; + DAT_BOOLEAN rqst_cb_allocated = DAT_FALSE; + DAT_BOOLEAN recv_cb_allocated = DAT_FALSE; + DAT_BOOLEAN rqst_iov_allocated = DAT_FALSE; + DAT_BOOLEAN recv_iov_allocated = DAT_FALSE; + + /* Flag indicating we've used (assigned to QP) a new one of these. */ + DAT_BOOLEAN qp_used = DAT_FALSE; + DAT_BOOLEAN rqst_cb_used = DAT_FALSE; + DAT_BOOLEAN recv_cb_used = DAT_FALSE; + DAT_BOOLEAN rqst_iov_used = DAT_FALSE; + DAT_BOOLEAN recv_iov_used = DAT_FALSE; + + dat_status = dapli_ep_modify_validate_parameters ( ep_handle, + ep_param_mask, + ep_param, + &ia, + &ep1, + &ep_attr1); + if ( DAT_SUCCESS != dat_status) + { + goto bail; + } + + /* + * Setup the alloc_ep with the appropriate parameters (primarily + * for allocating the QP. + */ + alloc_ep = *ep1; + alloc_ep.param.ep_attr = ep_attr1; + if ( ep_param_mask & DAT_EP_FIELD_PZ_HANDLE ) + { + alloc_ep.param.pz_handle = ep_param->pz_handle; + } + + if ( ep_param_mask & DAT_EP_FIELD_RECV_EVD_HANDLE ) + { + alloc_ep.param.recv_evd_handle = ep_param->recv_evd_handle; + } + + if ( ep_param_mask & DAT_EP_FIELD_REQUEST_EVD_HANDLE ) + { + alloc_ep.param.request_evd_handle = ep_param->request_evd_handle; + } + + if ( ep_param_mask & DAT_EP_FIELD_CONNECT_EVD_HANDLE ) + { + alloc_ep.param.connect_evd_handle = ep_param->connect_evd_handle; + } + + /* + * Allocate everything that might be needed. + * We allocate separately, and into a different "holding" + * ep, since we a) want the copy of the old ep into the new ep to + * be atomic with the assignment back (under lock), b) want the + * assignment of the allocated materials to be after the copy of the + * old ep into the new ep, and c) don't want the allocation done + * under lock. + */ + dat_status = dapls_cb_create ( + &alloc_ep.req_buffer, + ep1, /* For pointer in buffer bool. */ + ep_attr1.max_request_dtos ); + if ( DAT_SUCCESS != dat_status ) + { + goto bail; + } + rqst_cb_allocated = DAT_TRUE; + + dat_status = dapls_cb_create ( + &alloc_ep.recv_buffer, + ep1, /* For pointer in buffer bool. */ + ep_attr1.max_recv_dtos ); + if ( DAT_SUCCESS != dat_status ) + { + goto bail; + } + recv_cb_allocated = DAT_TRUE; + + alloc_ep.send_iov_num = ep_attr1.max_request_iov; + + /* + * Ensure that send_iov_num is big enough for all types + * of send requests. + */ + if (ep_attr1.max_rdma_read_iov > alloc_ep.send_iov_num) + { + alloc_ep.send_iov_num = ep_attr1.max_rdma_read_iov; + } + if (ep_attr1.max_rdma_write_iov > alloc_ep.send_iov_num) + { + alloc_ep.send_iov_num = ep_attr1.max_rdma_write_iov; + } + alloc_ep.send_iov = dapl_os_alloc (alloc_ep.send_iov_num + * sizeof (ib_data_segment_t)); + if ( alloc_ep.send_iov == NULL ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + rqst_iov_allocated = DAT_TRUE; + + alloc_ep.recv_iov_num = ep_attr1.max_recv_iov; + alloc_ep.recv_iov = dapl_os_alloc (ep_attr1.max_recv_iov + * sizeof (ib_data_segment_t)); + if ( alloc_ep.recv_iov == NULL ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + recv_iov_allocated = DAT_TRUE; + + dat_status = dapls_ib_qp_alloc ( ia, &alloc_ep, ep1 ); + if ( dat_status != DAT_SUCCESS ) + { + goto bail; + } + qp_allocated = DAT_TRUE; + + /* + * Now we atomically modify the EP, under lock + * There's a lot of work done here, but there should be no + * allocation or blocking. + */ + dapl_os_lock ( &ep1->header.lock ); + + /* + * Revalidate parameters; make sure that races haven't + * changed anything important. + */ + dat_status = dapli_ep_modify_validate_parameters ( ep_handle, + ep_param_mask, + ep_param, + &ia, + &ep2, + &ep_attr2 ); + if ( DAT_SUCCESS != dat_status ) + { + dapl_os_unlock ( &ep2->header.lock ); + goto bail; + } + + /* + * All of the following should be impossible, if validation + * occurred. But they're important to the logic of this routine, + * so we check. + */ + dapl_os_assert ( ep1 == ep2 ); + dapl_os_assert ( ep_attr2.max_recv_dtos == ep_attr1.max_recv_dtos ); + dapl_os_assert ( ep_attr2.max_request_dtos == ep_attr1.max_request_dtos ); + dapl_os_assert ( ep_attr2.max_recv_iov == ep_attr1.max_recv_iov ); + dapl_os_assert ( ep_attr2.max_request_iov == ep_attr1.max_request_iov ); + + copy_of_old_ep = *ep2; + + /* + * Setup new ep. + */ + new_ep = *ep2; + new_ep.param.ep_attr = ep_attr2; + + /* + * We can initialize the PZ and EVD handles from the alloc_ep because + * the only thing that could have changed since we setup the alloc_ep + * is stuff changed by dapl_cr_accept, and neither PZ nor EVD is in that + * list. + */ + new_ep.param.pz_handle = alloc_ep.param.pz_handle; + new_ep.param.recv_evd_handle = alloc_ep.param.recv_evd_handle; + new_ep.param.request_evd_handle = alloc_ep.param.request_evd_handle; + new_ep.param.connect_evd_handle = alloc_ep.param.connect_evd_handle; + + /* Deal with each of the allocation fields. */ + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RECV_DTOS + && (ep_param->ep_attr.max_recv_dtos + != ep2->param.ep_attr.max_recv_dtos) ) + { + new_ep.recv_buffer = alloc_ep.recv_buffer; + recv_cb_used = DAT_TRUE; + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_DTOS + && (ep_param->ep_attr.max_request_dtos + != ep2->param.ep_attr.max_request_dtos) ) + { + new_ep.req_buffer = alloc_ep.req_buffer; + rqst_cb_used = DAT_TRUE; + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RECV_IOV + && new_ep.recv_iov_num != ep2->recv_iov_num ) + { + new_ep.recv_iov = alloc_ep.recv_iov; + recv_iov_used = DAT_TRUE; + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_IOV + && new_ep.send_iov_num != ep2->send_iov_num ) + { + new_ep.send_iov = alloc_ep.send_iov; + recv_iov_used = DAT_TRUE; + } + + /* + * We need to change the QP only if there already was a QP + * (leave things the way you found them!) and one of the + * following has changed: send/recv EVD, send/recv reqs/IOV max. + */ + if ( DAPL_QP_STATE_UNATTACHED != new_ep.qp_state + && (ep_param_mask + & (DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_IOV + | DAT_EP_FIELD_EP_ATTR_MAX_RECV_IOV + | DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_DTOS + | DAT_EP_FIELD_EP_ATTR_MAX_RECV_DTOS + | DAT_EP_FIELD_RECV_EVD_HANDLE + | DAT_EP_FIELD_REQUEST_EVD_HANDLE)) ) + { + /* + * We shouldn't be racing with connection establishment + * because the parameter validate routine should protect us, + * but it's an important enough point that we assert it. + */ + dapl_os_assert ( (ep2->param.ep_state + != DAT_EP_STATE_PASSIVE_CONNECTION_PENDING) + && (ep2->param.ep_state + != DAT_EP_STATE_ACTIVE_CONNECTION_PENDING) ); + + new_ep.qp_handle = alloc_ep.qp_handle; + new_ep.qpn = alloc_ep.qpn; + } + + /* + * The actual assignment, including modifying QP parameters. + * Modifying QP parameters needs to come first, as if it fails + * we need to exit. + */ + if ( DAPL_QP_STATE_UNATTACHED != new_ep.qp_state ) + { + dat_status = dapls_ib_qp_modify ( ia, ep2, &ep_attr2 ); + if ( dat_status != DAT_SUCCESS ) + { + dapl_os_unlock ( & ep2->header.lock ); + goto bail; + } + } + *ep2 = new_ep; + + dapl_os_unlock ( &ep2->header.lock ); + + /* + * Modify reference counts, incrementing new ones + * and then decrementing old ones (so if they're the same + * the refcount never drops to zero). + */ + tmp_pz = (DAPL_PZ *) new_ep.param.pz_handle; + if ( NULL != tmp_pz ) + { + dapl_os_atomic_inc (&tmp_pz->pz_ref_count); + } + + tmp_evd = (DAPL_EVD *) new_ep.param.recv_evd_handle; + if ( NULL != tmp_evd ) + { + dapl_os_atomic_inc (&tmp_evd->evd_ref_count); + } + + tmp_evd = (DAPL_EVD *) new_ep.param.request_evd_handle; + if ( NULL != tmp_evd ) + { + dapl_os_atomic_inc (&tmp_evd->evd_ref_count); + } + + tmp_evd = (DAPL_EVD *) new_ep.param.connect_evd_handle; + if ( NULL != tmp_evd ) + { + dapl_os_atomic_inc (&tmp_evd->evd_ref_count); + } + + /* decreament the old reference counts */ + tmp_pz = (DAPL_PZ *) copy_of_old_ep.param.pz_handle; + if ( NULL != tmp_pz ) + { + dapl_os_atomic_dec (&tmp_pz->pz_ref_count); + } + + tmp_evd = (DAPL_EVD *) copy_of_old_ep.param.recv_evd_handle; + if ( NULL != tmp_evd ) + { + dapl_os_atomic_dec (&tmp_evd->evd_ref_count); + } + + tmp_evd = (DAPL_EVD *) copy_of_old_ep.param.request_evd_handle; + if ( NULL != tmp_evd ) + { + dapl_os_atomic_dec (&tmp_evd->evd_ref_count); + } + + tmp_evd = (DAPL_EVD *) copy_of_old_ep.param.connect_evd_handle; + if ( NULL != tmp_evd ) + { + dapl_os_atomic_dec (&tmp_evd->evd_ref_count); + } + +bail: + if ( qp_allocated ) + { + DAT_RETURN local_dat_status; + if ( dat_status != DAT_SUCCESS || !qp_used ) + { + local_dat_status = dapls_ib_qp_free (ia, &alloc_ep ); + } + else + { + local_dat_status = dapls_ib_qp_free (ia, ©_of_old_ep ); + } + if (local_dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ep_modify: Failed to free QP; status %x\n", + local_dat_status); + } + } + + if ( rqst_cb_allocated ) + { + if ( dat_status != DAT_SUCCESS || !rqst_cb_used ) + { + dapls_cb_free ( &alloc_ep.req_buffer ); + } + else + { + dapls_cb_free ( ©_of_old_ep.req_buffer ); + } + } + + if ( recv_cb_allocated ) + { + if ( dat_status != DAT_SUCCESS || !recv_cb_used ) + { + dapls_cb_free ( &alloc_ep.recv_buffer ); + } + else + { + dapls_cb_free ( ©_of_old_ep.recv_buffer ); + } + } + + if ( rqst_iov_allocated ) + { + if ( dat_status != DAT_SUCCESS || !rqst_iov_used ) + { + dapl_os_free ( alloc_ep.send_iov, + (alloc_ep.send_iov_num + * sizeof (ib_data_segment_t))); + } + else + { + dapl_os_free ( copy_of_old_ep.send_iov, + (copy_of_old_ep.send_iov_num + * sizeof (ib_data_segment_t))); + } + } + + if ( recv_iov_allocated ) + { + if ( dat_status != DAT_SUCCESS || !recv_iov_used ) + { + dapl_os_free ( alloc_ep.recv_iov, + (alloc_ep.recv_iov_num + * sizeof (ib_data_segment_t))); + } + else + { + dapl_os_free ( copy_of_old_ep.recv_iov, + (copy_of_old_ep.recv_iov_num + * sizeof (ib_data_segment_t))); + } + } + return dat_status; +} + + +/* + * dapli_ep_modify_validate_parameters + * + * Validate parameters + * + * The space for the ep_attr_ptr parameter should be allocated by the + * consumer. Upon success, this parameter will contain the current ep + * attribute values with the requested modifications made. + * + */ + +static DAT_RETURN +dapli_ep_modify_validate_parameters ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_PARAM_MASK ep_param_mask, + IN const DAT_EP_PARAM *ep_param, + OUT DAPL_IA **ia_ptr, + OUT DAPL_EP **ep_ptr, + OUT DAT_EP_ATTR *ep_attr_ptr ) +{ + DAPL_IA *ia; + DAPL_EP *ep; + DAT_EP_ATTR ep_attr; + DAT_EP_ATTR ep_attr_limit; + DAT_EP_ATTR ep_attr_request; + DAT_RETURN dat_status; + + *ia_ptr = NULL; + *ep_ptr = NULL; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + ep = (DAPL_EP *) ep_handle; + ia = ep->header.owner_ia; + + /* + * Verify parameters valid in current EP state + */ + if ( ep_param_mask & (DAT_EP_FIELD_IA_HANDLE | + DAT_EP_FIELD_EP_STATE | + DAT_EP_FIELD_LOCAL_IA_ADDRESS_PTR | + DAT_EP_FIELD_LOCAL_PORT_QUAL | + DAT_EP_FIELD_REMOTE_IA_ADDRESS_PTR | + DAT_EP_FIELD_REMOTE_PORT_QUAL) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail; + } + + /* + * Can only change the PZ handle if we are UNCONNECTED or + * TENTATIVE_CONNECTION_PENDING (psp PROVIDER allocated EP) + */ + if ( (ep_param_mask & DAT_EP_FIELD_PZ_HANDLE) && + ( ep->param.ep_state != DAT_EP_STATE_UNCONNECTED && + ep->param.ep_state != DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep)); + goto bail; + } + + if ( (ep_param_mask & (DAT_EP_FIELD_RECV_EVD_HANDLE | + DAT_EP_FIELD_REQUEST_EVD_HANDLE | + DAT_EP_FIELD_CONNECT_EVD_HANDLE | + DAT_EP_FIELD_EP_ATTR_SERVICE_TYPE | + DAT_EP_FIELD_EP_ATTR_MAX_MESSAGE_SIZE | + DAT_EP_FIELD_EP_ATTR_MAX_RDMA_SIZE | + DAT_EP_FIELD_EP_ATTR_QOS | + DAT_EP_FIELD_EP_ATTR_REQUEST_COMPLETION_FLAGS | + DAT_EP_FIELD_EP_ATTR_RECV_COMPLETION_FLAGS | + DAT_EP_FIELD_EP_ATTR_MAX_RECV_DTOS | + DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_DTOS | + DAT_EP_FIELD_EP_ATTR_MAX_RECV_IOV | + DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_IOV )) && + ( ep->param.ep_state != DAT_EP_STATE_UNCONNECTED && + ep->param.ep_state != DAT_EP_STATE_RESERVED && + ep->param.ep_state != DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep)); + goto bail; + } + + /* + * Validate handles being modified + */ + if ( ep_param_mask & DAT_EP_FIELD_PZ_HANDLE ) + { + if (ep_param->pz_handle != NULL && + DAPL_BAD_HANDLE (ep_param->pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_RECV_EVD_HANDLE ) + { + if (ep_param->recv_evd_handle != NULL && + (DAPL_BAD_HANDLE (ep_param->recv_evd_handle, DAPL_MAGIC_EVD) || + ! ((DAPL_EVD *)ep_param->recv_evd_handle)->evd_flags & DAT_EVD_DTO_FLAG)) + + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_REQUEST_EVD_HANDLE ) + { + if (ep_param->request_evd_handle != NULL && + DAPL_BAD_HANDLE (ep_param->request_evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_CONNECT_EVD_HANDLE ) + { + if (ep_param->connect_evd_handle != NULL && + DAPL_BAD_HANDLE (ep_param->connect_evd_handle, DAPL_MAGIC_EVD) && + ! (((DAPL_EVD *)ep_param->connect_evd_handle)->evd_flags & DAT_EVD_CONNECTION_FLAG) ) + + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + } + + /* + * Validate the attributes against the HCA limits + */ + ep_attr = ep->param.ep_attr; + + dapl_os_memzero (&ep_attr_limit, sizeof (DAT_EP_ATTR)); + dat_status = dapls_ib_query_hca (ia->hca_ptr, NULL, &ep_attr_limit, NULL); + if ( dat_status != DAT_SUCCESS ) + { + goto bail; + } + + ep_attr_request = ep_param->ep_attr; + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_SERVICE_TYPE ) + { + if ( ep_attr_request.service_type != DAT_SERVICE_TYPE_RC ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_MESSAGE_SIZE ) + { + if ( ep_attr_request.max_mtu_size > ep_attr_limit.max_mtu_size) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_mtu_size = ep_attr_request.max_mtu_size; + } + } + + /* Do nothing if the DAT_EP_FIELD_EP_ATTR_MAX_RDMA_SIZE flag is */ + /* set. Each RDMA transport/provider may or may not have a limit */ + /* on the size of an RDMA DTO. For InfiniBand, this parameter is */ + /* validated in the implementation of the dapls_ib_qp_modify() */ + /* function. */ + /* */ + /* if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RDMA_SIZE ) */ + /* { */ + /* */ + /* } */ + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_QOS ) + { + /* Do nothing, not defined in the spec yet */ + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_RECV_COMPLETION_FLAGS ) + { + dat_status = dapl_ep_check_recv_completion_flags ( + ep_attr_request.recv_completion_flags); + if ( dat_status != DAT_SUCCESS ) + + { + goto bail; + } + else + { + ep_attr.recv_completion_flags = + ep_attr_request.recv_completion_flags; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_REQUEST_COMPLETION_FLAGS ) + { + dat_status = dapl_ep_check_request_completion_flags ( + ep_attr_request.request_completion_flags); + if ( dat_status != DAT_SUCCESS ) + { + goto bail; + } + else + { + ep_attr.request_completion_flags = + ep_attr_request.request_completion_flags; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RECV_DTOS ) + { + if ( ep_attr_request.max_recv_dtos > ep_attr_limit.max_recv_dtos || + ( ep_param->recv_evd_handle == DAT_HANDLE_NULL && + ( ep_attr_request.max_recv_dtos > 0 ) ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_recv_dtos = ep_attr_request.max_recv_dtos; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_DTOS ) + { + if ( ep_attr_request.max_request_dtos > ep_attr_limit.max_request_dtos || + ( ep_param->request_evd_handle == DAT_HANDLE_NULL && + ( ep_attr_request.max_request_dtos > 0 ) ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_request_dtos = ep_attr_request.max_request_dtos; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RECV_IOV ) + { + if ( ep_attr_request.max_recv_iov > ep_attr_limit.max_recv_iov) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_recv_iov = ep_attr_request.max_recv_iov; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_IOV ) + { + if ( ep_attr_request.max_request_iov > ep_attr_limit.max_request_iov) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_request_iov = ep_attr_request.max_request_iov; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RDMA_READ_IOV ) + { + if ( ep_attr_request.max_rdma_read_iov > ep_attr_limit.max_rdma_read_iov) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_rdma_read_iov = ep_attr_request.max_rdma_read_iov; + } + } + + if ( ep_param_mask & DAT_EP_FIELD_EP_ATTR_MAX_RDMA_WRITE_IOV ) + { + if ( ep_attr_request.max_rdma_write_iov > ep_attr_limit.max_rdma_write_iov) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + else + { + ep_attr.max_rdma_write_iov = ep_attr_request.max_rdma_write_iov; + } + } + + *ia_ptr = ia; + *ep_ptr = ep; + *ep_attr_ptr = ep_attr; + +bail: + return dat_status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read.c new file mode 100644 index 00000000..ea003631 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_post_rdma_read.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ep_util.h" + +/* + * dapl_ep_post_rdma_read + * + * DAPL Requirements Version xxx, 6.5.12 + * + * Request the xfer of all data specified by the remote_iov over the + * connection of ep handle Endpint into the local_iov + * + * Input: + * ep_handle + * num_segments + * local_iov + * user_cookie + * remote_iov + * completion_flags + * + * Output: + * None. + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_LENGTH_ERROR + * DAT_PROTECTION_VIOLATION + * DAT_PRIVILEGES_VIOLATION + */ +DAT_RETURN DAT_API +dapl_ep_post_rdma_read ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_post_rdma_read (%p, %d, %p, %p, %p, %x)\n", + ep_handle, + num_segments, + local_iov, + user_cookie.as_64, + remote_iov, + completion_flags); + DAPL_CNTR(DCNT_POST_RDMA_READ); + + dat_status = dapl_ep_post_send_req(ep_handle, + num_segments, + local_iov, + user_cookie, + remote_iov, + completion_flags, + DAPL_DTO_TYPE_RDMA_READ, + OP_RDMA_READ); + + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_ep_post_rdma_read () returns 0x%x\n", + dat_status); + + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read_to_rmr.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read_to_rmr.c new file mode 100644 index 00000000..8f164d76 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_read_to_rmr.c @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_post_rdma_read_rmr.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 2.0 API, Chapter 6, section 6 + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ep_util.h" + +/* + * dapl_ep_post_rdma_read_to_rmr + * + * DAPL Requirements Version xxx, 6.6.24 + * + * Requests the transfer of all the data specified by the remote_buffer + * over the connection of the ep_handle Endpoint into the local_iov + * specified by the RMR segments. + * + * Input: + * ep_handle + * num_segments + * local_iov + * user_cookie + * completion_flags + * invalidate flag + * RMR context + * + * Output: + * None + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + * DAT_LENGTH_ERROR + * DAT_PROTECTION_VIOLATION + * DAT_PRIVILEGES_VIOLATION + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_post_rdma_read_to_rmr ( + IN DAT_EP_HANDLE ep_handle, /* ep_handle */ + IN const DAT_RMR_TRIPLET *local, /* local_iov */ + IN DAT_DTO_COOKIE cookie, /* user_cookie */ + IN const DAT_RMR_TRIPLET *remote, /* remote_iov */ + IN DAT_COMPLETION_FLAGS flags) /* completion_flags */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_write.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_write.c new file mode 100644 index 00000000..4d2f29d3 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_rdma_write.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_post_rdma_write.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ep_util.h" + +/* + * dapl_ep_post_rdma_write + * + * DAPL Requirements Version xxx, 6.5.13 + * + * Request the xfer of all data specified by the local_iov over the + * connection of ep handle Endpint into the remote_iov + * + * Input: + * ep_handle + * num_segments + * local_iov + * user_cookie + * remote_iov + * compltion_flags + * + * Output: + * None. + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_LENGTH_ERROR + * DAT_PROTECTION_VIOLATION + * DAT_PRIVILEGES_VIOLATION + */ +DAT_RETURN DAT_API +dapl_ep_post_rdma_write ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags ) +{ + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_post_rdma_write (%p, %d, %p, %p, %p, %x)\n", + ep_handle, + num_segments, + local_iov, + user_cookie.as_64, + remote_iov, + completion_flags); + DAPL_CNTR(DCNT_POST_RDMA_WRITE); + + dat_status = dapl_ep_post_send_req(ep_handle, + num_segments, + local_iov, + user_cookie, + remote_iov, + completion_flags, + DAPL_DTO_TYPE_RDMA_WRITE, + OP_RDMA_WRITE); + + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_ep_post_rdma_write () returns 0x%x", + dat_status); + + + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_post_recv.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_recv.c new file mode 100644 index 00000000..77a684a1 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_recv.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_post_recv.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_cookie.h" +#include "dapl_adapter_util.h" + +/* + * dapl_ep_post_recv + * + * DAPL Requirements Version xxx, 6.5.11 + * + * Request to receive data over the connection of ep handle into + * local_iov + * + * Input: + * ep_handle + * num_segments + * local_iov + * user_cookie + * completion_flags + * + * Output: + * None. + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_PROTECTION_VIOLATION + * DAT_PROVILEGES_VIOLATION + */ +DAT_RETURN DAT_API +dapl_ep_post_recv ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags ) +{ + DAPL_EP *ep_ptr; + DAPL_COOKIE *cookie; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_post_recv (%p, %d, %p, %p, %x)\n", + ep_handle, + num_segments, + local_iov, + user_cookie.as_64, + completion_flags); + DAPL_CNTR (DCNT_POST_RECV); + + if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Synchronization ok since this buffer is only used for receive + * requests, which aren't allowed to race with each other. + */ + dat_status = dapls_dto_cookie_alloc (&ep_ptr->recv_buffer, + DAPL_DTO_TYPE_RECV, + user_cookie, + &cookie); + if ( DAT_SUCCESS != dat_status) + { + goto bail; + } + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc (&ep_ptr->recv_count); + + /* + * Invoke provider specific routine to post DTO + */ + dat_status = dapls_ib_post_recv (ep_ptr, cookie, num_segments, local_iov); + + if ( dat_status != DAT_SUCCESS ) + { + dapl_os_atomic_dec (&ep_ptr->recv_count); + dapls_cookie_dealloc (&ep_ptr->recv_buffer, cookie); + } + +bail: + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_ep_post_recv () returns 0x%x\n", + dat_status); + + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_post_send.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_send.c new file mode 100644 index 00000000..6055e705 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_send.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_post_send.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id: dapl_ep_post_send.c 1364 2005-10-31 18:30:14Z jlentini $ + **********************************************************************/ + +#include "dapl_ep_util.h" + +/* + * dapl_ep_post_send + * + * DAPL Requirements Version xxx, 6.5.10 + * + * Request a transfer of all the data from the local_iov over + * the connection of the ep handle Endpoint to the remote side. + * + * Input: + * ep_handle + * num_segments + * local_iov + * user_cookie + * completion_flags + * + * Output: + * None + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_PROTECTION_VIOLATION + * DAT_PRIVILEGES_VIOLATION + */ +DAT_RETURN DAT_API +dapl_ep_post_send ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags ) +{ + DAT_RMR_TRIPLET remote_iov = {0,0,0}; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_post_send (%p, %d, %p, %p, %x)\n", + ep_handle, + num_segments, + local_iov, + user_cookie.as_64, + completion_flags); + DAPL_CNTR(DCNT_POST_SEND); + + dat_status = dapl_ep_post_send_req(ep_handle, + num_segments, + local_iov, + user_cookie, + &remote_iov, + completion_flags, + DAPL_DTO_TYPE_SEND, + OP_SEND); + + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_ep_post_send () returns 0x%x\n", + dat_status); + + + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_post_send_invalidate.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_send_invalidate.c new file mode 100644 index 00000000..d9e049f9 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_post_send_invalidate.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_post_send_invalidate.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 2.0 API, Chapter 6, section 6 + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ep_util.h" + +/* + * dapl_ep_post_send_invalidate + * + * DAPL Requirements Version xxx, 6.6.21 + * + * Requests a transfer of all the data from the local_iov over the connection + * of the ep_handle Endpoint to the remote side and invalidates the Remote + * Memory Region context. + * + * Input: + * ep_handle + * num_segments + * local_iov + * user_cookie + * completion_flags + * invalidate flag + * RMR context + * + * Output: + * None + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + * DAT_PROTECTION_VIOLATION + * DAT_PRIVILEGES_VIOLATION + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_post_send_with_invalidate ( + IN DAT_EP_HANDLE ep_handle, /* ep_handle */ + IN DAT_COUNT num_segments, /* num_segments */ + IN DAT_LMR_TRIPLET *local_iov, /* local_iov */ + IN DAT_DTO_COOKIE user_cookie, /* user_cookie */ + IN DAT_COMPLETION_FLAGS flags, /* completion_flags */ + IN DAT_BOOLEAN invalidate_flag, /* invalidate_flag */ + IN DAT_RMR_CONTEXT rmr_context) /* RMR context */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_query.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_query.c new file mode 100644 index 00000000..47f495e4 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_query.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_query.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * dapl_ep_query + * + * DAPL Requirements Version xxx, 6.5.5 + * + * Provide the consumer parameters, including attributes and status of + * the Endpoint. + * + * Input: + * ep_handle + * ep_param_mask + * + * Output: + * ep_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_ep_query ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_PARAM_MASK ep_param_mask, + OUT DAT_EP_PARAM *ep_param ) +{ + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ep_query (%p, %x, %p)\n", + ep_handle, + ep_param_mask, + ep_param); + + dat_status = DAT_SUCCESS; + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + + if ( ep_param == NULL) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + /* + * Fill in according to user request + * + * N.B. Just slam all values into the user structure, there + * is nothing to be gained by checking for each bit. + */ + if ( ep_param_mask & DAT_EP_FIELD_ALL ) + { + /* only attempt to get remote IA address if consumer requested it */ + if ( ep_param_mask & DAT_EP_FIELD_REMOTE_IA_ADDRESS_PTR ) + { + if ( ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED ) + { + /* obtain the remote IP address */ + dat_status = dapls_ib_cm_remote_addr ((DAT_HANDLE)ep_handle, + &ep_ptr->remote_ia_address ); + } + ep_ptr->param.remote_ia_address_ptr = + (DAT_IA_ADDRESS_PTR) &ep_ptr->remote_ia_address; + } + *ep_param = ep_ptr->param; + } + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_recv_query.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_recv_query.c new file mode 100644 index 00000000..53b8b463 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_recv_query.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_recv_query.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.2 API, Chapter 6, section 6.11 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ep_util.h" + +/* + * dapl_ep_recv_query + * + * uDAPL Version 1.2, 6.6.11 + * + * Destroy an instance of the Endpoint + * + * Input: + * ep_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_recv_query ( + IN DAT_EP_HANDLE ep_handle, + OUT DAT_COUNT *nbufs_allocate, + OUT DAT_COUNT *bufs_alloc_span) +{ + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_ep_recv_query (%p, %p, %p)\n", + ep_handle, nbufs_allocate, bufs_alloc_span); + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + +bail: + return dat_status; + +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_reset.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_reset.c new file mode 100644 index 00000000..4e5bdcdd --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_reset.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_reset.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5.13 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" +#include "dapl_ring_buffer_util.h" + +/* + * dapl_ep_reset + * + * DAPL Requirements Version 1.1, 6.5.13 + * + * Reset the QP attached to this Endpoint, transitioning back to the + * INIT state + * + * Input: + * ep_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + */ +DAT_RETURN DAT_API +dapl_ep_reset ( + IN DAT_EP_HANDLE ep_handle) +{ + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + + if ( ep_ptr->param.ep_state != DAT_EP_STATE_UNCONNECTED + && ep_ptr->param.ep_state != DAT_EP_STATE_DISCONNECTED ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED ) + { + dapls_ib_reinit_ep ( ep_ptr ); + ep_ptr->param.ep_state = DAT_EP_STATE_UNCONNECTED; + } + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_set_watermark.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_set_watermark.c new file mode 100644 index 00000000..23534b54 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_set_watermark.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_set_watermark.c + * + * PURPOSE: Endpoint management + * Description: Interfaces in this file are completely described in + * the DAPL 1.2 API, Chapter 6, section 6.13 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ep_util.h" + +/* + * dapl_ep_set_watermark + * + * DAPL Requirements Version 1.2, 6.6.13 + * + * Sets the watermark values for an EP for generating asynchronous + * events. + * + * Input: + * ep_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_ep_set_watermark ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT soft_high_watermark, + IN DAT_COUNT hard_high_watermark) +{ + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_ep_set_watermark (%p, %d, %d)\n", + ep_handle, soft_high_watermark, hard_high_watermark); + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + dat_status = DAT_NOT_IMPLEMENTED; + +bail: + return dat_status; + +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_util.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_util.c new file mode 100644 index 00000000..4d00926d --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_util.c @@ -0,0 +1,676 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_util.c + * + * PURPOSE: Manage EP Info structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ep_util.h" +#include "dapl_ring_buffer_util.h" +#include "dapl_cookie.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" /* for callback routine */ + +/* + * Local definitions + */ +/* + * Default number of I/O operations on an end point + */ +#define IB_IO_DEFAULT 16 +/* + * Default number of scatter/gather entries available to a single + * post send/recv + */ +#define IB_IOV_DEFAULT 4 + +/* + * Default number of RDMA operations in progress at a time + */ +#define IB_RDMA_DEFAULT 8 + +extern void dapli_ep_default_attrs ( + IN DAPL_EP *ep_ptr ); + + + +char *dapl_get_ep_state_str(DAT_EP_STATE state) +{ +#ifdef DAPL_DBG + static char *state_str[DAT_EP_STATE_CONNECTED_MULTI_PATH+1] = { + "DAT_EP_STATE_UNCONNECTED", /* quiescent state */ + "DAT_EP_STATE_UNCONFIGURED_UNCONNECTED", + "DAT_EP_STATE_RESERVED", + "DAT_EP_STATE_UNCONFIGURED_RESERVED", + "DAT_EP_STATE_PASSIVE_CONNECTION_PENDING", + "DAT_EP_STATE_UNCONFIGURED_PASSIVE", + "DAT_EP_STATE_ACTIVE_CONNECTION_PENDING", + "DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING", + "DAT_EP_STATE_UNCONFIGURED_TENTATIVE", + "DAT_EP_STATE_CONNECTED", + "DAT_EP_STATE_DISCONNECT_PENDING", + "DAT_EP_STATE_DISCONNECTED", + "DAT_EP_STATE_COMPLETION_PENDING", + "DAT_EP_STATE_CONNECTED_SINGLE_PATH", + "DAT_EP_STATE_CONNECTED_MULTI_PATH" + }; + return state_str[state]; +#else + static char buf[12]; + sprintf(buf,"%d",state); + return buf; +#endif +} + + +/* + * dapl_ep_alloc + * + * alloc and initialize an EP INFO struct + * + * Input: + * IA INFO struct ptr + * + * Output: + * ep_ptr + * + * Returns: + * none + * + */ +DAPL_EP * +dapl_ep_alloc ( + IN DAPL_IA *ia_ptr, + IN const DAT_EP_ATTR *ep_attr ) +{ + DAPL_EP *ep_ptr; + + /* Allocate EP */ + ep_ptr = (DAPL_EP *)dapl_os_alloc (sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR)); + if ( ep_ptr == NULL ) + { + goto bail; + } + + /* zero the structure */ + dapl_os_memzero (ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR)); + + /* + * initialize the header + */ + ep_ptr->header.provider = ia_ptr->header.provider; + ep_ptr->header.magic = DAPL_MAGIC_EP; + ep_ptr->header.handle_type = DAT_HANDLE_TYPE_EP; + ep_ptr->header.owner_ia = ia_ptr; + ep_ptr->header.user_context.as_64 = 0; + ep_ptr->header.user_context.as_ptr = NULL; + + dapl_llist_init_entry (&ep_ptr->header.ia_list_entry); + dapl_os_lock_init (&ep_ptr->header.lock); + + /* + * Initialize the body + */ + /* + * Set up default parameters if the user passed in a NULL + */ + if ( ep_attr == NULL ) + { + dapli_ep_default_attrs (ep_ptr); + } + else + { + ep_ptr->param.ep_attr = *ep_attr; + } + + /* + * IBM OS API specific fields + */ + ep_ptr->qp_handle = IB_INVALID_HANDLE; + ep_ptr->qpn = 0; + ep_ptr->qp_state = DAPL_QP_STATE_UNATTACHED; + ep_ptr->cm_handle = IB_INVALID_HANDLE; + + dapl_os_atomic_set (&ep_ptr->req_count, 0); + dapl_os_atomic_set (&ep_ptr->recv_count, 0); + + if ( DAT_SUCCESS != dapls_cb_create ( + &ep_ptr->req_buffer, + ep_ptr, + ep_ptr->param.ep_attr.max_request_dtos) ) + { + dapl_ep_dealloc (ep_ptr); + ep_ptr = NULL; + goto bail; + } + + if ( DAT_SUCCESS != dapls_cb_create ( + &ep_ptr->recv_buffer, + ep_ptr, + ep_ptr->param.ep_attr.max_recv_dtos) ) + { + dapl_ep_dealloc (ep_ptr); + ep_ptr = NULL; + goto bail; + } + + ep_ptr->recv_iov_num = ep_ptr->param.ep_attr.max_recv_iov; + ep_ptr->send_iov_num = ep_ptr->param.ep_attr.max_request_iov; + + /* + * Ensure that send_iov_num is big enough for all types + * of send requests. + */ + if (ep_ptr->param.ep_attr.max_rdma_read_iov > ep_ptr->send_iov_num) + { + ep_ptr->send_iov_num = ep_ptr->param.ep_attr.max_rdma_read_iov; + } + if (ep_ptr->param.ep_attr.max_rdma_write_iov > ep_ptr->send_iov_num) + { + ep_ptr->send_iov_num = ep_ptr->param.ep_attr.max_rdma_write_iov; + } + + ep_ptr->recv_iov = dapl_os_alloc ( + ep_ptr->recv_iov_num * sizeof (ib_data_segment_t)); + + if ( NULL == ep_ptr->recv_iov ) + { + dapl_ep_dealloc (ep_ptr); + ep_ptr = NULL; + goto bail; + } + + ep_ptr->send_iov = dapl_os_alloc ( + ep_ptr->send_iov_num * sizeof (ib_data_segment_t)); + + if ( NULL == ep_ptr->send_iov ) + { + dapl_ep_dealloc (ep_ptr); + ep_ptr = NULL; + goto bail; + } + + dapls_io_trc_alloc (ep_ptr); + +bail: + return ep_ptr; +} + + +/* + * dapl_ep_dealloc + * + * Free the passed in EP structure. + * + * Input: + * entry point pointer + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ep_dealloc ( + IN DAPL_EP *ep_ptr ) +{ + dapl_os_assert (ep_ptr->header.magic == DAPL_MAGIC_EP); + + ep_ptr->header.magic = DAPL_MAGIC_INVALID;/* reset magic to prevent reuse */ + + dapls_cb_free ( &ep_ptr->req_buffer ); + dapls_cb_free ( &ep_ptr->recv_buffer ); + + if ( NULL != ep_ptr->recv_iov ) + { + dapl_os_free (ep_ptr->recv_iov, ep_ptr->recv_iov_num * sizeof (ib_data_segment_t)); + } + + if ( NULL != ep_ptr->send_iov ) + { + dapl_os_free (ep_ptr->send_iov, ep_ptr->send_iov_num * sizeof (ib_data_segment_t)); + } + + if ( NULL != ep_ptr->cxn_timer ) + { + dapl_os_free ( ep_ptr->cxn_timer, sizeof ( DAPL_OS_TIMER ) ); + } + +#if defined(_WIN32) || defined(_WIN64) + if ( ep_ptr->ibal_cm_handle ) + { + dapl_os_free ( ep_ptr->ibal_cm_handle, + sizeof ( *ep_ptr->ibal_cm_handle ) ); + ep_ptr->ibal_cm_handle = NULL; + } +#endif + dapl_os_free (ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR) ); +} + + +/* + * dapl_ep_default_attrs + * + * Set default values in the parameter fields + * + * Input: + * entry point pointer + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapli_ep_default_attrs ( + IN DAPL_EP *ep_ptr ) +{ + DAT_EP_ATTR ep_attr_limit; + DAT_EP_ATTR *ep_attr; + DAT_RETURN dat_status; + + ep_attr = &ep_ptr->param.ep_attr; + /* Set up defaults */ + dapl_os_memzero (ep_attr, sizeof (DAT_EP_ATTR)); + + /* mtu and rdma sizes fixed in IB as per IBTA 1.1, 9.4.3, 9.4.4, 9.7.7. */ + ep_attr->max_mtu_size = 0x80000000; + ep_attr->max_rdma_size = 0x80000000; + + ep_attr->qos = DAT_QOS_BEST_EFFORT; + ep_attr->service_type = DAT_SERVICE_TYPE_RC; + ep_attr->max_recv_dtos = IB_IO_DEFAULT; + ep_attr->max_request_dtos = IB_IO_DEFAULT; + ep_attr->max_recv_iov = IB_IOV_DEFAULT; + ep_attr->max_request_iov = IB_IOV_DEFAULT; + ep_attr->max_rdma_read_in = IB_RDMA_DEFAULT; + ep_attr->max_rdma_read_out= IB_RDMA_DEFAULT; + + /* + * Configure the EP as a standard completion type, which will be + * used by the EVDs. A threshold of 1 is the default state of an + * EVD. + */ + ep_attr->request_completion_flags = DAT_COMPLETION_EVD_THRESHOLD_FLAG; + ep_attr->recv_completion_flags = DAT_COMPLETION_EVD_THRESHOLD_FLAG; + /* + * Unspecified defaults: + * - ep_privileges: No RDMA capabilities + * - num_transport_specific_params: none + * - transport_specific_params: none + * - num_provider_specific_params: 0 + * - provider_specific_params: 0 + */ + + dat_status = dapls_ib_query_hca (ep_ptr->header.owner_ia->hca_ptr, + NULL, &ep_attr_limit, NULL); + /* check against HCA maximums */ + if (dat_status == DAT_SUCCESS) + { + ep_ptr->param.ep_attr.max_mtu_size = + DAPL_MIN(ep_ptr->param.ep_attr.max_mtu_size, + ep_attr_limit.max_mtu_size); + ep_ptr->param.ep_attr.max_rdma_size = + DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_size, + ep_attr_limit.max_rdma_size); + ep_ptr->param.ep_attr.max_recv_dtos = + DAPL_MIN(ep_ptr->param.ep_attr.max_recv_dtos, + ep_attr_limit.max_recv_dtos); + ep_ptr->param.ep_attr.max_request_dtos = + DAPL_MIN(ep_ptr->param.ep_attr.max_request_dtos, + ep_attr_limit.max_request_dtos); + ep_ptr->param.ep_attr.max_recv_iov = + DAPL_MIN(ep_ptr->param.ep_attr.max_recv_iov, + ep_attr_limit.max_recv_iov); + ep_ptr->param.ep_attr.max_request_iov = + DAPL_MIN(ep_ptr->param.ep_attr.max_request_iov, + ep_attr_limit.max_request_iov); + ep_ptr->param.ep_attr.max_rdma_read_in = + DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_in, + ep_attr_limit.max_rdma_read_in); + ep_ptr->param.ep_attr.max_rdma_read_out = + DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_out, + ep_attr_limit.max_rdma_read_out); + } +} + + +DAT_RETURN +dapl_ep_check_recv_completion_flags ( + DAT_COMPLETION_FLAGS flags ) +{ + + /* + * InfiniBand will not allow signal suppression for RECV completions, + * see the 1.0.1 spec section 10.7.3.1, 10.8.6. + * N.B. SIGNALLED has a different meaning in dapl than it does + * in IB; IB SIGNALLED is the same as DAPL SUPPRESS. DAPL + * SIGNALLED simply means the user will not get awakened when + * an EVD completes, even though the dapl handler is invoked. + */ + + if (flags & DAT_COMPLETION_SUPPRESS_FLAG) + { + return DAT_INVALID_PARAMETER; + } + + return DAT_SUCCESS; +} + +DAT_RETURN +dapl_ep_check_request_completion_flags ( + DAT_COMPLETION_FLAGS flags ) +{ + return DAT_SUCCESS; +} + + +DAT_RETURN +dapl_ep_post_send_req ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags, + IN DAPL_DTO_TYPE dto_type, + IN int op_type) +{ + DAPL_EP *ep_ptr; + DAPL_COOKIE *cookie; + DAT_RETURN dat_status; + + if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + goto bail; + } + + ep_ptr = (DAPL_EP *) ep_handle; + + /* + * Synchronization ok since this buffer is only used for send + * requests, which aren't allowed to race with each other. + */ + dat_status = dapls_dto_cookie_alloc ( + &ep_ptr->req_buffer, + dto_type, + user_cookie, + &cookie ); + if ( dat_status != DAT_SUCCESS ) + { + goto bail; + } + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc (&ep_ptr->req_count); + + /* + * Invoke provider specific routine to post DTO + */ + dat_status = dapls_ib_post_send ( ep_ptr, + op_type, + cookie, + num_segments, + local_iov, + remote_iov, + completion_flags ); + + if ( dat_status != DAT_SUCCESS ) + { + dapl_os_atomic_dec (&ep_ptr->req_count); + dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie); + } + +bail: + return dat_status; +} + + +/* + * dapli_ep_timeout + * + * If this routine is invoked before a connection occurs, generate an + * event + */ +void +dapls_ep_timeout ( + uintptr_t arg ) +{ + DAPL_EP *ep_ptr; + ib_cm_events_t ib_cm_event; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> dapls_ep_timeout! ep %lx\n", arg); + + ep_ptr = (DAPL_EP *)arg; + + /* reset the EP state */ + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + + /* Clean up the EP and put the underlying QP into the ERROR state. + * The disconnect_clean interface requires the provided dependent + *cm event number. + */ + ib_cm_event = dapls_ib_get_cm_event (DAT_CONNECTION_EVENT_DISCONNECTED); + dapls_ib_disconnect_clean ( ep_ptr, + DAT_TRUE, + ib_cm_event ); + + (void) dapls_evd_post_connection_event ( + (DAPL_EVD *)ep_ptr->param.connect_evd_handle, + DAT_CONNECTION_EVENT_TIMED_OUT, + (DAT_HANDLE) ep_ptr, + 0, + 0); +} + + +/* + * dapls_ep_state_subtype + * + * Return the INVALID_STATE connection subtype associated with an + * INVALID_STATE on an EP. Strictly for error reporting. + */ +DAT_RETURN_SUBTYPE +dapls_ep_state_subtype ( + IN DAPL_EP *ep_ptr ) +{ + DAT_RETURN_SUBTYPE dat_status; + + switch ( ep_ptr->param.ep_state ) + { + case DAT_EP_STATE_UNCONNECTED: + { + dat_status = DAT_INVALID_STATE_EP_UNCONNECTED; + break; + } + case DAT_EP_STATE_RESERVED: + { + dat_status = DAT_INVALID_STATE_EP_RESERVED; + break; + } + case DAT_EP_STATE_PASSIVE_CONNECTION_PENDING: + { + dat_status = DAT_INVALID_STATE_EP_PASSCONNPENDING; + break; + } + case DAT_EP_STATE_ACTIVE_CONNECTION_PENDING: + { + dat_status = DAT_INVALID_STATE_EP_ACTCONNPENDING; + break; + } + case DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING: + { + dat_status = DAT_INVALID_STATE_EP_TENTCONNPENDING; + break; + } + case DAT_EP_STATE_CONNECTED: + { + dat_status = DAT_INVALID_STATE_EP_CONNECTED; + break; + } + case DAT_EP_STATE_DISCONNECT_PENDING: + { + dat_status = DAT_INVALID_STATE_EP_DISCPENDING; + break; + } + case DAT_EP_STATE_DISCONNECTED: + { + dat_status = DAT_INVALID_STATE_EP_DISCONNECTED; + break; + } + case DAT_EP_STATE_COMPLETION_PENDING: + { + dat_status = DAT_INVALID_STATE_EP_COMPLPENDING; + break; + } + + default: + { + dat_status = 0; + break; + } + } + + return dat_status; +} + +#ifdef DAPL_DBG_IO_TRC +/* allocate trace buffer */ +void +dapls_io_trc_alloc ( + DAPL_EP *ep_ptr) +{ + DAT_RETURN dat_status; + int i; + struct io_buf_track *ibt; + + ep_ptr->ibt_dumped = 0; /* bool to control how often we print */ + dat_status = dapls_rbuf_alloc (&ep_ptr->ibt_queue, DBG_IO_TRC_QLEN); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + ibt = (struct io_buf_track *)dapl_os_alloc (sizeof (struct io_buf_track) * DBG_IO_TRC_QLEN); + + if (dat_status != DAT_SUCCESS) + { + dapls_rbuf_destroy (&ep_ptr->ibt_queue); + goto bail; + } + ep_ptr->ibt_base = ibt; + dapl_os_memzero (ibt, sizeof (struct io_buf_track) * DBG_IO_TRC_QLEN); + + /* add events to free event queue */ + for (i = 0; i < DBG_IO_TRC_QLEN; i++) + { + dapls_rbuf_add (&ep_ptr->ibt_queue, ibt++); + } +bail: + return; +} +#endif /* DAPL_DBG_IO_TRC */ + +/* + * Generate a disconnect event on abruct close for older verbs providers + * that do not do it automatically. + */ + +void +dapl_ep_legacy_post_disconnect( + DAPL_EP *ep_ptr, + DAT_CLOSE_FLAGS disconnect_flags) +{ + ib_cm_events_t ib_cm_event; + DAPL_CR *cr_ptr; + + /* + * Acquire the lock and make sure we didn't get a callback + * that cleaned up. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + if (disconnect_flags == DAT_CLOSE_ABRUPT_FLAG && + ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING ) + { + /* + * If this is an ABRUPT close, the provider will not generate + * a disconnect message so we do it manually here. Just invoke + * the CM callback as it will clean up the appropriate + * data structures, reset the state, and generate the event + * on the way out. Obtain the provider dependent cm_event to + * pass into the callback for a disconnect. + */ + ib_cm_event = dapls_ib_get_cm_event (DAT_CONNECTION_EVENT_DISCONNECTED); + + cr_ptr = ep_ptr->cr_ptr; + dapl_os_unlock ( &ep_ptr->header.lock ); + + if (cr_ptr != NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_API | DAPL_DBG_TYPE_CM, + " dapl_ep_disconnect force callback on EP %p CM handle %x\n", + ep_ptr, cr_ptr->ib_cm_handle); + + dapls_cr_callback (cr_ptr->ib_cm_handle, + ib_cm_event, + NULL, + cr_ptr->sp_ptr); + } + else + { + dapl_evd_connection_callback (ep_ptr->cm_handle, + ib_cm_event, + NULL, + (void *) ep_ptr); + } + } + else + { + dapl_os_unlock ( &ep_ptr->header.lock ); + } +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_util.h b/trunk/ulp/dapl2/dapl/common/dapl_ep_util.h new file mode 100644 index 00000000..1361d768 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_util.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_ep_util.h + * + * PURPOSE: Utility defs & routines for the EP data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_EP_UTIL_H_ +#define _DAPL_EP_UTIL_H_ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* function prototypes */ + +extern DAPL_EP * +dapl_ep_alloc ( + IN DAPL_IA *ia, + IN const DAT_EP_ATTR *ep_attr ); + +extern void +dapl_ep_dealloc ( + IN DAPL_EP *ep_ptr ); + +extern DAT_RETURN +dapl_ep_check_recv_completion_flags ( + DAT_COMPLETION_FLAGS flags ); + +extern DAT_RETURN +dapl_ep_check_request_completion_flags ( + DAT_COMPLETION_FLAGS flags ); + +extern DAT_RETURN +dapl_ep_post_send_req ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags, + IN DAPL_DTO_TYPE dto_type, + IN int op_type ); + +void dapls_ep_timeout (uintptr_t arg ); + +DAT_RETURN_SUBTYPE +dapls_ep_state_subtype( + IN DAPL_EP *ep_ptr ); + +extern void +dapl_ep_legacy_post_disconnect( + DAPL_EP *ep_ptr, + DAT_CLOSE_FLAGS disconnect_flags); + +extern char *dapl_get_ep_state_str(DAT_EP_STATE state); + +#endif /* _DAPL_EP_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_connection_callb.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_connection_callb.c new file mode 100644 index 00000000..c7faa681 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_connection_callb.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_connection_callback.c + * + * PURPOSE: implements connection callbacks + * + * Description: Accepts asynchronous callbacks from the Communications Manager + * for EVDs that have been specified as the connection_evd. + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_ep_util.h" +#include "dapl_timer_util.h" + + +/* + * dapl_evd_connection_callback + * + * Connection callback function for ACTIVE connection requests; callbacks + * generated by the Connection Manager in response to issuing a + * connect call. + * + * Input: + * ib_cm_handle, + * ib_cm_event + * private_data_ptr + * context (evd) + * cr_pp + * + * Output: + * None + * + */ + +void +dapl_evd_connection_callback ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN const ib_cm_events_t ib_cm_event, + IN const void *private_data_ptr, + IN const void *context) +{ + DAPL_EP *ep_ptr; + DAPL_EVD *evd_ptr; + DAPL_PRIVATE *prd_ptr; + DAT_EVENT_NUMBER dat_event_num; + DAT_RETURN dat_status; + int private_data_size; + + + dapl_dbg_log ( + DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> dapl_evd_connection_callback: ctxt: %p event: %x cm_handle %p\n", + context, + ib_cm_event, + (void *) ib_cm_handle ); + + DAPL_CNTR(DCNT_EVD_CONN_CALLBACK); + + /* + * Determine the type of handle passed back to us in the context + * and sort out key parameters. + */ + if ( context == NULL || ((DAPL_HEADER *)context)->magic != DAPL_MAGIC_EP) + { + return; + } + + /* + * Active side of the connection, context is an EP and + * PSP is irrelevant. + */ + ep_ptr = (DAPL_EP *) context; + evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle; + + prd_ptr = (DAPL_PRIVATE *)private_data_ptr; + private_data_size = 0; + /* + * All operations effect the EP, so lock it once and unlock + * when necessary + */ + dapl_os_lock (&ep_ptr->header.lock); + + /* + * If a connection timer has been set up on this EP, cancel it now + */ + if ( ep_ptr->cxn_timer != NULL ) + { + dapls_timer_cancel ( ep_ptr->cxn_timer ); + dapl_os_free ( ep_ptr->cxn_timer, sizeof ( DAPL_OS_TIMER ) ); + ep_ptr->cxn_timer = NULL; + } + + /* Obtain the event number from the provider layer */ + dat_event_num = dapls_ib_get_dat_event (ib_cm_event, DAT_FALSE); + + switch (dat_event_num) + { + case DAT_CONNECTION_EVENT_ESTABLISHED: + { + /* If we don't have an EP at this point we are very screwed + * up + */ + if ( ep_ptr->param.ep_state != DAT_EP_STATE_ACTIVE_CONNECTION_PENDING) + { + /* If someone pulled the plug on the connection, just + * exit + */ + dapl_os_unlock ( &ep_ptr->header.lock ); + dat_status = DAT_SUCCESS; + break; + } + ep_ptr->param.ep_state = DAT_EP_STATE_CONNECTED; + ep_ptr->cm_handle = ib_cm_handle; + if (prd_ptr == NULL) + { + private_data_size = 0; + } + else + { + private_data_size = + dapls_ib_private_data_size (prd_ptr, DAPL_PDATA_CONN_REP); + } + + if (private_data_size > 0) + { + /* copy in the private data */ + dapl_os_memcpy ( ep_ptr->private.private_data, + prd_ptr->private_data, + DAPL_MIN (private_data_size, DAPL_MAX_PRIVATE_DATA_SIZE)); + } + dapl_os_unlock (&ep_ptr->header.lock); + + break; + } + case DAT_CONNECTION_EVENT_DISCONNECTED: + case DAT_CONNECTION_EVENT_PEER_REJECTED: + case DAT_CONNECTION_EVENT_UNREACHABLE: + case DAT_CONNECTION_EVENT_NON_PEER_REJECTED: + { + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + dapls_ib_disconnect_clean (ep_ptr, DAT_TRUE, ib_cm_event); + dapl_os_unlock (&ep_ptr->header.lock); + + break; + } + case DAT_CONNECTION_EVENT_BROKEN: + case DAT_CONNECTION_EVENT_TIMED_OUT: + { + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + dapls_ib_disconnect_clean (ep_ptr, DAT_FALSE, ib_cm_event); + dapl_os_unlock ( &ep_ptr->header.lock ); + + break; + } + case DAT_CONNECTION_REQUEST_EVENT: + default: + { + dapl_os_unlock (&ep_ptr->header.lock); + evd_ptr = NULL; + + dapl_os_assert (0); /* shouldn't happen */ + break; + } + } + + /* + * Post the event + * If the EP has been freed, the evd_ptr will be NULL + */ + if ( evd_ptr != NULL ) + { + dat_status = dapls_evd_post_connection_event ( + evd_ptr, + dat_event_num, + (DAT_HANDLE) ep_ptr, + private_data_size, /* 0 except for CONNECTED */ + ep_ptr->private.private_data ); + + if (dat_status != DAT_SUCCESS && + dat_event_num == DAT_CONNECTION_EVENT_ESTABLISHED) + { + /* We can't tell the user we are connected, something + * is wrong locally. Just kill the connection and + * reset the state to DISCONNECTED as we don't + * expect a callback on an ABRUPT disconnect. + */ + dapls_ib_disconnect (ep_ptr, DAT_CLOSE_ABRUPT_FLAG); + dapl_os_lock (&ep_ptr->header.lock); + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + dapl_os_unlock (&ep_ptr->header.lock); + } + } + + dapl_dbg_log (DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "dapl_evd_connection_callback () returns\n"); + +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_cq_async_error_callb.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_cq_async_error_callb.c new file mode 100644 index 00000000..c5d0cdac --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_cq_async_error_callb.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_cq_async_error_callback.c + * + * PURPOSE: implements CQ async_callbacks from verbs + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" + +/* + * dapl_evd_cq_async_error_callback + * + * The callback function registered with verbs for cq async errors + * + * Input: + * ib_cm_handle, + * ib_cm_event + * cause_ptr + * context (evd) + * + * Output: + * None + * + */ + +void +dapl_evd_cq_async_error_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_cq_handle_t ib_cq_handle, + IN ib_error_record_t * cause_ptr, + IN void * context) + +{ + DAPL_EVD *async_evd; + DAPL_EVD *evd; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION , + "dapl_evd_cq_async_error_callback (%p, %p, %p, %p)\n", + ib_hca_handle, + ib_cq_handle, + cause_ptr, + context); + + if ( NULL == context ) + { + dapl_os_panic ("NULL == context\n"); + } + + evd = (DAPL_EVD *) context; + async_evd = evd->header.owner_ia->async_error_evd; + + dat_status = dapls_evd_post_async_error_event( + async_evd, + DAT_ASYNC_ERROR_EVD_OVERFLOW, + (DAT_IA_HANDLE) async_evd->header.owner_ia); + + + if ( dat_status != DAT_SUCCESS ) + { + dapl_os_panic ("async EVD overflow\n"); + } + + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION , + "dapl_evd_cq_async_error_callback () returns\n"); +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_dequeue.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_dequeue.c new file mode 100644 index 00000000..6f5274bb --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_dequeue.c @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_dequeue.c + * + * PURPOSE: Event Management + * + * Description: Interfaces in this file are completely described in + * the uDAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ring_buffer_util.h" +#include "dapl_evd_util.h" + +/* + * dapl_evd_dequeue + * + * DAPL Requirements Version xxx, 6.3.2.7 + * + * Remove first element from an event dispatcher + * + * Input: + * evd_handle + * + * Output: + * event + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_QUEUE_EMPTY + */ + +DAT_RETURN DAT_API dapl_evd_dequeue ( + IN DAT_EVD_HANDLE evd_handle, + OUT DAT_EVENT *event) + +{ + DAPL_EVD *evd_ptr; + DAT_EVENT *local_event; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_evd_dequeue (%p, %p)\n", + evd_handle, + event); + DAPL_CNTR(DCNT_EVD_DEQUEUE); + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + + if (event == NULL) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + /* + * We need to dequeue under lock, as the IB OS Access API + * restricts us from having multiple threads in CQ poll, and the + * DAPL 1.1 API allows multiple threads in dat_evd_dequeue() + */ + dapl_os_lock ( &evd_ptr->header.lock ); + + /* + * Make sure there are no other waiters and the evd is active. + * Currently this means only the OPEN state is allowed. + */ + if (evd_ptr->evd_state != DAPL_EVD_STATE_OPEN || + evd_ptr->catastrophic_overflow) + { + dapl_os_unlock ( &evd_ptr->header.lock ); + dat_status = DAT_ERROR (DAT_INVALID_STATE,0); + goto bail; + } + + /* + * Try the EVD rbuf first; poll from the CQ only if that's empty. + * This keeps events in order if dat_evd_wait() has copied events + * from CQ to EVD. + */ + local_event = (DAT_EVENT *)dapls_rbuf_remove (&evd_ptr->pending_event_queue); + if (local_event != NULL) + { + *event = *local_event; + dat_status = dapls_rbuf_add (&evd_ptr->free_event_queue, + local_event); + DAPL_CNTR(DCNT_EVD_DEQUEUE_FOUND); + } + else if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) + { + dat_status = dapls_evd_cq_poll_to_event(evd_ptr, event); + DAPL_CNTR(DCNT_EVD_DEQUEUE_POLL); + } + else + { + dat_status = DAT_ERROR (DAT_QUEUE_EMPTY,0); + DAPL_CNTR(DCNT_EVD_DEQUEUE_NOT_FOUND); + } + + dapl_os_unlock ( &evd_ptr->header.lock ); + bail: + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_evd_dequeue () returns 0x%x\n", + dat_status); + + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_dto_callb.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_dto_callb.c new file mode 100644 index 00000000..b354e2a6 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_dto_callb.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_dto_callback.c + * + * PURPOSE: implements DTO callbacks from verbs + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_cno_util.h" +#include "dapl_cookie.h" +#include "dapl_adapter_util.h" + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + + +/********************************************************************* + * * + * Function Definitions * + * * + *********************************************************************/ + +/* + * dapl_evd_dto_callback + * + * Input: + * hca_handle_in, + * cq_handle_in, + * user_context_cq_p + * + * Output: + * none + * + * This is invoked for both DTO and MW bind completions. Strictly + * speaking it is an event callback rather than just a DTO callback. + * + */ + +void +dapl_evd_dto_callback ( + IN ib_hca_handle_t hca_handle, + IN ib_cq_handle_t cq_handle, + IN void* user_context) +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + DAPL_EVD_STATE state; + + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "dapl_evd_dto_callback(%p, %p, %p)\n", + hca_handle, + cq_handle, + user_context); + DAPL_CNTR(DCNT_EVD_DTO_CALLBACK); + + evd_ptr = (DAPL_EVD *) user_context; + + dapl_os_assert (hca_handle == evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle); + dapl_os_assert (evd_ptr->ib_cq_handle == cq_handle); + dapl_os_assert (evd_ptr->header.magic == DAPL_MAGIC_EVD); + + /* Read once. */ + state = *(volatile DAPL_EVD_STATE *) &evd_ptr->evd_state; + + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + "-- dapl_evd_dto_callback: CQ %p, state %x\n", + (void *)evd_ptr->ib_cq_handle, + state); + + /* + * This function does not dequeue from the CQ; only the consumer + * can do that. Instead, it wakes up waiters if any exist. + * It rearms the completion only if completions should always occur + * (specifically if a CNO is associated with the EVD and the + * EVD is enabled. + */ + + if (state == DAPL_EVD_STATE_WAITED) + { + /* + * If we could, it would be best to avoid this wakeup + * (and the context switch) unless the number of events/CQs + * waiting for the waiter was its threshold. We don't + * currently have the ability to determine that without + * dequeueing the events, and we can't do that for + * synchronization reasons (racing with the waiter waking + * up and dequeuing, sparked by other callbacks). + */ + + /* + * We don't need to worry about taking the lock for the + * wakeup because wakeups are sticky. + */ +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + dapls_ib_wait_object_wakeup (evd_ptr->cq_wait_obj_handle); + else +#endif + dapl_os_wait_object_wakeup (&evd_ptr->wait_object); + + } + else if (state == DAPL_EVD_STATE_OPEN) + { + DAPL_CNO *cno = evd_ptr->cno_ptr; + if (evd_ptr->evd_enabled && (evd_ptr->cno_ptr != NULL)) + { + /* + * Re-enable callback, *then* trigger. + * This guarantees we won't miss any events. + */ + dat_status = dapls_ib_completion_notify (hca_handle, + evd_ptr, + IB_NOTIFY_ON_NEXT_COMP); + + if ( DAT_SUCCESS != dat_status ) + { + (void) dapls_evd_post_async_error_event( + evd_ptr->header.owner_ia->async_error_evd, + DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR, + (DAT_IA_HANDLE) evd_ptr->header.owner_ia); + } + + dapl_internal_cno_trigger(cno, evd_ptr); + } + } + dapl_dbg_log (DAPL_DBG_TYPE_RTN, "dapl_evd_dto_callback () returns\n"); +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_free.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_free.c new file mode 100644 index 00000000..cbb40ac3 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_free.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_free.c + * + * PURPOSE: Event management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_ia_util.h" +#include "dapl_cno_util.h" /* for __KDAPL__ */ + +/* + * dapl_evd_free + * + * DAPL Requirements Version xxx, 6.3.2.2 + * + * Destroy a specific instance of the Event Dispatcher + * + * Input: + * evd_handle + * + * Output: + * None + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + */ +DAT_RETURN DAT_API dapl_evd_free ( + IN DAT_EVD_HANDLE evd_handle) + +{ + DAPL_EVD *evd_ptr; + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_evd_free (%p)\n", evd_handle); + DAPL_CNTR (DCNT_EVD_FREE); + + dat_status = DAT_SUCCESS; + evd_ptr = (DAPL_EVD *)evd_handle; + + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, 0); + goto bail; + } + + if (dapl_os_atomic_read (&evd_ptr->evd_ref_count) != 0) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_EVD_IN_USE); + goto bail; + } + + /* obtain the cno_ptr before the evd is released, which must occur + * before deallocating the CNO + */ + cno_ptr = evd_ptr->cno_ptr; + + dapl_ia_unlink_evd (evd_ptr->header.owner_ia, evd_ptr); + + dat_status = dapls_evd_dealloc (evd_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_ia_link_evd (evd_ptr->header.owner_ia, evd_ptr); + } + +#if defined(__KDAPL__) + if (cno_ptr != NULL) + { + if (dapl_os_atomic_read (&cno_ptr->cno_ref_count) > 0 || cno_ptr->cno_waiters > 0) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, + DAT_INVALID_STATE_EVD_IN_USE); + goto bail; + } + dapl_ia_unlink_cno (cno_ptr->header.owner_ia, cno_ptr); + dapl_cno_dealloc (cno_ptr); + } +#else + if (cno_ptr != NULL) + { + if (dapl_os_atomic_read(&cno_ptr->cno_ref_count) == 0 && cno_ptr->cno_waiters > 0) + { + /* + * Last reference on the CNO, trigger a notice. See + * uDAPL 1.1 spec 6.3.2.3 + */ + dapl_internal_cno_trigger (cno_ptr, NULL); + } + } +#endif /* defined(__KDAPL__) */ + +bail: + if ( dat_status ) + dapl_dbg_log (DAPL_DBG_TYPE_RTN, "dapl_evd_free () returns 0x%x\n", + dat_status); + + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_post_se.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_post_se.c new file mode 100644 index 00000000..9ebdfee3 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_post_se.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_post_se.c + * + * PURPOSE: Event Management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_ia_util.h" + +/* + * dapl_evd_post_se + * + * DAPL Requirements Version xxx, 6.3.2.7 + * + * Post a software event to the Event Dispatcher event queue. + * + * Input: + * evd_handle + * event + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ + + +DAT_RETURN DAT_API dapl_evd_post_se ( + DAT_EVD_HANDLE evd_handle, + const DAT_EVENT *event) + +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + /* Only post to EVDs that are specific to software events */ + if ( !(evd_ptr->evd_flags & DAT_EVD_SOFTWARE_FLAG) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG1); + goto bail; + } + + if (!event) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + if (event->event_number != DAT_SOFTWARE_EVENT) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + dat_status = dapls_evd_post_software_event( + evd_ptr, + DAT_SOFTWARE_EVENT, + event->event_data.software_event_data.pointer); + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_qp_async_error_callb.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_qp_async_error_callb.c new file mode 100644 index 00000000..c276827c --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_qp_async_error_callb.c @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_qp_async_error_callback.c + * + * PURPOSE: implements QP callbacks from verbs + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_qp_async_error_callback + * + * The callback function registered with verbs for qp async erros + * + * Input: + * ib_cm_handle, + * ib_cm_event + * cause_ptr + * context (evd) + * + * Output: + * None + * + */ + +void +dapl_evd_qp_async_error_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_qp_handle_t ib_qp_handle, + IN ib_error_record_t * cause_ptr, + IN void * context) + +{ + /* + * This is an affiliated error and hence should be able to + * supply us with exact information on the error type and QP. + * + * However the Mellanox and IBM APIs for registering this callback + * are different. + * + * The IBM API allows consumers to register the callback with + * + * ib_int32_t + * ib_set_qp_async_error_eh_us ( + * ib_hca_handle_t hca_handle, + * ib_qp_async_handler_t handler ) + * + * Notice that this function does not take a context. The context is + * specified per QP in the call to ib_qp_create_us(). + * + * In contrast the Mellanox API requires that the context be specified + * when the funciton is registered: + * + * VAPI_ret_t + * VAPI_set_async_event_handler ( + * IN VAPI_hca_hndl_t hca_hndl, + * IN VAPI_async_event_handler_t handler, + * IN void* private_data ) + * + * Therefore we always specify the context as the asyncronous EVD + * to be compatible with both APIs. + */ + + DAPL_IA *ia_ptr; + DAPL_EP *ep_ptr; + DAPL_EVD *async_evd; + DAT_EVENT_NUMBER async_event; + DAT_RETURN dat_status; + + ep_ptr = (DAPL_EP *) context; + ia_ptr = ep_ptr->header.owner_ia; + async_evd = (DAPL_EVD *) ia_ptr->async_error_evd; + + dapl_dbg_log ( + DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION, + "--> dapl_evd_qp_async_error_callback: ep %p qp %p (%x) state %d\n", + ep_ptr, + ep_ptr->qp_handle, + ep_ptr->qpn, + ep_ptr->param.ep_state); + + /* + * Transition to ERROR if we are connected; other states need to + * complete first (e.g. pending states) + */ + if ( ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED) + { + ep_ptr->param.ep_state = DAT_EP_STATE_ERROR; + } + + dapl_os_assert (async_evd != NULL); + + dat_status = dapls_ib_get_async_event(cause_ptr, &async_event); + if ( dat_status == DAT_SUCCESS ) + { + /* + * If dapls_ib_get_async_event is not successful, + * an event has been generated by the provide that + * we are not interested in. + */ + (void) dapls_evd_post_async_error_event( async_evd, + async_event, + async_evd->header.owner_ia); + } + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION, + "dapl_evd_qp_async_error_callback () returns\n"); +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_resize.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_resize.c new file mode 100644 index 00000000..2127c156 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_resize.c @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_resize.c + * + * PURPOSE: EVENT management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_ring_buffer_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_resize + * + * DAPL Requirements Version xxx, 6.3.2.5 + * + * Modify the size of the event queue of an Event Dispatcher + * + * Input: + * evd_handle + * evd_qlen + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_STATE + */ + +DAT_RETURN DAT_API dapl_evd_resize ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_COUNT evd_qlen ) +{ + DAPL_IA *ia_ptr; + DAPL_EVD *evd_ptr; + DAT_COUNT pend_cnt; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_evd_resize (%p, %d)\n", + evd_handle, evd_qlen); + + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE1); + goto bail; + } + + evd_ptr = (DAPL_EVD *)evd_handle; + ia_ptr = evd_ptr->header.owner_ia; + + if ( evd_qlen == evd_ptr->qlen ) + { + dat_status = DAT_SUCCESS; + goto bail; + } + + if ( evd_qlen > ia_ptr->hca_ptr->ia_attr.max_evd_qlen ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + dapl_os_lock(&evd_ptr->header.lock); + + /* Don't try to resize if we are actively waiting */ + if (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED) + { + dapl_os_unlock(&evd_ptr->header.lock); + dat_status = DAT_ERROR (DAT_INVALID_STATE,0); + goto bail; + } + + pend_cnt = dapls_rbuf_count(&evd_ptr->pending_event_queue); + if (pend_cnt > evd_qlen) { + dapl_os_unlock(&evd_ptr->header.lock); + dat_status = DAT_ERROR (DAT_INVALID_STATE,0); + goto bail; + } + + dat_status = dapls_ib_cq_resize(evd_ptr->header.owner_ia, + evd_ptr, + &evd_qlen); + if (dat_status != DAT_SUCCESS) + { + dapl_os_unlock(&evd_ptr->header.lock); + goto bail; + } + + dat_status = dapls_evd_event_realloc (evd_ptr, evd_qlen); + if (dat_status != DAT_SUCCESS) + { + dapl_os_unlock(&evd_ptr->header.lock); + goto bail; + } + + dapl_os_unlock(&evd_ptr->header.lock); + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_un_async_error_callb.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_un_async_error_callb.c new file mode 100644 index 00000000..5e636ff2 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_un_async_error_callb.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_un_async_error_callback.c + * + * PURPOSE: implements Unaffiliated callbacks from verbs + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_un_async_error_callback + * + * The callback function registered with verbs for unaffiliated async errors + * + * Input: + * ib_cm_handle, + * ib_cm_event + * cause_ptr + * context (evd) + * + * Output: + * None + * + */ + +void +dapl_evd_un_async_error_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_error_record_t * cause_ptr, + IN void * context) + +{ + DAPL_EVD *async_evd; + DAT_EVENT_NUMBER async_event; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION, + "dapl_evd_un_async_error_callback (%p, %p, %p)\n", + ib_hca_handle, + cause_ptr, + context); + + if ( NULL == context ) + { + dapl_os_panic ("NULL == context\n"); + return; + } + + async_evd = (DAPL_EVD *) context; + + dat_status = dapls_ib_get_async_event(cause_ptr, &async_event); + + if ( dat_status == DAT_SUCCESS ) + { + /* + * If dapls_ib_get_async_event is not successful, + * an event has been generated by the provider that + * we are not interested in. E.g. LINK_UP. + */ + dapls_evd_post_async_error_event( async_evd, + async_event, + async_evd->header.owner_ia); + } + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION, + "dapl_evd_un_async_error_callback () returns\n"); +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_util.c b/trunk/ulp/dapl2/dapl/common/dapl_evd_util.c new file mode 100644 index 00000000..66d99ab6 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_util.c @@ -0,0 +1,1423 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_util.c + * + * PURPOSE: Manage EVD Info structure + * + * $Id: dapl_evd_util.c 1410 2006-07-19 17:12:02Z ardavis $ + **********************************************************************/ + +#include "dapl_evd_util.h" +#include "dapl_ia_util.h" +#include "dapl_cno_util.h" +#include "dapl_ring_buffer_util.h" +#include "dapl_adapter_util.h" +#include "dapl_cookie.h" +#include "dapl.h" + +STATIC _INLINE_ void dapli_evd_eh_print_cqe ( + IN ib_work_completion_t *cqe); + +DAT_RETURN dapli_evd_event_alloc ( + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT qlen); + + +char *dapl_event_str( IN DAT_EVENT_NUMBER event_num ) +{ +#if defined(DAPL_DBG) + struct dat_event_str { char *str; DAT_EVENT_NUMBER num;}; + static struct dat_event_str events[] = { + {"DAT_DTO_COMPLETION_EVENT", DAT_DTO_COMPLETION_EVENT}, + {"DAT_RMR_BIND_COMPLETION_EVENT", DAT_RMR_BIND_COMPLETION_EVENT}, + {"DAT_CONNECTION_REQUEST_EVENT", DAT_CONNECTION_REQUEST_EVENT}, + {"DAT_CONNECTION_EVENT_ESTABLISHED", DAT_CONNECTION_EVENT_ESTABLISHED}, + {"DAT_CONNECTION_EVENT_PEER_REJECTED", DAT_CONNECTION_EVENT_PEER_REJECTED}, + {"DAT_CONNECTION_EVENT_NON_PEER_REJECTED", DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, + {"DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR", DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR}, + {"DAT_CONNECTION_EVENT_DISCONNECTED", DAT_CONNECTION_EVENT_DISCONNECTED}, + {"DAT_CONNECTION_EVENT_BROKEN", DAT_CONNECTION_EVENT_BROKEN}, + {"DAT_CONNECTION_EVENT_TIMED_OUT", DAT_CONNECTION_EVENT_TIMED_OUT}, + {"DAT_CONNECTION_EVENT_UNREACHABLE", DAT_CONNECTION_EVENT_UNREACHABLE}, + {"DAT_ASYNC_ERROR_EVD_OVERFLOW", DAT_ASYNC_ERROR_EVD_OVERFLOW}, + {"DAT_ASYNC_ERROR_IA_CATASTROPHIC", DAT_ASYNC_ERROR_IA_CATASTROPHIC}, + {"DAT_ASYNC_ERROR_EP_BROKEN", DAT_ASYNC_ERROR_EP_BROKEN}, + {"DAT_ASYNC_ERROR_TIMED_OUT", DAT_ASYNC_ERROR_TIMED_OUT}, + {"DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR", DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR}, + {"DAT_HA_DOWN_TO_1", DAT_HA_DOWN_TO_1}, + {"DAT_HA_UP_TO_MULTI_PATH", DAT_HA_UP_TO_MULTI_PATH}, + {"DAT_SOFTWARE_EVENT", DAT_SOFTWARE_EVENT}, +#ifdef DAT_EXTENSIONS + {"DAT_EXTENSION_EVENT", DAT_EXTENSION_EVENT}, + {"DAT_IB_EXTENSION_RANGE_BASE", DAT_IB_EXTENSION_RANGE_BASE}, + {"DAT_IW_EXTENSION_RANGE_BASE", DAT_IW_EXTENSION_RANGE_BASE}, +#endif /* DAT_EXTENSIONS */ + {NULL,0}, + }; + int i; + + for(i=0; events[i].str; i++) + { + if (events[i].num == event_num) + return events[i].str; + } + return "Unknown DAT event?"; +#else + static char str[16]; + sprintf(str,"%x",event_num); + return str; +#endif +} + +/* + * dapls_evd_internal_create + * + * actually create the evd. this is called after all parameter checking + * has been performed in dapl_ep_create. it is also called from dapl_ia_open + * to create the default async evd. + * + * Input: + * ia_ptr + * cno_ptr + * qlen + * evd_flags + * + * Output: + * evd_ptr_ptr + * + * Returns: + * none + * + */ + +DAT_RETURN +dapls_evd_internal_create ( + DAPL_IA *ia_ptr, + DAPL_CNO *cno_ptr, + DAT_COUNT min_qlen, + DAT_EVD_FLAGS evd_flags, + DAPL_EVD **evd_ptr_ptr) +{ + DAPL_EVD *evd_ptr; + DAT_COUNT cq_len; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + *evd_ptr_ptr = NULL; + cq_len = min_qlen; + + evd_ptr = dapls_evd_alloc (ia_ptr, + cno_ptr, + evd_flags, + min_qlen); + if (!evd_ptr) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + /* + * If we are dealing with event streams besides a CQ event stream, + * be conservative and set producer side locking. Otherwise, no. + */ + evd_ptr->evd_producer_locking_needed = + ((evd_flags & ~ (DAT_EVD_DTO_FLAG|DAT_EVD_RMR_BIND_FLAG)) != 0); + + /* Before we setup any callbacks, transition state to OPEN. */ + evd_ptr->evd_state = DAPL_EVD_STATE_OPEN; + + if (evd_flags & DAT_EVD_ASYNC_FLAG) + { + /* + * There is no cq associate with async evd. Set it to invalid + */ + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; + + } + else if ( 0 != (evd_flags & ~ (DAT_EVD_SOFTWARE_FLAG + | DAT_EVD_CONNECTION_FLAG + | DAT_EVD_CR_FLAG) ) ) + { +#if defined(_VENDOR_IBAL_) + /* + * The creation of CQ required a PD (PZ) associated with it and + * we do not have a PD here; therefore, the work-around is that we + * will postpone the creation of the cq till the creation of QP which + * this cq will associate with. + */ + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; +#else + dat_status = dapls_ib_cq_alloc (ia_ptr, + evd_ptr, + &cq_len); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + /* Now reset the cq_len in the attributes, it may have changed */ + evd_ptr->qlen = cq_len; + + dat_status = + dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_CQ_COMPLETION, + evd_ptr, + (ib_async_handler_t)dapl_evd_dto_callback, + evd_ptr); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + dat_status = dapls_set_cq_notify (ia_ptr, evd_ptr); + + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + +#endif /* _VENDOR_IBAL_ */ + } + + /* We now have an accurate count of events, so allocate them into + * the EVD + */ + dat_status = dapli_evd_event_alloc (evd_ptr, cq_len); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + dapl_ia_link_evd (ia_ptr, evd_ptr); + *evd_ptr_ptr = evd_ptr; + +bail: + if (dat_status != DAT_SUCCESS) + { + if (evd_ptr) + { + dapls_evd_dealloc (evd_ptr); + } + } + + return dat_status; +} + +/* + * dapls_evd_alloc + * + * alloc and initialize an EVD struct + * + * Input: + * ia + * + * Output: + * evd_ptr + * + * Returns: + * none + * + */ +DAPL_EVD * +dapls_evd_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_ptr, + IN DAT_EVD_FLAGS evd_flags, + IN DAT_COUNT qlen) +{ + DAPL_EVD *evd_ptr; + + /* Allocate EVD */ + evd_ptr = (DAPL_EVD *)dapl_os_alloc (sizeof (DAPL_EVD)); + if (!evd_ptr) + { + goto bail; + } + + /* zero the structure */ + dapl_os_memzero (evd_ptr, sizeof (DAPL_EVD)); + + /* + * initialize the header + */ + evd_ptr->header.provider = ia_ptr->header.provider; + evd_ptr->header.magic = DAPL_MAGIC_EVD; + evd_ptr->header.handle_type = DAT_HANDLE_TYPE_EVD; + evd_ptr->header.owner_ia = ia_ptr; + evd_ptr->header.user_context.as_64 = 0; + evd_ptr->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&evd_ptr->header.ia_list_entry); + dapl_os_lock_init (&evd_ptr->header.lock); + + /* + * Initialize the body + */ + evd_ptr->evd_state = DAPL_EVD_STATE_INITIAL; + evd_ptr->evd_flags = evd_flags; + evd_ptr->evd_enabled = DAT_TRUE; + evd_ptr->evd_waitable = DAT_TRUE; + evd_ptr->evd_producer_locking_needed = 1;/* Conservative value. */ + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; + dapl_os_atomic_set (&evd_ptr->evd_ref_count, 0); + evd_ptr->catastrophic_overflow = DAT_FALSE; + evd_ptr->qlen = qlen; + evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; /* FIXME: should be DAPL_EVD_STATE_INIT */ + dapl_os_wait_object_init (&evd_ptr->wait_object); + +#ifdef CQ_WAIT_OBJECT + /* Create CQ wait object; no CNO and data stream type */ + if (( cno_ptr == NULL ) && + ((evd_flags & ~ (DAT_EVD_DTO_FLAG|DAT_EVD_RMR_BIND_FLAG)) == 0 )) + { + dapls_ib_wait_object_create (evd_ptr, &evd_ptr->cq_wait_obj_handle); + } +#endif + + evd_ptr->cno_active_count = 0; + if ( cno_ptr != NULL ) + { + /* Take a reference count on the CNO */ + dapl_os_atomic_inc (&cno_ptr->cno_ref_count); + } + evd_ptr->cno_ptr = cno_ptr; + +bail: + return evd_ptr; +} + + +/* + * dapls_evd_event_alloc + * + * alloc events into an EVD. + * + * Input: + * evd_ptr + * qlen + * + * Output: + * NONE + * + * Returns: + * DAT_SUCCESS + * ERROR + * + */ +DAT_RETURN +dapli_evd_event_alloc ( + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT qlen) +{ + DAT_EVENT *event_ptr; + DAT_COUNT i; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + /* Allocate EVENTs */ + event_ptr = (DAT_EVENT *) dapl_os_alloc (evd_ptr->qlen * sizeof (DAT_EVENT)); + if (event_ptr == NULL) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + evd_ptr->events = event_ptr; + + /* allocate free event queue */ + dat_status = dapls_rbuf_alloc (&evd_ptr->free_event_queue, qlen); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + /* allocate pending event queue */ + dat_status = dapls_rbuf_alloc (&evd_ptr->pending_event_queue, qlen); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + /* add events to free event queue */ + for (i = 0; i < evd_ptr->qlen; i++) + { + dapls_rbuf_add (&evd_ptr->free_event_queue, (void *)event_ptr); + event_ptr++; + } + + evd_ptr->cq_notified = DAT_FALSE; + evd_ptr->cq_notified_when = 0; + evd_ptr->threshold = 0; + +bail: + return dat_status; +} + + +/* + * dapls_evd_event_realloc + * + * realloc events into an EVD. + * + * Input: + * evd_ptr + * qlen + * + * Output: + * NONE + * + * Returns: + * DAT_SUCCESS + * ERROR + * + */ +DAT_RETURN +dapls_evd_event_realloc ( + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT qlen) +{ + DAT_EVENT *events; + DAT_COUNT old_qlen; + DAT_COUNT i; + intptr_t diff; + DAT_RETURN dat_status; + + /* Allocate EVENTs */ + events = (DAT_EVENT *) dapl_os_realloc (evd_ptr->events, + qlen * sizeof (DAT_EVENT)); + if ( NULL == events ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + diff = events - evd_ptr->events; + evd_ptr->events = events; + + old_qlen = evd_ptr->qlen; + evd_ptr->qlen = qlen; + + /* reallocate free event queue */ + dat_status = dapls_rbuf_realloc (&evd_ptr->free_event_queue, qlen); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + dapls_rbuf_adjust (&evd_ptr->free_event_queue, diff); + + /* reallocate pending event queue */ + dat_status = dapls_rbuf_realloc (&evd_ptr->pending_event_queue, qlen); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + dapls_rbuf_adjust (&evd_ptr->pending_event_queue, diff); + + /* + * add new events to free event queue. + */ + for (i = old_qlen; i < qlen; i++) + { + dapls_rbuf_add (&evd_ptr->free_event_queue, (void *) &events[i]); + } + +bail: + return dat_status; +} + +/* + * dapls_evd_dealloc + * + * Free the passed in EVD structure. If an error occurs, this function + * will clean up all of the internal data structures and report the + * error. + * + * Input: + * evd_ptr + * + * Output: + * none + * + * Returns: + * status + * + */ +DAT_RETURN +dapls_evd_dealloc ( + IN DAPL_EVD *evd_ptr ) +{ + DAT_RETURN dat_status; + DAPL_IA *ia_ptr; + + dat_status = DAT_SUCCESS; + + dapl_os_assert (evd_ptr->header.magic == DAPL_MAGIC_EVD); + dapl_os_assert (dapl_os_atomic_read (&evd_ptr->evd_ref_count) == 0); + + /* + * Destroy the CQ first, to keep any more callbacks from coming + * up from it. + */ + if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) + { + ia_ptr = evd_ptr->header.owner_ia; + + dat_status = dapls_ib_cq_free (ia_ptr, evd_ptr); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + } + + /* + * We should now be safe to invalidate the EVD; reset the + * magic to prevent reuse. + */ + evd_ptr->header.magic = DAPL_MAGIC_INVALID; + + /* Release reference on the CNO if it exists */ + if ( evd_ptr->cno_ptr != NULL ) + { + dapl_os_atomic_dec ( &evd_ptr->cno_ptr->cno_ref_count ); + evd_ptr->cno_ptr = NULL; + } + + /* If the ring buffer allocation failed, then the dapls_rbuf_destroy */ + /* function will detect that the ring buffer's internal data (ex. base */ + /* pointer) are invalid and will handle the situation appropriately */ + dapls_rbuf_destroy (&evd_ptr->free_event_queue); + dapls_rbuf_destroy (&evd_ptr->pending_event_queue); + + if (evd_ptr->events) + { + dapl_os_free (evd_ptr->events, evd_ptr->qlen * sizeof (DAT_EVENT)); + } + + dapl_os_wait_object_destroy (&evd_ptr->wait_object); + +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + { + dapls_ib_wait_object_destroy (evd_ptr->cq_wait_obj_handle); + } +#endif + + dapl_os_free (evd_ptr, sizeof (DAPL_EVD)); + +bail: + return dat_status; +} + + +/* + * dapli_evd_eh_print_cqe + * + * Input: + * cqe_ptr + * + * Output: + * none + * + * Prints out a CQE for debug purposes + * + */ + +void +dapli_evd_eh_print_cqe ( + IN ib_work_completion_t *cqe_ptr) +{ +#ifdef DAPL_DBG + static char *optable[] = + { + "OP_RDMA_WRITE", + "OP_RDMA_WRITE_IMM", + "OP_SEND", + "OP_SEND_IMM", + "OP_RDMA_READ", + "OP_COMP_AND_SWAP", + "OP_FETCH_AND_ADD", + "OP_RECEIVE", + "OP_RECEIVE_IMM", + "OP_BIND_MW", + 0 + }; + + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t >>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<\n"); + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t dapl_evd_dto_callback : CQE \n"); + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t\t work_req_id %lli\n", + DAPL_GET_CQE_WRID (cqe_ptr)); + if (DAPL_GET_CQE_STATUS (cqe_ptr) == 0) + { + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t\t op_type: %s\n", + optable[DAPL_GET_CQE_OPTYPE (cqe_ptr)]); + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t\t bytes_num %d\n", + DAPL_GET_CQE_BYTESNUM (cqe_ptr)); + } + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t\t status %d\n", + DAPL_GET_CQE_STATUS (cqe_ptr)); + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "\t >>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<\n"); +#endif + return; +} + + + +/* + * Event posting code follows. + */ + +/* + * These next two functions (dapli_evd_get_event and dapli_evd_post_event) + * are a pair. They are always called together, from one of the functions + * at the end of this file (dapl_evd_post_*_event). + * + * Note that if producer side locking is enabled, the first one takes the + * EVD lock and the second releases it. + */ + +/* dapli_evd_get_event + * + * Get an event struct from the evd. The caller should fill in the event + * and call dapl_evd_post_event. + * + * If there are no events available, an overflow event is generated to the + * async EVD handler. + * + * If this EVD required producer locking, a successful return implies + * that the lock is held. + * + * Input: + * evd_ptr + * + * Output: + * event + * + */ + +static DAT_EVENT * +dapli_evd_get_event ( + DAPL_EVD *evd_ptr) +{ + DAT_EVENT *event; + + if (evd_ptr->evd_producer_locking_needed) + { + dapl_os_lock (&evd_ptr->header.lock); + } + + event = (DAT_EVENT *)dapls_rbuf_remove (&evd_ptr->free_event_queue); + + /* Release the lock if it was taken and the call failed. */ + if (!event && evd_ptr->evd_producer_locking_needed) + { + dapl_os_unlock (&evd_ptr->header.lock); + } + + return event; +} + +/* dapli_evd_post_event + * + * Post the to the evd. If possible, invoke the evd's CNO. + * Otherwise post the event on the pending queue. + * + * If producer side locking is required, the EVD lock must be held upon + * entry to this function. + * + * Input: + * evd_ptr + * event + * + * Output: + * none + * + */ + +static void +dapli_evd_post_event ( + IN DAPL_EVD *evd_ptr, + IN const DAT_EVENT *event_ptr) +{ + DAT_RETURN dat_status; + DAPL_CNO *cno_to_trigger = NULL; + + dapl_dbg_log (DAPL_DBG_TYPE_EVD, "%s: Called with event %s\n", + __FUNCTION__, dapl_event_str(event_ptr->event_number)); + + dat_status = dapls_rbuf_add (&evd_ptr->pending_event_queue, + (void *)event_ptr); + dapl_os_assert (dat_status == DAT_SUCCESS); + + dapl_os_assert (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED + || evd_ptr->evd_state == DAPL_EVD_STATE_OPEN); + + if (evd_ptr->evd_state == DAPL_EVD_STATE_OPEN) + { + /* No waiter. Arrange to trigger a CNO if it exists. */ + + if (evd_ptr->evd_enabled) + { + cno_to_trigger = evd_ptr->cno_ptr; + } + if (evd_ptr->evd_producer_locking_needed) + { + dapl_os_unlock (&evd_ptr->header.lock); + } + } + else + { + /* + * We're in DAPL_EVD_STATE_WAITED. Take the lock if + * we don't have it, recheck, and signal. + */ + if (!evd_ptr->evd_producer_locking_needed) + { + dapl_os_lock (&evd_ptr->header.lock); + } + + if (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED + && (dapls_rbuf_count (&evd_ptr->pending_event_queue) + >= evd_ptr->threshold)) + { + dapl_os_unlock (&evd_ptr->header.lock); + +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + dapls_ib_wait_object_wakeup (evd_ptr->cq_wait_obj_handle); + else +#endif + dapl_os_wait_object_wakeup (&evd_ptr->wait_object); + } + else + { + dapl_os_unlock (&evd_ptr->header.lock); + } + } + + if (cno_to_trigger != NULL) + { + dapl_internal_cno_trigger (cno_to_trigger, evd_ptr); + } +} + +/* dapli_evd_post_event_nosignal + * + * Post the to the evd. Do not do any wakeup processing. + * This function should only be called if it is known that there are + * no waiters that it is appropriate to wakeup on this EVD. An example + * of such a situation is during internal dat_evd_wait() processing. + * + * If producer side locking is required, the EVD lock must be held upon + * entry to this function. + * + * Input: + * evd_ptr + * event + * + * Output: + * none + * + */ + +static void +dapli_evd_post_event_nosignal ( + IN DAPL_EVD *evd_ptr, + IN const DAT_EVENT *event_ptr) +{ + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_EVD, "%s: Called with event %s\n", + __FUNCTION__, dapl_event_str(event_ptr->event_number)); + + dat_status = dapls_rbuf_add (&evd_ptr->pending_event_queue, + (void *)event_ptr); + dapl_os_assert (dat_status == DAT_SUCCESS); + + dapl_os_assert (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED + || evd_ptr->evd_state == DAPL_EVD_STATE_OPEN); + + if (evd_ptr->evd_producer_locking_needed) + { + dapl_os_unlock (&evd_ptr->header.lock); + } +} + +/* dapli_evd_format_overflow_event + * + * format an overflow event for posting + * + * Input: + * evd_ptr + * event_ptr + * + * Output: + * none + * + */ +static void +dapli_evd_format_overflow_event ( + IN DAPL_EVD *evd_ptr, + OUT DAT_EVENT *event_ptr) +{ + DAPL_IA *ia_ptr; + + ia_ptr = evd_ptr->header.owner_ia; + + event_ptr->evd_handle = (DAT_EVD_HANDLE)evd_ptr; + event_ptr->event_number = DAT_ASYNC_ERROR_EVD_OVERFLOW; + event_ptr->event_data.asynch_error_event_data.dat_handle = (DAT_HANDLE)ia_ptr; +} + +/* dapli_evd_post_overflow_event + * + * post an overflow event + * + * Input: + * async_evd_ptr + * evd_ptr + * + * Output: + * none + * + */ +static void +dapli_evd_post_overflow_event ( + IN DAPL_EVD *async_evd_ptr, + IN DAPL_EVD *overflow_evd_ptr) +{ + DAT_EVENT *overflow_event; + + /* The overflow_evd_ptr mght be the same as evd. + * In that case we've got a catastrophic overflow. + */ + if (async_evd_ptr == overflow_evd_ptr) + { + async_evd_ptr->catastrophic_overflow = DAT_TRUE; + async_evd_ptr->evd_state = DAPL_EVD_STATE_DEAD; + return; + } + + overflow_event = dapli_evd_get_event (overflow_evd_ptr); + if (!overflow_event) + { + /* this is not good */ + overflow_evd_ptr->catastrophic_overflow = DAT_TRUE; + overflow_evd_ptr->evd_state = DAPL_EVD_STATE_DEAD; + return; + } + dapli_evd_format_overflow_event (overflow_evd_ptr, overflow_event); + dapli_evd_post_event (overflow_evd_ptr, overflow_event); + + return; +} + +static DAT_EVENT * +dapli_evd_get_and_init_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number) +{ + DAT_EVENT *event_ptr; + + event_ptr = dapli_evd_get_event (evd_ptr); + if (NULL == event_ptr) + { + dapli_evd_post_overflow_event ( + evd_ptr->header.owner_ia->async_error_evd, + evd_ptr); + } + else + { + event_ptr->evd_handle = (DAT_EVD_HANDLE) evd_ptr; + event_ptr->event_number = event_number; + } + + return event_ptr; +} + +DAT_RETURN +dapls_evd_post_cr_arrival_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_SP_HANDLE sp_handle, + DAT_IA_ADDRESS_PTR ia_address_ptr, + DAT_CONN_QUAL conn_qual, + DAT_CR_HANDLE cr_handle) +{ + DAT_EVENT *event_ptr; + event_ptr = dapli_evd_get_and_init_event (evd_ptr, event_number); + /* + * Note event lock may be held on successful return + * to be released by dapli_evd_post_event(), if provider side locking + * is needed. + */ + + if (event_ptr == NULL) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + } + + event_ptr->event_data.cr_arrival_event_data.sp_handle = sp_handle; + event_ptr->event_data.cr_arrival_event_data.local_ia_address_ptr + = ia_address_ptr; + event_ptr->event_data.cr_arrival_event_data.conn_qual = conn_qual; + event_ptr->event_data.cr_arrival_event_data.cr_handle = cr_handle; + + dapli_evd_post_event (evd_ptr, event_ptr); + + return DAT_SUCCESS; +} + + +DAT_RETURN +dapls_evd_post_connection_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT private_data_size, + IN DAT_PVOID private_data) +{ + DAT_EVENT *event_ptr; + event_ptr = dapli_evd_get_and_init_event (evd_ptr, event_number); + /* + * Note event lock may be held on successful return + * to be released by dapli_evd_post_event(), if provider side locking + * is needed. + */ + + if (event_ptr == NULL) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + } + + event_ptr->event_data.connect_event_data.ep_handle = ep_handle; + event_ptr->event_data.connect_event_data.private_data_size + = private_data_size; + event_ptr->event_data.connect_event_data.private_data = private_data; + + dapli_evd_post_event (evd_ptr, event_ptr); + + return DAT_SUCCESS; +} + + +DAT_RETURN +dapls_evd_post_async_error_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_IA_HANDLE ia_handle) +{ + DAT_EVENT *event_ptr; + event_ptr = dapli_evd_get_and_init_event (evd_ptr, event_number); + /* + * Note event lock may be held on successful return + * to be released by dapli_evd_post_event(), if provider side locking + * is needed. + */ + + if (event_ptr == NULL) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + } + + event_ptr->event_data.asynch_error_event_data.dat_handle = (DAT_HANDLE)ia_handle; + + dapli_evd_post_event (evd_ptr, event_ptr); + + return DAT_SUCCESS; +} + + +DAT_RETURN +dapls_evd_post_software_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_PVOID pointer) +{ + DAT_EVENT *event_ptr; + event_ptr = dapli_evd_get_and_init_event (evd_ptr, event_number); + /* + * Note event lock may be held on successful return + * to be released by dapli_evd_post_event(), if provider side locking + * is needed. + */ + + if (event_ptr == NULL) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + } + + event_ptr->event_data.software_event_data.pointer = pointer; + + dapli_evd_post_event (evd_ptr, event_ptr); + + return DAT_SUCCESS; +} + + +/* + * dapls_evd_post_generic_event + * + * Post a generic event type. Not used by all providers + * + * Input: + * evd_ptr + * event_number + * data + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * + */ +DAT_RETURN +dapls_evd_post_generic_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_EVENT_DATA *data) +{ + DAT_EVENT *event_ptr; + + event_ptr = dapli_evd_get_and_init_event (evd_ptr, event_number); + /* + * Note event lock may be held on successful return + * to be released by dapli_evd_post_event(), if provider side locking + * is needed. + */ + + if (event_ptr == NULL) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + } + + event_ptr->event_data = *data; + + dapli_evd_post_event (evd_ptr, event_ptr); + + return DAT_SUCCESS; +} + +/* + * dapli_evd_cqe_to_event + * + * Convert a CQE into an event structure. + * + * Input: + * evd_ptr + * cqe_ptr + * + * Output: + * event_ptr + * + * Returns: + * none + * + */ +static void +dapli_evd_cqe_to_event ( + IN DAPL_EVD *evd_ptr, + IN void *cqe_ptr, + OUT DAT_EVENT *event_ptr) +{ + DAPL_EP *ep_ptr; + DAPL_COOKIE *cookie; + DAT_DTO_COMPLETION_STATUS dto_status; + DAPL_COOKIE_BUFFER *buffer; + + /* + * All that can be relied on if the status is bad is the status + * and WRID. + */ + dto_status = dapls_ib_get_dto_status (cqe_ptr); + + cookie = (DAPL_COOKIE *) (uintptr_t) DAPL_GET_CQE_WRID (cqe_ptr); + dapl_os_assert ( (NULL != cookie) ); + + ep_ptr = cookie->ep; + dapl_os_assert ( (NULL != ep_ptr) ); + if (ep_ptr->header.magic != DAPL_MAGIC_EP) + { + /* ep may have been freed, just return */ + return; + } + + dapls_io_trc_update_completion (ep_ptr, cookie, dto_status); + + event_ptr->evd_handle = (DAT_EVD_HANDLE) evd_ptr; + + switch (cookie->type) + { + case DAPL_COOKIE_TYPE_DTO: + { +#ifdef DAT_EXTENSIONS + /* Extended via request post or message receive */ + if ((cookie->val.dto.type == DAPL_DTO_TYPE_EXTENSION) || + (cookie->val.dto.type == DAPL_DTO_TYPE_RECV && + DAPL_GET_CQE_OPTYPE(cqe_ptr) != OP_RECEIVE)) + { + dapls_cqe_to_event_extension(ep_ptr, cookie, + cqe_ptr, event_ptr); + dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie); + break; + } +#endif + + if ( DAPL_DTO_TYPE_RECV == cookie->val.dto.type ) + { + dapl_os_atomic_dec (&ep_ptr->recv_count); + buffer = &ep_ptr->recv_buffer; + } + else + { + dapl_os_atomic_dec (&ep_ptr->req_count); + buffer = &ep_ptr->req_buffer; + } + + event_ptr->event_number = DAT_DTO_COMPLETION_EVENT; + event_ptr->event_data.dto_completion_event_data.ep_handle = + cookie->ep; + event_ptr->event_data.dto_completion_event_data.user_cookie = + cookie->val.dto.cookie; + event_ptr->event_data.dto_completion_event_data.status = dto_status; + + if ( cookie->val.dto.type == DAPL_DTO_TYPE_SEND || + cookie->val.dto.type == DAPL_DTO_TYPE_RDMA_WRITE ) + { + /* Get size from DTO; CQE value may be off. */ + event_ptr->event_data.dto_completion_event_data.transfered_length = + cookie->val.dto.size; + } + else + { + event_ptr->event_data.dto_completion_event_data.transfered_length = + DAPL_GET_CQE_BYTESNUM (cqe_ptr); + } + + dapls_cookie_dealloc (buffer, cookie); + break; + } + + case DAPL_COOKIE_TYPE_RMR: + { + dapl_os_atomic_dec (&ep_ptr->req_count); + + event_ptr->event_number = DAT_RMR_BIND_COMPLETION_EVENT; + + event_ptr->event_data.rmr_completion_event_data.rmr_handle = + cookie->val.rmr.rmr; + event_ptr->event_data.rmr_completion_event_data.user_cookie = + cookie->val.rmr.cookie; + if (dto_status == DAT_DTO_SUCCESS) + { + event_ptr->event_data.rmr_completion_event_data.status = + DAT_RMR_BIND_SUCCESS; + dapl_os_assert ((DAPL_GET_CQE_OPTYPE (cqe_ptr)) == OP_BIND_MW); + } + else + { + dapl_dbg_log (DAPL_DBG_TYPE_DTO_COMP_ERR, + " MW bind completion ERROR: %d: op %#x ep: %p\n", + dto_status, + DAPL_GET_CQE_OPTYPE (cqe_ptr), ep_ptr); + event_ptr->event_data.rmr_completion_event_data.status = + DAT_RMR_OPERATION_FAILED; + dapl_os_atomic_dec(&cookie->val.rmr.rmr->lmr->lmr_ref_count); + } + + dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie); + break; + } + default: + { + dapl_os_assert (!"Invalid Operation type"); + break; + } + } /* end switch */ + + /* + * Most error DTO ops result in disconnecting the EP. See + * IBTA Vol 1.1, Chapter 10,Table 68, for expected effect on + * state. + */ + if ((dto_status != DAT_DTO_SUCCESS) && + (dto_status != DAT_DTO_ERR_FLUSHED)) + { + DAPL_EVD *evd_ptr; + + /* + * If we are connected, generate disconnect and generate an + * event. We may be racing with other disconnect ops, so we + * need to check. We may also be racing CM connection events, + * requiring us to check for connection pending states too. + */ + dapl_os_lock ( &ep_ptr->header.lock ); + if (ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED || + ep_ptr->param.ep_state == DAT_EP_STATE_ACTIVE_CONNECTION_PENDING || + ep_ptr->param.ep_state == DAT_EP_STATE_PASSIVE_CONNECTION_PENDING|| + ep_ptr->param.ep_state == DAT_EP_STATE_COMPLETION_PENDING ) + + { + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; + dapl_os_unlock ( &ep_ptr->header.lock ); + dapls_io_trc_dump (ep_ptr, cqe_ptr, dto_status); + + /* Let the other side know we have disconnected */ + (void) dapls_ib_disconnect (ep_ptr, DAT_CLOSE_ABRUPT_FLAG); + + /* ... and clean up the local side */ + evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle; + if (evd_ptr != NULL) + { + dapls_evd_post_connection_event (evd_ptr, + DAT_CONNECTION_EVENT_BROKEN, + (DAT_HANDLE) ep_ptr, + 0, + 0); + } + } + else + { + dapl_os_unlock ( &ep_ptr->header.lock ); + } + + dapl_dbg_log (DAPL_DBG_TYPE_DTO_COMP_ERR, + " DTO completion ERROR: %d: op %#x (ep disconnected)\n", + DAPL_GET_CQE_STATUS (cqe_ptr), + DAPL_GET_CQE_OPTYPE (cqe_ptr)); + } +} + +/* + * dapls_evd_copy_cq + * + * Copy all entries on a CQ associated with the EVD onto that EVD + * Up to caller to handle races, if any. Note that no EVD waiters will + * be awoken by this copy. + * + * Input: + * evd_ptr + * + * Output: + * None + * + * Returns: + * none + * + */ +void +dapls_evd_copy_cq ( + DAPL_EVD *evd_ptr) +{ + ib_work_completion_t cur_cqe; + DAT_RETURN dat_status; + DAT_EVENT *event; + + if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) + { + /* Nothing to do if no CQ. */ + return; + } + + while (1) + { + dat_status = dapls_ib_completion_poll (evd_ptr->header.owner_ia->hca_ptr, + evd_ptr, + &cur_cqe); + + if (dat_status != DAT_SUCCESS) + { + break; + } + + /* For debugging. */ + dapli_evd_eh_print_cqe (&cur_cqe); + + /* + * Can use DAT_DTO_COMPLETION_EVENT because dapli_evd_cqe_to_event + * will overwrite. + */ + + event = dapli_evd_get_and_init_event ( + evd_ptr, DAT_DTO_COMPLETION_EVENT ); + if (event == NULL) + { + /* We've already attempted the overflow post; return. */ + return; + } + + dapli_evd_cqe_to_event ( evd_ptr, &cur_cqe, event ); + + dapli_evd_post_event_nosignal ( evd_ptr, event ); + } + + if ( DAT_GET_TYPE (dat_status) != DAT_QUEUE_EMPTY ) + { + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + "dapls_evd_copy_cq: dapls_ib_completion_poll returned 0x%x\n", + dat_status); + dapl_os_assert (!"Bad return from dapls_ib_completion_poll"); + } +} + +/* + * dapls_evd_cq_poll_to_event + * + * Attempt to dequeue a single CQE from a CQ and turn it into + * an event. + * + * Input: + * evd_ptr + * + * Output: + * event + * + * Returns: + * Status of operation + * + */ +DAT_RETURN +dapls_evd_cq_poll_to_event ( + IN DAPL_EVD *evd_ptr, + OUT DAT_EVENT *event) +{ + DAT_RETURN dat_status; + ib_work_completion_t cur_cqe; + + dat_status = dapls_ib_completion_poll (evd_ptr->header.owner_ia->hca_ptr, + evd_ptr, + &cur_cqe); + if (dat_status == DAT_SUCCESS) + { + /* For debugging. */ + dapli_evd_eh_print_cqe (&cur_cqe); + + dapli_evd_cqe_to_event (evd_ptr, &cur_cqe, event); + } + + return dat_status; +} +#ifdef DAPL_DBG_IO_TRC +/* + * Update I/O completions in the I/O trace buffer. I/O is posted to + * the buffer, then we find it here using the cookie and mark it + * completed with the completion status + */ +void +dapls_io_trc_update_completion ( + DAPL_EP *ep_ptr, + DAPL_COOKIE *cookie, + DAT_DTO_COMPLETION_STATUS dto_status) +{ + int i; + static unsigned int c_cnt = 1; + + for (i = 0; i < DBG_IO_TRC_QLEN; i++) + { + if (ep_ptr->ibt_base[i].cookie == cookie) + { + ep_ptr->ibt_base[i].status = dto_status; + ep_ptr->ibt_base[i].done = c_cnt++; + } + } +} + + +/* + * Dump the I/O trace buffers + */ +void +dapls_io_trc_dump ( + DAPL_EP *ep_ptr, + void *cqe_ptr, + DAT_DTO_COMPLETION_STATUS dto_status) +{ + struct io_buf_track *ibt; + int i; + int cnt; + + dapl_os_printf ("DISCONNECTING: dto_status = %x\n", dto_status); + dapl_os_printf (" OpType = %x\n", + DAPL_GET_CQE_OPTYPE (cqe_ptr)); + dapl_os_printf (" Bytes = %x\n", + DAPL_GET_CQE_BYTESNUM (cqe_ptr)); + dapl_os_printf (" WRID (cookie) = %llx\n", + DAPL_GET_CQE_WRID (cqe_ptr)); + + if (ep_ptr->ibt_dumped == 0) + { + + dapl_os_printf ("EP %p (qpn %d) I/O trace buffer\n", + ep_ptr, ep_ptr->qpn); + + ep_ptr->ibt_dumped = 1; + ibt = (struct io_buf_track *)dapls_rbuf_remove (&ep_ptr->ibt_queue); + cnt = DBG_IO_TRC_QLEN; + while (ibt != NULL && cnt > 0) + { + dapl_os_printf ("%2d. %3s (%2d, %d) OP: %x cookie %p wqe %p rmv_target_addr %llx rmv_rmr_context %x\n", + cnt, ibt->done == 0 ? "WRK" : "DON", ibt->status, ibt->done, + ibt->op_type, ibt->cookie, ibt->wqe, + ibt->remote_iov.target_address, + ibt->remote_iov.rmr_context); + for (i = 0; i < 3; i++) + { + if (ibt->iov[i].segment_length != 0) + { + dapl_os_printf (" (%4llx, %8x, %8llx)\n", + ibt->iov[i].segment_length, + ibt->iov[i].lmr_context, + ibt->iov[i].virtual_address); + } + } + ibt = (struct io_buf_track *)dapls_rbuf_remove (&ep_ptr->ibt_queue); + cnt--; + } + } +} +#endif /* DAPL_DBG_IO_TRC */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_evd_util.h b/trunk/ulp/dapl2/dapl/common/dapl_evd_util.h new file mode 100644 index 00000000..0fe29222 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_evd_util.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_evd_util.h + * + * PURPOSE: Utility defs & routines for the EVD data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_EVD_UTIL_H_ +#define _DAPL_EVD_UTIL_H_ + +#include "dapl.h" + +DAT_RETURN +dapls_evd_internal_create ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_ptr, + IN DAT_COUNT min_qlen, + IN DAT_EVD_FLAGS evd_flags, + OUT DAPL_EVD **evd_ptr_ptr) ; + +DAPL_EVD * +dapls_evd_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_ptr, + IN DAT_EVD_FLAGS evd_flags, + IN DAT_COUNT qlen) ; + +DAT_RETURN +dapls_evd_dealloc ( + IN DAPL_EVD *evd_ptr) ; + +DAT_RETURN dapls_evd_event_realloc ( + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT qlen); + +/* + * Each of these functions will retrieve a free event from + * the specified EVD, fill in the elements of that event, and + * post the event back to the EVD. If there is no EVD available, + * an overflow event will be posted to the async EVD associated + * with the EVD. + * + * DAT_INSUFFICIENT_RESOURCES will be returned on overflow, + * DAT_SUCCESS otherwise. + */ + +DAT_RETURN +dapls_evd_post_cr_arrival_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_SP_HANDLE sp_handle, + DAT_IA_ADDRESS_PTR ia_address_ptr, + DAT_CONN_QUAL conn_qual, + DAT_CR_HANDLE cr_handle); + +DAT_RETURN +dapls_evd_post_connection_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT private_data_size, + IN DAT_PVOID private_data); + +DAT_RETURN +dapls_evd_post_async_error_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_IA_HANDLE ia_handle); + +DAT_RETURN +dapls_evd_post_software_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_PVOID pointer); + +DAT_RETURN +dapls_evd_post_generic_event ( + IN DAPL_EVD *evd_ptr, + IN DAT_EVENT_NUMBER event_number, + IN DAT_EVENT_DATA *data); + +/************************************* + * dapl internal callbacks functions * + *************************************/ + +/* connection verb callback */ +extern void dapl_evd_connection_callback ( + IN dp_ib_cm_handle_t ib_cm_handle, + IN const ib_cm_events_t ib_cm_events, + IN const void *instant_data_p, + IN const void * context ); + +/* dto verb callback */ +extern void dapl_evd_dto_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_cq_handle_t ib_cq_handle, + IN void* context); + +/* async verb callbacks */ +extern void dapl_evd_un_async_error_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_error_record_t * cause_ptr, + IN void * context); + +extern void dapl_evd_cq_async_error_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_cq_handle_t ib_cq_handle, + IN ib_error_record_t * cause_ptr, + IN void * context); + +extern void dapl_evd_qp_async_error_callback ( + IN ib_hca_handle_t ib_hca_handle, + IN ib_qp_handle_t ib_qp_handle, + IN ib_error_record_t * cause_ptr, + IN void * context); + +extern void dapls_evd_copy_cq ( + DAPL_EVD *evd_ptr); + +extern DAT_RETURN dapls_evd_cq_poll_to_event ( + IN DAPL_EVD *evd_ptr, + OUT DAT_EVENT *event); + +#endif diff --git a/trunk/ulp/dapl2/dapl/common/dapl_get_consumer_context.c b/trunk/ulp/dapl2/dapl/common/dapl_get_consumer_context.c new file mode 100644 index 00000000..04a4d51c --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_get_consumer_context.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_get_consumer_context.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_get_consumer_context + * + * DAPL Requirements Version xxx, 6.2.2.2 + * + * Gets the consumer context from the specified dat_object + * + * Input: + * dat_handle + * + * Output: + * context + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_get_consumer_context ( + IN DAT_HANDLE dat_handle, + OUT DAT_CONTEXT *context ) +{ + DAT_RETURN dat_status; + DAPL_HEADER *header; + + dat_status = DAT_SUCCESS; + + header = (DAPL_HEADER *)dat_handle; + if ( ((header) == NULL) || + DAPL_BAD_PTR(header) || + (header->magic != DAPL_MAGIC_IA && + header->magic != DAPL_MAGIC_EVD && + header->magic != DAPL_MAGIC_EP && + header->magic != DAPL_MAGIC_LMR && + header->magic != DAPL_MAGIC_RMR && + header->magic != DAPL_MAGIC_PZ && + header->magic != DAPL_MAGIC_PSP && + header->magic != DAPL_MAGIC_RSP && + header->magic != DAPL_MAGIC_CR)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + if ( context == NULL || DAPL_BAD_PTR(header) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + *context = header->user_context; + +bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_get_handle_type.c b/trunk/ulp/dapl2/dapl/common/dapl_get_handle_type.c new file mode 100644 index 00000000..13f7d615 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_get_handle_type.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_get_handle_type.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_get_handle_type + * + * DAPL Requirements Version xxx, 6.2.2.6 + * + * Gets the handle type for the given dat_handle + * + * Input: + * dat_handle + * + * Output: + * handle_type + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ + +DAT_RETURN DAT_API +dapl_get_handle_type ( + IN DAT_HANDLE dat_handle, + OUT DAT_HANDLE_TYPE *handle_type ) +{ + DAT_RETURN dat_status; + DAPL_HEADER *header; + + dat_status = DAT_SUCCESS; + + header = (DAPL_HEADER *)dat_handle; + if ( ((header) == NULL) || + DAPL_BAD_PTR(header) || + (header->magic != DAPL_MAGIC_IA && + header->magic != DAPL_MAGIC_EVD && + header->magic != DAPL_MAGIC_EP && + header->magic != DAPL_MAGIC_LMR && + header->magic != DAPL_MAGIC_RMR && + header->magic != DAPL_MAGIC_PZ && + header->magic != DAPL_MAGIC_PSP && + header->magic != DAPL_MAGIC_RSP && + header->magic != DAPL_MAGIC_CR)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + *handle_type = header->handle_type; + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_hash.c b/trunk/ulp/dapl2/dapl/common/dapl_hash.c new file mode 100644 index 00000000..289dd562 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_hash.c @@ -0,0 +1,538 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_hash.c + * + * PURPOSE: Hash Table + * Description: + * + * Provides a generic hash table with chaining. + * + * $Id:$ + **********************************************************************/ + +#include "dapl_hash.h" + +/********************************************************************* + * * + * Structures * + * * + *********************************************************************/ + +/* + * A hash table element + */ +typedef struct DAPL_HASH_ELEM +{ + struct DAPL_HASH_ELEM * next_element; + DAPL_HASH_KEY key; + void * datum; +} DAPL_HASH_ELEM; + +/* + * The hash table + */ +struct dapl_hash_table +{ + unsigned long num_entries; + unsigned long tbl_size; + DAPL_HASH_ELEM *table; + DAPL_OS_LOCK lock; + /* + * statistics - we tally on insert operations, counting + * the number of entries in the whole hash table, as + * well as the length of chains we walk to insert. This + * ignores empty buckets, giving us data on overall table + * occupancy, as well as max/average chain length for + * the buckets used. If our hash function results in + * hot buckets, this will show it. + */ + uint64_t hash_tbl_inserts; /* total inserts ops */ + uint64_t hash_tbl_max; /* max in entire table */ + uint64_t hash_tbl_total; /* total in table */ + uint64_t hash_chn_max; /* longest chain */ + uint64_t hash_chn_total; /* total non-0 lenghts */ +}; + + +/********************************************************************* + * * + * Defines * + * * + *********************************************************************/ + +/* datum value in empty table slots (use 0UL or ~0UL as appropriate) */ +#define NO_DATUM_VALUE ((void *) 0UL) +#define NO_DATUM(value) ((value) == NO_DATUM_VALUE) + +/* Lookup macro (which falls back to function to rehash) */ +#define DAPL_HASHLOOKUP( p_table, in_key, out_datum, bucket_head) \ + do { \ + DAPL_HASH_KEY save_key = in_key; \ + DAPL_HASH_ELEM *element = \ + &((p_table)->table)[DAPL_DOHASH(in_key,(p_table)->tbl_size)]; \ + in_key = save_key; \ + if (NO_DATUM(element->datum)) { \ + (bucket_head) = (void *)0; \ + } else if (element->key == (DAPL_HASH_KEY) (in_key)) { \ + (out_datum) = element->datum; \ + (bucket_head) = (void *)element; \ + } else if (element->next_element) { \ + dapli_hash_rehash(element, \ + (in_key), \ + (void **)&(out_datum), \ + (DAPL_HASH_ELEM **)&(bucket_head)); \ + } else { \ + (bucket_head) = (void *)0; \ + }\ + } while (0) + + +/********************************************************************* + * * + * Internal Functions * + * * + *********************************************************************/ + +/* + * Rehash the key (used by add and lookup functions) + * + * Inputs: element element to rehash key + * key, datum datum for key head + * head head for list + */ +static void +dapli_hash_rehash ( + DAPL_HASH_ELEM * element, + DAPL_HASH_KEY key, + void **datum, + DAPL_HASH_ELEM ** head) +{ + /* + * assume we looked at the contents of element already, + * and start with the next element. + */ + dapl_os_assert (element->next_element); + dapl_os_assert (!NO_DATUM (element->datum)); + + *head = element; + while (1) + { + element = element->next_element; + if (!element) + { + break; + } + if (element->key == key) + { + *datum = element->datum; + return; + } + } + *head = 0; +} + +/* + * Add a new key to the hash table + * + * Inputs: + * table, key and datum to be added + * allow_dup - DAT_TRUE if dups are allowed + * Outputs: + * report_dup - should you care to know + * Returns: + * DAT_TRUE on success + */ +static DAT_BOOLEAN +dapli_hash_add ( + DAPL_HASH_TABLEP p_table, + DAPL_HASH_KEY key, + void *datum, + DAT_BOOLEAN allow_dup, + DAT_BOOLEAN * report_dup) +{ + void *olddatum; + DAPL_HASH_KEY hashValue, save_key = key; + DAPL_HASH_ELEM *found; + DAT_BOOLEAN status = DAT_FALSE; + unsigned int chain_len = 0; + + if (report_dup) + { + (*report_dup) = DAT_FALSE; + } + + if (NO_DATUM (datum)) + { + /* + * Reserved value used for datum + */ + dapl_dbg_log ( + DAPL_DBG_TYPE_ERR, + "dapli_hash_add () called with magic NO_DATA value (%p) " + "used as datum!\n", datum); + return DAT_FALSE; + } + + DAPL_HASHLOOKUP (p_table, key, olddatum, found); + if (found) + { + /* + * key exists already + */ + if (report_dup) + { + *report_dup = DAT_TRUE; + } + + if (!allow_dup) + { + dapl_dbg_log ( + DAPL_DBG_TYPE_ERR, + "dapli_hash_add () called with duplicate key (" F64x ")\n", + key); + return DAT_FALSE; + } + } + + hashValue = DAPL_DOHASH (key, p_table->tbl_size); + key = save_key; + if (NO_DATUM (p_table->table[hashValue].datum)) + { + /* + * Empty head - just fill it in + */ + p_table->table[hashValue].key = key; + p_table->table[hashValue].datum = datum; + p_table->table[hashValue].next_element = 0; + p_table->num_entries++; + status = DAT_TRUE; + } + else + { + DAPL_HASH_ELEM *newelement = (DAPL_HASH_ELEM *) + dapl_os_alloc (sizeof (DAPL_HASH_ELEM)); + /* + * Add an element to the end of the chain + */ + if (newelement) + { + DAPL_HASH_ELEM *lastelement; + newelement->key = key; + newelement->datum = datum; + newelement->next_element = 0; + for (lastelement = &p_table->table[hashValue]; + lastelement->next_element; + lastelement = lastelement->next_element) + { + /* Walk to the end of the chain */ + chain_len++; + } + lastelement->next_element = newelement; + p_table->num_entries++; + status = DAT_TRUE; + } + else + { + /* allocation failed - should not happen */ + status = DAT_FALSE; + } + } + + /* + * Tally up our counters. chain_len is one less than current chain + * length. + */ + chain_len++; + p_table->hash_tbl_inserts++; + p_table->hash_tbl_total += p_table->num_entries; + p_table->hash_chn_total += chain_len; + if (p_table->num_entries > p_table->hash_tbl_max) + { + p_table->hash_tbl_max = p_table->num_entries; + } + if (chain_len > p_table->hash_chn_max) + { + p_table->hash_chn_max = chain_len; + } + + return status; +} + + +/* + * Remove element from hash bucket + * + * Inputs: + * element, key to be deleted + * Returns: + * DAT_TRUE on success + */ +static DAT_BOOLEAN +dapl_hash_delete_element (DAPL_HASH_ELEM * element, + DAPL_HASH_KEY key, + DAPL_HASH_DATA *p_datum) +{ + DAPL_HASH_ELEM *curelement; + DAPL_HASH_ELEM *lastelement; + + lastelement = NULL; + for (curelement = element; + curelement; + lastelement = curelement, curelement = curelement->next_element) + { + if (curelement->key == key) + { + if (p_datum) + { + *p_datum = curelement->datum; + } + if (lastelement) + { + /* + * curelement was malloc'd; free it + */ + lastelement->next_element = curelement->next_element; + dapl_os_free ((void *) curelement, sizeof (DAPL_HASH_ELEM)); + } + else + { + /* + * curelement is static list head + */ + DAPL_HASH_ELEM *n = curelement->next_element; + if (n) + { + /* + * If there is a next element, copy its contents into the + * head and free the original next element. + */ + curelement->key = n->key; + curelement->datum = n->datum; + curelement->next_element = n->next_element; + dapl_os_free ((void *) n, sizeof (DAPL_HASH_ELEM)); + } + else + { + curelement->datum = NO_DATUM_VALUE; + } + } + break; + } + } + + return ( curelement != NULL ? DAT_TRUE : DAT_FALSE ); +} + + +/********************************************************************* + * * + * External Functions * + * * + *********************************************************************/ + + +/* + * Create a new hash table with at least 'table_size' hash buckets. + */ +DAT_RETURN +dapls_hash_create ( + IN DAT_COUNT table_size, + OUT DAPL_HASH_TABLE **pp_table) +{ + DAPL_HASH_TABLE *p_table; + DAT_COUNT table_length = table_size * sizeof (DAPL_HASH_ELEM); + DAT_RETURN dat_status; + DAT_COUNT i; + + dapl_os_assert (pp_table); + dat_status = DAT_SUCCESS; + + /* Allocate hash table */ + p_table = dapl_os_alloc (sizeof (DAPL_HASH_TABLE)); + if (NULL == p_table) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + /* Init hash table, allocate and init and buckets */ + dapl_os_memzero (p_table, sizeof (DAPL_HASH_TABLE)); + p_table->tbl_size = table_size; + p_table->table = (DAPL_HASH_ELEM *) dapl_os_alloc (table_length); + if (NULL == p_table->table) + { + dapl_os_free (p_table, sizeof (DAPL_HASH_TABLE)); + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_lock_init (&p_table->lock); + for (i = 0; i < table_size; i++) + { + p_table->table[i].datum = NO_DATUM_VALUE; + p_table->table[i].key = 0; + p_table->table[i].next_element = 0; + } + + *pp_table = p_table; + + bail: + return DAT_SUCCESS; +} + + +/* + * Destroy a hash table + */ +DAT_RETURN +dapls_hash_free ( + IN DAPL_HASH_TABLE *p_table) +{ + dapl_os_assert (p_table && p_table->table); + + dapl_os_lock_destroy (&p_table->lock); + dapl_os_free (p_table->table, sizeof (DAPL_HASH_ELEM) * p_table->tbl_size); + dapl_os_free (p_table, sizeof (DAPL_HASH_TABLE)); + + return DAT_SUCCESS; +} + + +/* + * Returns the number of elements stored in the table + */ + +DAT_RETURN +dapls_hash_size ( + IN DAPL_HASH_TABLE *p_table, + OUT DAT_COUNT *p_size) +{ + dapl_os_assert (p_table && p_size); + + *p_size = p_table->num_entries; + + return DAT_SUCCESS; +} + + +/* + * Inserts the specified data into the table with the given key. + * Duplicates are not expected, and return in error, having done nothing. + */ + +DAT_RETURN +dapls_hash_insert ( + IN DAPL_HASH_TABLE *p_table, + IN DAPL_HASH_KEY key, + IN DAPL_HASH_DATA data) +{ + DAT_RETURN dat_status; + + dapl_os_assert (p_table); + dat_status = DAT_SUCCESS; + + dapl_os_lock (&p_table->lock); + if (!dapli_hash_add (p_table, key, data, DAT_FALSE, NULL)) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + } + dapl_os_unlock (&p_table->lock); + + return dat_status; +} + + +/* + * Searches for the given key. If found, + * DAT_SUCCESS is returned and the associated + * data is returned in the DAPL_HASH_DATA + * pointer if that pointer is not NULL. + */ +DAT_RETURN +dapls_hash_search ( + IN DAPL_HASH_TABLE *p_table, + IN DAPL_HASH_KEY key, + OUT DAPL_HASH_DATA *p_data) +{ + DAT_RETURN dat_status; + void *olddatum; + DAPL_HASH_ELEM *found; + + dapl_os_assert (p_table); + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,0); + + dapl_os_lock (&p_table->lock); + DAPL_HASHLOOKUP (p_table, key, olddatum, found); + dapl_os_unlock (&p_table->lock); + + if (found) + { + if (p_data) + { + *p_data = olddatum; + } + dat_status = DAT_SUCCESS; + } + + return dat_status; +} + + +DAT_RETURN +dapls_hash_remove ( + IN DAPL_HASH_TABLE *p_table, + IN DAPL_HASH_KEY key, + OUT DAPL_HASH_DATA *p_data) +{ + DAT_RETURN dat_status; + DAPL_HASH_KEY hashValue, save_key = key; + + dapl_os_assert (p_table); + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,0); + + if (p_table->num_entries == 0) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dapls_hash_remove () called on empty hash table!\n"); + return dat_status; + } + + hashValue = DAPL_DOHASH (key, p_table->tbl_size); + key = save_key; + dapl_os_lock (&p_table->lock); + if (dapl_hash_delete_element (&p_table->table[hashValue], key, p_data)) + { + p_table->num_entries--; + dat_status = DAT_SUCCESS; + } + dapl_os_unlock (&p_table->lock); + + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_hash.h b/trunk/ulp/dapl2/dapl/common/dapl_hash.h new file mode 100644 index 00000000..b0531e95 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_hash.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_hash.h + * + * PURPOSE: Utility defs & routines for the hash data structure + * + * $Id:$ + **********************************************************************/ + +#ifndef _DAPL_HASH_H_ +#define _DAPL_HASH_H_ + +#include "dapl.h" + + +/********************************************************************* + * * + * Defines * + * * + *********************************************************************/ + +/* + * Hash table size. + * + * Default is small; use the larger sample values for hash tables + * known to be heavily used. The sample values chosen are the + * largest primes below 2^8, 2^9, and 2^10. + */ +#define DAPL_DEF_HASHSIZE 251 +#define DAPL_MED_HASHSIZE 509 +#define DAPL_LRG_HASHSIZE 1021 + +#define DAPL_HASH_TABLE_DEFAULT_CAPACITY DAPL_DEF_HASHSIZE + +/* The hash function */ +#if defined(__KDAPL__) +#define DAPL_DOHASH(key,hashsize) dapl_os_mod64(key,hashsize) +#else +#define DAPL_DOHASH(key,hashsize) ((uint64_t)((key) % (hashsize))) +#endif /* defined(__KDAPL__) */ + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +extern DAT_RETURN +dapls_hash_create( + IN DAT_COUNT capacity, + OUT DAPL_HASH_TABLE **pp_table); + +extern DAT_RETURN +dapls_hash_free( + IN DAPL_HASH_TABLE *p_table); + +extern DAT_RETURN +dapls_hash_size( + IN DAPL_HASH_TABLE *p_table, + OUT DAT_COUNT *p_size); + +extern DAT_RETURN +dapls_hash_insert( + IN DAPL_HASH_TABLE *p_table, + IN DAPL_HASH_KEY key, + IN DAPL_HASH_DATA data); + +extern DAT_RETURN +dapls_hash_search( + IN DAPL_HASH_TABLE *p_table, + IN DAPL_HASH_KEY key, + OUT DAPL_HASH_DATA *p_data); + +extern DAT_RETURN +dapls_hash_remove( + IN DAPL_HASH_TABLE *p_table, + IN DAPL_HASH_KEY key, + OUT DAPL_HASH_DATA *p_data); + + +#endif /* _DAPL_HASH_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_hca_util.c b/trunk/ulp/dapl2/dapl/common/dapl_hca_util.c new file mode 100644 index 00000000..9cbb92f2 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_hca_util.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_hca_util.c + * + * PURPOSE: Manage HCA structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_provider.h" +#include "dapl_hca_util.h" +#include "dapl_hash.h" + + +/* + * dapl_hca_alloc + * + * alloc and initialize an HCA struct + * + * Input: + * name + * port + * + * Output: + * hca_ptr + * + * Returns: + * none + * + */ +DAPL_HCA * +dapl_hca_alloc ( + char *name, + char *port ) +{ + DAPL_HCA *hca_ptr; + + hca_ptr = dapl_os_alloc (sizeof (DAPL_HCA)); + if ( NULL != hca_ptr ) + { + dapl_os_memzero (hca_ptr, sizeof (DAPL_HCA)); + + if ( DAT_SUCCESS == dapls_hash_create ( + DAPL_HASH_TABLE_DEFAULT_CAPACITY, &hca_ptr->lmr_hash_table) ) + { + dapl_os_lock_init(&hca_ptr->lock); + dapl_llist_init_head(&hca_ptr->ia_list_head); + + hca_ptr->name = dapl_os_strdup(name); + hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; + hca_ptr->port_num = dapl_os_strtol(port, NULL, 0); + if (hca_ptr->name == NULL) + { + dapl_os_free (hca_ptr, sizeof (DAPL_HCA)); + hca_ptr = NULL; + } + } + else + { + dapl_os_free (hca_ptr, sizeof (DAPL_HCA)); + hca_ptr = NULL; + } + } + return (hca_ptr); +} + +/* + * dapl_hca_free + * + * free an IA INFO struct + * + * Input: + * hca_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_hca_free ( DAPL_HCA *hca_ptr ) +{ + (void) dapls_hash_free ( hca_ptr->lmr_hash_table ); + dapl_os_free(hca_ptr->name, dapl_os_strlen(hca_ptr->name)+1); + dapl_os_free (hca_ptr, sizeof (DAPL_HCA)); +} + +/* + * dapl_hca_link_ia + * + * Add an ia to the HCA structure + * + * Input: + * hca_ptr + * ia_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_hca_link_ia ( + IN DAPL_HCA *hca_ptr, + IN DAPL_IA *ia_ptr) +{ + dapl_os_lock (&hca_ptr->lock); + dapl_llist_add_head (&hca_ptr->ia_list_head, + &ia_ptr->hca_ia_list_entry, + ia_ptr); + dapl_os_unlock (&hca_ptr->lock); +} + +/* + * dapl_hca_unlink_ia + * + * Remove an ia from the hca info structure + * + * Input: + * hca_ptr + * ia_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_hca_unlink_ia ( + IN DAPL_HCA *hca_ptr, + IN DAPL_IA *ia_ptr) +{ + dapl_os_lock (&hca_ptr->lock); + /* + * If an error occurred when we were opening the IA it + * will not be linked on the list; don't unlink an unlinked + * list! + */ + if ( ! dapl_llist_is_empty (&hca_ptr->ia_list_head) ) + { + dapl_llist_remove_entry (&hca_ptr->ia_list_head, + &ia_ptr->hca_ia_list_entry); + } + dapl_os_unlock (&hca_ptr->lock); +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_hca_util.h b/trunk/ulp/dapl2/dapl/common/dapl_hca_util.h new file mode 100644 index 00000000..e04d8776 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_hca_util.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_hca_util.h + * + * PURPOSE: Utility defs & routines for the HCA data structure + * + * $Id:$ + **********************************************************************/ + +#ifndef _DAPL_HCA_UTIL_H_ +#define _DAPL_HCA_UTIL_H_ + +#include "dapl.h" + +DAPL_HCA * +dapl_hca_alloc ( char *name, + char *port ) ; + +void +dapl_hca_free ( DAPL_HCA *hca_ptr ) ; + +void +dapl_hca_link_ia ( + IN DAPL_HCA *hca_ptr, + IN DAPL_IA *ia_info ) ; + +void +dapl_hca_unlink_ia ( + IN DAPL_HCA *hca_ptr, + IN DAPL_IA *ia_info ) ; + + +#endif diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ia_close.c b/trunk/ulp/dapl2/dapl/common/dapl_ia_close.c new file mode 100644 index 00000000..dec10780 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ia_close.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ia_close.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" + +/* + * dapl_ia_close + * + * DAPL Requirements Version xxx, 6.2.1.2 + * + * Close a provider, clean up resources, etc. + * + * Input: + * ia_handle + * + * Output: + * none + * + * Return Values: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_ia_close ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_CLOSE_FLAGS ia_flags) +{ + DAPL_IA *ia_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ia_close (%p, %d)\n", + ia_handle, + ia_flags); + + ia_ptr = (DAPL_IA *)ia_handle; + + if (DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + + if ( DAT_CLOSE_ABRUPT_FLAG == ia_flags ) + { + dat_status = dapl_ia_abrupt_close (ia_ptr); + } + else if ( DAT_CLOSE_GRACEFUL_FLAG == ia_flags ) + { + dat_status = dapl_ia_graceful_close (ia_ptr); + } + else + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + } + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ia_ha.c b/trunk/ulp/dapl2/dapl/common/dapl_ia_ha.c new file mode 100644 index 00000000..c728aec5 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ia_ha.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ia_ha.c + * + * PURPOSE: Interface Adapter High Availability - optional feature + * Description: Described in DAPL 2.0 API, Chapter 5, section 9 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_provider.h" +#include "dapl_evd_util.h" +#include "dapl_hca_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_ia_ha + * + * DAPL Requirements Version xxx, 5.9 + * + * Queries for provider HA support + * + * Input: + * ia_handle + * provider name + * + * Output: + * answer + * + * Returns: + * DAT_SUCCESS + * DAT_MODEL_NOT_SUPPORTED + */ + +DAT_RETURN DAT_API +dapl_ia_ha ( + IN DAT_IA_HANDLE ia_handle, /* ia_handle */ + IN const DAT_NAME_PTR provider, /* provider */ + OUT DAT_BOOLEAN *answer) /* answer */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ia_open.c b/trunk/ulp/dapl2/dapl/common/dapl_ia_open.c new file mode 100644 index 00000000..9eba640b --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ia_open.c @@ -0,0 +1,510 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ia_open.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_provider.h" +#include "dapl_evd_util.h" +#include "dapl_hca_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + + +/* + * LOCAL PROTOTYPES + */ +#ifdef IBHOSTS_NAMING +void dapli_assign_hca_ip_address ( + DAPL_HCA *hca_ptr, + char *device_name); +#endif /* IBHOSTS_NAMING */ + +void dapli_hca_cleanup ( + DAPL_HCA *hca_ptr, + DAT_BOOLEAN dec_ref ); + + +/* + * dapl_ia_open + * + * DAPL Requirements Version xxx, 6.2.1.1 + * + * Open a provider and return a handle. The handle enables the user + * to invoke operations on this provider. + * + * The dat_ia_open call is actually part of the DAT registration module. + * That function maps the DAT_NAME parameter of dat_ia_open to a DAT_PROVIDER, + * and calls this function. + * + * Input: + * provider + * async_evd_qlen + * async_evd_handle_ptr + * + * Output: + * async_evd_handle + * ia_handle + * + * Return Values: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_PROVIDER_NOT_FOUND (returned by dat registry if necessary) + */ +DAT_RETURN DAT_API +dapl_ia_open ( + IN const DAT_NAME_PTR name, + IN DAT_COUNT async_evd_qlen, + INOUT DAT_EVD_HANDLE *async_evd_handle_ptr, + OUT DAT_IA_HANDLE *ia_handle_ptr) +{ + DAT_RETURN dat_status; + DAT_PROVIDER *provider; + DAPL_HCA *hca_ptr; + DAPL_IA *ia_ptr; + DAPL_EVD *evd_ptr; + + dat_status = DAT_SUCCESS; + hca_ptr = NULL; + ia_ptr = NULL; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ia_open (%s, %d, %p, %p)\n", + name, + async_evd_qlen, + async_evd_handle_ptr, + ia_handle_ptr); + + dat_status = dapl_provider_list_search (name, &provider); + if ( DAT_SUCCESS != dat_status ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG1); + goto bail; + } + + /* ia_handle_ptr and async_evd_handle_ptr cannot be NULL */ + if (ia_handle_ptr == NULL) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; + } + if (async_evd_handle_ptr == NULL) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + /* initialize the caller's OUT param */ + *ia_handle_ptr = DAT_HANDLE_NULL; + + /* get the hca_ptr */ + hca_ptr = (DAPL_HCA *) provider->extension; + + /* + * Open the HCA if it has not been done before. + */ + dapl_os_lock (&hca_ptr->lock); + if (hca_ptr->ib_hca_handle == IB_INVALID_HANDLE ) + { + /* register with the HW */ + dat_status = dapls_ib_open_hca (hca_ptr->name, + hca_ptr); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dapls_ib_open_hca failed %x\n", dat_status); + dapl_os_unlock (&hca_ptr->lock); + goto bail; + } + + /* + * Obtain the IP address associated with this name and HCA. + */ +#ifdef IBHOSTS_NAMING + dapli_assign_hca_ip_address (hca_ptr, name); +#endif /* IBHOSTS_NAMING */ + + /* + * Obtain IA attributes from the HCA to limit certain + * operations. + * If using DAPL_ATS naming, ib_query_hca will also set the ip + * address. + */ + dat_status = dapls_ib_query_hca (hca_ptr, + &hca_ptr->ia_attr, + NULL, + &hca_ptr->hca_address); + if (dat_status != DAT_SUCCESS) + { + dapli_hca_cleanup (hca_ptr, DAT_FALSE); + dapl_os_unlock (&hca_ptr->lock); + goto bail; + } + } + + /* Take a reference on the hca_handle */ + dapl_os_atomic_inc (& hca_ptr->handle_ref_count ); + dapl_os_unlock (&hca_ptr->lock); + + /* Allocate and initialize ia structure */ + ia_ptr = dapl_ia_alloc (provider, hca_ptr); + if (!ia_ptr) + { + dapl_os_lock (&hca_ptr->lock); + dapli_hca_cleanup (hca_ptr, DAT_TRUE); + dapl_os_unlock (&hca_ptr->lock); + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + /* we need an async EVD for this IA + * use the one passed in (if non-NULL) or create one + */ + + evd_ptr = (DAPL_EVD *) *async_evd_handle_ptr; + if (evd_ptr) + { + if (DAPL_BAD_HANDLE (evd_ptr, DAPL_MAGIC_EVD) || + ! (evd_ptr->evd_flags & DAT_EVD_ASYNC_FLAG)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_ASYNC); + goto bail; + } + + /* InfiniBand allows only 1 asychronous event handler per HCA */ + /* (see InfiniBand Spec, release 1.1, vol I, section 11.5.2, */ + /* page 559). */ + /* */ + /* We only need to make sure that this EVD's CQ belongs to */ + /* the same HCA as is being opened. */ + + if ( evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle != + hca_ptr->ib_hca_handle ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_ASYNC); + goto bail; + } + + ia_ptr->cleanup_async_error_evd = DAT_FALSE; + ia_ptr->async_error_evd = evd_ptr; + } + else + { + /* Verify we have >0 length, and let the provider check the size */ + if ( async_evd_qlen <= 0 ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail; + } + dat_status = dapls_evd_internal_create (ia_ptr, + NULL, /* CNO ptr */ + async_evd_qlen, + DAT_EVD_ASYNC_FLAG, + &evd_ptr); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + dapl_os_atomic_inc (& evd_ptr->evd_ref_count); + + dapl_os_lock ( &hca_ptr->lock ); + if ( hca_ptr->async_evd != (DAPL_EVD *) 0 ) + { +#if 0 + /* + * The async EVD for this HCA has already been assigned. + * It's an error to try and assign another one. + * + * However, we need to somehow allow multiple IAs + * off of the same HCA. The right way to do this + * is by dispatching events off the HCA to the appropriate + * IA, but we aren't there yet. So for now we create + * the EVD but don't connect it to anything. + */ + dapl_os_atomic_dec ( &evd_ptr->evd_ref_count ); + dapl_evd_free ( evd_ptr ); + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; +#endif + dapl_os_unlock ( &hca_ptr->lock ); + } + else + { + hca_ptr->async_evd = evd_ptr; + dapl_os_unlock ( &hca_ptr->lock ); + + /* Register the handlers associated with the async EVD. */ + dat_status = dapls_ia_setup_callbacks ( ia_ptr, evd_ptr ); + if ( dat_status != DAT_SUCCESS ) + { + /* Assign the EVD so it gets cleaned up */ + ia_ptr->cleanup_async_error_evd = DAT_TRUE; + ia_ptr->async_error_evd = evd_ptr; + goto bail; + } + } + + ia_ptr->cleanup_async_error_evd = DAT_TRUE; + ia_ptr->async_error_evd = evd_ptr; + } + + dat_status = DAT_SUCCESS; + *ia_handle_ptr = ia_ptr; + *async_evd_handle_ptr = evd_ptr; + +bail: + if (dat_status != DAT_SUCCESS) + { + if (ia_ptr) + { + /* This will release the async EVD if needed. */ + dapl_ia_close (ia_ptr, DAT_CLOSE_ABRUPT_FLAG); + } + } + + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_ia_open () returns 0x%x\n", + dat_status); + + return dat_status; +} + + +/* + * dapli_hca_cleanup + * + * Clean up partially allocated HCA stuff. Strictly to make cleanup + * simple. + */ +void +dapli_hca_cleanup ( + DAPL_HCA *hca_ptr, + DAT_BOOLEAN dec_ref ) +{ + dapls_ib_close_hca (hca_ptr); + hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; + if ( dec_ref == DAT_TRUE ) + { + dapl_os_atomic_dec (& hca_ptr->handle_ref_count ); + } +} + + +#ifdef IBHOSTS_NAMING + +char *dapli_get_adapter_num ( + char *device_name); + +void dapli_setup_dummy_addr ( + IN DAPL_HCA *hca_ptr, + IN char *hca_name); +/* + * dapli_assign_hca_ip_address + * + * Obtain the IP address of the passed in name, which represents a + * port on the hca. There are three methods here to obtain the + * appropriate IP address, each with their own shortcoming: + * 1) IPOIB_NAMING. Requires the implementation of the IPoIB + * interface defined in include/dapl/ipoib_names.h. This is + * not the recommended interface as IPoIB is limited at + * the point we need to obtain an IP address on the + * passive side of a connection. The code supporting this + * implementation has been removed. + * + * 2) IBHOSTS. An entry exists in DNS and in the /etc/dapl/ibhosts + * file. The immediate drawback here is that we must dictate + * how to name the interface, which is a stated DAPL non-goal. + * In the broader perspective, this method requires us to xmit + * the IP address in the private data of a connection, which has + * other fun problems. This is the default method and is known to + * work, but it has problems. + * + * 3) Obtain the IP address from the driver, which has registered + * the address with the SA for retrieval. + * + * + * Input: + * hca_ptr Pointer to HCA structure + * device_name Name of device as reported by the provider + * + * Output: + * none + * + * Returns: + * char * to string number + */ +void +dapli_assign_hca_ip_address ( + DAPL_HCA *hca_ptr, + char *device_name) +{ + char *adapter_num; +#define NAMELEN 128 + struct addrinfo *addr; + char hostname[NAMELEN]; + char *str; + int rc; + + /* + * Obtain the IP address of the adapter. This is a simple + * scheme that creates a name that must appear available to + * DNS, e.g. it must be in the local site DNS or in the local + * /etc/hosts file, etc. + * + * -ib + * + * This scheme obviously doesn't work with adapters from + * multiple vendors, but will suffice in common installations. + */ + + rc = gethostname (hostname, NAMELEN); + /* + * Strip off domain info if it exists (e.g. mynode.mydomain.com) + */ + for (str = hostname; *str && *str != '.'; ) + { + str++; + } + if ( *str == '.' ) + { + *str = '\0'; + } + strcat (hostname, "-ib"); + adapter_num = dapli_get_adapter_num (device_name); + strcat (hostname, adapter_num); + + rc = dapls_osd_getaddrinfo (hostname, &addr); + + if ( rc != 0 ) + { + /* Not registered in DNS, provide a dummy value */ + dapli_setup_dummy_addr (hca_ptr, hostname); + } + else + { + hca_ptr->hca_address = * ((DAT_SOCK_ADDR6 *)addr->ai_addr); + } +} + + +/* + * dapli_stup_dummy_addr + * + * Set up a dummy local address for the HCA. Things are not going + * to work too well if this happens. + * We call this routine if: + * - remote host adapter name is not in DNS + * - IPoIB implementation is not correctly set up + * - Similar nonsense. + * + * Input: + * hca_ptr + * rhost_name Name of remote adapter + * + * Output: + * none + * + * Returns: + * none + */ +void +dapli_setup_dummy_addr ( + IN DAPL_HCA *hca_ptr, + IN char *rhost_name ) +{ + struct sockaddr_in *si; + + /* Not registered in DNS, provide a dummy value */ + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "WARNING: <%s> not registered in DNS, using dummy IP value\n", + rhost_name); + si = (struct sockaddr_in *)&hca_ptr->hca_address; + hca_ptr->hca_address.sin6_family = AF_INET; + si->sin_addr.s_addr = 0x01020304; +} + + +/* + * dapls_get_adapter_num + * + * Given a device name, return a string of the device number + * + * Input: + * device_name Name of device as reported by the provider + * + * Output: + * none + * + * Returns: + * char * to string number + */ +char * +dapli_get_adapter_num ( + char *device_name) +{ + static char *zero = "0"; + char *p; + + /* + * Optimisticaly simple algorithm: the device number appears at + * the end of the device name string. Device that do not end + * in a number are by default "0". + */ + + for (p = device_name; *p; p++) + { + if ( isdigit (*p) ) + { + return p; + } + } + + return zero; +} +#endif /* IBHOSTS_NAMING */ + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ia_query.c b/trunk/ulp/dapl2/dapl/common/dapl_ia_query.c new file mode 100644 index 00000000..ec48ee55 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ia_query.c @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ia_query.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_vendor.h" + +/* + * dapl_ia_query + * + * DAPL Requirements Version xxx, 6.2.1.3 + * + * Provide the consumer with Interface Adapter and Provider parameters. + * + * Input: + * ia_handle + * ia_mask + * provider_mask + * + * Output: + * async_evd_handle + * ia_parameters + * provider_parameters + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_ia_query ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_EVD_HANDLE *async_evd_handle, + IN DAT_IA_ATTR_MASK ia_attr_mask, + OUT DAT_IA_ATTR *ia_attr, + IN DAT_PROVIDER_ATTR_MASK provider_attr_mask, + OUT DAT_PROVIDER_ATTR *provider_attr ) +{ + DAPL_IA *ia_ptr; + DAT_RETURN dat_status; + struct evd_merge_type { + DAT_BOOLEAN array[6][6]; + } *evd_merge; + DAT_BOOLEAN val; + int i; + int j; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_ia_query (%p, %p, 0x%llx, %p, 0x%x, %p)\n", + ia_handle, + async_evd_handle, + ia_attr_mask, + ia_attr, + provider_attr_mask, + provider_attr); + + ia_ptr = (DAPL_IA *)ia_handle; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + + if ( NULL != async_evd_handle ) + { + *async_evd_handle = ia_ptr->async_error_evd; + } + + if ( ia_attr_mask & DAT_IA_FIELD_ALL ) + { + if ( NULL == ia_attr ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; + } + + /* + * Obtain parameters from the HCA. Protect against multiple + * IAs beating on the HCA at the same time. + */ + dat_status = dapls_ib_query_hca (ia_ptr->hca_ptr, ia_attr, NULL, NULL); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + } + + if ( ia_attr_mask & ~DAT_IA_FIELD_ALL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + if ( provider_attr_mask & DAT_PROVIDER_FIELD_ALL ) + { + if ( NULL == provider_attr ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG6); + goto bail; + } + + strncpy ( provider_attr->provider_name, + ia_ptr->header.provider->device_name, + DAT_NAME_MAX_LENGTH ); + provider_attr->provider_version_major = VN_PROVIDER_MAJOR; + provider_attr->provider_version_minor = VN_PROVIDER_MINOR; + provider_attr->dapl_version_major = DAT_VERSION_MAJOR; + provider_attr->dapl_version_minor = DAT_VERSION_MINOR; + provider_attr->lmr_mem_types_supported = + DAT_MEM_TYPE_VIRTUAL | DAT_MEM_TYPE_LMR; +#if VN_MEM_SHARED_VIRTUAL_SUPPORT > 0 && !defined(__KDAPL__) + provider_attr->lmr_mem_types_supported |= DAT_MEM_TYPE_SHARED_VIRTUAL; +#endif + provider_attr->iov_ownership_on_return = DAT_IOV_CONSUMER; + provider_attr->dat_qos_supported = DAT_QOS_BEST_EFFORT; + provider_attr->completion_flags_supported = DAT_COMPLETION_DEFAULT_FLAG; + provider_attr->is_thread_safe = DAT_FALSE; + /* + * N.B. The second part of the following equation will evaluate + * to 0 unless IBHOSTS_NAMING is enabled. + */ + provider_attr->max_private_data_size = + dapls_ib_private_data_size (NULL, DAPL_PDATA_CONN_REQ) - + (sizeof (DAPL_PRIVATE) - DAPL_MAX_PRIVATE_DATA_SIZE); + provider_attr->supports_multipath = DAT_FALSE; + provider_attr->ep_creator = DAT_PSP_CREATES_EP_NEVER; + provider_attr->optimal_buffer_alignment = DAT_OPTIMAL_ALIGNMENT; + /* The value of pz_support may vary by transport */ + provider_attr->num_provider_specific_attr = 0; + provider_attr->provider_specific_attr = NULL; +#if !defined(__KDAPL__) + provider_attr->pz_support = DAT_PZ_UNIQUE; +#endif /* !KDAPL */ + + /* + * Query for provider specific attributes + */ + dapls_query_provider_specific_attr(provider_attr); + + /* + * Set up evd_stream_merging_supported options. Note there is + * one bit per allowable combination, using the ordinal + * position of the DAT_EVD_FLAGS as positions in the + * array. e.g. + * [0][0] is DAT_EVD_SOFTWARE_FLAG | DAT_EVD_SOFTWARE_FLAG, + * [0][1] is DAT_EVD_SOFTWARE_FLAG | DAT_EVD_CR_FLAG, and + * [2][4] is DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG + * + * Most combinations are true, so initialize the array that way. + * Then finish by resetting the bad combinations. + * + * DAT_EVD_ASYNC_FLAG is not supported. InfiniBand only allows + * a single asynchronous event handle per HCA, and the first + * dat_ia_open forces the creation of the only one that can be + * used. We disallow the user from creating an ASYNC EVD here. + */ + + evd_merge = (struct evd_merge_type *)&provider_attr->evd_stream_merging_supported[0][0]; + val = DAT_TRUE; + for ( i = 0; i < 6; i++) + { + if (i > 4) + { + /* ASYNC EVD is 5, so entire row will be 0 */ + val = DAT_FALSE; + } + for ( j = 0; j < 5; j++) + { + evd_merge->array[i][j] = val; + } + /* Set the ASYNC_EVD column to FALSE */ + evd_merge->array[i][5] = DAT_FALSE; + } + +#ifndef DAPL_MERGE_CM_DTO + /* + * If an implementation supports CM and DTO completions on + * the same EVD then DAPL_MERGE_CM_DTO should be set to + * skip the following code + */ + /* DAT_EVD_DTO_FLAG | DAT_EVD_CONNECTION_FLAG */ + evd_merge->array[2][3] = DAT_FALSE; + /* DAT_EVD_CONNECTION_FLAG | DAT_EVD_DTO_FLAG */ + evd_merge->array[3][2] = DAT_FALSE; +#endif /* DAPL_MERGE_CM_DTO */ + } + +bail: + if (dat_status != DAT_SUCCESS) + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_ia_query () returns 0x%x\n", dat_status); + + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ia_util.c b/trunk/ulp/dapl2/dapl/common/dapl_ia_util.c new file mode 100644 index 00000000..7551a23e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ia_util.c @@ -0,0 +1,1248 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ia_util.c + * + * PURPOSE: Manage IA Info structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_hca_util.h" +#include "dapl_ia_util.h" +#include "dapl_sp_util.h" +#include "dapl_evd_util.h" +#include "dapl_cno_util.h" +#include "dapl_cr_util.h" +#include "dapl_adapter_util.h" + +/* Internal prototype */ +void dapli_ia_release_hca ( + DAPL_HCA *hca_ptr ); + + +/* + * dapl_ia_alloc + * + * alloc and initialize an IA INFO struct + * + * Input: + * none + * + * Output: + * ia_ptr + * + * Returns: + * none + * + */ +DAPL_IA * +dapl_ia_alloc ( DAT_PROVIDER * provider, DAPL_HCA * hca_ptr ) +{ + DAPL_IA * ia_ptr; + + /* Allocate IA */ + ia_ptr = (DAPL_IA *) dapl_os_alloc (sizeof (DAPL_IA)); + if (ia_ptr == NULL) + { + return (NULL); + } + + /* zero the structure */ + dapl_os_memzero (ia_ptr, sizeof (DAPL_IA)); + + /* + * initialize the header + */ + ia_ptr->header.provider = provider; + ia_ptr->header.magic = DAPL_MAGIC_IA; + ia_ptr->header.handle_type = DAT_HANDLE_TYPE_IA; + ia_ptr->header.owner_ia = ia_ptr; + ia_ptr->header.user_context.as_64 = 0; + ia_ptr->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&ia_ptr->header.ia_list_entry); + dapl_os_lock_init (&ia_ptr->header.lock); + + /* + * initialize the body + */ + ia_ptr->hca_ptr = hca_ptr; + ia_ptr->async_error_evd = NULL; + ia_ptr->cleanup_async_error_evd = DAT_FALSE; + dapl_llist_init_entry (&ia_ptr->hca_ia_list_entry); + dapl_llist_init_head (&ia_ptr->ep_list_head); + dapl_llist_init_head (&ia_ptr->lmr_list_head); + dapl_llist_init_head (&ia_ptr->rmr_list_head); + dapl_llist_init_head (&ia_ptr->pz_list_head); + dapl_llist_init_head (&ia_ptr->evd_list_head); + dapl_llist_init_head (&ia_ptr->cno_list_head); + dapl_llist_init_head (&ia_ptr->rsp_list_head); + dapl_llist_init_head (&ia_ptr->psp_list_head); + + dapl_hca_link_ia (hca_ptr, ia_ptr); + + return (ia_ptr); +} + + +/* + * dapl_ia_abrupt_close + * + * Performs an abrupt close of the IA + * + * Input: + * ia_ptr + * + * Output: + * none + * + * Returns: + * status + * + */ + +DAT_RETURN +dapl_ia_abrupt_close ( + IN DAPL_IA *ia_ptr ) +{ + DAT_RETURN dat_status; + DAPL_EP *ep_ptr, *next_ep_ptr; + DAPL_LMR *lmr_ptr, *next_lmr_ptr; + DAPL_RMR *rmr_ptr, *next_rmr_ptr; + DAPL_PZ *pz_ptr, *next_pz_ptr; + DAPL_EVD *evd_ptr, *next_evd_ptr; + DAPL_CNO *cno_ptr, *next_cno_ptr; + DAPL_SP *sp_ptr, *next_sp_ptr; /* for PSP and RSP queues */ + DAPL_CR *cr_ptr, *next_cr_ptr; + DAPL_HCA *hca_ptr; + + dat_status = DAT_SUCCESS; + + /* + * clear all the data structures associated with the IA. + * this must be done in order (rmr,rsp) before (ep lmr psp) before + * (pz evd) + * + * Note that in all the following we can leave the loop either + * when we run out of entries, or when we get back to the head + * if we end up skipping an entry. + */ + + rmr_ptr = (dapl_llist_is_empty (&ia_ptr->rmr_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->rmr_list_head)); + while (rmr_ptr != NULL) + { + next_rmr_ptr = dapl_llist_next_entry (&ia_ptr->rmr_list_head, + &rmr_ptr->header.ia_list_entry); + dat_status = dapl_rmr_free (rmr_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): rmr_free(%p) returns %x\n", + rmr_ptr, + dat_status ); + } + rmr_ptr = next_rmr_ptr; + } + + sp_ptr = (dapl_llist_is_empty (&ia_ptr->rsp_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->rsp_list_head)); + while (sp_ptr != NULL) + { + next_sp_ptr = dapl_llist_next_entry (&ia_ptr->rsp_list_head, + &sp_ptr->header.ia_list_entry); + dat_status = dapl_rsp_free (sp_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): rsp_free(%p) returns %x\n", + sp_ptr, + dat_status ); + } + sp_ptr = next_sp_ptr; + } + + ep_ptr = (dapl_llist_is_empty (&ia_ptr->ep_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->ep_list_head)); + while (ep_ptr != NULL) + { + next_ep_ptr = dapl_llist_next_entry (&ia_ptr->ep_list_head, + &ep_ptr->header.ia_list_entry); + /* + * Issue a disconnect if the EP needs it + */ + if ( ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED || + ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED || + ep_ptr->param.ep_state == DAT_EP_STATE_ACTIVE_CONNECTION_PENDING || + ep_ptr->param.ep_state == DAT_EP_STATE_COMPLETION_PENDING || + ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING ) + { + dat_status = dapl_ep_disconnect (ep_ptr, DAT_CLOSE_ABRUPT_FLAG); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): ep_disconnect(%p) returns %x\n", + ep_ptr, + dat_status ); + } + } + /* force the EP into error state to force flush all posted DTOs. */ + { + DAT_EP_ATTR ep_attr; + DAT_NAMED_ATTR ep_state; + + dapl_os_memzero (&ep_attr, sizeof (DAT_EP_ATTR)); + ep_state.name = (char *)IB_QP_STATE; + ep_state.value = (char *)DAPL_QP_STATE_ERROR; + ep_attr.ep_provider_specific_count = 1; + ep_attr.ep_provider_specific = &ep_state; + + (void) dapls_ib_qp_modify (ia_ptr, + ep_ptr, + &ep_attr ); + } + + dat_status = dapl_ep_free (ep_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): ep_free(%p) returns %x\n", + ep_ptr, + dat_status ); + } + ep_ptr = next_ep_ptr; + } + + lmr_ptr = (dapl_llist_is_empty (&ia_ptr->lmr_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->lmr_list_head)); + while (lmr_ptr != NULL) + { + next_lmr_ptr = dapl_llist_next_entry (&ia_ptr->lmr_list_head, + &lmr_ptr->header.ia_list_entry); + dat_status = dapl_lmr_free (lmr_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): lmr_free(%p) returns %x\n", + lmr_ptr, + dat_status ); + } + lmr_ptr = next_lmr_ptr; + } + + sp_ptr = (dapl_llist_is_empty (&ia_ptr->psp_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->psp_list_head)); + while (sp_ptr != NULL) + { + /* + * Shut down the PSP so we get no further callbacks. There + * should be no competing threads after this. + */ + dat_status = dapls_ib_remove_conn_listener (ia_ptr, + sp_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): psp cannot remove listener, returns %x\n", + dat_status); + } + + next_sp_ptr = dapl_llist_next_entry (&ia_ptr->psp_list_head, + &sp_ptr->header.ia_list_entry); + + /* Remove CR's from this PSP and clean them up */ + cr_ptr = dapl_llist_is_empty (&sp_ptr->cr_list_head) ? NULL : + dapl_llist_peek_head (&sp_ptr->cr_list_head); + while (cr_ptr != NULL) + { + next_cr_ptr = dapl_llist_next_entry (&sp_ptr->cr_list_head, + &cr_ptr->header.ia_list_entry); + /* Remove the CR from the queue & cleanup*/ + dapl_os_lock (&sp_ptr->header.lock); + dapl_sp_remove_cr (sp_ptr, cr_ptr); + dapl_os_unlock (&sp_ptr->header.lock); + + dapls_cr_free (cr_ptr); + cr_ptr = next_cr_ptr; + } + + dat_status = dapl_psp_free (sp_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): psp_free(%p) returns %x\n", + sp_ptr, + dat_status ); + } + + sp_ptr = next_sp_ptr; + } + + pz_ptr = (dapl_llist_is_empty (&ia_ptr->pz_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->pz_list_head)); + while (pz_ptr != NULL) + { + next_pz_ptr = dapl_llist_next_entry (&ia_ptr->pz_list_head, + &pz_ptr->header.ia_list_entry); + dat_status = dapl_pz_free (pz_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): pz_free(%p) returns %x\n", + pz_ptr, + dat_status ); + } + pz_ptr = next_pz_ptr; + } + + /* + * EVDs are tricky; we want to release all except for the async + * EVD. That EVD needs to stick around until after we close the + * HCA, to accept any async events that occur. So we cycle through + * the list with dapl_llist_next_entry instead of dapl_llist_is_empty. + */ + evd_ptr = (dapl_llist_is_empty (&ia_ptr->evd_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->evd_list_head)); + while (evd_ptr != NULL) + { + next_evd_ptr = dapl_llist_next_entry (&ia_ptr->evd_list_head, + &evd_ptr->header.ia_list_entry); + if (evd_ptr == ia_ptr->async_error_evd) + { +#if !defined(__KDAPL__) + /* Don't delete the EVD, but break any CNO connections. */ + dapl_evd_disable (evd_ptr); + dapl_evd_modify_cno (evd_ptr, DAT_HANDLE_NULL); +#endif /* __KDAPL__ */ + } + else + { + /* it isn't the async EVD; delete it. */ + dat_status = dapl_evd_free (evd_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): evd_free(%p) returns %x\n", + evd_ptr, + dat_status ); + } + } + evd_ptr = next_evd_ptr; + } + + cno_ptr = (dapl_llist_is_empty (&ia_ptr->cno_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->cno_list_head)); + while (cno_ptr != NULL) + { + next_cno_ptr = dapl_llist_next_entry (&ia_ptr->cno_list_head, + &cno_ptr->header.ia_list_entry); +#if !defined(__KDAPL__) + if (cno_ptr->cno_waiters > 0) + { + /* Notify the waiter the IA is going away: see uDAPL 1.1 spec, + * 6.3.2.3 + */ + dapl_internal_cno_trigger (cno_ptr, NULL); + } + /* clean up the CNO */ + dat_status = dapl_cno_free (cno_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): cno_free(%p) returns %x\n", + cno_ptr, + dat_status ); + } +#endif /* __KDAPL__ */ + cno_ptr = next_cno_ptr; + } + + hca_ptr = ia_ptr->hca_ptr; + + /* + * Free the async EVD, shutting down callbacks from the HCA. + */ + if ( ia_ptr->async_error_evd && + (DAT_TRUE == ia_ptr->cleanup_async_error_evd) ) + { + dat_status = dapls_ia_teardown_callbacks ( ia_ptr ); + + dapl_os_atomic_dec (& ia_ptr->async_error_evd->evd_ref_count); + dat_status = dapl_evd_free (ia_ptr->async_error_evd); + + if (DAT_SUCCESS != dat_status) + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "ia_close(ABRUPT): evd_free(%p) returns %x\n", + ia_ptr->async_error_evd, + dat_status ); + } + + ia_ptr->async_error_evd = NULL; + } + + /* + * Release our reference on the hca_handle. If we are the last + * one, close it + */ + dapli_ia_release_hca (hca_ptr); + + dapls_ia_free (ia_ptr); + + return DAT_SUCCESS; /* Abrupt close can't fail. */ +} + + +/* + * dapl_ia_graceful_close + * + * Performs an graceful close of the IA + * + * Input: + * ia_ptr + * + * Output: + * none + * + * Returns: + * status + * + */ + +DAT_RETURN +dapl_ia_graceful_close ( + IN DAPL_IA *ia_ptr ) +{ + DAT_RETURN dat_status; + DAT_RETURN cur_dat_status; + DAPL_EVD *evd_ptr; + DAPL_LLIST_ENTRY *entry; + DAPL_HCA *hca_ptr; + + dat_status = DAT_SUCCESS; + + if ( !dapl_llist_is_empty (&ia_ptr->rmr_list_head) || + !dapl_llist_is_empty (&ia_ptr->rsp_list_head) || + !dapl_llist_is_empty (&ia_ptr->ep_list_head) || + !dapl_llist_is_empty (&ia_ptr->lmr_list_head) || + !dapl_llist_is_empty (&ia_ptr->psp_list_head) || + !dapl_llist_is_empty (&ia_ptr->pz_list_head) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_IA_IN_USE); + goto bail; + } + + /* if the async evd does not need to be cleaned up */ + /* (ie. it was not created by dapl_ia_open) */ + /* then the evd list should be empty */ + if ( DAT_FALSE == ia_ptr->cleanup_async_error_evd ) + { + if ( !dapl_llist_is_empty (&ia_ptr->evd_list_head) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_IA_IN_USE); + goto bail; + } + } + /* else the async evd should be the only evd in */ + /* the list. */ + else + { + evd_ptr = (dapl_llist_is_empty (&ia_ptr->evd_list_head) + ? NULL : dapl_llist_peek_head (&ia_ptr->evd_list_head)); + + if ( evd_ptr != NULL && + ! (evd_ptr->evd_flags & DAT_EVD_ASYNC_FLAG) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_IA_IN_USE); + goto bail; + } + + entry = ia_ptr->evd_list_head; + + /* if the async evd is not the only element in the list */ + if ( entry->blink != entry->flink ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_IA_IN_USE); + goto bail; + } + + /* + * If the async evd has a non-unary ref count (i.e. it's in + * use by someone besides us. + */ + if ( dapl_os_atomic_read (&evd_ptr->evd_ref_count) != 1 ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_IA_IN_USE); + goto bail; + } + } + + /* + * We've validated the call; now we can start the teardown. + * Because we're in the IA close routine, we're safe from races with DAPL + * consumers on this IA (operate/destroy races are disallowed in + * DAPL). + */ + hca_ptr = ia_ptr->hca_ptr; + + /* Tear down the async EVD if needed, first shutting down callbacks. */ + if ( ia_ptr->async_error_evd && + (DAT_TRUE == ia_ptr->cleanup_async_error_evd) ) + { + cur_dat_status = dapls_ia_teardown_callbacks ( ia_ptr ); + if ( DAT_SUCCESS != cur_dat_status ) + { + dat_status = cur_dat_status; + } + dapl_os_atomic_dec (& ia_ptr->async_error_evd->evd_ref_count); + cur_dat_status = dapl_evd_free (ia_ptr->async_error_evd); + if ( DAT_SUCCESS != cur_dat_status ) + { + dat_status = cur_dat_status; + } + + ia_ptr->async_error_evd = NULL; + } + + dapli_ia_release_hca (hca_ptr); + + dapls_ia_free (ia_ptr); + +bail: + return dat_status; +} + +/* + * Release a reference on the HCA handle. If it is 0, close the + * handle. Manipulate under lock to prevent races with threads trying to + * open the HCA. + */ +void +dapli_ia_release_hca ( + DAPL_HCA *hca_ptr ) +{ + dapl_os_lock (&hca_ptr->lock); + dapl_os_atomic_dec (& hca_ptr->handle_ref_count ); + if ( dapl_os_atomic_read (&hca_ptr->handle_ref_count) == 0 ) + { + dapls_ib_close_hca (hca_ptr); + hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; + hca_ptr->async_evd = NULL; + } + dapl_os_unlock (&hca_ptr->lock); +} + + +/* + * dapls_ia_free + * + * free an IA INFO struct + * + * Input: + * ia_ptr + * + * Output: + * one + * + * Returns: + * none + * + */ +void +dapls_ia_free ( DAPL_IA *ia_ptr ) +{ + dapl_os_assert (ia_ptr->header.magic == DAPL_MAGIC_IA); + + dapl_os_assert (ia_ptr->async_error_evd == NULL); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->lmr_list_head)); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->rmr_list_head)); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->ep_list_head)); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->evd_list_head)); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->cno_list_head)); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->psp_list_head)); + dapl_os_assert (dapl_llist_is_empty (&ia_ptr->rsp_list_head)); + + /* + * deinitialize the header + */ + dapl_hca_unlink_ia (ia_ptr->hca_ptr, ia_ptr); + ia_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ + dapl_os_lock_destroy (&ia_ptr->header.lock); + + dapl_os_free (ia_ptr, sizeof (DAPL_IA)); +} + +/* + * dapl_ia_link_ep + * + * Add an ep to the IA structure + * + * Input: + * ia_ptr + * ep_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_ep ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->ep_list_head, + &ep_ptr->header.ia_list_entry, + ep_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_ep + * + * Remove an ep from the ia info structure + * + * Input: + * ia_ptr + * ep_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_ep ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->ep_list_head, + &ep_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_srq + * + * Add an srq to the IA structure + * + * Input: + * ia_ptr + * srq_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_srq ( + IN DAPL_IA *ia_ptr, + IN DAPL_SRQ *srq_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->srq_list_head, + &srq_ptr->header.ia_list_entry, + srq_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_srq + * + * Remove an srq from the ia info structure + * + * Input: + * ia_ptr + * srq_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_srq ( + IN DAPL_IA *ia_ptr, + IN DAPL_SRQ *srq_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->srq_list_head, + &srq_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_lmr + * + * Add an lmr to the IA structure + * + * Input: + * ia_ptr + * lmr_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_lmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_LMR *lmr_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->lmr_list_head, + &lmr_ptr->header.ia_list_entry, + lmr_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_lmr + * + * Remove an lmr from the ia info structure + * + * Input: + * ia_ptr + * lmr_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_lmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_LMR *lmr_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->lmr_list_head, + &lmr_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_rmr + * + * Add an rmr to the IA structure + * + * Input: + * ia_ptr + * rmr_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_rmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_RMR *rmr_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->rmr_list_head, + &rmr_ptr->header.ia_list_entry, + rmr_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_rmr + * + * Remove an rmr from the ia info structure + * + * Input: + * ia_ptr + * rmr_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_rmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_RMR *rmr_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->rmr_list_head, + &rmr_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_pz + * + * Add an pz to the IA structure + * + * Input: + * ia_ptr + * pz_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_pz ( + IN DAPL_IA *ia_ptr, + IN DAPL_PZ *pz_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->pz_list_head, + &pz_ptr->header.ia_list_entry, + pz_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_pz + * + * Remove an pz from the ia info structure + * + * Input: + * ia_ptr + * pz_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_pz ( + IN DAPL_IA *ia_ptr, + IN DAPL_PZ *pz_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->pz_list_head, + &pz_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_evd + * + * Add an evd to the IA structure + * + * Input: + * ia_ptr + * evd_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_evd ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->evd_list_head, + &evd_ptr->header.ia_list_entry, + evd_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_evd + * + * Remove an evd from the ia info structure + * + * Input: + * ia_ptr + * evd_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_evd ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->evd_list_head, + &evd_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_cno + * + * Add an cno to the IA structure + * + * Input: + * ia_ptr + * cno_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_cno ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->cno_list_head, + &cno_ptr->header.ia_list_entry, + cno_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_unlink_cno + * + * Remove an cno from the ia info structure + * + * Input: + * ia_ptr + * cno_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_unlink_cno ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (&ia_ptr->cno_list_head, + &cno_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapl_ia_link_psp + * + * Add an psp to the IA structure + * + * Input: + * ia_ptr + * sp_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_psp ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->psp_list_head, + &sp_ptr->header.ia_list_entry, + sp_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * daps_ia_unlink_sp + * + * Remove an sp from the appropriate ia rsp or psp queue + * + * Input: + * ia_ptr + * sp_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapls_ia_unlink_sp ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_ptr) +{ + DAPL_LLIST_HEAD *list_head; + + if ( sp_ptr->header.handle_type == DAT_HANDLE_TYPE_PSP ) + { + list_head = &ia_ptr->psp_list_head; + } + else + { + dapl_os_assert (sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP); + list_head = &ia_ptr->rsp_list_head; + } + + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_remove_entry (list_head, + &sp_ptr->header.ia_list_entry); + dapl_os_unlock (&ia_ptr->header.lock); +} + +/* + * dapls_ia_sp_search + * + * Find an RSP or PSP on the IA list with a matching conn_qual value + * + * Input: + * ia_ptr + * sp_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +DAPL_SP * +dapls_ia_sp_search ( + IN DAPL_IA *ia_ptr, + IN DAT_CONN_QUAL conn_qual, + IN DAT_BOOLEAN is_psp ) +{ + DAPL_SP *sp_ptr; + DAPL_LLIST_HEAD *list_head; + + if ( is_psp ) + { + list_head = &ia_ptr->psp_list_head; + } + else + { + list_head = &ia_ptr->rsp_list_head; + } + + dapl_os_lock (&ia_ptr->header.lock); + + sp_ptr = (dapl_llist_is_empty (list_head) ? NULL : + dapl_llist_peek_head (list_head)); + + while (sp_ptr != NULL) + { + if ( sp_ptr->conn_qual == conn_qual ) + { + break; + } + sp_ptr = dapl_llist_next_entry (list_head, + &sp_ptr->header.ia_list_entry); + } + + dapl_os_unlock (&ia_ptr->header.lock); + + return sp_ptr; +} + + +/* + * dapl_ia_link_rsp + * + * Add an rsp to the IA structure + * + * Input: + * ia_ptr + * sp_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_ia_link_rsp ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_ptr) +{ + dapl_os_lock (&ia_ptr->header.lock); + dapl_llist_add_head (&ia_ptr->rsp_list_head, + &sp_ptr->header.ia_list_entry, + sp_ptr); + dapl_os_unlock (&ia_ptr->header.lock); +} + + +DAT_RETURN +dapls_ia_setup_callbacks ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *async_evd_ptr ) +{ + DAT_RETURN dat_status = DAT_SUCCESS; + + /* unaffiliated handler */ + dat_status = + dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_UNAFILIATED, + NULL, + (ib_async_handler_t)dapl_evd_un_async_error_callback, + async_evd_ptr); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ib_set_un_async_error_eh failed %d\n", dat_status); + goto bail; + } + + /* affiliated cq handler */ + dat_status = dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_CQ_ERROR, + NULL, + (ib_async_handler_t)dapl_evd_cq_async_error_callback, + async_evd_ptr); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ib_set_cq_async_error_eh failed %d\n", dat_status); + goto bail; + } + + /* affiliated qp handler */ + dat_status = dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_QP_ERROR, + NULL, + (ib_async_handler_t)dapl_evd_qp_async_error_callback, + ia_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ib_set_qp_async_error_eh failed %d\n", dat_status); + goto bail; + } + +bail: + return dat_status; +} + +DAT_RETURN +dapls_ia_teardown_callbacks ( + IN DAPL_IA *ia_ptr) +{ + DAT_RETURN dat_status = DAT_SUCCESS; + + /* unaffiliated handler */ + dat_status = + dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_UNAFILIATED, + NULL, + (ib_async_handler_t) 0, + NULL); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ib_set_un_async_error_eh failed %d\n", dat_status); + goto bail; + } + + /* affiliated cq handler */ + dat_status = dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_CQ_ERROR, + NULL, + (ib_async_handler_t) 0, + NULL); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ib_set_cq_async_error_eh failed %d\n", dat_status); + goto bail; + } + + /* affiliated qp handler */ + dat_status = dapls_ib_setup_async_callback ( + ia_ptr, + DAPL_ASYNC_QP_ERROR, + NULL, + (ib_async_handler_t) 0, + NULL); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ib_set_qp_async_error_eh failed %d\n", dat_status); + goto bail; + } + +bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ia_util.h b/trunk/ulp/dapl2/dapl/common/dapl_ia_util.h new file mode 100644 index 00000000..13675bf8 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ia_util.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_ia_util.h + * + * PURPOSE: Utility defs & routines for the IA data structure + * + * $Id:$ + **********************************************************************/ + +#ifndef _DAPL_IA_UTIL_H_ +#define _DAPL_IA_UTIL_H_ + +#include "dapl.h" + +DAPL_IA * +dapl_ia_alloc ( + DAT_PROVIDER *provider, + DAPL_HCA *hca_ptr) ; + +DAT_RETURN +dapl_ia_abrupt_close ( + IN DAPL_IA *ia_ptr ) ; + +DAT_RETURN +dapl_ia_graceful_close ( + IN DAPL_IA *ia_ptr ) ; + +void +dapls_ia_free ( DAPL_IA *ia_ptr ) ; + +void +dapl_ia_link_ep ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_info ) ; + +void +dapl_ia_unlink_ep ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_info ) ; + +void +dapl_ia_link_srq ( + IN DAPL_IA *ia_ptr, + IN DAPL_SRQ *srq_ptr ) ; + +void +dapl_ia_unlink_srq ( + IN DAPL_IA *ia_ptr, + IN DAPL_SRQ *srq_ptr ) ; + +void +dapl_ia_link_lmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_LMR *lmr_info ) ; + +void +dapl_ia_unlink_lmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_LMR *lmr_info ) ; + +void +dapl_ia_link_rmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_RMR *rmr_info ) ; + +void +dapl_ia_unlink_rmr ( + IN DAPL_IA *ia_ptr, + IN DAPL_RMR *rmr_info ) ; + +void +dapl_ia_link_pz ( + IN DAPL_IA *ia_ptr, + IN DAPL_PZ *pz_info ) ; + +void +dapl_ia_unlink_pz ( + IN DAPL_IA *ia_ptr, + IN DAPL_PZ *pz_info ) ; + +void +dapl_ia_link_evd ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_info ) ; + +void +dapl_ia_unlink_evd ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_info ) ; + +void +dapl_ia_link_cno ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_info ) ; + +void +dapl_ia_unlink_cno ( + IN DAPL_IA *ia_ptr, + IN DAPL_CNO *cno_info ) ; + +void +dapl_ia_link_psp ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_info ) ; + +void +dapls_ia_unlink_sp ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_info ) ; + +void +dapl_ia_link_rsp ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_info ) ; + +DAPL_SP * +dapls_ia_sp_search ( + IN DAPL_IA *ia_ptr, + IN DAT_CONN_QUAL conn_qual, + IN DAT_BOOLEAN is_psp ) ; + +DAT_RETURN +dapls_ia_setup_callbacks ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *async_evd_ptr ); + +DAT_RETURN +dapls_ia_teardown_callbacks ( + IN DAPL_IA *ia_ptr ); + +#endif diff --git a/trunk/ulp/dapl2/dapl/common/dapl_init.h b/trunk/ulp/dapl2/dapl/common/dapl_init.h new file mode 100644 index 00000000..0430e5b9 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_init.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_init.h + * + * PURPOSE: Prototypes for library-interface init and fini functions + * + * $Id:$ + * + **********************************************************************/ + + +#ifndef _DAPL_INIT_H_ +#define _DAPL_INIT_H_ + +extern void DAT_API +DAT_PROVIDER_INIT_FUNC_NAME ( + IN const DAT_PROVIDER_INFO *, + IN const char * ); /* instance data */ + +extern void DAT_API +DAT_PROVIDER_FINI_FUNC_NAME ( + IN const DAT_PROVIDER_INFO * ); + +extern void +dapl_init ( void ) ; + +extern void +dapl_fini ( void ) ; + +#endif diff --git a/trunk/ulp/dapl2/dapl/common/dapl_llist.c b/trunk/ulp/dapl2/dapl/common/dapl_llist.c new file mode 100644 index 00000000..3e6c7c10 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_llist.c @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_llist.c + * + * PURPOSE: Manage doubly linked lists within the DAPL Reference Implementation + * + * A link list head points to the first member of a linked list, but + * is itself not a member of the list. + * + * +---------------------------------------------+ + * | entry entry entry | + * HEAD -> | +-------+ +-------+ +-------+ | + * +--> | flink | --> | flink | --> | flink | >--+ + * | data | | data | | data | + * +--< | blink | <-- | blink | <-- | blink | <--| + * | +-------+ +-------+ +-------+ | + * | | + * +---------------------------------------------+ + * + * Note: Each of the remove functions takes an assertion failure if + * an element cannot be removed from the list. + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_llist_init_head() + * + * Purpose: initialize a linked list head + */ +void +dapl_llist_init_head (DAPL_LLIST_HEAD * head) +{ + *head = NULL; +} + +/* + * dapl_llist_init_entry() + * + * Purpose: initialize a linked list entry + */ +void +dapl_llist_init_entry (DAPL_LLIST_ENTRY * entry) +{ + entry->blink = NULL; + entry->flink = NULL; + entry->data = 0; + entry->list_head = NULL; +} + +/* + * dapl_llist_is_empty() + * + * Purpose: returns TRUE if the linked list is empty + */ +DAT_BOOLEAN +dapl_llist_is_empty (DAPL_LLIST_HEAD * head) +{ + return (*head == NULL); +} + +/* + * dapl_llist_add_head() + * + * Purpose: Add an entry to the head of a linked list + */ +void +dapl_llist_add_head (DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry, + void * data) +{ + DAPL_LLIST_ENTRY *first; + + /* deal with empty list */ + if (dapl_llist_is_empty (head)) + { + entry->flink = entry; + entry->blink = entry; + } + else + { + first = *head; + entry->flink = first; + entry->blink = first->blink; + first->blink->flink = entry; + first->blink = entry; + } + + *head = entry; + entry->data = data; + entry->list_head = head; +} + +/* + * dapl_llist_add_tail() + * + * Purpose: Add an entry to the tail of a linked list + */ +void +dapl_llist_add_tail (DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry, + void * data) +{ + DAPL_LLIST_ENTRY *last; + + /* deal with empty list */ + if (dapl_llist_is_empty (head)) + { + *head = entry; + entry->flink = entry; + entry->blink = entry; + } + else + { + last = (*head)->blink; + entry->flink = last->flink; + entry->blink = last; + last->flink->blink = entry; + last->flink = entry; + } + entry->data = data; + entry->list_head = head; +} + + +/* + * dapl_llist_add_entry() + * + * Purpose: Add an entry before an element in the list. Does + * not verify the list or the validity of the entries passed in. + */ +void +dapl_llist_add_entry (DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry, + DAPL_LLIST_ENTRY * new_entry, + void * data) +{ + DAPL_LLIST_ENTRY *first; + DAPL_LLIST_ENTRY *prev; + + /* get the head pointer */ + first = *head; + + /* link the entry into the queue */ + prev = entry->blink; + entry->blink = new_entry; + prev->flink = new_entry; + + new_entry->flink = entry; + new_entry->blink = prev; + + new_entry->data = data; + new_entry->list_head = head; + + /* If the old entry was the head of the list, adjust */ + if (first == entry) + { + (*head) = new_entry; + } + +} + +/* + * dapl_llist_remove_head() + * + * Purpose: Remove the first entry of a linked list + */ +void * +dapl_llist_remove_head (DAPL_LLIST_HEAD *head) +{ + DAPL_LLIST_ENTRY *first; + + dapl_os_assert (!dapl_llist_is_empty (head)); + first = *head; + *head = first->flink; + + first->flink->blink = first->blink; + first->blink->flink = first->flink; + + if (first->flink == first) + { + *head = NULL; + } + /* clean up the links for good measure */ + first->flink = NULL; + first->blink = NULL; + first->list_head = NULL; + return (first->data); +} + +/* + * dapl_llist_remove_tail() + * + * Purpose: Remove the last entry of a linked list + */ +void * +dapl_llist_remove_tail (DAPL_LLIST_HEAD *head) +{ + DAPL_LLIST_ENTRY *last; + + dapl_os_assert (!dapl_llist_is_empty (head)); + last = (*head)->blink; + + last->blink->flink = last->flink; + last->flink->blink = last->blink; + + if (last->flink == last) + { + *head = NULL; + } + /* clean up the links for good measure */ + last->flink = NULL; + last->blink = NULL; + last->list_head = NULL; + + return (last->data); +} + +/* + * dapl_llist_remove_entry() + * + * Purpose: Remove the specified entry from a linked list + */ +void * +dapl_llist_remove_entry (DAPL_LLIST_HEAD *head, DAPL_LLIST_ENTRY *entry) +{ + DAPL_LLIST_ENTRY *first; + + dapl_os_assert (!dapl_llist_is_empty (head)); + first = *head; + + /* if it's the first entry, pull it off */ + if (first == entry) + { + (*head) = first->flink; + /* if it was the only entry, kill the list */ + if (first->flink == first) + { + (*head) = NULL; + } + } +#ifdef VERIFY_LINKED_LIST + else + { + DAPL_LLIST_ENTRY *try_entry; + + try_entry = first->flink; + for (;;) + { + if (try_entry == first) + { + /* not finding the element on the list is a BAD thing */ + dapl_os_assert (0); + break; + } + if (try_entry == entry) + { + break; + } + try_entry = try_entry->flink; + } + } +#endif /* VERIFY_LINKED_LIST */ + + dapl_os_assert ( entry->list_head == head ); + entry->list_head = NULL; + + entry->flink->blink = entry->blink; + entry->blink->flink = entry->flink; + entry->flink = NULL; + entry->blink = NULL; + + return (entry->data); +} + +/* + * dapl_llist_peek_head + */ + +void * +dapl_llist_peek_head (DAPL_LLIST_HEAD *head) +{ + DAPL_LLIST_ENTRY *first; + + dapl_os_assert (!dapl_llist_is_empty (head)); + first = *head; + return (first->data); +} + + +/* + * dapl_llist_next_entry + * + * Obtain the next entry in the list, return NULL when we get to the + * head + */ + +void * +dapl_llist_next_entry (IN DAPL_LLIST_HEAD *head, + IN DAPL_LLIST_ENTRY *cur_ent) +{ + DAPL_LLIST_ENTRY *next; + + dapl_os_assert (!dapl_llist_is_empty (head)); + if ( cur_ent == NULL ) + { + next = *head; + } + else + { + next = cur_ent->flink; + if ( next == *head ) + { + return NULL; + } + } + return (next->data); +} + +#ifdef DAPL_DBG +/* + * dapl_llist_debug_print_list() + * + * Purpose: Prints the linked list for debugging + */ +void +dapl_llist_debug_print_list (DAPL_LLIST_HEAD *head) +{ + DAPL_LLIST_ENTRY * entry; + DAPL_LLIST_ENTRY * first; + first = *head; + if (!first) + { + dapl_os_printf ("EMPTY_LIST\n"); + return; + } + dapl_os_printf ("HEAD %p\n", *head); + dapl_os_printf ("Entry %p %p %p %p\n", + first, + first->flink, + first->blink, + first->data); + entry = first->flink; + while (entry != first) + { + dapl_os_printf ("Entry %p %p %p %p\n", + entry, + entry->flink, + entry->blink, + entry->data); + entry = entry->flink; + } +} + +#endif /* DAPL_DBG */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_lmr_free.c b/trunk/ulp/dapl2/dapl/common/dapl_lmr_free.c new file mode 100644 index 00000000..5e7cefb0 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_lmr_free.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_lmr_free.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl_lmr_util.h" +#include "dapl_adapter_util.h" +#include "dapl_ia_util.h" + +/* + * dapl_lmr_free + * + * DAPL Requirements Version xxx, 6.6.3.2 + * + * Destroy an instance of the Local Memory Region + * + * Input: + * lmr_handle + * + * Output: + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + */ + +DAT_RETURN DAT_API +dapl_lmr_free ( + IN DAT_LMR_HANDLE lmr_handle ) +{ + DAPL_LMR *lmr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_lmr_free (%p)\n", lmr_handle); + + if ( DAPL_BAD_HANDLE (lmr_handle, DAPL_MAGIC_LMR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_LMR); + goto bail; + } + + lmr = (DAPL_LMR *) lmr_handle; + + switch (lmr->param.mem_type) + { +#if defined(__KDAPL__) + case DAT_MEM_TYPE_PHYSICAL: +#else + case DAT_MEM_TYPE_SHARED_VIRTUAL: +#endif /* defined(__KDAPL__) */ + /* fall through */ + case DAT_MEM_TYPE_VIRTUAL: + case DAT_MEM_TYPE_LMR: + { + DAPL_PZ *pz; + + if ( 0 != dapl_os_atomic_read (&lmr->lmr_ref_count) ) + { + return DAT_INVALID_STATE; + } + + dat_status = dapls_hash_remove (lmr->header.owner_ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, NULL); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + dat_status = dapls_ib_mr_deregister (lmr); + + if (dat_status == DAT_SUCCESS) + { + + pz = (DAPL_PZ *) lmr->param.pz_handle; + dapl_os_atomic_dec (&pz->pz_ref_count); + + dapl_lmr_dealloc (lmr); + } + else + { + /* + * Deregister failed; put it back in the + * hash table. + */ + dapls_hash_insert (lmr->header.owner_ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, lmr); + } + + break; + } +#if defined(__KDAPL__) + case DAT_MEM_TYPE_PLATFORM: + case DAT_MEM_TYPE_IA: + case DAT_MEM_TYPE_BYPASS: + { + return DAT_ERROR (DAT_NOT_IMPLEMENTED,0); + } +#endif /* defined(__KDAPL__) */ + default: + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG1); + break; + } + } + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_lmr_query.c b/trunk/ulp/dapl2/dapl/common/dapl_lmr_query.c new file mode 100644 index 00000000..f9cd4683 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_lmr_query.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_lmr_query.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_lmr_query + * + * Provide the LMR arguments. + * + * Input: + * lmr_handle + * lmr_param_mask + * lmr_param + * + * Output: + * lmr_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_lmr_query ( + IN DAT_LMR_HANDLE lmr_handle, + IN DAT_LMR_PARAM_MASK lmr_param_mask, + IN DAT_LMR_PARAM *lmr_param ) +{ + DAPL_LMR *lmr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_lmr_query (%p, 0x%x, %p)\n", + lmr_handle, + lmr_param_mask, + lmr_param); + + if ( DAPL_BAD_HANDLE (lmr_handle, DAPL_MAGIC_LMR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_LMR); + goto bail; + } + if (NULL == lmr_param) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + dat_status = DAT_SUCCESS; + lmr = (DAPL_LMR *) lmr_handle; + + dapl_os_memcpy (lmr_param, &lmr->param, sizeof (DAT_LMR_PARAM)); + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_read.c b/trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_read.c new file mode 100644 index 00000000..aa6fe74e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_read.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_lmr_sync_rdma_read.c + * + * PURPOSE: Interface Adapter management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" + +/* + * dat_lmr_sync_rdma_read + * + * Ensure a region of memory is consistent by locally flushing + * non-coherent cache + * + * Input: + * ia_handle + * local_segments Array of buffer segments + * num_segments Number of segments in local_segments + * + * Output: + * none + * + * Return Values: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_lmr_sync_rdma_read ( + IN DAT_IA_HANDLE ia_handle, + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments ) +{ + DAPL_IA *ia_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dat_lmr_sync_rdma_read (%p, %p, %ld)\n", + ia_handle, local_segments, num_segments); + + ia_ptr = (DAPL_IA *)ia_handle; + + if (DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + + dat_status = dapl_os_sync_rdma_read (local_segments, num_segments); + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_write.c b/trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_write.c new file mode 100644 index 00000000..32694c96 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_lmr_sync_rdma_write.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_lmr_sync_rdma_write.c + * + * PURPOSE: Interface Adapter management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" + +/* + * dat_lmr_sync_rdma_write + * + * Ensure a region of memory is consistent by locally flushing + * non-coherent cache + * + * Input: + * ia_handle + * local_segments Array of buffer segments + * num_segments Number of segments in local_segments + * + * Output: + * none + * + * Return Values: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_lmr_sync_rdma_write ( + IN DAT_IA_HANDLE ia_handle, + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + DAPL_IA *ia_ptr; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dat_lmr_sync_rdma_write (%p, %p, %ld)\n", + ia_handle, local_segments, num_segments); + + ia_ptr = (DAPL_IA *)ia_handle; + + if (DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + + dat_status = dapl_os_sync_rdma_write (local_segments, num_segments); + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_lmr_util.c b/trunk/ulp/dapl2/dapl/common/dapl_lmr_util.c new file mode 100644 index 00000000..50adb4b8 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_lmr_util.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_lmr_util.c + * + * PURPOSE: Memory management support routines + * Description: Support routines for LMR functions + * + * $Id:$ + **********************************************************************/ + +#include "dapl_lmr_util.h" +#include "dapl_ia_util.h" + +DAPL_LMR * +dapl_lmr_alloc ( + IN DAPL_IA *ia, + IN DAT_MEM_TYPE mem_type, + IN DAT_REGION_DESCRIPTION region_desc, + IN DAT_VLEN length, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_MEM_PRIV_FLAGS mem_priv) +{ + DAPL_LMR *lmr; + + /* Allocate LMR */ + lmr = (DAPL_LMR *) dapl_os_alloc (sizeof (DAPL_LMR)); + if (NULL == lmr) + { + return (NULL); + } + + /* zero the structure */ + dapl_os_memzero (lmr, sizeof (DAPL_LMR)); + + /* + * initialize the header + */ + lmr->header.provider = ia->header.provider; + lmr->header.magic = DAPL_MAGIC_LMR; + lmr->header.handle_type = DAT_HANDLE_TYPE_LMR; + lmr->header.owner_ia = ia; + lmr->header.user_context.as_64 = 0; + lmr->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&lmr->header.ia_list_entry); + dapl_ia_link_lmr (ia, lmr); + dapl_os_lock_init (&lmr->header.lock); + + /* + * initialize the body + */ + lmr->param.ia_handle = (DAT_IA_HANDLE) ia; + lmr->param.mem_type = mem_type; + lmr->param.region_desc = region_desc; + lmr->param.length = length; + lmr->param.pz_handle = pz_handle; + lmr->param.mem_priv = mem_priv; + dapl_os_atomic_set (&lmr->lmr_ref_count, 0); + + return (lmr); +} + +void +dapl_lmr_dealloc ( + IN DAPL_LMR *lmr) +{ + lmr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ + dapl_ia_unlink_lmr (lmr->header.owner_ia, lmr); + dapl_os_lock_destroy (&lmr->header.lock); + + dapl_os_free ((void *) lmr, sizeof (DAPL_LMR)); +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_lmr_util.h b/trunk/ulp/dapl2/dapl/common/dapl_lmr_util.h new file mode 100644 index 00000000..01c301b8 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_lmr_util.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_lmr_util.h + * + * PURPOSE: Utility defs & routines for the LMR data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_LMR_UTIL_H_ +#define _DAPL_LMR_UTIL_H_ + +#include "dapl_mr_util.h" + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +extern DAPL_LMR * +dapl_lmr_alloc ( + IN DAPL_IA *ia, + IN DAT_MEM_TYPE mem_type, + IN DAT_REGION_DESCRIPTION region_desc, + IN DAT_VLEN length, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_MEM_PRIV_FLAGS mem_priv); + +extern void +dapl_lmr_dealloc ( + IN DAPL_LMR *lmr); + + +#endif /* _DAPL_LMR_UTIL_H_*/ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_mr_util.c b/trunk/ulp/dapl2/dapl/common/dapl_mr_util.c new file mode 100644 index 00000000..e28bd71f --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_mr_util.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_mr_util.h" + +/********************************************************************** + * + * MODULE: dapl_mr_util.c + * + * PURPOSE: Common Memory Management functions and data structures + * + * $Id:$ + **********************************************************************/ + +/********************************************************************* + * * + * Function Definitions * + * * + *********************************************************************/ + +/* + * dapl_mr_get_address + * + * Returns the memory address associated with the given memory descriptor + * + * Input: + * desc memory descriptor + * type type of memory represented by desc + * + * Output: + * None + * + */ + +DAT_VADDR +dapl_mr_get_address ( + IN DAT_REGION_DESCRIPTION desc, + IN DAT_MEM_TYPE type) +{ + switch (type) + { + case DAT_MEM_TYPE_VIRTUAL: + { + return (DAT_VADDR) (uintptr_t) desc.for_va; + } + case DAT_MEM_TYPE_LMR: + { + DAPL_LMR *lmr; + + lmr = (DAPL_LMR *) desc.for_lmr_handle; + + /* Since this function is recoursive we cannot inline it */ + return dapl_mr_get_address (lmr->param.region_desc, + lmr->param.mem_type); + } +#if defined(__KDAPL__) + case DAT_MEM_TYPE_PHYSICAL: + { + return desc.for_pa; + } +#else + case DAT_MEM_TYPE_SHARED_VIRTUAL: + { + /* multi-cast necessary to convert a pvoid to a DAT_VADDR on + * all architectures + */ + return (DAT_VADDR) (uintptr_t) desc.for_shared_memory.virtual_address; + } +#endif /* defined(__KDAPL__) */ + default: + { + /* + * The following kDAPL memory types have not been implemented: + * DAT_MEM_TYPE_PLATFORM + * DAT_MEM_TYPE_IA + * DAT_MEM_TYPE_BYPASS + */ + dapl_os_assert (0); + return 0; + } + } +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_mr_util.h b/trunk/ulp/dapl2/dapl/common/dapl_mr_util.h new file mode 100644 index 00000000..6dd9680b --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_mr_util.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_mr_util.h + * + * PURPOSE: Utility defs & routines for memory registration functions + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_MR_UTIL_H_ +#define _DAPL_MR_UTIL_H_ + +#include "dapl.h" +#include "dapl_hash.h" + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +extern DAT_VADDR +dapl_mr_get_address ( + IN DAT_REGION_DESCRIPTION desc, + IN DAT_MEM_TYPE type); + +STATIC _INLINE_ DAT_BOOLEAN +dapl_mr_bounds_check ( + IN DAT_VADDR addr_a, + IN DAT_VLEN length_a, + IN DAT_VADDR addr_b, + IN DAT_VLEN length_b); + + +/********************************************************************* + * * + * Inline Functions * + * * + *********************************************************************/ + +/* + * dapl_mr_bounds_check + * + * Returns true if region B is contained within region A + * and false otherwise + * + */ + +STATIC _INLINE_ DAT_BOOLEAN +dapl_mr_bounds_check ( + IN DAT_VADDR addr_a, + IN DAT_VLEN length_a, + IN DAT_VADDR addr_b, + IN DAT_VLEN length_b) +{ + if ( (addr_a <= addr_b) && + (addr_b + length_b) <= (addr_a + length_a)) + { + return DAT_TRUE; + } + else + { + return DAT_FALSE; + } +} + +#endif /* _DAPL_MR_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_name_service.c b/trunk/ulp/dapl2/dapl/common/dapl_name_service.c new file mode 100644 index 00000000..ebdb5b8c --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_name_service.c @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_name_service.c + * + * PURPOSE: Provide simple, file base name services in the absence + * of DNS hooks for a particular transport type. If an + * InfiniBand implementation supports IPoIB, this should + * not be used. + * + * Description: Interfaces in this file are completely described in + * dapl_name_service.h + * + * $Id:$ + **********************************************************************/ + +/* + * Include files for setting up a network name + */ +#include "dapl.h" +#include "dapl_name_service.h" + +/* + * Prototypes + */ +#ifdef IBHOSTS_NAMING +DAT_RETURN dapli_ns_create_gid_map (void); + +DAT_RETURN dapli_ns_add_address ( + IN DAPL_GID_MAP *gme); +#endif /* IBHOSTS_NAMING */ + + +/* + * dapls_ns_init + * + * Initialize naming services + * + * Input: + * none + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN +dapls_ns_init (void) +{ + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; +#ifdef IBHOSTS_NAMING + dat_status = dapli_ns_create_gid_map (); +#endif /* IBHOSTS_NAMING */ + + return dat_status; +} + + +#ifdef IBHOSTS_NAMING + +#define MAX_GID_ENTRIES 32 +DAPL_GID_MAP g_gid_map_table[MAX_GID_ENTRIES]; + +#ifdef _WIN32 +#define MAP_FILE "c:/WINNT/system32/drivers/etc/ibhosts" +#else +#define MAP_FILE "/etc/dapl/ibhosts" +#endif /* !defined WIN32 */ + + +/* + * dapli_ns_create_gid_map() + * + * Read the MAP_FILE to obtain host names and GIDs. + * Create a table containing IP addresses and GIDs which can + * be used for lookups. + * + * This implementation is a simple method providing name services + * when more advanced mechanisms do not exist. The proper way + * to obtain these mappings is to use a name service such as is + * provided by IPoIB on InfiniBand. + * + * Input: + * device_name Name of device as reported by the provider + * + * Output: + * none + * + * Returns: + * char * to string number + */ +DAT_RETURN +dapli_ns_create_gid_map (void) +{ + FILE *f; + ib_gid_t gid; + char hostname[128]; + int rc; + struct addrinfo *addr; + struct sockaddr_in *si; + DAPL_GID_MAP gmt; + + f = fopen (MAP_FILE, "r"); + if ( f == NULL ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "ERROR: Must have file <%s> for IP/GID mappings\n", + MAP_FILE); + return DAT_ERROR (DAT_INTERNAL_ERROR, 0); + } + + rc = fscanf ( f, "%s " F64x " " F64x , hostname, &gid.gid_prefix, &gid.guid); + while ( rc != EOF ) + { + rc = dapls_osd_getaddrinfo (hostname, &addr); + + if ( rc != 0 ) + { + /* + * hostname not registered in DNS, provide a dummy value + */ + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "WARNING: <%s> not registered in DNS, using dummy IP value\n", + hostname); + /*dapl_os_memcpy(hca_ptr->hca_address.sa_data, "0x01020304", 4);*/ + gmt.ip_address = 0x01020304; + } + else + { + /* + * Load into the ip/gid mapping table + */ + si = (struct sockaddr_in *)addr->ai_addr; + if ( AF_INET == addr->ai_addr->sa_family ) + { + gmt.ip_address = si->sin_addr.s_addr; + } + else + { + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "WARNING: <%s> Address family not supported, using dummy IP value\n", + hostname); + gmt.ip_address = 0x01020304; + } + dapls_osd_freeaddrinfo (addr); + } + gmt.gid.gid_prefix = gid.gid_prefix; + gmt.gid.guid = gid.guid; + + dapli_ns_add_address (&gmt); + rc = fscanf ( f, "%s " F64x " " F64x , hostname, &gid.gid_prefix, &gid.guid); + } + + fclose (f); + + return DAT_SUCCESS; +} + + +/* + * dapli_ns_add_address + * + * Add a table entry to the gid_map_table. + * + * Input: + * remote_ia_address remote IP address + * gid pointer to output gid + * + * Output: + * gid filled in GID + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + */ +DAT_RETURN +dapli_ns_add_address ( + IN DAPL_GID_MAP *gme) +{ + DAPL_GID_MAP *gmt; + int count; + + gmt = g_gid_map_table; + + for ( count = 0, gmt = g_gid_map_table; gmt->ip_address; gmt++ ) + { + count++; + } + + if ( count > MAX_GID_ENTRIES ) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, 0); + } + + *gmt = *gme; + + return DAT_SUCCESS; +} + + +/* + * dapls_ns_lookup_address + * + * Look up the provided IA_ADDRESS in the gid_map_table. Return + * the gid if found. + * + * Input: + * remote_ia_address remote IP address + * gid pointer to output gid + * + * Output: + * gid filled in GID + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + */ +DAT_RETURN +dapls_ns_lookup_address ( + IN DAPL_IA *ia_ptr, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + OUT ib_gid_t *gid) +{ + DAPL_GID_MAP *gmt; + struct sockaddr_in *si; + + ia_ptr = ia_ptr; /* unused here */ + + si = (struct sockaddr_in *)remote_ia_address; + + for ( gmt = g_gid_map_table; gmt->ip_address; gmt++ ) + { + if (gmt->ip_address == si->sin_addr.s_addr) + { + gid->guid = gmt->gid.guid; + gid->gid_prefix = gmt->gid.gid_prefix; + + return DAT_SUCCESS; + } + } + + return DAT_ERROR (DAT_INVALID_PARAMETER, 0); +} + +#endif /* IBHOSTS_NAMING */ + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_name_service.h b/trunk/ulp/dapl2/dapl/common/dapl_name_service.h new file mode 100644 index 00000000..33d02e4e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_name_service.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_name_service.h + * + * PURPOSE: Utility defs & routines supporting name services + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_NAME_SERVICE_H_ +#define _DAPL_NAME_SERVICE_H_ + +#include "dapl.h" + +/* + * Prototypes for name service routines + */ + +DAT_RETURN dapls_ns_init (void); + +#ifdef IBHOSTS_NAMING +DAT_RETURN dapls_ns_lookup_address ( + IN DAPL_IA *ia_ptr, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + OUT ib_gid_t *gid); +#else + +/* routine is a no op if not using local name services */ +#define dapls_ns_lookup_address(ia,ria,gid) DAT_SUCCESS + +#endif /* IBHOSTS_NAMING */ + +#endif /* _DAPL_NAME_SERVICE_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_provider.c b/trunk/ulp/dapl2/dapl/common/dapl_provider.c new file mode 100644 index 00000000..431cf5ad --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_provider.c @@ -0,0 +1,494 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_provider.c + * + * PURPOSE: Provider function table + * Description: DAT Interfaces to this provider + * + * $Id:$ + **********************************************************************/ + +#include "dapl_provider.h" + +extern DAT_RETURN dapl_not_implemented (void); + +/********************************************************************* + * * + * Global Data * + * * + *********************************************************************/ + +DAPL_PROVIDER_LIST g_dapl_provider_list; + + +/* + * the function table for this provider + */ + + +#if defined(__KDAPL__) +DAT_PROVIDER g_dapl_provider_template = +{ + NULL, + 0, + &dapl_ia_open, + &dapl_ia_query, + &dapl_ia_close, + &dapl_ia_memtype_hint, /* unimplemented */ + + &dapl_set_consumer_context, /* untested */ + &dapl_get_consumer_context, /* untested */ + &dapl_get_handle_type, /* untested */ + + &dapl_cr_query, + &dapl_cr_accept, + &dapl_cr_reject, + &dapl_cr_handoff, + + &dapl_evd_kcreate, + &dapl_evd_kquery, /* untested */ + &dapl_evd_modify_upcall, + &dapl_evd_resize, /* unimplemented */ + &dapl_evd_post_se, /* untested */ + &dapl_evd_dequeue, + &dapl_evd_free, + + &dapl_ep_create, + &dapl_ep_query, + &dapl_ep_modify, /* untested */ + &dapl_ep_connect, + &dapl_ep_dup_connect, /* untested */ + &dapl_ep_disconnect, + &dapl_ep_post_send, + &dapl_ep_post_recv, + &dapl_ep_post_rdma_read, + &dapl_ep_post_rdma_write, + &dapl_ep_get_status, + &dapl_ep_free, + + &dapl_lmr_kcreate, + &dapl_lmr_query, + &dapl_lmr_free, + + &dapl_rmr_create, + &dapl_rmr_query, /* untested */ + &dapl_rmr_bind, + &dapl_rmr_free, + + &dapl_psp_create, + &dapl_psp_query, /* untested */ + &dapl_psp_free, + + &dapl_rsp_create, + &dapl_rsp_query, /* untested */ + &dapl_rsp_free, + + &dapl_pz_create, + &dapl_pz_query, /* untested */ + &dapl_pz_free, + + /* dat-1.1 */ + &dapl_psp_create_any, + &dapl_ep_reset, + + /* dat-1.2 */ + &dapl_lmr_sync_rdma_read, + &dapl_lmr_sync_rdma_write, + + &dapl_ep_create_with_srq, + &dapl_ep_recv_query, + &dapl_ep_set_watermark, + &dapl_srq_create, + &dapl_srq_free, + &dapl_srq_post_recv, + &dapl_srq_query, + &dapl_srq_resize, + &dapl_srq_set_lw +}; +#else +/* + * uDAPL version of the provider jump table + */ +DAT_PROVIDER g_dapl_provider_template = +{ + NULL, + 0, + &dapl_ia_open, + &dapl_ia_query, + &dapl_ia_close, + + &dapl_set_consumer_context, + &dapl_get_consumer_context, + &dapl_get_handle_type, + + &dapl_cno_create, + &dapl_cno_modify_agent, + &dapl_cno_query, + &dapl_cno_free, + &dapl_cno_wait, + + &dapl_cr_query, + &dapl_cr_accept, + &dapl_cr_reject, + &dapl_cr_handoff, + + &dapl_evd_create, + &dapl_evd_query, + &dapl_evd_modify_cno, + &dapl_evd_enable, + &dapl_evd_disable, + &dapl_evd_wait, + &dapl_evd_resize, + &dapl_evd_post_se, + &dapl_evd_dequeue, + &dapl_evd_free, + + &dapl_ep_create, + &dapl_ep_query, + &dapl_ep_modify, + &dapl_ep_connect, + &dapl_ep_dup_connect, + &dapl_ep_disconnect, + &dapl_ep_post_send, + &dapl_ep_post_recv, + &dapl_ep_post_rdma_read, + &dapl_ep_post_rdma_write, + &dapl_ep_get_status, + &dapl_ep_free, + + &dapl_lmr_create, + &dapl_lmr_query, + &dapl_lmr_free, + + &dapl_rmr_create, + &dapl_rmr_query, + &dapl_rmr_bind, + &dapl_rmr_free, + + &dapl_psp_create, + &dapl_psp_query, + &dapl_psp_free, + + &dapl_rsp_create, + &dapl_rsp_query, + &dapl_rsp_free, + + &dapl_pz_create, + &dapl_pz_query, + &dapl_pz_free, + + &dapl_psp_create_any, + &dapl_ep_reset, + &dapl_evd_set_unwaitable, + &dapl_evd_clear_unwaitable, + + /* dat-1.2 */ + &dapl_lmr_sync_rdma_read, + &dapl_lmr_sync_rdma_write, + + &dapl_ep_create_with_srq, + &dapl_ep_recv_query, + &dapl_ep_set_watermark, + &dapl_srq_create, + &dapl_srq_free, + &dapl_srq_post_recv, + &dapl_srq_query, + &dapl_srq_resize, + &dapl_srq_set_lw, + + /* dat-2.0 */ + &dapl_csp_create, + &dapl_csp_query, + &dapl_csp_free, + &dapl_ep_common_connect, + &dapl_rmr_create_for_ep, + &dapl_ep_post_send_with_invalidate, + &dapl_ep_post_rdma_read_to_rmr, + &dapl_cno_fd_create, + &dapl_cno_trigger, + &dapl_ia_ha + +#ifdef DAT_EXTENSIONS + ,&dapl_extensions +#endif +}; +#endif /* __KDAPL__ */ + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +static DAT_BOOLEAN +dapl_provider_list_key_cmp ( + const char *name_a, + const char *name_b ); + + +/********************************************************************* + * * + * Function Definitions * + * * + *********************************************************************/ + +DAT_RETURN +dapl_provider_list_create ( void ) +{ + DAT_RETURN status; + + status = DAT_SUCCESS; + + /* create the head node */ + g_dapl_provider_list.head = dapl_os_alloc (sizeof (DAPL_PROVIDER_LIST_NODE)); + if ( NULL == g_dapl_provider_list.head ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_memzero (g_dapl_provider_list.head, sizeof (DAPL_PROVIDER_LIST_NODE)); + + /* create the tail node */ + g_dapl_provider_list.tail = dapl_os_alloc (sizeof (DAPL_PROVIDER_LIST_NODE)); + if ( NULL == g_dapl_provider_list.tail ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_memzero (g_dapl_provider_list.tail, sizeof (DAPL_PROVIDER_LIST_NODE)); + + g_dapl_provider_list.head->next = g_dapl_provider_list.tail; + g_dapl_provider_list.tail->prev = g_dapl_provider_list.head; + g_dapl_provider_list.size = 0; + +bail: + if ( DAT_SUCCESS != status ) + { + if ( NULL != g_dapl_provider_list.head ) + { + dapl_os_free (g_dapl_provider_list.head, sizeof (DAPL_PROVIDER_LIST_NODE)); + } + + if ( NULL != g_dapl_provider_list.tail ) + { + dapl_os_free (g_dapl_provider_list.tail, sizeof (DAPL_PROVIDER_LIST_NODE)); + } + } + + return status; +} + + +DAT_RETURN +dapl_provider_list_destroy ( void ) +{ + DAPL_PROVIDER_LIST_NODE *cur_node; + + while ( NULL != g_dapl_provider_list.head ) + { + cur_node = g_dapl_provider_list.head; + g_dapl_provider_list.head = cur_node->next; + + dapl_os_free (cur_node, sizeof (DAPL_PROVIDER_LIST_NODE)); + } + + return DAT_SUCCESS; +} + + +DAT_COUNT +dapl_provider_list_size ( void ) +{ + return g_dapl_provider_list.size; +} + + +DAT_RETURN +dapl_provider_list_insert ( + IN const char *name, + IN DAT_PROVIDER **p_data ) +{ + DAPL_PROVIDER_LIST_NODE *cur_node, *prev_node, *next_node; + DAT_RETURN status; + unsigned int len; + + status = DAT_SUCCESS; + + cur_node = dapl_os_alloc (sizeof (DAPL_PROVIDER_LIST_NODE)); + + if ( NULL == cur_node ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + len = dapl_os_strlen (name); + + if ( DAT_NAME_MAX_LENGTH <= len ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + /* insert node at end of list to preserve registration order*/ + prev_node = g_dapl_provider_list.tail->prev; + next_node = g_dapl_provider_list.tail; + + dapl_os_memcpy (cur_node->name, name, len); + cur_node->name[len] = '\0'; + cur_node->data = g_dapl_provider_template; + cur_node->data.device_name = cur_node->name; + + cur_node->next = next_node; + cur_node->prev = prev_node; + + prev_node->next = cur_node; + next_node->prev = cur_node; + + g_dapl_provider_list.size++; + + if ( NULL != p_data ) + { + *p_data = &cur_node->data; + } + +bail: + if ( DAT_SUCCESS != status ) + { + if ( NULL != cur_node ) + { + dapl_os_free (cur_node, sizeof (DAPL_PROVIDER_LIST_NODE)); + } + } + + return status; +} + + +DAT_RETURN +dapl_provider_list_search ( + IN const char *name, + OUT DAT_PROVIDER **p_data ) +{ + DAPL_PROVIDER_LIST_NODE *cur_node; + DAT_RETURN status; + + status = DAT_ERROR (DAT_PROVIDER_NOT_FOUND, DAT_NAME_NOT_REGISTERED); + + for (cur_node = g_dapl_provider_list.head->next; + g_dapl_provider_list.tail != cur_node; + cur_node = cur_node->next) + { + if ( dapl_provider_list_key_cmp (cur_node->name, name) ) + { + if ( NULL != p_data ) + { + *p_data = &cur_node->data; + } + + status = DAT_SUCCESS; + goto bail; + } + } + +bail: + return status; +} + + +DAT_RETURN +dapl_provider_list_remove ( + IN const char *name ) +{ + DAPL_PROVIDER_LIST_NODE *cur_node, *prev_node, *next_node; + DAT_RETURN status; + + status = DAT_ERROR (DAT_PROVIDER_NOT_FOUND, DAT_NAME_NOT_REGISTERED); + + for (cur_node = g_dapl_provider_list.head->next; + g_dapl_provider_list.tail != cur_node; + cur_node = cur_node->next) + { + if ( dapl_provider_list_key_cmp (cur_node->name, name) ) + { + prev_node = cur_node->prev; + next_node = cur_node->next; + + prev_node->next = next_node; + next_node->prev = prev_node; + + dapl_os_free (cur_node, sizeof (DAPL_PROVIDER_LIST_NODE)); + + g_dapl_provider_list.size--; + + status = DAT_SUCCESS; + goto bail; + } + } + +bail: + return status; +} + + +DAT_BOOLEAN +dapl_provider_list_key_cmp ( + const char *name_a, + const char *name_b ) +{ + unsigned int len; + + len = dapl_os_strlen (name_a); + + if ( dapl_os_strlen (name_b) != len ) + { + return DAT_FALSE; + } + else if ( dapl_os_memcmp (name_a, name_b, len) ) + { + return DAT_FALSE; + } + else + { + return DAT_TRUE; + } +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_provider.h b/trunk/ulp/dapl2/dapl/common/dapl_provider.h new file mode 100644 index 00000000..688636fe --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_provider.h @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_provider.h + * + * PURPOSE: Provider function table + * Description: DAT Interfaces to this provider + * + * $Id:$ + **********************************************************************/ + +#ifndef _DAPL_PROVIDER_H_ +#define _DAPL_PROVIDER_H_ + +#include "dapl.h" + + +/********************************************************************* + * * + * Structures * + * * + *********************************************************************/ + +typedef struct DAPL_PROVIDER_LIST_NODE +{ + char name[DAT_NAME_MAX_LENGTH]; + DAT_PROVIDER data; + struct DAPL_PROVIDER_LIST_NODE *next; + struct DAPL_PROVIDER_LIST_NODE *prev; +} DAPL_PROVIDER_LIST_NODE; + + +typedef struct DAPL_PROVIDER_LIST +{ + DAPL_PROVIDER_LIST_NODE *head; + DAPL_PROVIDER_LIST_NODE *tail; + DAT_COUNT size; +} DAPL_PROVIDER_LIST; + + +/********************************************************************* + * * + * Global Data * + * * + *********************************************************************/ + +extern DAPL_PROVIDER_LIST g_dapl_provider_list; +extern DAT_PROVIDER g_dapl_provider_template; +extern int g_dapl_loopback_connection; + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +extern DAT_RETURN +dapl_provider_list_create( void ); + +extern DAT_RETURN +dapl_provider_list_destroy( void ); + +extern DAT_COUNT +dapl_provider_list_size( void ); + +extern DAT_RETURN +dapl_provider_list_insert( + IN const char *name, + OUT DAT_PROVIDER **p_data ); + +extern DAT_RETURN +dapl_provider_list_search( + IN const char *name, + OUT DAT_PROVIDER **p_data ); + +extern DAT_RETURN +dapl_provider_list_remove( + IN const char *name ); + + +#endif /* _DAPL_PROVIDER_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_psp_create.c b/trunk/ulp/dapl2/dapl/common/dapl_psp_create.c new file mode 100644 index 00000000..ce5a7be0 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_psp_create.c @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_psp_create.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_sp_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_psp_create + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.1.1 + * + * Create a persistent Public Service Point that can receive multiple + * requests for connections and generate multiple connection request + * instances that will be delivered to the specified Event Dispatcher + * in a notification event. + * + * Input: + * ia_handle + * conn_qual + * evd_handle + * psp_flags + * + * Output: + * psp_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_CONN_QUAL_IN_USE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_psp_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_CONN_QUAL conn_qual, + IN DAT_EVD_HANDLE evd_handle, + IN DAT_PSP_FLAGS psp_flags, + OUT DAT_PSP_HANDLE *psp_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_SP *sp_ptr; + DAPL_EVD *evd_ptr; + DAT_BOOLEAN sp_found; + DAT_RETURN dat_status; + + ia_ptr = (DAPL_IA *)ia_handle; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EVD_CR); + goto bail; + } + + if ( psp_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG5); + goto bail; + } + + evd_ptr = (DAPL_EVD *)evd_handle; + if ( ! (evd_ptr->evd_flags & DAT_EVD_CR_FLAG) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EVD_CR); + goto bail; + } + + if (psp_flags != DAT_PSP_CONSUMER_FLAG && + psp_flags != DAT_PSP_PROVIDER_FLAG) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4); + goto bail; + } + + + /* + * See if we have a quiescent listener to use for this PSP, else + * create one and set it listening + */ + sp_ptr = dapls_ia_sp_search (ia_ptr, conn_qual, DAT_TRUE); + sp_found = DAT_TRUE; + if (sp_ptr == NULL) + { + /* Allocate PSP */ + sp_found = DAT_FALSE; + sp_ptr = dapls_sp_alloc ( ia_ptr, DAT_TRUE ); + if ( sp_ptr == NULL ) + { + dat_status = + DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + } + else if (sp_ptr->listening == DAT_TRUE) + { + dat_status = DAT_ERROR (DAT_CONN_QUAL_IN_USE, 0); + goto bail; + } + + /* + * Fill out the args for a PSP + */ + sp_ptr->conn_qual = conn_qual; + sp_ptr->evd_handle = evd_handle; + sp_ptr->psp_flags = psp_flags; + sp_ptr->ep_handle = NULL; + + /* + * Take a reference on the EVD handle + */ + dapl_os_atomic_inc (& ((DAPL_EVD *)evd_handle)->evd_ref_count); + + /* + * Set up a listener for a connection. Connections can arrive + * even before this call returns! + */ + sp_ptr->state = DAPL_SP_STATE_PSP_LISTENING; + sp_ptr->listening = DAT_TRUE; + + /* + * If this is a new sp we need to add it to the IA queue, and set up + * a conn_listener. + */ + if (sp_found == DAT_FALSE) + { + /* Link it onto the IA before enabling it to receive conn + * requests + */ + dapl_ia_link_psp (ia_ptr, sp_ptr); + + dat_status = dapls_ib_setup_conn_listener ( ia_ptr, + conn_qual, + sp_ptr ); + + if ( dat_status != DAT_SUCCESS ) + { + /* + * Have a problem setting up the connection, something + * wrong! Decrements the EVD refcount & release it. + */ + dapl_os_atomic_dec (& ((DAPL_EVD *)evd_handle)->evd_ref_count); + sp_ptr->evd_handle = NULL; + dapls_ia_unlink_sp ( ia_ptr, sp_ptr ); + dapls_sp_free_sp ( sp_ptr ); + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> dapl_psp_create setup_conn_listener failed: %x\n", + dat_status); + + goto bail; + } + } + + /* + * Return handle to the user + */ + *psp_handle = (DAT_PSP_HANDLE)sp_ptr; + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_psp_create_any.c b/trunk/ulp/dapl2/dapl/common/dapl_psp_create_any.c new file mode 100644 index 00000000..cea17f16 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_psp_create_any.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_psp_create.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_sp_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_psp_create_any + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.3.3 + * + * Create a persistent Public Service Point that can receive multiple + * requests for connections and generate multiple connection request + * instances that will be delivered to the specified Event Dispatcher + * in a notification event. Differs from dapl_psp_create() in that + * the conn_qual is selected by the implementation and returned to + * the user. + * + * Input: + * ia_handle + * evd_handle + * psp_flags + * + * Output: + * conn_qual + * psp_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * DAT_CONN_QUAL_IN_USE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_psp_create_any ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_CONN_QUAL *conn_qual, + IN DAT_EVD_HANDLE evd_handle, + IN DAT_PSP_FLAGS psp_flags, + OUT DAT_PSP_HANDLE *psp_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_SP *sp_ptr; + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + static DAT_CONN_QUAL hint_conn_qual = 1000; /* seed value */ + DAT_CONN_QUAL lcl_conn_qual; + DAT_CONN_QUAL limit_conn_qual; + + ia_ptr = (DAPL_IA *)ia_handle; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EVD_CR); + goto bail; + } + + if ( psp_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG5); + goto bail; + } + if (conn_qual == NULL) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + evd_ptr = (DAPL_EVD *)evd_handle; + if ( ! (evd_ptr->evd_flags & DAT_EVD_CR_FLAG) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EVD_CR); + goto bail; + } + + if (psp_flags != DAT_PSP_CONSUMER_FLAG && + psp_flags != DAT_PSP_PROVIDER_FLAG) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4); + goto bail; + } + + /* Allocate PSP */ + sp_ptr = dapls_sp_alloc ( ia_ptr, DAT_TRUE ); + if ( sp_ptr == NULL ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + /* + * Fill out the args for a PSP + */ + sp_ptr->evd_handle = evd_handle; + sp_ptr->psp_flags = psp_flags; + sp_ptr->ep_handle = NULL; + + /* + * Take a reference on the EVD handle + */ + dapl_os_atomic_inc (& ((DAPL_EVD *)evd_handle)->evd_ref_count); + + /* Link it onto the IA */ + dapl_ia_link_psp (ia_ptr, sp_ptr); + + /* + * Set up a listener for a connection. Connections can arrive + * even before this call returns! + */ + sp_ptr->state = DAPL_SP_STATE_PSP_LISTENING; + sp_ptr->listening = DAT_TRUE; + + limit_conn_qual = 0; + lcl_conn_qual = hint_conn_qual; + dat_status = ~DAT_SUCCESS; + + while (dat_status != DAT_SUCCESS) + { + dat_status = dapls_ib_setup_conn_listener ( ia_ptr, + lcl_conn_qual, + sp_ptr ); + + lcl_conn_qual++; + + if ( dat_status == DAT_CONN_QUAL_IN_USE ) + { + /* + * If we have a big number of tries and we still haven't + * found a service_ID we can use, bail out with an error, + * something is wrong! + */ + if ( limit_conn_qual++ > 100000) + { + dat_status = DAT_CONN_QUAL_UNAVAILABLE; + break; + } + } + } + hint_conn_qual = lcl_conn_qual; + + if ( dat_status != DAT_SUCCESS ) + { + /* + * Have a problem setting up the connection, something wrong! + */ + dapl_os_atomic_dec (& ((DAPL_EVD *)evd_handle)->evd_ref_count); + sp_ptr->evd_handle = NULL; + dapls_ia_unlink_sp ( ia_ptr, sp_ptr ); + dapls_sp_free_sp ( sp_ptr ); + + dapl_os_printf ("--> dapl_psp_create cannot set up conn listener: %x\n", dat_status); + + goto bail; + } + + sp_ptr->conn_qual = lcl_conn_qual - 1; + + /* + * Return handle to the user + */ + *conn_qual = sp_ptr->conn_qual; + *psp_handle = (DAT_PSP_HANDLE)sp_ptr; + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_psp_free.c b/trunk/ulp/dapl2/dapl/common/dapl_psp_free.c new file mode 100644 index 00000000..a376ed2e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_psp_free.c @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_psp_free.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_sp_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_psp_free + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.1.2 + * + * Destroy a specific instance of a Service Point. + * + * Input: + * psp_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_psp_free ( + IN DAT_PSP_HANDLE psp_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_SP *sp_ptr; + DAT_RETURN dat_status; + DAPL_SP_STATE save_state; + + sp_ptr = (DAPL_SP *) psp_handle; + dat_status = DAT_SUCCESS; + /* + * Verify handle + */ + dapl_dbg_log (DAPL_DBG_TYPE_CM, ">>> dapl_psp_free %p\n", psp_handle); + + if ( DAPL_BAD_HANDLE (sp_ptr, DAPL_MAGIC_PSP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_PSP); + goto bail; + } + + /* ia_ptr = (DAPL_IA *)sp_ptr->header.owner_ia; */ + ia_ptr = sp_ptr->header.owner_ia; + + /* + * Remove the connection listener if it has been established + * and there are no current connections in progress. + * If we defer removing the sp it becomes something of a zombie + * container until the last connection is disconnected, after + * which it will be cleaned up. + */ + dapl_os_lock (&sp_ptr->header.lock); + + sp_ptr->listening = DAT_FALSE; + + /* Release reference on EVD. If an error was encountered in a previous + * free the evd_handle will be NULL + */ + if (sp_ptr->evd_handle) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)sp_ptr->evd_handle)->evd_ref_count); + sp_ptr->evd_handle = NULL; + } + + /* + * Release the base resource if there are no outstanding + * connections; else the last disconnect on this PSP will free it + * up. The PSP is used to contain CR records for each connection, + * which contain information necessary to disconnect. + */ + + if ( (sp_ptr->state == DAPL_SP_STATE_PSP_LISTENING || + sp_ptr->state == DAPL_SP_STATE_PSP_PENDING) && + sp_ptr->cr_list_count == 0 ) + { + save_state = sp_ptr->state; + sp_ptr->state = DAPL_SP_STATE_FREE; + dapl_os_unlock (&sp_ptr->header.lock); + + dat_status = dapls_ib_remove_conn_listener ( ia_ptr, + sp_ptr ); + if (dat_status != DAT_SUCCESS) + { + /* revert to entry state on error */ + sp_ptr->state = save_state; + goto bail; + } + dapls_ia_unlink_sp ( ia_ptr, sp_ptr ); + dapls_sp_free_sp ( sp_ptr ); + } + else + { + /* The PSP is now in the pending state, where it will sit until + * the last connection terminates or the app uses the same + * ServiceID again, which will reactivate it. + */ + sp_ptr->state = DAPL_SP_STATE_PSP_PENDING; + dapl_os_unlock (&sp_ptr->header.lock); + dapl_dbg_log (DAPL_DBG_TYPE_CM, ">>> dapl_psp_free: PSP PENDING\n"); + } + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_psp_query.c b/trunk/ulp/dapl2/dapl/common/dapl_psp_query.c new file mode 100644 index 00000000..f7914936 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_psp_query.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_psp_query.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_psp_query + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.1.3 + * + * Provide arguments of the public service points + * + * Input: + * psp_handle + * psp_args_mask + * + * Output: + * psp_args + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_psp_query ( + IN DAT_PSP_HANDLE psp_handle, + IN DAT_PSP_PARAM_MASK psp_args_mask, + OUT DAT_PSP_PARAM *psp_param ) +{ + DAPL_SP *sp_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (psp_handle, DAPL_MAGIC_PSP) || + ((DAPL_SP *)psp_handle)->listening != DAT_TRUE ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_PSP); + goto bail; + } + + if (NULL == psp_param) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + sp_ptr = (DAPL_SP *) psp_handle; + + /* + * Fill in the PSP params + */ + psp_param->ia_handle = sp_ptr->header.owner_ia; + psp_param->conn_qual = sp_ptr->conn_qual; + psp_param->evd_handle = sp_ptr->evd_handle; + psp_param->psp_flags = sp_ptr->psp_flags; + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_pz_create.c b/trunk/ulp/dapl2/dapl/common/dapl_pz_create.c new file mode 100644 index 00000000..12fd22cf --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_pz_create.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_pz_create.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_pz_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_pz_create + * + * Create an instance of a protection zone + * + * Input: + * ia_handle + * + * Output: + * pz_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVLAID_HANDLE + */ +DAT_RETURN DAT_API +dapl_pz_create ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_PZ_HANDLE *pz_handle) +{ + DAPL_IA *ia; + DAPL_PZ *pz; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_pz_create (%p, %p)\n", + ia_handle, + pz_handle); + + dat_status = DAT_SUCCESS; + if ( DAPL_BAD_HANDLE (ia_handle, DAPL_MAGIC_IA) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + + if (NULL == pz_handle) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + ia = (DAPL_IA *) ia_handle; + + pz = dapl_pz_alloc (ia); + if ( NULL == pz ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_status = dapls_ib_pd_alloc (ia, pz); + if ( DAT_SUCCESS != dat_status ) + { + dapl_pz_dealloc (pz); + pz = NULL; + } + + *pz_handle = pz; + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_pz_free.c b/trunk/ulp/dapl2/dapl/common/dapl_pz_free.c new file mode 100644 index 00000000..a890c565 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_pz_free.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_pz_free.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_pz_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_pz_free + * + * Remove an instance of a protection zone + * + * Input: + * pz_handle + * + * Output: + * None. + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * DAT_INVALID_HANDLE + */ +DAT_RETURN DAT_API +dapl_pz_free ( + IN DAT_PZ_HANDLE pz_handle) +{ + DAPL_PZ *pz; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_pz_free (%p)\n", pz_handle); + + dat_status = DAT_SUCCESS; + if ( DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_PZ); + goto bail; + } + + pz = (DAPL_PZ *) pz_handle; + + if ( 0 != dapl_os_atomic_read (&pz->pz_ref_count) ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_PZ_IN_USE); + goto bail; + } + + dat_status = dapls_ib_pd_free (pz); + if ( DAT_SUCCESS == dat_status ) + { + dapl_pz_dealloc (pz); + } + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_pz_query.c b/trunk/ulp/dapl2/dapl/common/dapl_pz_query.c new file mode 100644 index 00000000..3d4f1b7d --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_pz_query.c @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_pz_query.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_pz_query + * + * Return the ia associated with the protection zone pz + * + * Input: + * pz_handle + * pz_param_mask + * + * Output: + * pz_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_pz_query ( + IN DAT_PZ_HANDLE pz_handle, + IN DAT_PZ_PARAM_MASK pz_param_mask, + OUT DAT_PZ_PARAM *pz_param) +{ + DAPL_PZ *pz; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_pz_query (%p, %x, %p)\n", + pz_handle, + pz_param_mask, + pz_param); + + if ( DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_PZ); + goto bail; + } + if (NULL == pz_param) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + dat_status = DAT_SUCCESS; + pz = (DAPL_PZ *) pz_handle; + + /* Since the DAT_PZ_ARGS values are easily accessible, */ + /* don't bother checking the DAT_PZ_ARGS_MASK value */ + pz_param->ia_handle = (DAT_IA_HANDLE) pz->header.owner_ia; + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_pz_util.c b/trunk/ulp/dapl2/dapl/common/dapl_pz_util.c new file mode 100644 index 00000000..d5aa6ffd --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_pz_util.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_pz_util.c + * + * PURPOSE: Manage PZ structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl_pz_util.h" +#include "dapl_ia_util.h" + +/* + * dapl_pz_alloc + * + * alloc and initialize an PZ struct + * + * Input: + * none + * + * Output: + * pz_ptr + * + * Returns: + * none + * + */ +DAPL_PZ * +dapl_pz_alloc ( + IN DAPL_IA *ia) +{ + DAPL_PZ *pz; + + /* Allocate PZ */ + pz = (DAPL_PZ *) dapl_os_alloc (sizeof (DAPL_PZ)); + if (NULL == pz) + { + return (NULL); + } + + /* zero the structure */ + dapl_os_memzero (pz, sizeof (DAPL_PZ)); + + /* + * initialize the header + */ + pz->header.provider = ia->header.provider; + pz->header.magic = DAPL_MAGIC_PZ; + pz->header.handle_type = DAT_HANDLE_TYPE_PZ; + pz->header.owner_ia = ia; + pz->header.user_context.as_64 = 0; + pz->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&pz->header.ia_list_entry); + dapl_ia_link_pz (ia, pz); + dapl_os_lock_init (&pz->header.lock); + + /* + * initialize the body + */ + dapl_os_atomic_set (&pz->pz_ref_count, 0); + + return (pz); +} + +/* + * dapl_pz_free + * + * free an PZ struct + * + * Input: + * pz_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_pz_dealloc ( + IN DAPL_PZ *pz) +{ + pz->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ + dapl_ia_unlink_pz (pz->header.owner_ia, pz); + dapl_os_lock_destroy (&pz->header.lock); + + dapl_os_free (pz, sizeof (DAPL_PZ)); +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_pz_util.h b/trunk/ulp/dapl2/dapl/common/dapl_pz_util.h new file mode 100644 index 00000000..73ff2cb9 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_pz_util.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_pz_util.h + * + * PURPOSE: Utility defs & routines for the PZ data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_PZ_UTIL_H_ +#define _DAPL_PZ_UTIL_H_ + +#include "dapl.h" + +extern DAPL_PZ * +dapl_pz_alloc ( + IN DAPL_IA *ia); + +extern void +dapl_pz_dealloc ( + IN DAPL_PZ *pz); + +#endif /* _DAPL_PZ_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.c b/trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.c new file mode 100644 index 00000000..3a9bacff --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.c @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ring_buffer_util.c + * + * PURPOSE: Ring buffer management + * Description: Support and management functions for ring buffers + * + * $Id:$ + **********************************************************************/ + +#include "dapl_ring_buffer_util.h" + + +/* + * dapls_rbuf_alloc + * + * Given a DAPL_RING_BUFFER, initialize it and provide memory for + * the ringbuf itself. A passed in size will be adjusted to the next + * largest power of two number to simplify management. + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * size number of elements to allocate & manage + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_rbuf_alloc ( + INOUT DAPL_RING_BUFFER *rbuf, + IN DAT_COUNT size ) +{ + unsigned int rsize; /* real size */ + + /* The circular buffer must be allocated one too large. + * This eliminates any need for a distinct counter, as + * having the two pointers equal always means "empty" -- never "full" + */ + size++; + + /* Put size on a power of 2 boundary */ + rsize = 1; + while ( (DAT_COUNT)rsize < size) + { + rsize <<= 1; + } + + rbuf->base = (void *) dapl_os_alloc (rsize * sizeof (void *)); + if ( rbuf->base != NULL ) + { + rbuf->lim = rsize - 1; + dapl_os_atomic_set (&rbuf->head, 0); + dapl_os_atomic_set (&rbuf->tail, 0); + } + else + { + return DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY; + } + + return DAT_SUCCESS; +} + + +/* + * dapls_rbuf_realloc + * + * Resizes a DAPL_RING_BUFFER. This function is not thread safe; + * adding or removing elements from a ring buffer while resizing + * will have indeterminate results. + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * size number of elements to allocate & manage + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_rbuf_realloc ( + INOUT DAPL_RING_BUFFER *rbuf, + IN DAT_COUNT size ) +{ + DAPL_RING_BUFFER new_rbuf; + void *entry; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + /* decreasing the size or retaining the old size is not allowed */ + if (size <= rbuf->lim + 1) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail; + } + + /* + * !This is NOT ATOMIC! + * Simple algorithm: Allocate a new ring buffer, take everything + * out of the old one and put it in the new one, and release the + * old base buffer. + */ + dat_status = dapls_rbuf_alloc (&new_rbuf, size); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + while ( (entry = dapls_rbuf_remove(rbuf)) != NULL) + { + /* We know entries will fit so ignore the return code */ + (void)dapls_rbuf_add (&new_rbuf, entry); + } + + /* release the old base buffer */ + dapl_os_free (rbuf->base, (rbuf->lim + 1) * sizeof (void *)); + + *rbuf = new_rbuf; + + bail: + return dat_status; +} + + +/* + * dapls_rbuf_destroy + * + * Release the buffer and reset pointers to a DAPL_RING_BUFFER + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapls_rbuf_destroy ( + IN DAPL_RING_BUFFER *rbuf) +{ + if ( (NULL == rbuf) || + (NULL == rbuf->base) ) + { + return; + } + + dapl_os_free (rbuf->base, (rbuf->lim + 1) * sizeof (void *)); + rbuf->base = NULL; + rbuf->lim = 0; + + return; +} + +/* + * dapls_rbuf_add + * + * Add an entry to the ring buffer + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * entry entry to add + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES (queue full) + * + */ +DAT_RETURN +dapls_rbuf_add ( + IN DAPL_RING_BUFFER *rbuf, + IN void *entry) +{ + int pos; + int val; + + while ( ((dapl_os_atomic_read (&rbuf->head) + 1) & rbuf->lim) != (dapl_os_atomic_read (&rbuf->tail) & rbuf->lim) ) + { + pos = dapl_os_atomic_read (&rbuf->head); + val = dapl_os_atomic_assign (&rbuf->head, pos, pos + 1); + if ( val == pos ) + { + pos = (pos + 1) & rbuf->lim; /* verify in range */ + rbuf->base[pos] = entry; + return DAT_SUCCESS; + } + } + + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + +} + + +/* + * dapls_rbuf_remove + * + * Remove an entry from the ring buffer + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * + * Output: + * entry entry removed from the ring buffer + * + * Returns: + * DAT_SUCCESS + * DAT_QUEUE_EMPTY + + */ +void * +dapls_rbuf_remove ( + IN DAPL_RING_BUFFER *rbuf ) +{ + int pos; + int val; + + while ( dapl_os_atomic_read (&rbuf->head) != dapl_os_atomic_read (&rbuf->tail) ) + { + pos = dapl_os_atomic_read (&rbuf->tail); + val = dapl_os_atomic_assign (&rbuf->tail, pos, pos + 1); + if ( val == pos ) + { + pos = (pos + 1) & rbuf->lim; /* verify in range */ + + return (rbuf->base[pos]); + } + } + + return NULL; + +} + + +/* + * dapli_rbuf_count + * + * Return the number of entries in use in the ring buffer + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * + * + * Output: + * none + * + * Returns: + * count of entries + * + */ +DAT_COUNT +dapls_rbuf_count ( + IN DAPL_RING_BUFFER *rbuf ) +{ + DAT_COUNT count; + int head; + int tail; + + head = dapl_os_atomic_read (&rbuf->head) & rbuf->lim; + tail = dapl_os_atomic_read (&rbuf->tail) & rbuf->lim; + if ( head > tail ) + { + count = head - tail; + } + else + { + /* add 1 to lim as it is a mask, number of entries - 1 */ + count = (rbuf->lim + 1 - tail + head) & rbuf->lim; + } + + return count; +} + + +/* + * dapls_rbuf_adjust + * + * Adjusts the addresses of all elements stored in the + * ring buffer by a constant. This is useful for situations + * in which the memory area for the elements being stored + * has been reallocated (see dapl_evd_realloc() and helper + * functions). + * + * Input: + * rbuf pointer to DAPL_RING_BUFFER + * offset offset to adjust elemnt addresss by, + * used for addresses of type void * + * Output: + * none + * + * Returns: + * none + */ +void +dapls_rbuf_adjust ( + IN DAPL_RING_BUFFER *rbuf, + IN intptr_t offset) +{ + int pos; + + pos = dapl_os_atomic_read (&rbuf->head); + while ( pos != dapl_os_atomic_read (&rbuf->tail) ) + { + rbuf->base[pos] = (void*)((char*)rbuf->base[pos] + offset); + pos = (pos + 1) & rbuf->lim; /* verify in range */ + } +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.h b/trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.h new file mode 100644 index 00000000..6461082e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_ring_buffer_util.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_ring_buffer_util.h + * + * PURPOSE: Utility defs & routines for the ring buffer data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_RING_BUFFER_H_ +#define _DAPL_RING_BUFFER_H_ + +#include "dapl.h" + +/* + * Prototypes + */ +DAT_RETURN dapls_rbuf_alloc ( + DAPL_RING_BUFFER *rbuf, + DAT_COUNT size ); + +DAT_RETURN dapls_rbuf_realloc ( + DAPL_RING_BUFFER *rbuf, + DAT_COUNT size ); + +void dapls_rbuf_destroy ( + DAPL_RING_BUFFER *rbuf); + +DAT_RETURN dapls_rbuf_add ( + DAPL_RING_BUFFER *rbuf, + void *entry); + +void * dapls_rbuf_remove ( + DAPL_RING_BUFFER *rbuf); + +DAT_COUNT dapls_rbuf_count ( + DAPL_RING_BUFFER *rbuf ); + +void dapls_rbuf_adjust ( + IN DAPL_RING_BUFFER *rbuf, + IN intptr_t offset); + + +/* + * Simple functions + */ +#define dapls_rbuf_empty(rbuf) (rbuf->head == rbuf->tail) + + +#endif /* _DAPL_RING_BUFFER_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rmr_bind.c b/trunk/ulp/dapl2/dapl/common/dapl_rmr_bind.c new file mode 100644 index 00000000..f783911e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rmr_bind.c @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rmr_bind.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_rmr_util.h" +#include "dapl_ep_util.h" +#include "dapl_cookie.h" +#include "dapl_adapter_util.h" + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +STATIC _INLINE_ DAT_RETURN +dapli_rmr_bind_fuse ( + IN DAPL_RMR *rmr, + IN const DAT_LMR_TRIPLET *lmr_triplet, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN DAPL_EP *ep_ptr, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags, + OUT DAT_RMR_CONTEXT *rmr_context ); + +STATIC _INLINE_ DAT_RETURN +dapli_rmr_bind_unfuse ( + IN DAPL_RMR *rmr, + IN DAPL_EP *ep_ptr, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags); + + +/********************************************************************* + * * + * Function Definitions * + * * + *********************************************************************/ + +DAT_RETURN +dapli_rmr_bind_fuse ( + IN DAPL_RMR *rmr, + IN const DAT_LMR_TRIPLET* lmr_triplet, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN DAPL_EP *ep_ptr, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags, + OUT DAT_RMR_CONTEXT *rmr_context ) +{ + DAPL_LMR *lmr; + DAPL_COOKIE *cookie; + DAT_RETURN dat_status; + DAT_BOOLEAN is_signaled; + + dat_status = dapls_hash_search (rmr->header.owner_ia->hca_ptr->lmr_hash_table, + lmr_triplet->lmr_context, + (DAPL_HASH_DATA *)&lmr); + if ( DAT_SUCCESS != dat_status) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail; + } + + /* if the ep in unconnected return an error. IB requires that the */ + /* QP be connected to change a memory window binding since: */ + /* */ + /* - memory window bind operations are WQEs placed on a QP's */ + /* send queue */ + /* */ + /* - QP's only process WQEs on the send queue when the QP is in */ + /* the RTS state */ + if (DAT_EP_STATE_CONNECTED != ep_ptr->param.ep_state) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + if ( DAT_FALSE == dapl_mr_bounds_check ( + dapl_mr_get_address (lmr->param.region_desc, lmr->param.mem_type), + lmr->param.length, + lmr_triplet->virtual_address, + lmr_triplet->segment_length) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail; + } + + /* If the LMR, RMR, and EP are not in the same PZ, there is an error */ + if ( (ep_ptr->param.pz_handle != lmr->param.pz_handle) || + (ep_ptr->param.pz_handle != rmr->param.pz_handle) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; + } + + if ( !dapl_rmr_validate_completion_flag (DAT_COMPLETION_SUPPRESS_FLAG, + ep_ptr->param.ep_attr.request_completion_flags, + completion_flags) || + !dapl_rmr_validate_completion_flag (DAT_COMPLETION_UNSIGNALLED_FLAG, + ep_ptr->param.ep_attr.request_completion_flags, + completion_flags) || + !dapl_rmr_validate_completion_flag (DAT_COMPLETION_BARRIER_FENCE_FLAG, + ep_ptr->param.ep_attr.request_completion_flags, + completion_flags) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; + } + + dat_status = dapls_rmr_cookie_alloc (&ep_ptr->req_buffer, + rmr, + user_cookie, + &cookie); + if ( DAT_SUCCESS != dat_status ) + { + goto bail; + } + + is_signaled = (completion_flags & DAT_COMPLETION_SUPPRESS_FLAG) ? DAT_FALSE : DAT_TRUE; + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc (&ep_ptr->req_count); + + dat_status = dapls_ib_mw_bind (rmr, + lmr, + ep_ptr, + cookie, + lmr_triplet->virtual_address, + lmr_triplet->segment_length, + mem_priv, + is_signaled); + if ( DAT_SUCCESS != dat_status ) + { + dapl_os_atomic_dec (&ep_ptr->req_count); + dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie); + goto bail; + } + + dapl_os_atomic_inc (&lmr->lmr_ref_count); + + /* if the RMR was previously bound */ + if ( NULL != rmr->lmr ) + { + dapl_os_atomic_dec (&rmr->lmr->lmr_ref_count); + } + + rmr->param.mem_priv = mem_priv; + rmr->param.lmr_triplet = *lmr_triplet; + rmr->ep = ep_ptr; + rmr->lmr = lmr; + + if ( NULL != rmr_context ) + { + *rmr_context = rmr->param.rmr_context; + } + +bail: + return dat_status; +} + + +DAT_RETURN +dapli_rmr_bind_unfuse ( + IN DAPL_RMR *rmr, + IN DAPL_EP *ep_ptr, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + DAPL_COOKIE *cookie; + DAT_RETURN dat_status; + DAT_BOOLEAN is_signaled; + + dat_status = DAT_SUCCESS; + /* + * if the ep in unconnected return an error. IB requires that the + * QP be connected to change a memory window binding since: + * + * - memory window bind operations are WQEs placed on a QP's + * send queue + * + * - QP's only process WQEs on the send queue when the QP is in + * the RTS state + */ + if (DAT_EP_STATE_CONNECTED != ep_ptr->param.ep_state) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, dapls_ep_state_subtype (ep_ptr)); + goto bail1; + } + + /* If the RMR and EP are not in the same PZ, there is an error */ + if ( ep_ptr->param.pz_handle != rmr->param.pz_handle ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail1; + } + + if ( !dapl_rmr_validate_completion_flag (DAT_COMPLETION_SUPPRESS_FLAG, + ep_ptr->param.ep_attr.request_completion_flags, + completion_flags) || + !dapl_rmr_validate_completion_flag (DAT_COMPLETION_UNSIGNALLED_FLAG, + ep_ptr->param.ep_attr.request_completion_flags, + completion_flags) || + !dapl_rmr_validate_completion_flag (DAT_COMPLETION_BARRIER_FENCE_FLAG, + ep_ptr->param.ep_attr.request_completion_flags, + completion_flags) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + goto bail1; + } + + dat_status = dapls_rmr_cookie_alloc (&ep_ptr->req_buffer, + rmr, + user_cookie, + &cookie); + if ( DAT_SUCCESS != dat_status ) + { + goto bail1; + } + + is_signaled = (completion_flags & DAT_COMPLETION_UNSIGNALLED_FLAG) ? DAT_FALSE : DAT_TRUE; + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc (&ep_ptr->req_count); + + dat_status = dapls_ib_mw_unbind (rmr, + ep_ptr, + cookie, + is_signaled); + if ( DAT_SUCCESS != dat_status ) + { + dapl_os_atomic_dec (&ep_ptr->req_count); + dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie); + goto bail1; + } + + /* if the RMR was previously bound */ + if ( NULL != rmr->lmr ) + { + dapl_os_atomic_dec (&rmr->lmr->lmr_ref_count); + } + + rmr->param.mem_priv = DAT_MEM_PRIV_NONE_FLAG; + rmr->param.lmr_triplet.lmr_context = 0; + rmr->param.lmr_triplet.virtual_address = 0; + rmr->param.lmr_triplet.segment_length = 0; + rmr->ep = ep_ptr; + rmr->lmr = NULL; + +bail1: + return dat_status; +} + + +/* + * dapl_rmr_bind + * + * DAPL Requirements Version xxx, 6.6.4.4 + * + * Bind the RMR to the specified memory region within the LMR and + * provide a new rmr_context value. + * + * Input: + * Output: + */ +DAT_RETURN DAT_API +dapl_rmr_bind ( + IN DAT_RMR_HANDLE rmr_handle, + IN DAT_LMR_HANDLE lmr_handle, + IN const DAT_LMR_TRIPLET *lmr_triplet, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN DAT_VA_TYPE va_type, + IN DAT_EP_HANDLE ep_handle, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags, + OUT DAT_RMR_CONTEXT *rmr_context ) +{ + DAPL_RMR *rmr; + DAPL_EP *ep_ptr; + + if ( DAPL_BAD_HANDLE (rmr_handle, DAPL_MAGIC_RMR) ) + { + return DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RMR); + } + if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) ) + { + return DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + + rmr = (DAPL_RMR *) rmr_handle; + ep_ptr = (DAPL_EP *) ep_handle; + + /* if the rmr should be bound */ + if (0 != lmr_triplet->segment_length) + { + return dapli_rmr_bind_fuse (rmr, + lmr_triplet, + mem_priv, + ep_ptr, + user_cookie, + completion_flags, + rmr_context); + } + else /* the rmr should be unbound */ + { + return dapli_rmr_bind_unfuse (rmr, + ep_ptr, + user_cookie, + completion_flags); + } +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rmr_create.c b/trunk/ulp/dapl2/dapl/common/dapl_rmr_create.c new file mode 100644 index 00000000..4c4db556 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rmr_create.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rmr_create.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl_rmr_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_rmr_create + * + * Create a remote memory region for the specified protection zone + * + * Input: + * pz_handle + * + * Output: + * rmr_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_rmr_create ( + IN DAT_PZ_HANDLE pz_handle, + OUT DAT_RMR_HANDLE *rmr_handle) +{ + DAPL_PZ *pz; + DAPL_RMR *rmr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_PZ); + goto bail; + } + + pz = (DAPL_PZ *) pz_handle; + + rmr = dapl_rmr_alloc (pz); + + if ( rmr == NULL ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_status = dapls_ib_mw_alloc (rmr); + + if ( dat_status != DAT_SUCCESS ) + { + dapl_rmr_dealloc (rmr); + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY_REGION); + goto bail; + } + + dapl_os_atomic_inc (&pz->pz_ref_count); + + *rmr_handle = rmr; + + bail: + return dat_status; +} + +/* + * dapl_rmr_create_for_ep + * + * DAPL Requirements Version 2.0, 6.7.3.x + * + * Creates an RMR that is specific to a single connection at a time. + * + * This operation is relatively heavy. The created RMR can be bound to a + * memory region within the LMR through a lightweight dat_rmr_bind + * operation for EPs that use the pz_handle that generates rmr_context. + * + * If the operation fails (does not return DAT_SUCCESS), the return values + * of rmr_handle are undefined and Consumers should not use it. + * pz_handle provide Consumers a way to restrict access to an RMR by + * authorized connections only. + * + * + * Input: + * pz_handle + * + * Output: + * rmr_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_HANDLE + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_rmr_create_for_ep ( + IN DAT_PZ_HANDLE pz_handle, /* pz_handle */ + OUT DAT_RMR_HANDLE *rmr_handle) /* rmr_handle */ +{ + return DAT_MODEL_NOT_SUPPORTED; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rmr_free.c b/trunk/ulp/dapl2/dapl/common/dapl_rmr_free.c new file mode 100644 index 00000000..329cbc6a --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rmr_free.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rmr_free.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl_rmr_util.h" +#include "dapl_adapter_util.h" +#include "dapl_ia_util.h" + +/* + * dapl_rmr_free + * + * Destroy an instance of the Remote Memory Region + * + * Input: + * rmr_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_rmr_free ( + IN DAT_RMR_HANDLE rmr_handle ) +{ + DAPL_RMR *rmr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (rmr_handle, DAPL_MAGIC_RMR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_RMR); + goto bail; + } + + rmr = (DAPL_RMR *) rmr_handle; + + /* + * If the user did not perform an unbind op, release + * counts here. + */ + if ( rmr->param.lmr_triplet.virtual_address != 0 ) + { + dapl_os_atomic_dec (&rmr->lmr->lmr_ref_count); + rmr->param.lmr_triplet.virtual_address = 0; + } + + dat_status = dapls_ib_mw_free (rmr); + + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + dapl_os_atomic_dec (&rmr->pz->pz_ref_count); + + dapl_rmr_dealloc (rmr); + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rmr_query.c b/trunk/ulp/dapl2/dapl/common/dapl_rmr_query.c new file mode 100644 index 00000000..408a1cb8 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rmr_query.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rmr_query.c + * + * PURPOSE: Memory management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 6 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_rmr_query + * + * DAPL Requirements Version xxx, 6.6.4.3 + * + * Provide the RMR arguments. + * + * Input: + * rmr_handle + * rmr_args_mask + * + * Output: + * rmr_args + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_rmr_query ( + IN DAT_RMR_HANDLE rmr_handle, + IN DAT_RMR_PARAM_MASK rmr_param_mask, + IN DAT_RMR_PARAM *rmr_param ) +{ + DAPL_RMR *rmr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (rmr_handle, DAPL_MAGIC_RMR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_RMR); + goto bail; + } + if (NULL == rmr_param) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + rmr = (DAPL_RMR *) rmr_handle; + + /* If the RMR is unbound, there is no LMR triplet associated with */ + /* this RMR. If the consumer requests this field, return an error. */ + if ( (rmr_param_mask & DAT_RMR_FIELD_LMR_TRIPLET) && (NULL == rmr->lmr) ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + dapl_os_memcpy (rmr_param, &rmr->param, sizeof (DAT_RMR_PARAM)); + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rmr_util.c b/trunk/ulp/dapl2/dapl/common/dapl_rmr_util.c new file mode 100644 index 00000000..213721b7 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rmr_util.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_rmr_util.h" +#include "dapl_ia_util.h" + +DAPL_RMR * +dapl_rmr_alloc( + IN DAPL_PZ *pz) +{ + DAPL_RMR *rmr; + + /* Allocate LMR */ + rmr = (DAPL_RMR *) dapl_os_alloc(sizeof(DAPL_RMR)); + if (NULL == rmr) + { + return (NULL); + } + + /* zero the structure */ + dapl_os_memzero(rmr, sizeof(DAPL_RMR)); + + /* + * initialize the header + */ + rmr->header.provider = pz->header.provider; + rmr->header.magic = DAPL_MAGIC_RMR; + rmr->header.handle_type = DAT_HANDLE_TYPE_RMR; + rmr->header.owner_ia = pz->header.owner_ia; + rmr->header.user_context.as_64 = 0; + rmr->header.user_context.as_ptr = 0; + dapl_llist_init_entry (&rmr->header.ia_list_entry); + dapl_ia_link_rmr(rmr->header.owner_ia, rmr); + dapl_os_lock_init(&rmr->header.lock); + + /* + * initialize the body + */ + rmr->param.ia_handle = (DAT_IA_HANDLE) pz->header.owner_ia; + rmr->param.pz_handle = (DAT_PZ_HANDLE) pz; + rmr->param.lmr_triplet.lmr_context = 0; + rmr->param.lmr_triplet.virtual_address = 0; + rmr->param.lmr_triplet.segment_length = 0; + + rmr->param.mem_priv = 0; + rmr->pz = pz; + rmr->lmr = NULL; + + return (rmr); +} + + +void +dapl_rmr_dealloc( + IN DAPL_RMR *rmr) +{ + rmr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ + + dapl_ia_unlink_rmr(rmr->header.owner_ia, rmr); + dapl_os_lock_destroy(&rmr->header.lock); + + dapl_os_free((void *) rmr, sizeof(DAPL_RMR)); +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rmr_util.h b/trunk/ulp/dapl2/dapl/common/dapl_rmr_util.h new file mode 100644 index 00000000..3484ea05 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rmr_util.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_rmr_util.h + * + * PURPOSE: Utility defs & routines for the RMR data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_RMR_UTIL_H_ +#define _DAPL_RMR_UTIL_H_ + +#include "dapl_mr_util.h" + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +extern DAPL_RMR * +dapl_rmr_alloc ( + IN DAPL_PZ *pz); + +extern void +dapl_rmr_dealloc ( + IN DAPL_RMR *rmr); + +STATIC _INLINE_ DAT_BOOLEAN +dapl_rmr_validate_completion_flag ( + IN DAT_COMPLETION_FLAGS mask, + IN DAT_COMPLETION_FLAGS allow, + IN DAT_COMPLETION_FLAGS request); + + +/********************************************************************* + * * + * Inline Functions * + * * + *********************************************************************/ + +STATIC _INLINE_ DAT_BOOLEAN +dapl_rmr_validate_completion_flag ( + IN DAT_COMPLETION_FLAGS mask, + IN DAT_COMPLETION_FLAGS allow, + IN DAT_COMPLETION_FLAGS request) +{ + if ( (mask & request ) && !(mask & allow) ) + { + return DAT_FALSE; + } + else + { + return DAT_TRUE; + } +} + +#endif /* _DAPL_RMR_UTIL_H_*/ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rsp_create.c b/trunk/ulp/dapl2/dapl/common/dapl_rsp_create.c new file mode 100644 index 00000000..5ea1bd16 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rsp_create.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rsp_create.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_sp_util.h" +#include "dapl_ia_util.h" +#include "dapl_ep_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_rsp_create + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.3.4.1 + * + * Create a Resereved Service Point with the specified Endpoint + * that generates at most one Connection Request that is + * delivered to the specified Event Dispatcher in a notification + * event + * + * Input: + * ia_handle + * conn_qual + * ep_handle + * evd_handle + * + * Output: + * rsp_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + * DAT_CONN_QUAL_IN_USE + */ +DAT_RETURN DAT_API +dapl_rsp_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_CONN_QUAL conn_qual, + IN DAT_EP_HANDLE ep_handle, + IN DAT_EVD_HANDLE evd_handle, + OUT DAT_RSP_HANDLE *rsp_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_SP *sp_ptr; + DAPL_EVD *evd_ptr; + DAPL_EP *ep_ptr; + DAT_BOOLEAN sp_found; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + ia_ptr = (DAPL_IA *)ia_handle; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + ">>> dapl_rsp_free conn_qual: %x EP: %p\n", + conn_qual, ep_handle); + + if ( DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EP); + goto bail; + } + if ( DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EVD_CR); + goto bail; + } + + if ( rsp_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG5); + goto bail; + } + + ep_ptr = (DAPL_EP *) ep_handle; + if ( ep_ptr->param.ep_state != DAT_EP_STATE_UNCONNECTED ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,dapls_ep_state_subtype (ep_ptr)); + goto bail; + } + + evd_ptr = (DAPL_EVD *)evd_handle; + if ( ! (evd_ptr->evd_flags & DAT_EVD_CR_FLAG) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_EVD_CR); + goto bail; + } + + sp_ptr = dapls_ia_sp_search (ia_ptr, conn_qual, DAT_FALSE); + sp_found = DAT_TRUE; + if (sp_ptr == NULL) + { + sp_found = DAT_FALSE; + + /* Allocate RSP */ + sp_ptr = dapls_sp_alloc ( ia_ptr, DAT_FALSE ); + if ( sp_ptr == NULL ) + { + dat_status = + DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + } + + /* + * Fill out the RSP args + */ + sp_ptr->conn_qual = conn_qual; + sp_ptr->evd_handle = evd_handle; + sp_ptr->psp_flags = 0; + sp_ptr->ep_handle = ep_handle; + + /* + * Take a reference on the EVD handle + */ + dapl_os_atomic_inc (& ((DAPL_EVD *)evd_handle)->evd_ref_count); + + /* + * Update the EP state indicating the provider now owns it + */ + ep_ptr->param.ep_state = DAT_EP_STATE_RESERVED; + + /* + * Set up a listener for a connection. Connections can arrive + * even before this call returns! + */ + sp_ptr->state = DAPL_SP_STATE_RSP_LISTENING; + sp_ptr->listening = DAT_TRUE; + + if (sp_found == DAT_FALSE) + { + /* Link it onto the IA */ + dapl_ia_link_rsp (ia_ptr, sp_ptr); + + dat_status = dapls_ib_setup_conn_listener ( ia_ptr, + conn_qual, + sp_ptr ); + + if ( dat_status != DAT_SUCCESS ) + { + /* + * Have a problem setting up the connection, something + * wrong! Decrements the EVD refcount & release it. Set + * the state to FREE, so we know the call failed. + */ + dapl_os_atomic_dec (& ((DAPL_EVD *)evd_handle)->evd_ref_count); + sp_ptr->evd_handle = NULL; + sp_ptr->state = DAPL_SP_STATE_FREE; + dapls_ia_unlink_sp (ia_ptr, sp_ptr); + dapls_sp_free_sp (sp_ptr); + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> dapl_rsp_create setup_conn_listener failed: %x\n", + dat_status); + + goto bail; + } + } + + /* + * Return handle to the user + */ + *rsp_handle = (DAT_RSP_HANDLE)sp_ptr; + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ + + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rsp_free.c b/trunk/ulp/dapl2/dapl/common/dapl_rsp_free.c new file mode 100644 index 00000000..0e37a73d --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rsp_free.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rsp_free.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_sp_util.h" +#include "dapl_ia_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_rsp_free + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.3.5 + * + * Destroy a specific instance of a Reserved Service Point. + * + * Input: + * rsp_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ +DAT_RETURN DAT_API +dapl_rsp_free ( + IN DAT_RSP_HANDLE rsp_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_SP *sp_ptr; + DAPL_EP *ep_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + sp_ptr = (DAPL_SP *) rsp_handle; + /* + * Verify handle + */ + dapl_dbg_log (DAPL_DBG_TYPE_CM, + ">>> dapl_rsp_free %p\n", + rsp_handle); + if ( DAPL_BAD_HANDLE (sp_ptr, DAPL_MAGIC_RSP ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_RSP); + goto bail; + } + + /* ia_ptr = (DAPL_IA *)sp_ptr->header.owner_ia; */ + ia_ptr = sp_ptr->header.owner_ia; + + /* + * Remove the connection listener if there are no connections. If + * we defer removing the sp it becomes something of a zombie + * container until disconnection, after which it will be cleaned up. + */ + dapl_os_lock (&sp_ptr->header.lock); + + /* + * Make sure we don't leave a dangling EP. If the state is still + * RESERVED then the RSP still owns it. + */ + ep_ptr = (DAPL_EP *)sp_ptr->ep_handle; + if ( ep_ptr != NULL && ep_ptr->param.ep_state == DAT_EP_STATE_RESERVED ) + { + ep_ptr->param.ep_state = DAT_EP_STATE_UNCONNECTED; + } + sp_ptr->ep_handle = NULL; + + /* Release reference on EVD. If an error was encountered in a previous + * free the evd_handle will be NULL + */ + if (sp_ptr->evd_handle) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)sp_ptr->evd_handle)->evd_ref_count); + sp_ptr->evd_handle = NULL; + } + + /* + * Release the base resource if there are no outstanding connections; + * else the last disconnect on this RSP will free it up. The RSP + * is used to contain CR records for each connection, which + * contain information necessary to disconnect. + * sp_ptr->listening will be DAT_TRUE if there has never been a + * connection event, and DAT_FALSE if a connection attempt resulted + * in a reject. + */ + if ( sp_ptr->cr_list_count == 0 ) + { + /* This RSP has never been used. Clean it up */ + sp_ptr->listening = DAT_FALSE; + sp_ptr->state = DAPL_SP_STATE_FREE; + dapl_os_unlock (&sp_ptr->header.lock); + + dat_status = dapls_ib_remove_conn_listener ( ia_ptr, + sp_ptr ); + if (dat_status != DAT_SUCCESS) + { + sp_ptr->state = DAPL_SP_STATE_RSP_LISTENING; + goto bail; + } + dapls_ia_unlink_sp ( ia_ptr, sp_ptr ); + dapls_sp_free_sp ( sp_ptr ); + } + else + { + /* The RSP is now in the pending state, where it will sit until + * the connection terminates or the app uses the same + * ServiceID again, which will reactivate it. + */ + sp_ptr->state = DAPL_SP_STATE_RSP_PENDING; + dapl_os_unlock (&sp_ptr->header.lock); + } + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_rsp_query.c b/trunk/ulp/dapl2/dapl/common/dapl_rsp_query.c new file mode 100644 index 00000000..1fc1b9c9 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_rsp_query.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_rsp_query.c + * + * PURPOSE: Connection management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_rsp_query + * + * uDAPL: User Direct Access Program Library Version 1.1, 6.4.1.6 + * + * Provide arguments of the reserved service points + * + * Input: + * rsp_handle + * rsp_args_mask + * + * Output: + * rsp_args + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_rsp_query ( + IN DAT_RSP_HANDLE rsp_handle, + IN DAT_RSP_PARAM_MASK rsp_mask, + OUT DAT_RSP_PARAM *rsp_param ) +{ + DAPL_SP *sp_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (rsp_handle, DAPL_MAGIC_RSP) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_RSP); + goto bail; + } + + if (NULL == rsp_param) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + sp_ptr = (DAPL_SP *) rsp_handle; + + /* + * Fill in the RSP params + */ + rsp_param->ia_handle = sp_ptr->header.owner_ia; + rsp_param->conn_qual = sp_ptr->conn_qual; + rsp_param->evd_handle = sp_ptr->evd_handle; + rsp_param->ep_handle = sp_ptr->ep_handle; + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * c-brace-offset: -4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_set_consumer_context.c b/trunk/ulp/dapl2/dapl/common/dapl_set_consumer_context.c new file mode 100644 index 00000000..e965fb52 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_set_consumer_context.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_set_consumer_context.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_set_consumer_context + * + * DAPL Requirements Version xxx, 6.2.2.1 + * + * Set a consumer context in the provided dat_handle + * + * Input: + * dat_handle + * context + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ +DAT_RETURN DAT_API +dapl_set_consumer_context ( + IN DAT_HANDLE dat_handle, + IN DAT_CONTEXT context ) +{ + DAT_RETURN dat_status; + DAPL_HEADER *header; + + dat_status = DAT_SUCCESS; + + header = (DAPL_HEADER *)dat_handle; + if ( ((header) == NULL) || + DAPL_BAD_PTR(header) || + (header->magic != DAPL_MAGIC_IA && + header->magic != DAPL_MAGIC_EVD && + header->magic != DAPL_MAGIC_EP && + header->magic != DAPL_MAGIC_LMR && + header->magic != DAPL_MAGIC_RMR && + header->magic != DAPL_MAGIC_PZ && + header->magic != DAPL_MAGIC_PSP && + header->magic != DAPL_MAGIC_RSP && + header->magic != DAPL_MAGIC_CR)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + header->user_context = context; + +bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/common/dapl_sp_util.c b/trunk/ulp/dapl2/dapl/common/dapl_sp_util.c new file mode 100644 index 00000000..19bd3ce4 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_sp_util.c @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_sp_util.c + * + * PURPOSE: Manage PSP Info structure + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ep_util.h" +#include "dapl_sp_util.h" +#include "dapl_cr_util.h" + + +/* + * Local definitions + */ + + +/* + * dapl_sp_alloc + * + * alloc and initialize a PSP INFO struct + * + * Input: + * IA INFO struct ptr + * + * Output: + * sp_ptr + * + * Returns: + * NULL + * pointer to sp info struct + * + */ +DAPL_SP * +dapls_sp_alloc ( + IN DAPL_IA *ia_ptr, + IN DAT_BOOLEAN is_psp ) +{ + DAPL_SP *sp_ptr; + + /* Allocate EP */ + sp_ptr = (DAPL_SP *)dapl_os_alloc (sizeof (DAPL_SP)); + if ( sp_ptr == NULL ) + { + return (NULL); + } + + /* zero the structure */ + dapl_os_memzero (sp_ptr, sizeof (DAPL_SP)); + + /* + * initialize the header + */ + sp_ptr->header.provider = ia_ptr->header.provider; + if ( is_psp ) + { + sp_ptr->header.magic = DAPL_MAGIC_PSP; + sp_ptr->header.handle_type = DAT_HANDLE_TYPE_PSP; + } + else + { + sp_ptr->header.magic = DAPL_MAGIC_RSP; + sp_ptr->header.handle_type = DAT_HANDLE_TYPE_RSP; + } + sp_ptr->header.owner_ia = ia_ptr; + sp_ptr->header.user_context.as_64 = 0; + sp_ptr->header.user_context.as_ptr = NULL; + dapl_llist_init_entry (&sp_ptr->header.ia_list_entry); + dapl_os_lock_init (&sp_ptr->header.lock); + +#if defined(_WIN32) || defined(_WIN64) + dapl_os_wait_object_init( &sp_ptr->wait_object ); +#endif + /* + * Initialize the Body (set to NULL above) + */ + dapl_llist_init_head (&sp_ptr->cr_list_head); + + return ( sp_ptr ); +} + + +/* + * dapl_sp_free + * + * Free the passed in PSP structure. + * + * Input: + * entry point pointer + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapls_sp_free_sp ( + IN DAPL_SP *sp_ptr ) +{ + dapl_os_assert (sp_ptr->header.magic == DAPL_MAGIC_PSP || + sp_ptr->header.magic == DAPL_MAGIC_RSP); + dapl_os_assert (dapl_llist_is_empty (&sp_ptr->cr_list_head)); + +#if defined(_WIN32) || defined(_WIN64) + dapl_os_wait_object_destroy( &sp_ptr->wait_object ); +#endif + dapl_os_lock (&sp_ptr->header.lock); + sp_ptr->header.magic = DAPL_MAGIC_INVALID;/* reset magic to prevent reuse */ + dapl_os_unlock (&sp_ptr->header.lock); + dapl_os_free (sp_ptr, sizeof (DAPL_SP)); +} + + +/* + * dapl_cr_link_cr + * + * Add a cr to a PSP structure + * + * Input: + * sp_ptr + * cr_ptr + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_sp_link_cr ( + IN DAPL_SP *sp_ptr, + IN DAPL_CR *cr_ptr ) +{ + dapl_os_lock (&sp_ptr->header.lock); + dapl_llist_add_tail (&sp_ptr->cr_list_head, + &cr_ptr->header.ia_list_entry, + cr_ptr); + sp_ptr->cr_list_count++; + dapl_os_unlock (&sp_ptr->header.lock); +} + + +/* + * dapl_sp_search_cr + * + * Search for a CR on the PSP cr_list with a matching cm_handle. When + * found, remove it from the list and update fields. + * + * Must be called with the sp_ptr lock taken. + * + * Input: + * sp_ptr + * ib_cm_handle + * + * Output: + * none + * + * Returns: + * cr_ptr_fnd Pointer to matching DAPL_CR + * + */ +DAPL_CR * +dapl_sp_search_cr ( + IN DAPL_SP *sp_ptr, + IN dp_ib_cm_handle_t ib_cm_handle ) +{ + DAPL_CR *cr_ptr; + DAPL_CR *cr_ptr_fnd; + + if (dapl_llist_is_empty (&sp_ptr->cr_list_head)) + { + return NULL; + } + cr_ptr_fnd = NULL; + cr_ptr = (DAPL_CR *) dapl_llist_peek_head (&sp_ptr->cr_list_head); + dapl_os_assert( cr_ptr ); + + do + { + if ( cr_ptr->ib_cm_handle == ib_cm_handle ) + { + cr_ptr_fnd = cr_ptr; + + break; + } + cr_ptr = cr_ptr->header.ia_list_entry.flink->data; + } while ((void *)cr_ptr != (void *)sp_ptr->cr_list_head->data); + + return cr_ptr_fnd; +} + + + +/* + * dapl_sp_remove_cr + * + * Remove the CR from the PSP. Done prior to freeing the CR resource. + * + * Must be called with the sp_ptr lock taken. + * + * Input: + * sp_ptr + * cr_ptr + * + * Output: + * none + * + * Returns: + * void + * + */ +void +dapl_sp_remove_cr ( + IN DAPL_SP *sp_ptr, + IN DAPL_CR *cr_ptr ) +{ + if ( dapl_llist_is_empty (&sp_ptr->cr_list_head) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "**dapl_sp_remove_cr: removing from empty queue! sp %p\n", + sp_ptr ); + return; + } + + dapl_llist_remove_entry (&sp_ptr->cr_list_head, + &cr_ptr->header.ia_list_entry); + sp_ptr->cr_list_count--; +} + + + +/* + * dapl_sp_remove_ep + * + * Remove a CR from a PSP, given an EP. + * + * + * Input: + * ep_ptr + * + * Output: + * none + * + * Returns: + * void + * + */ +void +dapl_sp_remove_ep ( + IN DAPL_EP *ep_ptr ) +{ + DAPL_SP *sp_ptr; + DAPL_CR *cr_ptr; + + cr_ptr = ep_ptr->cr_ptr; + + if (cr_ptr != NULL) + { + sp_ptr = cr_ptr->sp_ptr; + + dapl_os_lock (&sp_ptr->header.lock); + + /* Remove the CR from the queue */ + dapl_sp_remove_cr (sp_ptr, cr_ptr); + + dapl_os_unlock (&sp_ptr->header.lock); + + /* free memory outside of the lock */ + dapls_cr_free (cr_ptr); + + return; + } +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_sp_util.h b/trunk/ulp/dapl2/dapl/common/dapl_sp_util.h new file mode 100644 index 00000000..d10ba5d2 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_sp_util.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_sp_util.h + * + * PURPOSE: Utility defs & routines for the PSP & RSP data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_PSP_UTIL_H_ +#define _DAPL_PSP_UTIL_H_ + +DAPL_SP *dapls_sp_alloc ( + IN DAPL_IA *ia_ptr, + IN DAT_BOOLEAN is_psp ); + +void dapls_sp_free_sp ( + IN DAPL_SP *sp_ptr ); + +void dapl_sp_link_cr ( + IN DAPL_SP *sp_ptr, + IN DAPL_CR *cr_ptr ); + +DAPL_CR *dapl_sp_search_cr ( + IN DAPL_SP *sp_ptr, + IN dp_ib_cm_handle_t ib_cm_handle ); + +void dapl_sp_remove_cr ( + IN DAPL_SP *sp_ptr, + IN DAPL_CR *cr_ptr ); + +void dapl_sp_remove_ep ( + IN DAPL_EP *ep_ptr ); + +#endif /* _DAPL_PSP_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_create.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_create.c new file mode 100644 index 00000000..21295e48 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_create.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_srq_create.c + * + * PURPOSE: Shared Receive Queue management + * Description: Interfaces in this file are completely described in + * the DAPL 1.2 API, Chapter 6, section 5.1 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_srq_util.h" +#include "dapl_adapter_util.h" + + +/* + * dapl_srq_create + * + * DAPL Version 1.2, 6.5.1 + * + * Create an instance of a Shared Receive Queue that is provided to the + * consumer at srq_handle. + * + * Input: + * ia_handle + * pz_handle + * srq_attr + * + * Output: + * srq_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * ?DAT_INVALID_ATTRIBUTE?? + * DAT_MODEL_NOT_SUPPORTED + */ +DAT_RETURN DAT_API +dapl_srq_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_SRQ_ATTR *srq_attr, + OUT DAT_SRQ_HANDLE *srq_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_SRQ *srq_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_srq_create (%p, %p, %p, %p)\n", + ia_handle, + pz_handle, + srq_attr, + srq_handle); + + ia_ptr = (DAPL_IA *)ia_handle; + + /* + * Verify parameters + */ + if ( DAPL_BAD_HANDLE (ia_ptr, DAPL_MAGIC_IA) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + + /* + * Verify non-required parameters. + * N.B. Assumption: any parameter that can be + * modified by dat_ep_modify() is not strictly + * required when the EP is created + */ + if ( pz_handle != NULL && + DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + goto bail; + } + + if ( srq_handle == NULL ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); + goto bail; + } + if (DAPL_BAD_PTR(srq_attr)) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + /* Allocate SRQ */ + srq_ptr = dapl_srq_alloc (ia_ptr, srq_attr); + if (srq_ptr == NULL) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + srq_ptr->param.ia_handle = (DAT_IA_HANDLE) ia_ptr; + srq_ptr->param.srq_state = DAT_SRQ_STATE_OPERATIONAL; + srq_ptr->param.pz_handle = pz_handle; + + /* + * XXX Allocate provider resource here!!! + */ + /* XXX */ dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + /* XXX */ dapl_srq_dealloc (srq_ptr); + /* XXX */ goto bail; + + /* Link it onto the IA */ + dapl_ia_link_srq (ia_ptr, srq_ptr); + + *srq_handle = srq_ptr; + +bail: + return dat_status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_free.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_free.c new file mode 100644 index 00000000..cecda0c8 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_free.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_srq_free.c + * + * PURPOSE: Shared Receive Queue management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5.5 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_srq_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_srq_free + * + * DAPL Version 1.2, 6.5.5 + * + * Destroy an instance of an SRQ + * + * Input: + * srq_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + */ +DAT_RETURN DAT_API +dapl_srq_free ( + IN DAT_SRQ_HANDLE srq_handle) +{ + DAPL_SRQ *srq_ptr; + DAPL_IA *ia_ptr; + DAT_SRQ_PARAM *param; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_srq_free (%p)\n", srq_handle); + + srq_ptr = (DAPL_SRQ *) srq_handle; + param = &srq_ptr->param; + + /* + * Verify parameter & state + */ + if ( DAPL_BAD_HANDLE (srq_ptr, DAPL_MAGIC_SRQ ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + goto bail; + } + + if (dapl_os_atomic_read (&srq_ptr->srq_ref_count) != 0) + { + /* + * The DAPL 1.2 spec says to return DAT_SRQ_IN_USE, which does + * not exist. Have filed the following as an errata. + */ + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_SRQ_IN_USE); + goto bail; + } + + ia_ptr = srq_ptr->header.owner_ia; + + /* + * Do verification of parameters and the state change atomically. + */ + dapl_os_lock ( &srq_ptr->header.lock ); + + /* Remove the SRQ from the IA */ + dapl_ia_unlink_srq ( ia_ptr, srq_ptr ); + + dapl_os_unlock (&srq_ptr->header.lock); + + /* + * Finish tearing everything down. + */ + + /* + * Take care of the transport resource + */ + + /* XXX Put provider code here!!! */ + + /* Free the resource */ + dapl_srq_dealloc (srq_ptr); + +bail: + return dat_status; + +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_post_recv.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_post_recv.c new file mode 100644 index 00000000..72c07fb4 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_post_recv.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_srq_post_recv.c + * + * PURPOSE: Shared Receive Queue management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 5.8 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_cookie.h" +#include "dapl_adapter_util.h" + +/* + * dapl_srq_post_recv + * + * DAPL Requirements Version 1.2, 6.5.8 + * + * Post a receive buffer that can be used by any incoming + * message by any connected EP using the SRQ. Request to receive data + * over a connection of any ep handle into local_iov + * + * Input: + * srq_handle + * num_segments + * local_iov + * user_cookie + * + * Output: + * None. + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + * DAT_PROTECTION_VIOLATION + * DAT_PROVILEGES_VIOLATION + */ +DAT_RETURN DAT_API +dapl_srq_post_recv ( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie) +{ + DAPL_SRQ *srq_ptr; + DAPL_COOKIE *cookie; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_srq_post_recv (%p, %d, %p, %p)\n", + srq_handle, + num_segments, + local_iov, + user_cookie.as_64); + + if ( DAPL_BAD_HANDLE (srq_handle, DAPL_MAGIC_SRQ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + goto bail; + } + + srq_ptr = (DAPL_SRQ *) srq_handle; + + /* + * Synchronization ok since this buffer is only used for receive + * requests, which aren't allowed to race with each other. The + * app must syncronize access to the SRQ. + */ + dat_status = dapls_dto_cookie_alloc (&srq_ptr->recv_buffer, + DAPL_DTO_TYPE_RECV, + user_cookie, + &cookie); + if ( DAT_SUCCESS != dat_status) + { + goto bail; + } + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc (&srq_ptr->recv_count); + + /* + * Invoke provider specific routine to post DTO + */ + /* XXX Put code here XXX */ + /* XXX */ dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + + if ( dat_status != DAT_SUCCESS ) + { + dapl_os_atomic_dec (&srq_ptr->recv_count); + dapls_cookie_dealloc (&srq_ptr->recv_buffer, cookie); + } + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_query.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_query.c new file mode 100644 index 00000000..47ee3ccc --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_query.c @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_srq_query.c + * + * PURPOSE: Shared Receive Queue management + * + * Description: Interfaces in this file are completely described in + * the DAPL 1.2 API, Chapter 6, section 5.6 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_srq_query + * + * DAPL Requirements Version 1.2, 6.5.6 + * + * Return SRQ parameters to the consumer + * + * Input: + * srq_handle + * srq_param_mask + * + * Output: + * srq_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_srq_query ( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_SRQ_PARAM_MASK srq_param_mask, + OUT DAT_SRQ_PARAM *srq_param) +{ + DAPL_SRQ *srq_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_srq_query (%p, %x, %p)\n", + srq_handle, + srq_param_mask, + srq_param); + + if ( DAPL_BAD_HANDLE (srq_handle, DAPL_MAGIC_SRQ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + goto bail; + } + if (srq_param == NULL) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + srq_ptr = (DAPL_SRQ *) srq_handle; + + /* + * XXX Need to calculate available_dto_count and outstanding_dto_count + */ + srq_ptr->param.available_dto_count = DAT_VALUE_UNKNOWN; + srq_ptr->param.outstanding_dto_count = DAT_VALUE_UNKNOWN; + + *srq_param = srq_ptr->param; + + bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_resize.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_resize.c new file mode 100644 index 00000000..f119d772 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_resize.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_srq_resize.c + * + * PURPOSE: Shared Receive Queue management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 5.7 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_srq_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_srq_resize + * + * DAPL Requirements Version 1.2, 6.5.7 + * + * Modify the size of the event queue of a Shared Receive Queue + * + * Input: + * srq_handle + * srq_max_recv_dto + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_STATE + */ + +DAT_RETURN DAT_API +dapl_srq_resize ( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_COUNT srq_max_recv_dto ) +{ + DAPL_IA *ia_ptr; + DAPL_SRQ *srq_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_srq_resize (%p, %d)\n", + srq_handle, srq_max_recv_dto); + + if (DAPL_BAD_HANDLE (srq_handle, DAPL_MAGIC_SRQ)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + goto bail; + } + + srq_ptr = (DAPL_SRQ *)srq_handle; + ia_ptr = srq_ptr->header.owner_ia; + + /* + * Check for nonsense requests per the spec + */ + if (srq_max_recv_dto <= srq_ptr->param.low_watermark) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_NO_SUBTYPE); + goto bail; + } + + /* XXX Put implementation here XXX */ + + /* XXX */ dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_set_lw.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_set_lw.c new file mode 100644 index 00000000..c2544750 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_set_lw.c @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_srq_set_lw.c + * + * PURPOSE: Shared Receive Queue management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 5.4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_srq_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_srq_set_lw + * + * DAPL Requirements Version 1.2, 6.5.4 + * + * Set the low water mark for an SRQ and arm the SRQ to generate an + * event if it is reached. + * + * Input: + * srq_handle + * srq_max_recv_dto + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + * DAT_MODEL_NOT_SUPPORTED + */ + +DAT_RETURN DAT_API +dapl_srq_set_lw ( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_COUNT low_watermark ) +{ + DAPL_SRQ *srq_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_srq_set_lw (%p, %d)\n", + srq_handle, low_watermark); + + if (DAPL_BAD_HANDLE (srq_handle, DAPL_MAGIC_SRQ)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + goto bail; + } + + srq_ptr = (DAPL_SRQ *)srq_handle; + + /* XXX Put implementation here XXX */ + + /* XXX */ dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + + bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_util.c b/trunk/ulp/dapl2/dapl/common/dapl_srq_util.c new file mode 100644 index 00000000..ef36ca8c --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_util.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ep_util.c + * + * PURPOSE: Shared Receive Queue management and support + * + * $Id:$ + **********************************************************************/ + +#include "dapl_srq_util.h" +#include "dapl_ia_util.h" +#include "dapl_cookie.h" + +/* + * dapl_srq_alloc + * + * alloc and initialize an SRQ struct + * + * Input: + * IA INFO struct ptr + * SRQ ATTR ptr + * + * Output: + * none + * + * Returns: + * pointer to srq + * + */ +DAPL_SRQ * +dapl_srq_alloc ( + IN DAPL_IA *ia_ptr, + IN const DAT_SRQ_ATTR *srq_attr ) +{ + DAPL_SRQ *srq_ptr; + + /* Allocate SRQ */ + srq_ptr = (DAPL_SRQ *)dapl_os_alloc (sizeof (DAPL_SRQ)); + + if ( srq_ptr == NULL ) + { + goto bail; + } + + /* zero the structure */ + dapl_os_memzero (srq_ptr, sizeof (DAPL_SRQ)); + + /* + * initialize the header + */ + srq_ptr->header.provider = ia_ptr->header.provider; + srq_ptr->header.magic = DAPL_MAGIC_SRQ; + srq_ptr->header.handle_type = DAT_HANDLE_TYPE_SRQ; + srq_ptr->header.owner_ia = ia_ptr; + srq_ptr->header.user_context.as_64 = 0; + srq_ptr->header.user_context.as_ptr = NULL; + dapl_os_atomic_set (&srq_ptr->srq_ref_count, 0); + + dapl_llist_init_entry (&srq_ptr->header.ia_list_entry); + dapl_os_lock_init (&srq_ptr->header.lock); + + /* + * Initialize the body. + * XXX Assume srq_attrs is required + */ + srq_ptr->param.max_recv_dtos = srq_attr->max_recv_dtos; + srq_ptr->param.max_recv_iov = srq_attr->max_recv_iov; + srq_ptr->param.low_watermark = srq_attr->low_watermark; + + /* Get a cookie buffer to track outstanding recvs */ + if ( DAT_SUCCESS != dapls_cb_create ( + &srq_ptr->recv_buffer, + (DAPL_EP *) srq_ptr, /* just saves the value */ + srq_ptr->param.max_recv_dtos) ) + { + dapl_srq_dealloc (srq_ptr); + srq_ptr = NULL; + goto bail; + } + +bail: + return srq_ptr; +} + + +/* + * dapl_srq_dealloc + * + * Free the passed in SRQ structure. + * + * Input: + * SRQ pointer + * + * Output: + * none + * + * Returns: + * none + * + */ +void +dapl_srq_dealloc ( + IN DAPL_SRQ *srq_ptr ) +{ + dapl_os_assert (srq_ptr->header.magic == DAPL_MAGIC_SRQ); + + srq_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ + dapl_ia_unlink_srq (srq_ptr->header.owner_ia, srq_ptr); + dapls_cb_free (&srq_ptr->recv_buffer); + dapl_os_lock_destroy (&srq_ptr->header.lock); + + dapl_os_free (srq_ptr, sizeof (DAPL_SRQ)); +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_srq_util.h b/trunk/ulp/dapl2/dapl/common/dapl_srq_util.h new file mode 100644 index 00000000..5be6e279 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_srq_util.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_srq_util.h + * + * PURPOSE: Utility defs & routines for the SRQ data structure + * + * $Id:$ + * + **********************************************************************/ + +#ifndef _DAPL_SRQ_UTIL_H_ +#define _DAPL_SRQ_UTIL_H_ + +#include "dapl.h" + +/* function prototypes */ + +extern DAPL_SRQ * +dapl_srq_alloc ( + IN DAPL_IA *ia, + IN const DAT_SRQ_ATTR *srq_attr ); + +extern void +dapl_srq_dealloc ( + IN DAPL_SRQ *srq_ptr ); + +#endif /* _DAPL_SRQ_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_timer_util.c b/trunk/ulp/dapl2/dapl/common/dapl_timer_util.c new file mode 100644 index 00000000..0267780f --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_timer_util.c @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_timer_util.c + * + * PURPOSE: DAPL timer management + * Description: Routines to add and cancel timer records. A timer record + * is put on the global timer queue. If the timer thread is + * not running, start it. The timer thread will sleep + * until a timer event or until a process wakes it up + * to notice a new timer is available; we use a DAPL_WAIT_OBJ + * for synchronization. + * + * If a timer is cancelled, it is simlpy removed from the + * queue. The timer may wakeup and notice there is no timer + * record to awaken at this time, so it will reset for the + * next entry. When there are no timer records to manage, + * the timer thread just sleeps until awakened. + * + * This file also contains the timer handler thread, + * embodied in dapls_timer_thread(). + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_timer_util.h" + + +struct timer_head +{ + DAPL_LLIST_HEAD timer_list_head; + DAPL_OS_LOCK lock; + DAPL_OS_WAIT_OBJECT wait_object; + DAPL_OS_THREAD timeout_thread_handle; +} g_daplTimerHead; + +typedef struct timer_head DAPL_TIMER_HEAD; + + +void dapls_timer_thread (void *arg ); + +void +dapls_timer_init () +{ + /* + * Set up the timer thread elements. The timer thread isn't + * started until it is actually needed + */ + g_daplTimerHead.timer_list_head = NULL; + dapl_os_lock_init ( &g_daplTimerHead.lock ); + dapl_os_wait_object_init ( &g_daplTimerHead.wait_object ); + g_daplTimerHead.timeout_thread_handle = 0; +} + + +/* + * dapls_timer_set + * + * Set a timer. The timer will invoke the specified function + * after a number of useconds expires. + * + * Input: + * timer User provided timer structure + * func Function to invoke when timer expires + * data Argument passed to func() + * expires microseconds until timer fires + * + * Returns: + * no return value + * + */ +DAT_RETURN +dapls_timer_set ( + IN DAPL_OS_TIMER *timer, + IN void (*func) (uintptr_t), + IN void *data, + IN DAPL_OS_TIMEVAL expires ) +{ + DAPL_OS_TIMER *list_ptr; + DAPL_OS_TIMEVAL cur_time; + DAT_BOOLEAN wakeup_tmo_thread; + + DAPL_CNTR(DCNT_TIMER_SET); + /* + * Start the timer thread the first time we need a timer + */ + if ( g_daplTimerHead.timeout_thread_handle == 0 ) + { + dapl_os_thread_create ( dapls_timer_thread, + &g_daplTimerHead, + &g_daplTimerHead.timeout_thread_handle ); + } + + dapl_llist_init_entry ( &timer->list_entry); + wakeup_tmo_thread = DAT_FALSE; + dapl_os_get_time ( &cur_time ); + timer->expires = cur_time + expires; /* calculate future time */ + timer->function = func; + timer->data = data; + + /* + * Put the element on the queue: sorted by wakeup time, eariliest + * first. + */ + dapl_os_lock ( &g_daplTimerHead.lock ); + /* + * Deal with 3 cases due to our list structure: + * 1) list is empty: become the list head + * 2) New timer is sooner than list head: become the list head + * 3) otherwise, sort the timer into the list, no need to wake + * the timer thread up + */ + if ( dapl_llist_is_empty ( &g_daplTimerHead.timer_list_head) ) + { + /* Case 1: add entry to head of list */ + dapl_llist_add_head ( &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&timer->list_entry, + timer ); + wakeup_tmo_thread = DAT_TRUE; + } + else + { + list_ptr = (DAPL_OS_TIMER *) + dapl_llist_peek_head (&g_daplTimerHead.timer_list_head); + + if ( timer->expires < list_ptr->expires ) + { + /* Case 2: add entry to head of list */ + dapl_llist_add_head ( &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&timer->list_entry, + timer ); + wakeup_tmo_thread = DAT_TRUE; + } + else + { + /* Case 3: figure out where entry goes in sorted list */ + list_ptr = dapl_llist_next_entry ( + &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&list_ptr->list_entry); + + while (list_ptr != NULL) + { + if ( timer->expires < list_ptr->expires ) + { + dapl_llist_add_entry ( &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&list_ptr->list_entry, + (DAPL_LLIST_ENTRY *)&timer->list_entry, + timer ); + break; + + } + list_ptr = dapl_llist_next_entry ( + &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&list_ptr->list_entry); + } + if (list_ptr == NULL) + { + /* entry goes to the end of the list */ + dapl_llist_add_tail ( &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&timer->list_entry, + timer ); + } + } + + } + dapl_os_unlock ( &g_daplTimerHead.lock ); + + if (wakeup_tmo_thread == DAT_TRUE) + { + dapl_os_wait_object_wakeup (&g_daplTimerHead.wait_object); + } + + return DAT_SUCCESS; +} + + +/* + * dapls_os_timer_cancel + * + * Cancel a timer. Simply deletes the timer with no function invocations + * + * Input: + * timer User provided timer structure + * + * Returns: + * no return value + */ +void +dapls_timer_cancel ( + IN DAPL_OS_TIMER *timer) +{ + DAPL_CNTR(DCNT_TIMER_CANCEL); + dapl_os_lock ( &g_daplTimerHead.lock ); + /* + * make sure the entry has not been removed by another thread + */ + if ( ! dapl_llist_is_empty ( &g_daplTimerHead.timer_list_head ) && + timer->list_entry.list_head == &g_daplTimerHead.timer_list_head ) + { + dapl_llist_remove_entry ( &g_daplTimerHead.timer_list_head, + (DAPL_LLIST_ENTRY *)&timer->list_entry ); + } + /* + * If this was the first entry on the queue we could awaken the + * thread and have it reset the list; but it will just wake up + * and find that the timer entry has been removed, then go back + * to sleep, so don't bother. + */ + dapl_os_unlock ( &g_daplTimerHead.lock ); +} + + +/* + * dapls_timer_thread + * + * Core worker thread dealing with all timers. Basic algorithm: + * - Sleep until work shows up + * - Take first element of sorted timer list and wake + * invoke the callback if expired + * - Sleep for the timeout period if not expired + * + * Input: + * timer_head Timer head structure to manage timer lists + * + * Returns: + * no return value + */ +void +dapls_timer_thread ( + void *arg ) +{ + DAPL_OS_TIMER *list_ptr; + DAPL_OS_TIMEVAL cur_time; + DAT_RETURN dat_status; + DAPL_TIMER_HEAD *timer_head; + + timer_head = arg; + + for (;;) + { + if ( dapl_llist_is_empty ( &timer_head->timer_list_head ) ) + { + dat_status = dapl_os_wait_object_wait (&timer_head->wait_object, + DAT_TIMEOUT_INFINITE ); + } + + /* + * Lock policy: + * While this thread is accessing the timer list, it holds the + * lock. Otherwise, it doesn't. + */ + dapl_os_lock ( &timer_head->lock ); + while ( ! dapl_llist_is_empty ( &timer_head->timer_list_head) ) + { + list_ptr = (DAPL_OS_TIMER *) + dapl_llist_peek_head ( &g_daplTimerHead.timer_list_head ); + dapl_os_get_time ( &cur_time ); + + if ( list_ptr->expires <= cur_time ) + { + /* + * Remove the entry from the list. Sort out how much + * time we need to sleep for the next one + */ + list_ptr = dapl_llist_remove_head ( &timer_head->timer_list_head ); + dapl_os_unlock ( &timer_head->lock ); + + /* + * Invoke the user callback + */ + list_ptr->function ( (uintptr_t)list_ptr->data ); + /* timer structure was allocated by caller, we don't + * free it here. + */ + + /* reacquire the lock */ + dapl_os_lock ( &timer_head->lock ); + } + else + { + dapl_os_unlock ( &timer_head->lock ); + dat_status = + dapl_os_wait_object_wait ( &timer_head->wait_object, + (DAT_TIMEOUT) (list_ptr->expires - cur_time) ); + dapl_os_lock ( &timer_head->lock ); + } + } + /* + * release the lock before going back to the top to sleep + */ + dapl_os_unlock ( &timer_head->lock ); + + if ( dat_status == DAT_INTERNAL_ERROR ) + { + /* + * XXX What do we do here? + */ + } + } /* for (;;) */ +} + + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/common/dapl_timer_util.h b/trunk/ulp/dapl2/dapl/common/dapl_timer_util.h new file mode 100644 index 00000000..b46a9841 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/common/dapl_timer_util.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_timer_util.h + * + * PURPOSE: DAPL timer management + * Description: support for dapl_timer.h + * + * $Id:$ + **********************************************************************/ + +void dapls_timer_init ( void ); + +DAT_RETURN dapls_timer_set ( + IN DAPL_OS_TIMER *timer, + IN void (*func) (uintptr_t), + IN void *data, + IN DAPL_OS_TIMEVAL expires ); + +void dapls_timer_cancel ( + IN DAPL_OS_TIMER *timer); diff --git a/trunk/ulp/dapl2/dapl/dirs b/trunk/ulp/dapl2/dapl/dirs new file mode 100644 index 00000000..2d7badc4 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/dirs @@ -0,0 +1 @@ +DIRS=udapl diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cm.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cm.c new file mode 100644 index 00000000..61762794 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cm.c @@ -0,0 +1,2320 @@ + +/* + * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_cm.c + * + * PURPOSE: IB Connection routines for access to IBAL APIs + * + * $Id: dapl_ibal_cm.c 584 2007-02-07 13:12:18Z sleybo $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" +#include "dapl_sp_util.h" +#include "dapl_ep_util.h" +#include "dapl_ia_util.h" +#include "dapl_ibal_util.h" +#include "dapl_name_service.h" +#include "dapl_ibal_name_service.h" + +#define IB_INFINITE_SERVICE_LEASE 0xFFFFFFFF +#define DAPL_ATS_SERVICE_ID ATS_SERVICE_ID //0x10000CE100415453 +#define DAPL_ATS_NAME ATS_NAME +#define HCA_IPV6_ADDRESS_LENGTH 16 + +int g_dapl_loopback_connection = 0; +extern dapl_ibal_root_t dapl_ibal_root; + +/* + * Prototypes + */ +static void +dapli_ib_sa_query_cb ( + IN ib_query_rec_t *p_query_rec ); + +static char * +dapli_get_ip_addr_str ( + IN DAT_SOCK_ADDR6 *ipa, + OUT char *str ); + + +char * +dapli_ib_cm_event_str(ib_cm_events_t e) +{ +#ifdef DBG + char *cp; + static char *event_str[13] = { + "IB_CME_CONNECTED", + "IB_CME_DISCONNECTED", + "IB_CME_DISCONNECTED_ON_LINK_DOWN", + "IB_CME_CONNECTION_REQUEST_PENDING", + "IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA", + "IB_CME_DESTINATION_REJECT", + "IB_CME_DESTINATION_REJECT_PRIVATE_DATA", + "IB_CME_DESTINATION_UNREACHABLE", + "IB_CME_TOO_MANY_CONNECTION_REQUESTS", + "IB_CME_LOCAL_FAILURE", + "IB_CME_REPLY_RECEIVED", + "IB_CME_REPLY_RECEIVED_PRIVATE_DATA", + "IB_CM_LOCAL_FAILURE" + }; + + if (e > IB_CM_LOCAL_FAILURE || e < IB_CME_CONNECTED) + cp = "BAD EVENT"; + else + cp = event_str[e]; + + return cp; +#else + static char num[8]; + sprintf(num,"%d",e); + return num; +#endif +} + +#ifndef NO_NAME_SERVICE + +static void +dapli_ib_reg_svc_cb ( + IN ib_reg_svc_rec_t *p_reg_svc_rec ) +{ + DAPL_HCA *hca_ptr; + + hca_ptr = (DAPL_HCA * __ptr64) p_reg_svc_rec->svc_context; + + dapl_os_assert (hca_ptr); + + if (IB_SUCCESS == p_reg_svc_rec->req_status) + { + hca_ptr->ib_trans.name_service_handle = (void *) p_reg_svc_rec->h_reg_svc; + + dapl_dbg_log ( DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> DiRScb: register to SA successfully for port %d\n", + hca_ptr->port_num); + + dapl_dbg_log ( DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> DiRScb: reg_service_handle %lx\n", + hca_ptr->ib_trans.name_service_handle); + + } + else + { + hca_ptr->ib_trans.name_service_handle = IB_INVALID_HANDLE; + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DiRScb: Register service to SA failed = %s\n", + ib_get_err_str(p_reg_svc_rec->req_status)); + } + +} + + + +DAT_RETURN +dapls_ib_ns_map_gid ( + IN DAPL_HCA *hca_ptr, + IN DAT_IA_ADDRESS_PTR p_ia_address, + OUT GID *p_gid) +{ + ib_user_query_t user_query; + dapl_ibal_ca_t *p_ca; + dapl_ibal_port_t *p_active_port; + ib_service_record_t service_rec; + ib_api_status_t ib_status; + ib_query_req_t query_req; + DAT_SOCK_ADDR6 ipv6_addr; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (NULL == p_ca) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsNMG: There is no HCA = %d\n", __LINE__); + return (DAT_INVALID_HANDLE); + } + + /* + * We are using the first active port in the list for + * communication. We have to get back here when we decide to support + * fail-over and high-availability. + */ + p_active_port = dapli_ibal_get_port ( p_ca, (uint8_t)hca_ptr->port_num ); + + if (NULL == p_active_port) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsNMG: Port %d is not available = %d\n", + hca_ptr->port_num, __LINE__); + return (DAT_INVALID_STATE); + } + + if (p_active_port->p_attr->lid == 0) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsNMG: Port %d has no LID " + "assigned; can not operate\n", + p_active_port->p_attr->port_num); + return (DAT_INVALID_STATE); + } + else + { + if (!dapl_os_memcmp (p_ia_address, + &hca_ptr->hca_address, + HCA_IPV6_ADDRESS_LENGTH)) + { + /* + * We are operating in the LOOPBACK mode + */ + p_gid->guid = + p_active_port->p_attr->p_gid_table[0].unicast.interface_id; + p_gid->gid_prefix = + p_active_port->p_attr->p_gid_table[0].unicast.prefix; + return DAT_SUCCESS; + } + else if (p_active_port->p_attr->link_state != IB_LINK_ACTIVE) + { + /* + * Port is DOWN; can not send or recv messages + */ + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"--> DsNMG: Port %d is DOWN; can " + "not send to fabric\n", + p_active_port->p_attr->port_num); + return (DAT_INVALID_STATE); + } + } + + dapl_os_memzero (&user_query, sizeof (ib_user_query_t)); + dapl_os_memzero (&service_rec, sizeof (ib_service_record_t)); + dapl_os_memzero (&query_req, sizeof (ib_query_req_t)); + dapl_os_memzero (&ipv6_addr, sizeof (DAT_SOCK_ADDR6)); + + if (p_ia_address->sa_family == AF_INET) + { + dapl_os_memcpy (&ipv6_addr.sin6_addr.s6_addr[12], + &((struct sockaddr_in *)p_ia_address)->sin_addr.s_addr, + 4); +#ifdef DAPL_DBG + { + char ipa[20]; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> DsNMG: Remote ia_address %s\n", + dapli_get_ip_addr_str( + (DAT_SOCK_ADDR6*)p_ia_address, ipa)); + } +#endif + + } + else + { + /* + * Assume IPv6 address + */ + dapl_os_assert (p_ia_address->sa_family == AF_INET6); + dapl_os_memcpy (ipv6_addr.sin6_addr.s6_addr, + ((DAT_SOCK_ADDR6 *)p_ia_address)->sin6_addr.s6_addr, + HCA_IPV6_ADDRESS_LENGTH); +#ifdef DAPL_DBG + { + int i; + uint8_t *tmp = ipv6_addr.sin6_addr.s6_addr; + + dapl_dbg_log ( DAPL_DBG_TYPE_CM, + "--> DsNMG: Remote ia_address - "); + + for ( i = 1; i < HCA_IPV6_ADDRESS_LENGTH; i++) + { + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "%x:", + tmp[i-1] ); + } + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "%x\n", + tmp[i-1] ); + } +#endif + + } + + /* + * query SA for GID + */ + //service_rec.service_id = CL_HTON64 (DAPL_ATS_SERVICE_ID); + dapl_os_memcpy ( service_rec.service_name, + ATS_NAME, + __min(sizeof(ATS_NAME),sizeof(ib_svc_name_t)) ); + dapl_os_memcpy (&service_rec.service_data8[0], + ipv6_addr.sin6_addr.s6_addr, + HCA_IPV6_ADDRESS_LENGTH); + service_rec.service_lease = IB_INFINITE_SERVICE_LEASE; + service_rec.service_pkey = IB_DEFAULT_PKEY; + + user_query.method = IB_MAD_METHOD_GETTABLE; + user_query.attr_id = IB_MAD_ATTR_SERVICE_RECORD; + user_query.comp_mask = IB_SR_COMPMASK_SPKEY | + IB_SR_COMPMASK_SLEASE | + IB_SR_COMPMASK_SNAME | + IB_SR_COMPMASK_SDATA8_0 | + IB_SR_COMPMASK_SDATA8_1 | + IB_SR_COMPMASK_SDATA8_2 | + IB_SR_COMPMASK_SDATA8_3 | + IB_SR_COMPMASK_SDATA8_4 | + IB_SR_COMPMASK_SDATA8_5 | + IB_SR_COMPMASK_SDATA8_6 | + IB_SR_COMPMASK_SDATA8_7 | + IB_SR_COMPMASK_SDATA8_8 | + IB_SR_COMPMASK_SDATA8_9 | + IB_SR_COMPMASK_SDATA8_10 | + IB_SR_COMPMASK_SDATA8_11 | + IB_SR_COMPMASK_SDATA8_12 | + IB_SR_COMPMASK_SDATA8_13 | + IB_SR_COMPMASK_SDATA8_14 | + IB_SR_COMPMASK_SDATA8_15; + + user_query.attr_size = sizeof (ib_service_record_t); + user_query.p_attr = (void *)&service_rec; + + query_req.query_type = IB_QUERY_USER_DEFINED; + query_req.p_query_input = (void *)&user_query; + query_req.flags = IB_FLAGS_SYNC; /* this is a blocking call */ + query_req.timeout_ms = 1 * 1000; /* 1 second */ + query_req.retry_cnt = 5; + /* query SA using this port */ + query_req.port_guid = p_active_port->p_attr->port_guid; + query_req.query_context = (void *) &user_query; + query_req.pfn_query_cb = dapli_ib_sa_query_cb; + + ib_status = ib_query (dapl_ibal_root.h_al, &query_req, NULL); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"ns_map_gid: status %s @line = %d\n", + ib_get_err_str(ib_status), __LINE__); + return (dapl_ib_status_convert (ib_status)); + } + else if (service_rec.service_gid.unicast.interface_id == 0) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: query SA found no record\n",__FUNCTION__); + return DAT_INVALID_PARAMETER; + } + + /* + * return the GID + */ + p_gid->guid = service_rec.service_gid.unicast.interface_id; + p_gid->gid_prefix = service_rec.service_gid.unicast.prefix; + + return DAT_SUCCESS; +} + + + +DAT_RETURN +dapls_ib_ns_map_ipaddr ( + IN DAPL_HCA *hca_ptr, + IN GID gid, + OUT DAT_IA_ADDRESS_PTR p_ia_address) +{ + ib_user_query_t user_query; + dapl_ibal_ca_t *p_ca; + dapl_ibal_port_t *p_active_port; + ib_service_record_t service_rec; + ib_api_status_t ib_status; + ib_query_req_t query_req; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (NULL == p_ca) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsNMI: There is no HCA = %d\n", __LINE__); + return (DAT_INVALID_HANDLE); + } + + /* + * We are using the first active port in the list for + * communication. We have to get back here when we decide to support + * fail-over and high-availability. + */ + p_active_port = dapli_ibal_get_port ( p_ca, (uint8_t)hca_ptr->port_num ); + + if (NULL == p_active_port) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsNMI: Port %d is not available = %d\n", + hca_ptr->port_num, __LINE__); + return (DAT_INVALID_STATE); + } + + if (p_active_port->p_attr->lid == 0) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsNMI: Port %d has no LID " + "assigned; can not operate\n", + p_active_port->p_attr->port_num); + return (DAT_INVALID_STATE); + } + /*else + { + // + // We are operating in the LOOPBACK mode + // + if ((gid.gid_prefix == + p_active_port->p_attr->p_gid_table[0].unicast.prefix) && + (gid.guid == + p_active_port->p_attr->p_gid_table[0].unicast.interface_id)) + { + dapl_os_memcpy (((DAT_SOCK_ADDR6 *)p_ia_address)->sin6_addr.s6_addr, + hca_ptr->hca_address.sin6_addr.s6_addr, + HCA_IPV6_ADDRESS_LENGTH); + return DAT_SUCCESS; + } + + }*/ + if (p_active_port->p_attr->link_state != IB_LINK_ACTIVE) + { + /* + * Port is DOWN; can not send or recv messages + */ + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"--> DsNMI: Port %d is DOWN; " + "can not send/recv to/from fabric\n", + p_active_port->p_attr->port_num); + return (DAT_INVALID_STATE); + } + + dapl_os_memzero (&user_query, sizeof (ib_user_query_t)); + dapl_os_memzero (&service_rec, sizeof (ib_service_record_t)); + dapl_os_memzero (&query_req, sizeof (ib_query_req_t)); + + /* + * query SA for IPAddress + */ + //service_rec.service_id = CL_HTON64 (DAPL_ATS_SERVICE_ID); + dapl_os_memcpy( service_rec.service_name, + ATS_NAME, + __min (sizeof(ATS_NAME), sizeof(ib_svc_name_t))); + service_rec.service_gid.unicast.interface_id = gid.guid; + service_rec.service_gid.unicast.prefix = gid.gid_prefix; + service_rec.service_pkey = IB_DEFAULT_PKEY; + service_rec.service_lease = IB_INFINITE_SERVICE_LEASE; + + user_query.method = IB_MAD_METHOD_GETTABLE; + user_query.attr_id = IB_MAD_ATTR_SERVICE_RECORD; + + user_query.comp_mask = IB_SR_COMPMASK_SGID | + IB_SR_COMPMASK_SPKEY | + IB_SR_COMPMASK_SLEASE | + IB_SR_COMPMASK_SNAME; + + user_query.attr_size = sizeof (ib_service_record_t); + user_query.p_attr = (void *)&service_rec; + + query_req.query_type = IB_QUERY_USER_DEFINED; + query_req.p_query_input = (void *)&user_query; + query_req.flags = IB_FLAGS_SYNC; /* this is a blocking call */ + query_req.timeout_ms = 1 * 1000; /* 1 second */ + query_req.retry_cnt = 5; + /* query SA using this port */ + query_req.port_guid = p_active_port->p_attr->port_guid; + query_req.query_context = (void *) &user_query; + query_req.pfn_query_cb = dapli_ib_sa_query_cb; + + ib_status = ib_query (dapl_ibal_root.h_al, &query_req, NULL); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "ns_map_ipaddr: exits status %s @line = %d\n", + ib_get_err_str(ib_status), __LINE__); + return (dapl_ib_status_convert (ib_status)); + } + + /* *********************** + * return the IP_address + *************************/ + dapl_os_memcpy ( + (void *) &((struct sockaddr_in *)p_ia_address)->sin_addr.s_net, + (const void *)&service_rec.service_data8[ATS_IPV4_OFFSET], 4); + //HCA_IPV6_ADDRESS_LENGTH); + ((DAT_SOCK_ADDR6 *)p_ia_address)->sin6_family = AF_INET; + + return (DAT_SUCCESS); +} + + +/* + * dapls_ib_ns_create_gid_map() + * + * Register a ServiceRecord containing uDAPL_svc_id, IP address and GID to SA + * Other nodes can look it up by quering the SA + * + * Input: + * hca_ptr HCA device pointer + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN +dapls_ib_ns_create_gid_map ( + IN DAPL_HCA *hca_ptr) +{ + UNUSED_PARAM( hca_ptr ); + return (DAT_SUCCESS); +} + + +DAT_RETURN +dapls_ib_ns_remove_gid_map ( + IN DAPL_HCA *hca_ptr) +{ + UNUSED_PARAM( hca_ptr ); + return (DAT_SUCCESS); +} + +#endif /* NO_NAME_SERVICE */ + + +static void +dapli_ib_sa_query_cb ( + IN ib_query_rec_t *p_query_rec ) +{ + ib_api_status_t ib_status; + + if (IB_SUCCESS != p_query_rec->status) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: SA query callback failed %s\n", + __FUNCTION__, ib_get_err_str(p_query_rec->status)); + return; + } + + if (!p_query_rec->p_result_mad) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "%s: SA query callback [no mad] @line %d\n", + __FUNCTION__, __LINE__); + return; + } + + switch (p_query_rec->query_type) + { + case IB_QUERY_PATH_REC_BY_GIDS: + { + ib_path_rec_t *p_path_rec; + + p_path_rec = ib_get_query_path_rec (p_query_rec->p_result_mad, 0); + if (p_path_rec) + { + dapl_os_memcpy ((void * __ptr64) p_query_rec->query_context, + (void *) p_path_rec, + sizeof (ib_path_rec_t)); + dapl_dbg_log ( + DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "sa_query_cb: path {slid: 0x%x, dlid: 0x%x}\n", + p_path_rec->slid, p_path_rec->dlid); + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: return NULL? @line %d\n", + __FUNCTION__, __LINE__); + return; + } + break; + } + + case IB_QUERY_SVC_REC_BY_ID: + { + ib_service_record_t *p_svc_rec; + + p_svc_rec = ib_get_query_svc_rec (p_query_rec->p_result_mad, 0); + if (p_svc_rec) + { + dapl_os_memcpy ((void * __ptr64) p_query_rec->query_context, + (void *) p_svc_rec, + sizeof (ib_service_record_t)); + dapl_dbg_log ( + DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "%s: SER{0x%I64x, 0x%I64x}\n", __FUNCTION__, + cl_hton64 (p_svc_rec->service_gid.unicast.prefix), + cl_hton64 (p_svc_rec->service_gid.unicast.interface_id)); + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: return NULL? @line %d\n", + __FUNCTION__, __LINE__); + return; + } + break; + + } + + case IB_QUERY_USER_DEFINED: + { + ib_user_query_t *p_user_query; + + p_user_query = (ib_user_query_t * __ptr64) p_query_rec->query_context; + if (p_user_query) + { + switch (p_user_query->attr_id) + { + case IB_MAD_ATTR_SERVICE_RECORD: + { + ib_service_record_t *p_svc_rec; + + p_svc_rec = ib_get_query_svc_rec ( + p_query_rec->p_result_mad, 0); + if (p_svc_rec) + { + dapl_os_memcpy ((void *) p_user_query->p_attr, + (void *) p_svc_rec, + sizeof (ib_service_record_t)); + dapl_dbg_log ( + DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "%s: GID{0x" F64x ", 0x" F64x "} record count %d\n", + __FUNCTION__, + cl_hton64(p_svc_rec->service_gid.unicast.prefix), + cl_hton64(p_svc_rec->service_gid.unicast.interface_id), + p_query_rec->result_cnt); + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "%s: return NULL? @line %d\n", + __FUNCTION__, __LINE__); + return; + } + break; + + } + default: + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "%s: USER_DEFINED %d\n", + p_user_query->attr_id ); + break; + } + } + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: return NULL? @line %d\n", + __FUNCTION__, __LINE__); + return; + } + break; + } + + default: + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: unsupportedTYPE %d\n", + __FUNCTION__, p_query_rec->query_type); + break; + } + + } + + if ((ib_status = ib_put_mad (p_query_rec->p_result_mad)) != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: can not free MAD %s\n", + __FUNCTION__, ib_get_err_str(ib_status)); + } +} + + +static void +dapli_ibal_listen_err_cb ( + IN ib_listen_err_rec_t *p_listen_err_rec ) +{ + UNUSED_PARAM( p_listen_err_rec ); + + dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> %s: CM callback listen error\n", + "DiLEcb"); +} + +static void +dapli_ib_cm_apr_cb ( + IN ib_cm_apr_rec_t *p_cm_apr_rec ) +{ + UNUSED_PARAM( p_cm_apr_rec ); + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCAcb: CM callback APR (Alternate Path Request)\n"); +} + +static void +dapli_ib_cm_lap_cb ( + IN ib_cm_lap_rec_t *p_cm_lap_rec ) +{ + UNUSED_PARAM( p_cm_lap_rec ); + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCLcb: CM callback LAP (Load Alternate Path)\n"); +} + +/* + * Connection Disconnect Request callback + */ + +static void +dapli_ib_cm_dreq_cb ( + IN ib_cm_dreq_rec_t *p_cm_dreq_rec ) +{ + ib_cm_drep_t cm_drep; + DAPL_EP *ep_ptr; + int bail=10; + + dapl_os_assert (p_cm_dreq_rec); + + ep_ptr = (DAPL_EP * __ptr64) p_cm_dreq_rec->qp_context; + + if ( ep_ptr == NULL || + ep_ptr->header.magic == DAPL_MAGIC_INVALID ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCDcb: EP %lx invalid or FREED\n", ep_ptr); + return; + } + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> %s() QP %lx EP %lx state %s sent_discreq %d\n", + __FUNCTION__, ep_ptr->qp_handle, ep_ptr, + dapl_get_ep_state_str(ep_ptr->param.ep_state), + ep_ptr->sent_discreq ); + + dapl_os_lock (&ep_ptr->header.lock); + if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED + /*|| ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING + && ep_ptr->sent_discreq == DAT_FALSE)*/ ) + { + dapl_os_unlock (&ep_ptr->header.lock); + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCDcb: EP %lx QP %lx already Disconnected\n", + ep_ptr, ep_ptr->qp_handle); + return; + } + + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECT_PENDING; + + ep_ptr->recv_discreq = DAT_TRUE; + dapl_os_unlock (&ep_ptr->header.lock); + + dapl_os_memzero (&cm_drep, sizeof ( ib_cm_drep_t)); + + /* Could fail if we received reply from other side, no need to retry */ + /* Wait for any transaction in process holding reference */ + while ( dapl_os_atomic_read(&ep_ptr->req_count) && bail-- > 0 ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCDcb: WAIT for EP=%lx req_count(%d) != 0\n", + ep_ptr, dapl_os_atomic_read(&ep_ptr->req_count)); + dapl_os_sleep_usec (100); + } + + ib_cm_drep (p_cm_dreq_rec->h_cm_dreq, &cm_drep); + + /* CM puts QP in reset state */ + ep_ptr->qp_state = IB_QPS_RESET; + + if (ep_ptr->cr_ptr) + { + dapl_os_assert(ep_ptr->ibal_cm_handle->cid + == p_cm_dreq_rec->h_cm_dreq.cid); + /* passive side */ + dapls_cr_callback ( ep_ptr->cm_handle, + IB_CME_DISCONNECTED, + (void * __ptr64) p_cm_dreq_rec->p_dreq_pdata, + (void *) (((DAPL_CR *) ep_ptr->cr_ptr)->sp_ptr) ); + } + else + { + /* active side */ + dapl_evd_connection_callback ( + (dp_ib_cm_handle_t) &p_cm_dreq_rec->h_cm_dreq, + IB_CME_DISCONNECTED, + (void * __ptr64) + p_cm_dreq_rec->p_dreq_pdata, + p_cm_dreq_rec->qp_context ); + } +} + +/* + * Connection Disconnect Reply callback + */ + +static void +dapli_ib_cm_drep_cb ( + IN ib_cm_drep_rec_t *p_cm_drep_rec ) +{ + DAPL_EP *ep_ptr; + + dapl_os_assert (p_cm_drep_rec != NULL); + + ep_ptr = (DAPL_EP * __ptr64) p_cm_drep_rec->qp_context; + + if ( !ep_ptr || DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> %s: BAD EP Handle EP=%lx\n", __FUNCTION__,ep_ptr); + return; + } + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCDpcb: QP %lx EP %lx state %s cm_hdl %lx\n", + ep_ptr->qp_handle, ep_ptr, + dapl_get_ep_state_str(ep_ptr->param.ep_state), + ep_ptr->cm_handle); + + if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCDpcb: EP %lx QP %lx already Disconnected\n", + ep_ptr, ep_ptr->qp_handle); + return; + } + + if (ep_ptr->cm_handle == IB_INVALID_HANDLE ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> %s: Invalid EP->CM handle?\n", __FUNCTION__); + return; + } + + if (ep_ptr->cr_ptr) + { + /* passive connection side */ + dapls_cr_callback ( ep_ptr->cm_handle, + IB_CME_DISCONNECTED, + (void * __ptr64) p_cm_drep_rec->p_drep_pdata, + (void *) (((DAPL_CR *) ep_ptr->cr_ptr)->sp_ptr) ); + } + else + { + /* active connection side */ + dapl_evd_connection_callback ( + ep_ptr->cm_handle, + IB_CME_DISCONNECTED, + (void * __ptr64) p_cm_drep_rec->p_drep_pdata, + p_cm_drep_rec->qp_context ); + } +} + + +static void +dapli_ib_cm_rep_cb ( + IN ib_cm_rep_rec_t *p_cm_rep_rec ) +{ + ib_api_status_t ib_status; + ib_cm_rtu_t cm_rtu; + uint8_t cm_cb_op; + DAPL_PRIVATE *prd_ptr; + DAPL_EP *ep_ptr; + dapl_ibal_ca_t *p_ca; + + dapl_os_assert (p_cm_rep_rec != NULL); + + dapl_os_memzero (&cm_rtu, sizeof ( ib_cm_rtu_t )); + + dapl_os_assert ( ((DAPL_HEADER * __ptr64) + p_cm_rep_rec->qp_context)->magic == DAPL_MAGIC_EP ); + + ep_ptr = (DAPL_EP * __ptr64) p_cm_rep_rec->qp_context; + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCRpcb: EP = %lx local_max_rdma_read_in %d\n", + ep_ptr, p_cm_rep_rec->resp_res); + + p_ca = (dapl_ibal_ca_t *) + ep_ptr->header.owner_ia->hca_ptr->ib_hca_handle; + + cm_rtu.pfn_cm_apr_cb = dapli_ib_cm_apr_cb; + cm_rtu.pfn_cm_dreq_cb = dapli_ib_cm_dreq_cb; + cm_rtu.p_rtu_pdata = NULL; + cm_rtu.access_ctrl = IB_AC_LOCAL_WRITE|IB_AC_RDMA_WRITE|IB_AC_MW_BIND; + if ((ep_ptr->param.ep_attr.max_rdma_read_in > 0) || + (ep_ptr->param.ep_attr.max_rdma_read_out > 0)) + { + cm_rtu.access_ctrl |= IB_AC_RDMA_READ; + } + + cm_rtu.rq_depth = 0; + cm_rtu.sq_depth = 0; + + ib_status = ib_cm_rtu (p_cm_rep_rec->h_cm_rep, &cm_rtu); + + if (ib_status == IB_SUCCESS) + { + cm_cb_op = IB_CME_CONNECTED; + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCRpcb: EP %lx Connected req_count %d\n", + ep_ptr, dapl_os_atomic_read(&ep_ptr->req_count)); + } + else + { + cm_cb_op = IB_CME_LOCAL_FAILURE; + } + + prd_ptr = (DAPL_PRIVATE * __ptr64) p_cm_rep_rec->p_rep_pdata; + +#ifdef DAPL_DBG +#if 0 + { + int i; + + dapl_dbg_log ( DAPL_DBG_TYPE_EP, "--> DiCRpcb: private_data: "); + + for ( i = 0 ; i < IB_MAX_REP_PDATA_SIZE ; i++ ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_EP, + "0x%x ", prd_ptr->private_data[i]); + + } + dapl_dbg_log ( DAPL_DBG_TYPE_EP, "\n"); + + } +#endif +#endif + + dapl_evd_connection_callback ( + (dp_ib_cm_handle_t)&p_cm_rep_rec->h_cm_rep, + cm_cb_op, + (void *) prd_ptr, + (void * __ptr64) p_cm_rep_rec->qp_context); +} + + +static void +dapli_ib_cm_rej_cb ( + IN ib_cm_rej_rec_t *p_cm_rej_rec ) +{ + DAPL_EP *ep_ptr; + ib_cm_events_t cm_event; + + dapl_os_assert (p_cm_rej_rec); + + ep_ptr = (DAPL_EP * __ptr64) p_cm_rej_rec->qp_context; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCRjcb: EP = %lx QP = %lx rej reason = 0x%x\n", + ep_ptr,ep_ptr->qp_handle,CL_NTOH16(p_cm_rej_rec->rej_status)); + + switch (p_cm_rej_rec->rej_status) + { + case IB_REJ_INSUF_RESOURCES: + case IB_REJ_INSUF_QP: + case IB_REJ_INVALID_COMM_ID: + case IB_REJ_INVALID_COMM_INSTANCE: + case IB_REJ_INVALID_PKT_RATE: + case IB_REJ_INVALID_ALT_GID: + case IB_REJ_INVALID_ALT_LID: + case IB_REJ_INVALID_ALT_SL: + case IB_REJ_INVALID_ALT_TRAFFIC_CLASS: + case IB_REJ_INVALID_ALT_PKT_RATE: + case IB_REJ_INVALID_ALT_HOP_LIMIT: + case IB_REJ_INVALID_ALT_FLOW_LBL: + case IB_REJ_INVALID_GID: + case IB_REJ_INVALID_LID: + case IB_REJ_INVALID_SID: + case IB_REJ_INVALID_SL: + case IB_REJ_INVALID_TRAFFIC_CLASS: + case IB_REJ_PORT_REDIRECT: + case IB_REJ_INVALID_MTU: + case IB_REJ_INSUFFICIENT_RESP_RES: + case IB_REJ_INVALID_CLASS_VER: + case IB_REJ_INVALID_FLOW_LBL: + cm_event = IB_CME_DESTINATION_REJECT; + break; + + case IB_REJ_TIMEOUT: + cm_event = IB_CME_DESTINATION_UNREACHABLE; + dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> DiCRjcb: CR TIMEOUT\n"); + break; + + case IB_REJ_USER_DEFINED: + cm_event = IB_CME_DESTINATION_REJECT; + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCRjcb: user defined rej reason %s\n", + p_cm_rej_rec->p_ari); + break; + + default: + cm_event = IB_CME_LOCAL_FAILURE; + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DiCRjcb: with unknown status %x\n", + p_cm_rej_rec->rej_status); + break; + } + + /* FIXME - Vu + * We do not take care off the user defined rej reason with additional + * rejection information (p_ari) + */ + + if (ep_ptr->cr_ptr) + { + dapls_cr_callback ( ep_ptr->cm_handle, + cm_event, + (void * __ptr64) p_cm_rej_rec->p_rej_pdata, + (void *) ((DAPL_CR *) ep_ptr->cr_ptr)->sp_ptr); + } + else + { + dapl_evd_connection_callback ( + ep_ptr->cm_handle, + cm_event, + (void * __ptr64) p_cm_rej_rec->p_rej_pdata, + (void * __ptr64) p_cm_rej_rec->qp_context ); + } + +} + + + +static void +dapli_ib_cm_req_cb ( IN ib_cm_req_rec_t *p_cm_req_rec ) +{ + DAPL_SP *sp_ptr; + DAT_SOCK_ADDR6 dest_ia_addr; + dp_ib_cm_handle_t cm_handle; + + struct ibal_cr_data { + ib_cm_handle_t cm_hdl; + DAT_SOCK_ADDR6 dst_ip_addr; + } *crd; + + dapl_os_assert (p_cm_req_rec); + + sp_ptr = (DAPL_SP * __ptr64) p_cm_req_rec->context; + + dapl_os_assert (sp_ptr); + + /* + * The context pointer could have been cleaned up in a racing + * CM callback, check to see if we should just exit here + */ + if (sp_ptr->header.magic == DAPL_MAGIC_INVALID) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "%s: BAD-Magic in SP %lx, racing CM callback?\n", + __FUNCTION__, sp_ptr ); + return; + } + + dapl_os_assert ( sp_ptr->header.magic == DAPL_MAGIC_PSP || + sp_ptr->header.magic == DAPL_MAGIC_RSP ); + + /* preserve ibal's connection handle storage so we have a consistent + * pointer value. The reasons this is done dynamically instead of static + * allocation in an end_point is the pointer value is set in the SP list + * of CR's here and searched for from disconnect callbacks. If the pointer + * value changes, you never find the CR on the sp list... + * EP struct deallocation is where this memory is released or prior in the + * error case. + */ + crd = dapl_os_alloc ( sizeof(struct ibal_cr_data) ); + if ( !crd ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "%s: FAILED to alloc IB CM handle storage?\n", + __FUNCTION__); + return; + } + + cm_handle = &crd->cm_hdl; + dapl_os_memzero ( (void*)crd, sizeof(*crd) ); + + /* + * Save the cm_srvc_handle to avoid the race condition between + * the return of the ib_cm_listen and the notification of a conn req + */ + if (sp_ptr->cm_srvc_handle != p_cm_req_rec->h_cm_listen) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> DiCRqcb: cm_service_handle is changed\n"); + sp_ptr->cm_srvc_handle = p_cm_req_rec->h_cm_listen; + } + + dapl_os_memzero (&dest_ia_addr, sizeof (dest_ia_addr)); + +#ifdef NO_NAME_SERVICE + + { + DAPL_PRIVATE *prd_ptr; + + prd_ptr = (DAPL_PRIVATE *)p_cm_req_rec->p_req_pdata; + + dapl_os_memcpy ((void *)&dest_ia_addr, + (void *)&prd_ptr->hca_address, + sizeof (DAT_SOCK_ADDR6)); + } + +#else + + { + GID dest_gid; + + dapl_os_memzero (&dest_gid, sizeof (dest_gid)); + + dest_gid.guid = p_cm_req_rec->primary_path.dgid.unicast.interface_id; + dest_gid.gid_prefix = p_cm_req_rec->primary_path.dgid.unicast.prefix; + + if (DAT_SUCCESS != dapls_ns_map_ipaddr ( + sp_ptr->header.owner_ia->hca_ptr, + dest_gid, + (DAT_IA_ADDRESS_PTR)&dest_ia_addr)) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "cm_req_cb: SP = %lx failed mapping GID-IPaddr\n", + sp_ptr); + } + } + +#endif /* NO_NAME_SERVICE */ + +#if defined(DAPL_DBG) + { + char ipa[20]; + + //rval = ((struct sockaddr_in *) (&dest_ia_addr))->sin_addr.s_addr; + + dapl_dbg_log (DAPL_DBG_TYPE_CM|DAPL_DBG_TYPE_CALLBACK, + "%s: query SA for RemoteAddr: %s\n", + __FUNCTION__, + dapli_get_ip_addr_str( (DAT_SOCK_ADDR6*) + &dest_ia_addr, ipa) ); + } +#endif + + /* preserve CR cm handle data */ + dapl_os_memcpy( (void*)cm_handle, + (void*)&p_cm_req_rec->h_cm_req, + sizeof(ib_cm_handle_t) ); + + /* preserve remote IP address */ + dapl_os_memcpy( (void*)&crd->dst_ip_addr, + (void*)&dest_ia_addr, + sizeof(dest_ia_addr) ); + + /* FIXME - Vu + * We have NOT used/saved the primary and alternative path record + * ie. p_cm_req_rec->p_primary_path and p_cm_req_rec->p_alt_path + * We should cache some fields in path record in the Name Service DB + * such as: dgid, dlid + * Also we do not save resp_res (ie. max_oustanding_rdma_read/atomic) + * rnr_retry_cnt and flow_ctrl fields + */ + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "%s: SP %lx max_rdma_read %d PrivateData %lx\n", + __FUNCTION__, sp_ptr, p_cm_req_rec->resp_res, + p_cm_req_rec->p_req_pdata); + + dapls_cr_callback ( cm_handle, + IB_CME_CONNECTION_REQUEST_PENDING, + (void * __ptr64) p_cm_req_rec->p_req_pdata, + (void * __ptr64) sp_ptr ); +} + + +static void +dapli_ib_cm_mra_cb ( + IN ib_cm_mra_rec_t *p_cm_mra_rec ) +{ + UNUSED_PARAM( p_cm_mra_rec ); + dapl_dbg_log (DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> DiCMcb: CM callback MRA\n"); +} + +static void +dapli_ib_cm_rtu_cb ( + IN ib_cm_rtu_rec_t *p_cm_rtu_rec ) +{ + DAPL_EP *ep_ptr; + + dapl_os_assert (p_cm_rtu_rec != NULL); + + ep_ptr = (DAPL_EP * __ptr64) p_cm_rtu_rec->qp_context; + + dapl_dbg_log (DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, + "--> DiCRucb: EP %lx QP %lx\n", ep_ptr, ep_ptr->qp_handle); + + if (ep_ptr->cr_ptr) + { + DAPL_SP *sp_ptr; + + sp_ptr = ((DAPL_CR *) ep_ptr->cr_ptr)->sp_ptr; + + dapls_cr_callback ( ep_ptr->cm_handle, + IB_CME_CONNECTED, + (void * __ptr64) p_cm_rtu_rec->p_rtu_pdata, + (void *) sp_ptr); + + } + else + { + dapl_evd_connection_callback ( + ep_ptr->cm_handle, + IB_CME_CONNECTED, + (void * __ptr64) p_cm_rtu_rec->p_rtu_pdata, + (void *) ep_ptr); + } +} + +static void +dapli_ib_cm_cancel_cb( void *context ) +{ + DAPL_SP *sp_ptr; + + sp_ptr = (DAPL_SP *) context; + dapl_os_assert ( sp_ptr ); + if ( sp_ptr->header.magic == DAPL_MAGIC_PSP || + sp_ptr->header.magic == DAPL_MAGIC_RSP ) + { + dapl_os_wait_object_wakeup( &sp_ptr->wait_object ); + } +} + +/* + * dapls_ib_cm_remote_addr + * + * Obtain the remote IP address given a connection + * + * Input: + * cr_handle + * + * Output: + * remote_ia_address: where to place the remote address + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * + */ +DAT_RETURN +dapls_ib_cm_remote_addr ( + IN DAT_HANDLE dat_handle, + OUT DAT_SOCK_ADDR6 *remote_address ) +{ + + DAPL_HEADER *header; + void *vp; + char ipa[20]; + + header = (DAPL_HEADER *)dat_handle; + + if (header->magic == DAPL_MAGIC_EP) + { + vp = &((DAPL_EP *) dat_handle)->remote_ia_address; + } + else if (header->magic == DAPL_MAGIC_CR) + { + vp = &((DAPL_CR *) dat_handle)->remote_ia_address; + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_CM, + "%s: hdr->magic %x, dat_handle(%lx)\n", + __FUNCTION__, header->magic, dat_handle ); + return DAT_INVALID_HANDLE; + } + + dapl_os_memcpy( remote_address, vp, sizeof(DAT_SOCK_ADDR6) ); + + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "%s: returns %s\n", + __FUNCTION__, + dapli_get_ip_addr_str((DAT_SOCK_ADDR6*)remote_address,ipa) ); + + return DAT_SUCCESS; +} + + +/* + * dapls_ib_connect + * + * Initiate a connection with the passive listener on another node + * + * Input: + * ep_handle, + * remote_ia_address, + * remote_conn_qual, + * prd_size size of private data and structure + * prd_prt pointer to private data structure + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + IN DAT_CONN_QUAL remote_conn_qual, + IN DAT_COUNT prd_size, + IN DAPL_PRIVATE *prd_ptr ) +{ + DAPL_EP *ep_ptr; + DAPL_IA *ia_ptr; + ib_api_status_t ib_status; + dapl_ibal_port_t *p_active_port; + dapl_ibal_ca_t *p_ca; + ib_cm_req_t cm_req; + ib_path_rec_t path_rec; + GID dest_GID; + ib_query_req_t query_req; + ib_gid_pair_t gid_pair; + ib_service_record_t service_rec; + int retry_cnt; + DAT_RETURN dat_status; + + ep_ptr = (DAPL_EP *) ep_handle; + ia_ptr = ep_ptr->header.owner_ia; + ep_ptr->cr_ptr = NULL; + retry_cnt = 0; + dat_status = DAT_SUCCESS; + + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + + /* + * We are using the first active port in the list for + * communication. We have to get back here when we decide to support + * fail-over and high-availability. + */ + p_active_port = dapli_ibal_get_port ( p_ca, + (uint8_t)ia_ptr->hca_ptr->port_num ); + + if (NULL == p_active_port) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"--> DsC: Port %d not available %d\n", + ia_ptr->hca_ptr->port_num, __LINE__ ); + return (DAT_INVALID_STATE); + } + + dapl_os_memzero (&dest_GID, sizeof (GID)); + dapl_os_memzero (&cm_req, sizeof (ib_cm_req_t)); + dapl_os_memzero (&path_rec, sizeof (ib_path_rec_t)); + dapl_os_memzero (&service_rec, sizeof (ib_service_record_t)); + dapl_os_memzero (&query_req, sizeof (ib_query_req_t)); + dapl_os_memzero (&gid_pair, sizeof (ib_gid_pair_t)); + dapl_os_memzero (&ep_ptr->remote_ia_address, sizeof (DAT_SOCK_ADDR6)); + + dapl_os_memcpy (&ep_ptr->remote_ia_address, + remote_ia_address, + sizeof (ep_ptr->remote_ia_address)); + + +#ifdef NO_NAME_SERVICE + + if (DAT_SUCCESS != + (dat_status = dapls_ns_lookup_address ( + ia_ptr, + remote_ia_address, + &dest_GID ))) + { + /* + * Remote address not in the table, this is a + * strange return code! + */ + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"--> DsC: exits status = %x\n", dat_status); + return dat_status; + } + + dest_GID.guid = CL_HTON64 (dest_GID.guid); + dest_GID.gid_prefix = CL_HTON64 (dest_GID.gid_prefix); + +#else + + /* + * We query the SA to get the dest_gid with the + * {uDAPL_svc_id, IP-address} as the key to get GID. + */ + if (DAT_SUCCESS != + (dat_status = dapls_ns_map_gid (ia_ptr->hca_ptr, + remote_ia_address, + &dest_GID))) + + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsC: fail to map remote_ia_addr " + "(sa_family %d) to gid\n", + remote_ia_address->sa_family); + return dat_status; + } +#endif /* NO_NAME_SERVICE */ + + gid_pair.dest_gid.unicast.interface_id = dest_GID.guid; + gid_pair.dest_gid.unicast.prefix = dest_GID.gid_prefix; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "dapls_ib_connect: EP %lx QP %lx SERVER GID{0x" F64x + ", 0x" F64x "}\n", + ep_ptr, ep_ptr->qp_handle, + cl_hton64 (gid_pair.dest_gid.unicast.prefix), + cl_hton64 (gid_pair.dest_gid.unicast.interface_id)); + + gid_pair.src_gid = p_active_port->p_attr->p_gid_table[0]; +/* + if ((gid_pair.src_gid.unicast.interface_id == + gid_pair.dest_gid.unicast.interface_id ) && + (gid_pair.src_gid.unicast.prefix == + gid_pair.dest_gid.unicast.prefix )) + { + path_rec.dgid = gid_pair.dest_gid; + path_rec.sgid = gid_pair.src_gid; + path_rec.slid = path_rec.dlid = p_active_port->p_attr->lid; + path_rec.pkey = p_active_port->p_attr->p_pkey_table[0]; + path_rec.mtu = p_active_port->p_attr->mtu; + path_rec.pkt_life = 18; // 1 sec + path_rec.rate = IB_PATH_RECORD_RATE_10_GBS; + + } + else + { + */ /* + * Query SA to get the path record from pair of GIDs + */ + dapl_os_memzero (&query_req, sizeof (ib_query_req_t)); + query_req.query_type = IB_QUERY_PATH_REC_BY_GIDS; + query_req.p_query_input = (void *) &gid_pair; + query_req.flags = IB_FLAGS_SYNC; + query_req.timeout_ms = 1 * 1000; /* 1 second */ + query_req.retry_cnt = 3; + /* query SA using this port */ + query_req.port_guid = p_active_port->p_attr->port_guid; + query_req.query_context = (void *) &path_rec; + query_req.pfn_query_cb = dapli_ib_sa_query_cb; + + ib_status = ib_query (dapl_ibal_root.h_al, &query_req, NULL); + + if ((ib_status != IB_SUCCESS) || (!path_rec.dlid)) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"--> DsC: EP %lx QP %lx query " + "pair_gids status = %s\n", + ep_ptr, ep_ptr->qp_handle,ib_get_err_str(ib_status)); + return DAT_INVALID_PARAMETER; + } + + //} + + /* + * Tavor has a HW bug that causes bandwidth with 2K MTU to be less than + * with 1K MTU. Cap the MTU based on device ID to compensate for this. + */ + if( (p_ca->p_ca_attr->dev_id == 0x5A44) && + (ib_path_rec_mtu( &path_rec ) > IB_MTU_LEN_1024) ) + { + /* Local endpoint is Tavor - cap MTU to 1K for extra bandwidth. */ + path_rec.mtu &= IB_PATH_REC_SELECTOR_MASK; + path_rec.mtu |= IB_MTU_LEN_1024; + } + + /* + * prepare the Service ID from conn_qual + */ + cm_req.svc_id = remote_conn_qual; + cm_req.p_primary_path = &path_rec; + cm_req.p_alt_path = NULL; + cm_req.h_qp = ep_ptr->qp_handle; + cm_req.qp_type = IB_QPT_RELIABLE_CONN; + cm_req.p_req_pdata = (uint8_t *) prd_ptr; + cm_req.req_length = (uint8_t)prd_size; + /* cm retry to send this request messages, IB max of 4 bits */ + cm_req.max_cm_retries = 15; /* timer outside of call, s/be infinite */ + /* qp retry to send any wr */ + cm_req.retry_cnt = 5; + /* max num of oustanding RDMA read/atomic support */ + cm_req.resp_res = (uint8_t)ep_ptr->param.ep_attr.max_rdma_read_in; + /* max num of oustanding RDMA read/atomic will use */ + cm_req.init_depth = (uint8_t)ep_ptr->param.ep_attr.max_rdma_read_out; + + /* time wait before retrying a pkt after receiving a RNR NAK */ + cm_req.rnr_nak_timeout = 12; /* 163.84ms */ + + /* + * number of time local QP should retry after receiving RNR NACK before + * reporting an error + */ + cm_req.rnr_retry_cnt = 6; /* 7 is infinite */ + + cm_req.remote_resp_timeout = 16; /* 250ms */ + cm_req.local_resp_timeout = 16; /* 250ms */ + + cm_req.flow_ctrl = TRUE; + cm_req.flags = 0; + /* + * We do not use specific data buffer to check for specific connection + */ + cm_req.p_compare_buffer = NULL; + cm_req.compare_offset = 0; + cm_req.compare_length = 0; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> DsConn: EP=%lx QP=%lx rio=%d,%d pl=%d " + "mtu=%d slid=%#x dlid=%#x\n", + ep_ptr, ep_ptr->qp_handle, cm_req.resp_res, + cm_req.init_depth, ib_path_rec_pkt_life(&path_rec), + ib_path_rec_mtu(&path_rec), + cm_req.p_primary_path->slid, + cm_req.p_primary_path->dlid); + + /* + * We do not support peer_to_peer; therefore, we set pfn_cm_req_cb = NULL + */ + cm_req.pfn_cm_req_cb = NULL; + cm_req.pfn_cm_rep_cb = dapli_ib_cm_rep_cb; + cm_req.pfn_cm_rej_cb = dapli_ib_cm_rej_cb; + /* callback when a message received acknowledgement is received */ + cm_req.pfn_cm_mra_cb = dapli_ib_cm_mra_cb; + + ib_status = ib_cm_req (&cm_req); + + if ( ib_status != IB_SUCCESS ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsC: EP %lx QP %lx conn_request failed = %s\n", + ep_ptr, ep_ptr->qp_handle, ib_get_err_str(ib_status)); + + return (dapl_ib_status_convert (ib_status)); + } + + return DAT_SUCCESS; +} + + +/* + * dapls_ib_disconnect + * + * Disconnect an EP + * + * Input: + * ep_handle, + * disconnect_flags + * DAT_CLOSE_ABRUPT_FLAG - no callback + * DAT_CLOSE_GRACEFUL_FLAG - callback desired. + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_disconnect ( IN DAPL_EP *ep_ptr, + IN DAT_CLOSE_FLAGS disconnect_flags ) +{ + DAPL_IA *ia_ptr; + ib_api_status_t ib_status; + ib_cm_dreq_t cm_dreq; + + //UNUSED_PARAM( disconnect_flags ); + + dapl_os_assert(ep_ptr); + if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> %s: BAD EP Magic EP=%lx\n", __FUNCTION__,ep_ptr); + } + + if (ep_ptr->cm_handle == IB_INVALID_HANDLE ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> %s: Invalid EP->CM handle, OK.\n", __FUNCTION__); + return DAT_SUCCESS; + } + + dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> %s() DsD: EP %lx QP %lx ep_state %s " + "rx_drq %d tx_drq %d Close %s\n", __FUNCTION__, + ep_ptr, ep_ptr->qp_handle, + dapl_get_ep_state_str (ep_ptr->param.ep_state), + ep_ptr->recv_discreq, ep_ptr->sent_discreq, + (disconnect_flags == DAT_CLOSE_ABRUPT_FLAG + ? "Abrupt":"Graceful")); + + if ( disconnect_flags == DAT_CLOSE_ABRUPT_FLAG ) { + dapl_ep_legacy_post_disconnect(ep_ptr, disconnect_flags); + return DAT_SUCCESS; + } + + if ( ep_ptr->param.ep_state != DAT_EP_STATE_CONNECTED ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DsD: EP %lx NOT connected state %s\n", + ep_ptr, dapl_get_ep_state_str (ep_ptr->param.ep_state)); + } + ia_ptr = ep_ptr->header.owner_ia; + ib_status = IB_SUCCESS; + + dapl_os_memzero(&cm_dreq, sizeof(ib_cm_dreq_t)); + + cm_dreq.qp_type = IB_QPT_RELIABLE_CONN; + cm_dreq.h_qp = ep_ptr->qp_handle; + cm_dreq.pfn_cm_drep_cb = dapli_ib_cm_drep_cb; + + /* + * Currently we do not send any disconnect private data to + * the other endpoint because DAT 2.0 does not support it. + */ + cm_dreq.p_dreq_pdata = NULL; + cm_dreq.flags = IB_FLAGS_SYNC; + + if ( (ep_ptr->recv_discreq == DAT_FALSE) + && (ep_ptr->sent_discreq == DAT_FALSE) ) + { + ep_ptr->sent_discreq = DAT_TRUE; + + ib_status = ib_cm_dreq ( &cm_dreq ); + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DsD: EP %lx QP %lx DREQ SENT status %s\n", + ep_ptr, ep_ptr->qp_handle,ib_get_err_str(ib_status)); + } + + return dapl_ib_status_convert (ib_status); +} + + +/* + * dapl_ib_setup_conn_listener + * + * Have the CM set up a connection listener. + * + * Input: + * ibm_hca_handle HCA handle + * qp_handle QP handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_setup_conn_listener ( + IN DAPL_IA *ia_ptr, + IN DAT_UINT64 ServiceID, + IN DAPL_SP *sp_ptr ) +{ + ib_api_status_t ib_status; + ib_cm_listen_t cm_listen; + dapl_ibal_ca_t *p_ca; + dapl_ibal_port_t *p_active_port; + + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + + /* + * We are using the first active port in the list for + * communication. We have to get back here when we decide to support + * fail-over and high-availability. + */ + p_active_port = dapli_ibal_get_port ( p_ca, (uint8_t)ia_ptr->hca_ptr->port_num ); + + if (NULL == p_active_port) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"%s: SP %lx port %d not available\n", + __FUNCTION__, sp_ptr, ia_ptr->hca_ptr->port_num ); + return (DAT_INVALID_STATE); + } + + if (p_active_port->p_attr->lid == 0) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsSCL: SP %lx SID 0x" F64x " port %d\n", + sp_ptr, cl_hton64(ServiceID), + p_active_port->p_attr->port_num); + return (DAT_INVALID_STATE); + } + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "%s: SP %lx port %d GID{0x" F64x ", 0x" F64x "} and SID 0x" F64x "\n", + __FUNCTION__, + sp_ptr, p_active_port->p_attr->port_num, + cl_hton64 (p_active_port->p_attr->p_gid_table[0].unicast.prefix), + cl_hton64 (p_active_port->p_attr->p_gid_table[0].unicast.interface_id), + cl_hton64 (ServiceID)); + + dapl_os_memzero (&cm_listen, sizeof (ib_cm_listen_t)); + + /* + * Listen for all request on this specific CA + */ + cm_listen.ca_guid = (p_ca->p_ca_attr->ca_guid); + cm_listen.svc_id = ServiceID; + cm_listen.qp_type = IB_QPT_RELIABLE_CONN; + + /* + * We do not use specific data buffer to check for specific connection + */ + cm_listen.p_compare_buffer = NULL;//(uint8_t*)&sp_ptr->conn_qual; + cm_listen.compare_offset = 0;//IB_MAX_REQ_PDATA_SIZE - sizeof(DAT_CONN_QUAL); + cm_listen.compare_length = 0;//sizeof(DAT_CONN_QUAL); + + /* + * We can pick a port here for communication and the others are reserved + * for fail-over / high-availability - TBD + */ + cm_listen.port_guid = p_active_port->p_attr->port_guid; + cm_listen.lid = p_active_port->p_attr->lid; + cm_listen.pkey = p_active_port->p_attr->p_pkey_table[0]; + + /* + * Register request or mra callback functions + */ + cm_listen.pfn_cm_req_cb = dapli_ib_cm_req_cb; + + ib_status = ib_cm_listen ( dapl_ibal_root.h_al, + &cm_listen, + dapli_ibal_listen_err_cb, + (void *) sp_ptr, + &sp_ptr->cm_srvc_handle ); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "%s: SP %lx SID 0x" F64x " listen failed %s\n", + __FUNCTION__, sp_ptr, cl_hton64 (ServiceID), + ib_get_err_str(ib_status)); + } + + return dapl_ib_status_convert (ib_status); +} + + +/* + * dapl_ib_remove_conn_listener + * + * Have the CM remove a connection listener. + * + * Input: + * ia_handle IA handle + * ServiceID IB Channel Service ID + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_remove_conn_listener ( + IN DAPL_IA *ia_ptr, + IN DAPL_SP *sp_ptr ) +{ + ib_api_status_t ib_status; + DAT_RETURN dat_status = DAT_SUCCESS; + + UNUSED_PARAM( ia_ptr ); + + dapl_os_assert ( sp_ptr ); + + dapl_os_assert ( sp_ptr->header.magic == DAPL_MAGIC_PSP || + sp_ptr->header.magic == DAPL_MAGIC_RSP ); + + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "%s() cm_srvc_handle %lx\n", + __FUNCTION__, sp_ptr->cm_srvc_handle ); + + if (sp_ptr->cm_srvc_handle) + { + ib_status = ib_cm_cancel ( sp_ptr->cm_srvc_handle, + dapli_ib_cm_cancel_cb ); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsRCL: SP %lx ib_cm_cancel failed(0x%x) %s\n", + sp_ptr, sp_ptr->cm_srvc_handle, + ib_get_err_str(ib_status)); + sp_ptr->cm_srvc_handle = NULL; + return (DAT_INVALID_PARAMETER); + } + + /* wait object created/destroyed in common\dapl_sp_util.c */ + dat_status = dapl_os_wait_object_wait ( &sp_ptr->wait_object, + DAT_TIMEOUT_INFINITE ); + + if ( DAT_SUCCESS != dat_status ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsRCL: SP %lx (Inf) wait failed 0x%x\n", + sp_ptr, dat_status ); + } + sp_ptr->cm_srvc_handle = NULL; + } + + return dat_status; +} + +/* + * dapls_ib_reject_connection + * + * Perform necessary steps to reject a connection + * + * Input: + * cr_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_reject_connection ( IN dp_ib_cm_handle_t ib_cm_handle, + IN int reject_reason ) +{ + ib_api_status_t ib_status; + ib_cm_rej_t cm_rej; + static char *rej_table[] = + { + "INVALID_REJ_REASON", + "INVALID_REJ_REASON", + "INVALID_REJ_REASON", + "INVALID_REJ_REASON", + "INVALID_REJ_REASON", + "IB_CME_DESTINATION_REJECT", + "IB_CME_DESTINATION_REJECT_PRIVATE_DATA", + "IB_CME_DESTINATION_UNREACHABLE", + "IB_CME_TOO_MANY_CONNECTION_REQUESTS", + "IB_CME_LOCAL_FAILURE", + "IB_CM_LOCAL_FAILURE" + }; + +#define REJ_TABLE_SIZE IB_CM_LOCAL_FAILURE + + reject_reason = __min( reject_reason & 0xff, REJ_TABLE_SIZE); + + cm_rej.rej_status = IB_REJ_USER_DEFINED; + cm_rej.p_ari = (ib_ari_t *)&rej_table[reject_reason]; + cm_rej.ari_length = (uint8_t)strlen (rej_table[reject_reason]); + cm_rej.p_rej_pdata = NULL; + cm_rej.rej_length = 0; + + ib_status = ib_cm_rej ( *ib_cm_handle, &cm_rej); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsRjC: cm_handle %lx reject failed %s\n", + ib_cm_handle, ib_get_err_str(ib_status) ); + } + + return ( dapl_ib_status_convert ( ib_status ) ); + +} + + + +#if 0 +static void +dapli_query_qp( ib_qp_handle_t qp_handle, ib_qp_attr_t *qpa ) +{ + ib_api_status_t ib_status; + + ib_status = ib_query_qp ( qp_handle, qpa ); + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"ib_query_qp(%lx) '%s'\n", + qp_handle, ib_get_err_str(ib_status) ); + } +#if 1 + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "--> QP(%lx) state %s " + "type %d init %d acc %x\n", + qp_handle, + ib_get_port_state_str(qpa->state), + qpa->qp_type, + qpa->init_depth, + qpa->access_ctrl ); + } +#endif +} +#endif + +static char * +dapli_get_ip_addr_str(DAT_SOCK_ADDR6 *ipa, char *str) +{ + int rval; + + rval = ((struct sockaddr_in *)ipa)->sin_addr.s_addr; + + sprintf(str, "%d.%d.%d.%d", + (rval >> 0) & 0xff, + (rval >> 8) & 0xff, + (rval >> 16) & 0xff, + (rval >> 24) & 0xff); + return str; +} + + +/* + * dapls_ib_accept_connection + * + * Perform necessary steps to accept a connection + * + * Input: + * cr_handle + * ep_handle + * private_data_size - ignored as DAT layer sets 0 + * private_data - ignored as DAT layer sets NULL + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_accept_connection ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT p_size, + IN DAPL_PRIVATE *prd_ptr ) +{ + DAPL_CR *cr_ptr; + DAPL_EP *ep_ptr; + DAPL_IA *ia_ptr; + DAT_RETURN dat_status; + ib_api_status_t ib_status; + dapl_ibal_ca_t *p_ca; + dapl_ibal_port_t *p_active_port; + ib_cm_rep_t cm_rep; + ib_qp_attr_t qpa; + + cr_ptr = (DAPL_CR *) cr_handle; + ep_ptr = (DAPL_EP *) ep_handle; + ia_ptr = ep_ptr->header.owner_ia; + + if ( ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED ) + { + /* + * If we are lazy attaching the QP then we may need to + * hook it up here. Typically, we run this code only for + * DAT_PSP_PROVIDER_FLAG + */ + dat_status = dapls_ib_qp_alloc ( ia_ptr, ep_ptr, ep_ptr ); + + if ( dat_status != DAT_SUCCESS) + { + /* This is not a great error code, but all the spec allows */ + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIBAC: CR %lx EP %lx alloc QP failed 0x%x\n", + cr_ptr, ep_ptr, dat_status ); + return (dat_status); + } + } + + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + p_active_port = dapli_ibal_get_port ( p_ca, + (uint8_t)ia_ptr->hca_ptr->port_num ); + if (NULL == p_active_port) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIBAC: CR %lx EP %lx port %d is not available\n", + cr_ptr, ep_ptr, ia_ptr->hca_ptr->port_num); + return (DAT_INVALID_STATE); + } + + cr_ptr->param.local_ep_handle = ep_handle; + ep_ptr->cm_handle = cr_ptr->ib_cm_handle; + /* + * assume ownership, in that once the EP is released the dynamic + * memory containing the IBAL CM handle (ib_cm_handle_t) struct will + * be released; see dapl_ep_dealloc(). + */ + ep_ptr->ibal_cm_handle = cr_ptr->ib_cm_handle; + + /* set remote IP addr fields. IP addr data is deduced from Connection + * Request record (gid/lib) and stashed away for use here. DAPL 1.1 + * had an interface for passing the IP info down, interface went away + * in 2.0? + */ + dapl_os_memcpy( (void*)&ep_ptr->remote_ia_address, + (void*)(ep_ptr->cm_handle+1), + sizeof(DAT_SOCK_ADDR6) ); + + dapl_os_memcpy( (void*)&cr_ptr->remote_ia_address, + (void*)&ep_ptr->remote_ia_address, + sizeof(DAT_SOCK_ADDR6) ); + +#if defined(DAPL_DBG) && 0 + { + char ipa[20]; + + dapl_dbg_log (DAPL_DBG_TYPE_CM|DAPL_DBG_TYPE_CALLBACK, + "%s: EP(%lx) RemoteAddr: %s\n", + __FUNCTION__, ep_ptr, + dapli_get_ip_addr_str( + (DAT_SOCK_ADDR6*)&ep_ptr->remote_ia_address, ipa)); + } +#endif + + ep_ptr->qp_state = IB_QPS_INIT; + ep_ptr->cr_ptr = cr_ptr; + + dapl_os_memzero ( (void*)&cm_rep, sizeof (ib_cm_rep_t) ); + + cm_rep.h_qp = ep_ptr->qp_handle; + cm_rep.qp_type = IB_QPT_RELIABLE_CONN; + cm_rep.p_rep_pdata = (uint8_t *) cr_ptr->private_data; + cm_rep.rep_length = IB_REQ_PDATA_SIZE; + +#if defined(DAPL_DBG) && 0 + { + int i; + + dapl_dbg_log ( DAPL_DBG_TYPE_EP, "--> DsAC: private_data: "); + + for ( i = 0 ; i < IB_MAX_REP_PDATA_SIZE ; i++ ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_EP, + "0x%x ", prd_ptr->private_data[i]); + + } + dapl_dbg_log ( DAPL_DBG_TYPE_EP, "\n"); + + } +#endif + + cm_rep.pfn_cm_rej_cb = dapli_ib_cm_rej_cb; + cm_rep.pfn_cm_mra_cb = dapli_ib_cm_mra_cb; + cm_rep.pfn_cm_rtu_cb = dapli_ib_cm_rtu_cb; + cm_rep.pfn_cm_lap_cb = dapli_ib_cm_lap_cb; + cm_rep.pfn_cm_dreq_cb = dapli_ib_cm_dreq_cb; + + /* + * FIXME - Vu + * Pay attention to the attributes. + * Some of them are desirably set by DAT consumers + */ + /* + * We enable the qp associate with this connection ep all the access right + * We enable the flow_ctrl, retry till success + * We will limit the access right and flow_ctrl upon DAT consumers + * requirements + */ + cm_rep.access_ctrl = IB_AC_LOCAL_WRITE|IB_AC_RDMA_WRITE|IB_AC_MW_BIND; + if ((ep_ptr->param.ep_attr.max_rdma_read_in > 0) + || (ep_ptr->param.ep_attr.max_rdma_read_out > 0)) + { + cm_rep.access_ctrl |= IB_AC_RDMA_READ; + } + + cm_rep.sq_depth = 0; + cm_rep.rq_depth = 0; + cm_rep.init_depth = (uint8_t)ep_ptr->param.ep_attr.max_rdma_read_out; + cm_rep.flow_ctrl = TRUE; + cm_rep.flags = 0; + cm_rep.failover_accepted = IB_FAILOVER_ACCEPT_UNSUPPORTED; + cm_rep.target_ack_delay = 14; + cm_rep.rnr_nak_timeout = 12; + cm_rep.rnr_retry_cnt = 6; + cm_rep.pp_recv_failure = NULL; + cm_rep.p_recv_wr = NULL; + + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DsIBAC: cm_rep: acc %x init %d qp_type %x req_count %d\n", + cm_rep.access_ctrl, cm_rep.init_depth,cm_rep.qp_type, + dapl_os_atomic_read(&ep_ptr->req_count)); + + ib_status = ib_cm_rep ( *ep_ptr->cm_handle, &cm_rep ); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIBAC: EP %lx QP %lx CR reply failed '%s'\n", + ep_ptr, ep_ptr->qp_handle, ib_get_err_str(ib_status) ); + + /* errors not perculated back to CR callback which allocated the + * memory, free it here on error. + */ + dapl_os_free ( (void*)ep_ptr->ibal_cm_handle, sizeof(ib_cm_handle_t) ); + ep_ptr->ibal_cm_handle = NULL; + } + + return ( dapl_ib_status_convert ( ib_status ) ); +} + + + +/* + * dapls_ib_disconnect_clean + * + * Clean up outstanding connection data. This routine is invoked + * after the final disconnect callback has occurred. Only on the + * ACTIVE side of a connection. + * + * Input: + * ep_ptr DAPL_EP + * + * Output: + * none + * + * Returns: + * void + * + */ +void +dapls_ib_disconnect_clean ( + IN DAPL_EP *ep_ptr, + IN DAT_BOOLEAN active, + IN const ib_cm_events_t ib_cm_event ) +{ + DAPL_IA *ia_ptr; + ib_qp_attr_t qp_attr; + ib_api_status_t ib_status; + + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "%s(%s): cm_event: %s \n", __FUNCTION__, + (active?"A":"P"), dapli_ib_cm_event_str(ib_cm_event)); + + ia_ptr = ep_ptr->header.owner_ia; + + if ( ia_ptr == NULL || ia_ptr->header.magic != DAPL_MAGIC_IA ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_CM, + ">>>DSCONN_CLEAN(%s): cm_event: %s Invalid IA_ptr\n", + (active?"Act":"Pas"),dapli_ib_cm_event_str(ib_cm_event)); + return; + } + dapl_os_assert ( ep_ptr->header.magic == DAPL_MAGIC_EP ); + + ep_ptr->sent_discreq = DAT_FALSE; + ep_ptr->recv_discreq = DAT_FALSE; + if ( ep_ptr->ibal_cm_handle ) + { + dapl_os_free ( (void*)ep_ptr->ibal_cm_handle, sizeof(ib_cm_handle_t) ); + } + ep_ptr->ibal_cm_handle = NULL; + + /* + * Query the QP to get the current state */ + ib_status = ib_query_qp ( ep_ptr->qp_handle, &qp_attr ); + + if ( ib_status != IB_SUCCESS ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + ">>>DSCONN_CLEAN(%s): Query QP failed = %s\n", + (active?"Act":"Pas"),ib_get_err_str(ib_status) ); + return; + } + + ep_ptr->qp_state = qp_attr.state; + + dapl_dbg_log ( DAPL_DBG_TYPE_CM, ">>>DSCONN_CLEAN(%s): cm_event: %d " + "ep_ptr %lx ep_state %s qp_state %#x\n", + (active?"A":"P"), ib_cm_event, ep_ptr, + dapl_get_ep_state_str(ep_ptr->param.ep_state), + ep_ptr->qp_state ); + + if ( ep_ptr->qp_state != IB_QPS_ERROR && + ep_ptr->qp_state != IB_QPS_RESET && + ep_ptr->qp_state != IB_QPS_INIT ) + { + ep_ptr->qp_state = IB_QPS_ERROR; + dapls_modify_qp_state_to_error (ep_ptr->qp_handle); + } +} + + +#ifdef NOT_USED +/* + * dapls_ib_cr_handoff + * + * Hand off the connection request to another service point + * + * Input: + * cr_handle DAT_CR_HANDLE + * handoff_serv_id DAT_CONN_QUAL + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_cr_handoff ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_CONN_QUAL handoff_serv_id ) +{ + DAPL_CR *cr_ptr; + ib_api_status_t ib_status; + + cr_ptr = (DAPL_CR *) cr_handle; + + if (cr_ptr->ib_cm_handle->cid == 0xFFFFFFFF) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsCH: CR = %lx invalid cm handle\n", cr_ptr); + return DAT_INVALID_PARAMETER; + } + + if (cr_ptr->sp_ptr == IB_INVALID_HANDLE) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsCH: CR = %lx invalid psp handle\n", cr_ptr); + return DAT_INVALID_PARAMETER; + } + + ib_status = ib_cm_handoff (*cr_ptr->ib_cm_handle, handoff_serv_id); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsCH: CR = %lx handoff failed = %s\n", + cr_ptr, ib_get_err_str(ib_status) ); + + return dapl_ib_status_convert (ib_status); + } + + dapl_dbg_log ( DAPL_DBG_TYPE_CM, + "--> %s(): remove CR %lx from SP %lx Queue\n", + __FUNCTION__, cr_ptr, cr_ptr->sp_ptr); + /* Remove the CR from the queue */ + dapl_sp_remove_cr (cr_ptr->sp_ptr, cr_ptr); + + /* + * If this SP has been removed from service, free it + * up after the last CR is removed + */ + dapl_os_lock (&cr_ptr->sp_ptr->header.lock); + if ( cr_ptr->sp_ptr->listening != DAT_TRUE && + cr_ptr->sp_ptr->cr_list_count == 0 && + cr_ptr->sp_ptr->state != DAPL_SP_STATE_FREE ) + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, + "--> DsCH: CR = %lx disconnect dump SP = %lx \n", + cr_ptr, cr_ptr->sp_ptr); + /* Decrement the ref count on the EVD */ + if (cr_ptr->sp_ptr->evd_handle) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)cr_ptr->sp_ptr->evd_handle)->evd_ref_count); + cr_ptr->sp_ptr->evd_handle = NULL; + } + cr_ptr->sp_ptr->state = DAPL_SP_STATE_FREE; + dapl_os_unlock (&cr_ptr->sp_ptr->header.lock); + (void)dapls_ib_remove_conn_listener ( cr_ptr->sp_ptr->header.owner_ia, + cr_ptr->sp_ptr ); + dapls_ia_unlink_sp ( (DAPL_IA *)cr_ptr->sp_ptr->header.owner_ia, + cr_ptr->sp_ptr ); + dapls_sp_free_sp ( cr_ptr->sp_ptr ); + } + else + { + dapl_os_unlock (&cr_ptr->sp_ptr->header.lock); + } + + /* + * Clean up and dispose of the resource + */ + dapls_cr_free (cr_ptr); + + return (DAT_SUCCESS); +} +#endif + +/* + * dapls_ib_private_data_size + * + * Return the size of private data given a connection op type + * + * Input: + * prd_ptr private data pointer + * conn_op connection operation type + * + * If prd_ptr is NULL, this is a query for the max size supported by + * the provider, otherwise it is the actual size of the private data + * contained in prd_ptr. + * + * Infiniband has fixed size private data, so prd_ptr is ignored. + * + * Output: + * None + * + * Returns: + * length of private data + * + */ +int +dapls_ib_private_data_size ( + IN DAPL_PRIVATE *prd_ptr, + IN DAPL_PDATA_OP conn_op) +{ + int size; + + UNUSED_PARAM( prd_ptr ); + + switch (conn_op) + { + case DAPL_PDATA_CONN_REQ: + { + size = IB_MAX_REQ_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_REP: + { + size = IB_MAX_REP_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_REJ: + { + size = IB_MAX_REJ_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_DREQ: + { + size = IB_MAX_DREQ_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_DREP: + { + size = IB_MAX_DREP_PDATA_SIZE; + break; + } + default: + { + size = 0; + } + } /* end case */ + +#if defined(DAPL_DBG) && 0 + dapl_dbg_log (DAPL_DBG_TYPE_CM, "%s: returns %d\n", __FUNCTION__, size ); +#endif + + return size; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cq.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cq.c new file mode 100644 index 00000000..7ea580b2 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_cq.c @@ -0,0 +1,489 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_cq.c + * + * PURPOSE: CQ (Completion Qeueu) access routine using IBAL APIs + * + * $Id$ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" +#include "dapl_lmr_util.h" +#include "dapl_rmr_util.h" +#include "dapl_cookie.h" +#include "dapl_ring_buffer_util.h" + +#ifndef NO_NAME_SERVICE +#include "dapl_name_service.h" +#endif /* NO_NAME_SERVICE */ + + +static void +dapli_ibal_cq_async_error_callback ( IN ib_async_event_rec_t *p_err_rec ) +{ + DAPL_EVD *evd_ptr = (DAPL_EVD*)((void *)p_err_rec->context); + DAPL_EVD *async_evd_ptr; + DAPL_IA *ia_ptr; + dapl_ibal_ca_t *p_ca; + dapl_ibal_evd_cb_t *evd_cb; + + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCqAEC: CQ error %d for EVD context %lx\n", + p_err_rec->code, p_err_rec->context); + + if (DAPL_BAD_HANDLE (evd_ptr, DAPL_MAGIC_EVD)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCqAEC: invalid EVD %lx\n", evd_ptr); + return; + } + + ia_ptr = evd_ptr->header.owner_ia; + async_evd_ptr = ia_ptr->async_error_evd; + if (async_evd_ptr == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiCqAEC: can't find async_error_evd on %s HCA\n", + (ia_ptr->header.provider)->device_name ); + return; + } + + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiCqAEC: can't find %s HCA\n", + (ia_ptr->header.provider)->device_name); + return; + } + + /* find CQ error callback using ia_ptr for context */ + evd_cb = dapli_find_evd_cb_by_context ( async_evd_ptr, p_ca ); + if ((evd_cb == NULL) || (evd_cb->pfn_async_cq_err_cb == NULL)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCqAEC: no ERROR cb on %lx found \n", p_ca); + return; + } + + /* maps to dapl_evd_cq_async_error_callback(), context is EVD */ + evd_cb->pfn_async_cq_err_cb( (ib_hca_handle_t)p_ca, + (ib_error_record_t*)&p_err_rec->code, evd_ptr); + +} + + +/* + * dapli_ibal_cq_competion_callback + * + * Completion callback for a CQ + * + * Input: + * cq_context User context + * + * Output: + * none + * + * Returns: + */ +static void +dapli_ib_cq_completion_cb ( + IN const ib_cq_handle_t h_cq, + IN void *cq_context ) +{ + DAPL_EVD *evd_ptr; + dapl_ibal_ca_t *p_ca; + + evd_ptr = (DAPL_EVD *) cq_context; + + dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK, + "--> DiICCC: cq_completion_cb evd %lx CQ %lx\n", + evd_ptr, evd_ptr->ib_cq_handle); + + dapl_os_assert (evd_ptr != DAT_HANDLE_NULL); + + p_ca = (dapl_ibal_ca_t *) evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle; + + dapl_os_assert( h_cq == evd_ptr->ib_cq_handle ); + + dapl_evd_dto_callback ( (ib_hca_handle_t) p_ca, h_cq, cq_context); +} + + +/* + * dapl_ib_cq_late_alloc + * + * Alloc a CQ + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * cqlen minimum QLen + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_cq_late_alloc ( + IN ib_pd_handle_t pd_handle, + IN DAPL_EVD *evd_ptr ) +{ + ib_cq_create_t cq_create; + ib_api_status_t ib_status; + DAT_RETURN dat_status; + dapl_ibal_ca_t *ibal_ca; + + dat_status = DAT_SUCCESS; + cq_create.size = evd_ptr->qlen; + + ibal_ca = (dapl_ibal_ca_t*)evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle; + +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + { + cq_create.h_wait_obj = evd_ptr->cq_wait_obj_handle; + cq_create.pfn_comp_cb = NULL; + } + else +#endif + { + cq_create.h_wait_obj = NULL; + cq_create.pfn_comp_cb = dapli_ib_cq_completion_cb; + } + + ib_status = ib_create_cq ( + (ib_ca_handle_t)ibal_ca->h_ca, + &cq_create, + evd_ptr /* context */, + dapli_ibal_cq_async_error_callback, + &evd_ptr->ib_cq_handle); + + dat_status = dapl_ib_status_convert (ib_status); + + if ( dat_status != DAT_SUCCESS ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsICLA: failed to create CQ for EVD %lx\n",evd_ptr); + goto bail; + } + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsCQ_alloc: pd=%lx cq=%lx CQsz=%d EVD->Qln=%d\n", + pd_handle, evd_ptr->ib_cq_handle, + cq_create.size, evd_ptr->qlen); + + /* + * As long as the CQ size is >= the evd size, then things are OK; sez Arlin. + */ + if ( cq_create.size < (uint32_t)evd_ptr->qlen ) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsCQ_alloc: created CQ size(%d) < evd->qlen(%d)?\n", + cq_create.size, evd_ptr->qlen); + dat_status = dapl_ib_status_convert (IB_INVALID_CQ_SIZE); + } + +bail: + return dat_status; +} + +/* + * dapl_ib_cq_alloc + * + * Alloc a CQ + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * cqlen minimum QLen + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ + +#if 0 + +/* windows delays CQ creation as a PD (Protection Domain) is required + * and we do not have one at this juncture. The follow code is for future + * reference only. + */ + +DAT_RETURN +dapls_ib_cq_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT *cqlen ) +{ + dapl_dbg_log ( DAPL_DBG_TYPE_UTIL, + "dapls_ib_cq_alloc: evd %lx cqlen=%d \n", evd_ptr, *cqlen ); + + struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq; + +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + channel = evd_ptr->cq_wait_obj_handle; +#endif + + /* Call IB verbs to create CQ */ + evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, + *cqlen, + evd_ptr, + channel, + 0); + + if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) + return DAT_INSUFFICIENT_RESOURCES; + + /* arm cq for events */ + dapls_set_cq_notify(ia_ptr, evd_ptr); + + /* update with returned cq entry size */ + *cqlen = evd_ptr->ib_cq_handle->cqe; + + dapl_dbg_log ( DAPL_DBG_TYPE_UTIL, + "dapls_ib_cq_alloc: new_cq %lx cqlen=%d \n", + evd_ptr->ib_cq_handle, *cqlen ); + + return DAT_SUCCESS; +} +#endif + + +/* + * dapl_ib_cq_free + * + * Dealloc a CQ + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_cq_free ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr) +{ + ib_api_status_t ib_status; + + if ( ia_ptr == NULL || ia_ptr->header.magic != DAPL_MAGIC_IA ) + { + return DAT_INVALID_HANDLE; + } + + ib_status = ib_destroy_cq (evd_ptr->ib_cq_handle, + /* destroy_callback */ NULL); + + return dapl_ib_status_convert (ib_status); +} + +/* + * dapls_cq_resize + * + * Resize CQ completion notifications + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * cqlen minimum QLen + * + * Output: + * cqlen may round up for optimal memory boundaries + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INSUFFICIENT_RESOURCES + * + */ + +DAT_RETURN +dapls_ib_cq_resize ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr, + IN DAT_COUNT *cqlen ) +{ + ib_api_status_t ib_status = IB_SUCCESS; + + if ( ia_ptr == NULL || ia_ptr->header.magic != DAPL_MAGIC_IA ) + { + return DAT_INVALID_HANDLE; + } + /* + * Resize CQ only if CQ handle is valid, may be delayed waiting + * for PZ allocation with IBAL + */ +#if defined(_VENDOR_IBAL_) + if ( evd_ptr->ib_cq_handle != IB_INVALID_HANDLE ) +#endif /* _VENDOR_IBAL_ */ + { + ib_status = ib_modify_cq ( evd_ptr->ib_cq_handle, + (uint32_t *)cqlen ); + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + "ib_modify_cq ( new cqlen = %d, status=%d ) \n", + *cqlen, ib_status ); + } + return dapl_ib_status_convert (ib_status); +} + + +/* + * dapl_set_cq_notify + * + * Set up CQ completion notifications + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_set_cq_notify ( + IN DAPL_IA *ia_ptr, + IN DAPL_EVD *evd_ptr ) +{ + ib_api_status_t ib_status; + if ( ia_ptr == NULL || ia_ptr->header.magic != DAPL_MAGIC_IA ) + { + return DAT_INVALID_HANDLE; + } + ib_status = ib_rearm_cq ( + evd_ptr->ib_cq_handle, + FALSE /* next event but not solicited event */ ); + + return dapl_ib_status_convert (ib_status); +} + + +/* + * dapls_ib_cqd_create + * + * Set up CQ notification event thread + * + * Input: + * ia_handle HCA handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_cqd_create ( IN DAPL_HCA *p_hca ) +{ + /* + * We do not have CQD concept + */ + p_hca->ib_trans.ib_cqd_handle = IB_INVALID_HANDLE; + + return DAT_SUCCESS; +} + + +/* + * dapl_cqd_destroy + * + * Destroy CQ notification event thread + * + * Input: + * ia_handle IA handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_cqd_destroy ( IN DAPL_HCA *p_hca ) +{ + p_hca->ib_trans.ib_cqd_handle = IB_INVALID_HANDLE; + return (DAT_SUCCESS); +} + + + +DAT_RETURN +dapls_ib_n_completions_notify ( + IN ib_hca_handle_t hca_handle, + IN ib_cq_handle_t cq_handle, + IN uint32_t n_cqes ) +{ + ib_api_status_t ib_status; + UNREFERENCED_PARAMETER(hca_handle); + + ib_status = ib_rearm_n_cq ( + cq_handle, + n_cqes ); + + return dapl_ib_status_convert (ib_status); +} + + +DAT_RETURN +dapls_ib_peek_cq ( + IN ib_cq_handle_t cq_handle, + OUT uint32_t* p_n_cqes) +{ + ib_api_status_t ib_status; + + ib_status = ib_peek_cq ( cq_handle, p_n_cqes ); + + return dapl_ib_status_convert (ib_status); +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_dto.h b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_dto.h new file mode 100644 index 00000000..4953f015 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_dto.h @@ -0,0 +1,556 @@ + +/* + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_dto.h + * + * PURPOSE: Utility routines for data transfer operations using the + * IBAL APIs + * + * $Id: dapl_ibal_dto.h 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#ifndef _DAPL_IBAL_DTO_H +#define _DAPL_IBAL_DTO_H + +#include "dapl_ibal_util.h" + +//STATIC _INLINE_ int +//dapls_cqe_opcode_convert (ib_work_completion_t *cqe_p); + +extern DAT_RETURN +dapls_ib_cq_late_alloc ( + IN ib_pd_handle_t pd_handle, + IN DAPL_EVD *evd_ptr); + +#define DAPL_DEFAULT_DS_ENTRIES 8 + +#ifdef NOT_SUPPORTED +extern DAT_RETURN +dapls_ib_post_recv_defered ( + IN DAPL_EP *ep_ptr, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov); // dapl_ibal_util.c +#endif + +/* + * dapls_ib_post_recv + * + * Provider specific Post RECV function + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_post_recv ( + IN DAPL_EP *ep_ptr, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov) +{ + ib_api_status_t ib_status; + ib_recv_wr_t recv_wr, *failed_wr_p; + ib_local_ds_t ds_array[DAPL_DEFAULT_DS_ENTRIES], *ds_array_p; + DAT_COUNT i, total_len; + +#ifdef NOT_SUPPORTED + if (ep_ptr->qp_state == IB_QPS_INIT) + { + return dapls_ib_post_recv_defered ( ep_ptr, + cookie, + num_segments, + local_iov); + } +#endif + dapl_os_memzero(&recv_wr, sizeof(ib_recv_wr_t)); + recv_wr.wr_id = (DAT_UINT64) cookie; + recv_wr.num_ds = num_segments; + + if( num_segments <= DAPL_DEFAULT_DS_ENTRIES ) + { + ds_array_p = ds_array; + } + else + { + ds_array_p = dapl_os_alloc(num_segments*sizeof(ib_local_ds_t)); + } + recv_wr.ds_array = ds_array_p; + + if (NULL == ds_array_p) + { + return (DAT_INSUFFICIENT_RESOURCES); + } + + for (total_len = i = 0; i < num_segments; i++, ds_array_p++) + { + ds_array_p->length = (uint32_t)local_iov[i].segment_length; + ds_array_p->lkey = local_iov[i].lmr_context; + ds_array_p->vaddr = local_iov[i].virtual_address; + total_len += ds_array_p->length; + } + + if (cookie != NULL) + { + cookie->val.dto.size = total_len; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsPR: EP = %p QP = %p cookie= %p, num_seg= %d\n", + ep_ptr, ep_ptr->qp_handle, cookie, num_segments); + } + + recv_wr.p_next = NULL; + + ib_status = ib_post_recv( ep_ptr->qp_handle, &recv_wr, &failed_wr_p ); + + if( num_segments > DAPL_DEFAULT_DS_ENTRIES ) + dapl_os_free( recv_wr.ds_array, num_segments * sizeof(ib_local_ds_t) ); + + if (IB_SUCCESS == ib_status) + { + return DAT_SUCCESS; + } + else + { + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPR: post_recv status %s\n", + ib_get_err_str(ib_status)); + + /* + * Moving QP to error state; + */ + ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle); + ep_ptr->qp_state = IB_QPS_ERROR; + + return (dapl_ib_status_convert (ib_status)); + } +} + + +/* + * dapls_ib_post_send + * + * Provider specific Post SEND function + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_post_send ( + IN DAPL_EP *ep_ptr, + IN ib_send_op_type_t op_type, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + ib_api_status_t ib_status; + ib_send_wr_t send_wr, *failed_wr_p; + ib_local_ds_t ds_array[DAPL_DEFAULT_DS_ENTRIES], *ds_array_p; + DAT_COUNT i, total_len; + + if (ep_ptr->param.ep_state != DAT_EP_STATE_CONNECTED) + { + ib_qp_attr_t qp_attr; + ib_query_qp ( ep_ptr->qp_handle, &qp_attr ); + + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsPS: !CONN EP (%p) ep_state=%d " + "QP_state=%d\n", + ep_ptr, ep_ptr->param.ep_state, qp_attr.state ); + + return (DAT_ERROR(DAT_INVALID_STATE,DAT_INVALID_STATE_EP_DISCONNECTED)); + } + + dapl_os_memzero (&send_wr, sizeof(ib_send_wr_t)); + send_wr.wr_type = op_type; + send_wr.num_ds = num_segments; + + if( num_segments <= DAPL_DEFAULT_DS_ENTRIES ) + { + ds_array_p = ds_array; + } + else + { + ds_array_p = dapl_os_alloc( num_segments * sizeof(ib_local_ds_t) ); + } + send_wr.ds_array = ds_array_p; + + if (NULL == ds_array_p) + { + return (DAT_INSUFFICIENT_RESOURCES); + } + + total_len = 0; + + for (i = 0; i < num_segments; i++, ds_array_p++) + { + ds_array_p->length = (uint32_t)local_iov[i].segment_length; + ds_array_p->lkey = local_iov[i].lmr_context; + ds_array_p->vaddr = local_iov[i].virtual_address; + total_len += ds_array_p->length; + } + + if (cookie != NULL) + { + cookie->val.dto.size = total_len; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsPS: EVD=%p EP=%p QP=%p type=%d, sg=%d " + "ln=%d, ck=%p 0x" F64x "\n", + ep_ptr->param.request_evd_handle, ep_ptr, ep_ptr->qp_handle, + op_type, num_segments, total_len, + cookie, cookie->val.dto.cookie.as_64 ); + } + + send_wr.wr_id = (DAT_UINT64)cookie; + + /* RC for now */ + if (total_len > 0) + { + send_wr.remote_ops.vaddr = remote_iov->virtual_address; + send_wr.remote_ops.rkey = remote_iov->rmr_context; + } + + send_wr.send_opt = 0; + + send_wr.send_opt |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & + completion_flags) ? IB_SEND_OPT_FENCE : 0; + send_wr.send_opt |= (DAT_COMPLETION_SUPPRESS_FLAG & + completion_flags) ? 0 : IB_SEND_OPT_SIGNALED; + send_wr.send_opt |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & + completion_flags) ? IB_SEND_OPT_SOLICITED : 0; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPS: EP=%p QP=%p send_opt=0x%x," + "rem_addr=%p, rem_rkey=0x%x completion_flags=0x%x\n", + ep_ptr, ep_ptr->qp_handle, + send_wr.send_opt, (void *)(uintptr_t)send_wr.remote_ops.vaddr, + send_wr.remote_ops.rkey, completion_flags); + + send_wr.p_next = NULL; + + ib_status = ib_post_send( ep_ptr->qp_handle, &send_wr, &failed_wr_p ); + + if( num_segments > DAPL_DEFAULT_DS_ENTRIES ) + dapl_os_free( send_wr.ds_array, num_segments * sizeof(ib_local_ds_t) ); + + if (IB_SUCCESS == ib_status) + return DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPS: EP=%p post_send status = %s\n", + ep_ptr, ib_get_err_str(ib_status)); + /* + * Moving QP to error state; + */ + ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle); + ep_ptr->qp_state = IB_QPS_ERROR; + + return (dapl_ib_status_convert (ib_status)); +} + +/* + * dapls_ib_optional_prv_dat + * + * Allocate space for private data to be used in CR calls + * + * Input: + * cr_ptr CR handle + * event_data data provided by the provider callback function + * cr_pp Pointer for private data + * + * Output: + * cr_pp Area + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_optional_prv_dat ( + IN DAPL_CR *cr_ptr, + IN const void *event_data, + OUT DAPL_CR **cr_pp) +{ + DAT_RETURN dat_status = DAT_SUCCESS; + DAPL_PRIVATE *p_prv_data = (DAPL_PRIVATE *)event_data; + + if ( ! cr_ptr->param.private_data_size ) + { + cr_ptr->param.private_data_size = sizeof(cr_ptr->private_data); + cr_ptr->param.private_data = cr_ptr->private_data; + dapl_os_memcpy(cr_ptr->private_data, p_prv_data->private_data, cr_ptr->param.private_data_size ); + *cr_pp = (DAPL_CR *)cr_ptr->param.private_data; + } + return dat_status; +} + + +STATIC _INLINE_ int +dapls_cqe_opcode_convert (ib_work_completion_t *cqe_p) +{ + switch (((ib_work_completion_t *)cqe_p)->wc_type) + { + case IB_WC_SEND: + { + return (OP_SEND); + } + case IB_WC_RDMA_WRITE: + case IB_WC_RECV_RDMA_WRITE: + { + return (OP_RDMA_WRITE); + } + case IB_WC_RDMA_READ: + { + return (OP_RDMA_READ); + } + case IB_WC_COMPARE_SWAP: + { + return (OP_COMP_AND_SWAP); + } + case IB_WC_FETCH_ADD: + { + return (OP_FETCH_AND_ADD); + } + case IB_WC_MW_BIND: + { + return (OP_BIND_MW); + } + case IB_WC_RECV: + { + return (OP_RECEIVE); + } + default : + { + /* error */ + return (IB_ERROR); + } + } +} + + +/* map Work Completions to DAPL WR operations */ +STATIC _INLINE_ DAT_DTOS dapls_cqe_dtos_opcode(ib_work_completion_t *cqe_p) +{ + switch (cqe_p->wc_type) { + + case IB_WC_SEND: + return (DAT_DTO_SEND); + case IB_WC_MW_BIND: + return (DAT_DTO_BIND_MW); +#ifdef DAT_EXTENSIONS + case IB_WC_RDMA_WRITE: + case IB_WC_RECV_RDMA_WRITE: + if (cqe_p->recv.conn.immediate_data & IB_RECV_OPT_IMMEDIATE) + return (DAT_IB_DTO_RDMA_WRITE_IMMED); + else + return (DAT_DTO_RDMA_WRITE); + case IB_WC_COMPARE_SWAP: + return (DAT_IB_DTO_CMP_SWAP); + case IB_WC_FETCH_ADD: + return (DAT_IB_DTO_FETCH_ADD); + case IB_WC_RDMA_READ: + return (DAT_DTO_RDMA_READ); + case IB_WC_RECV: + if (cqe_p->recv.conn.immediate_data & IB_RECV_OPT_IMMEDIATE) + return (DAT_IB_DTO_RECV_IMMED); + else + return (DAT_DTO_RECEIVE); +#else + case IB_WC_RDMA_READ: + return (DAT_DTO_RDMA_READ); + case IB_WC_RDMA_WRITE: + case IB_WC_RECV_RDMA_WRITE: + return (DAT_DTO_RDMA_WRITE); + case IB_WC_RECV: + return (DAT_DTO_RECEIVE); +#endif + default: + return (0xff); + } +} +#define DAPL_GET_CQE_DTOS_OPTYPE(cqe_p) dapls_cqe_dtos_opcode(cqe_p) + +#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t *)cqe_p)->wr_id +#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode_convert(cqe_p) +#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t *)cqe_p)->length +#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t *)cqe_p)->status +#define DAPL_GET_CQE_IMMED_DATA(cqe_p) \ + ((ib_work_completion_t*)cqe_p)->recv.conn.immediate_data + + +#ifdef DAT_EXTENSIONS +/* + * dapls_ib_post_ext_send + * + * Provider specific extended Post SEND function for atomics + * OP_COMP_AND_SWAP and OP_FETCH_AND_ADD + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_post_ext_send ( + IN DAPL_EP *ep_ptr, + IN ib_send_op_type_t op_type, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT num_segs, + IN DAT_LMR_TRIPLET *local_iov, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_UINT32 immed_data, + IN DAT_UINT64 compare_add, + IN DAT_UINT64 swap, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + ib_api_status_t ib_status; + ib_data_segment_t ds_array[DAPL_DEFAULT_DS_ENTRIES], *ds_array_p; + ib_send_wr_t wr, *bad_wr; + DAT_COUNT i, total_len; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: ep %p op %d ck %p sgs", + "%d l_iov %p r_iov %p f %d\n", + ep_ptr, op_type, cookie, num_segs, local_iov, + remote_iov, completion_flags); + + if (num_segs <= DAPL_DEFAULT_DS_ENTRIES) + ds_array_p = ds_array; + else + ds_array_p = dapl_os_alloc(num_segs*sizeof(ib_data_segment_t)); + + if (NULL == ds_array_p) + return (DAT_INSUFFICIENT_RESOURCES); + + /* setup the work request */ + dapl_os_memzero (&wr, sizeof(ib_send_wr_t)); + wr.wr_type = op_type; + wr.wr_id = (uint64_t)(uintptr_t)cookie; + wr.ds_array = ds_array_p; + + total_len = 0; + + for (i = 0; i < num_segs; i++ ) { + if ( !local_iov[i].segment_length ) + continue; + + ds_array_p->vaddr = (uint64_t) local_iov[i].virtual_address; + ds_array_p->length = local_iov[i].segment_length; + ds_array_p->lkey = local_iov[i].lmr_context; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: lkey 0x%x va %p len %d\n", + ds_array_p->lkey, ds_array_p->vaddr, + ds_array_p->length ); + + total_len += ds_array_p->length; + wr.num_ds++; + ds_array_p++; + } + + if (cookie != NULL) + cookie->val.dto.size = total_len; + + switch (op_type) { + + case WR_RDMA_WRITE: + if ( immed_data == 0 ) + break; + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: rkey 0x%x va %#016Lx immed=0x%x\n", + remote_iov->rmr_context, + remote_iov->virtual_address, immed_data); + + wr.immediate_data = immed_data; + wr.send_opt |= IB_SEND_OPT_IMMEDIATE; + wr.remote_ops.vaddr = remote_iov->virtual_address; + wr.remote_ops.rkey = remote_iov->rmr_context; + break; + + case WR_COMPARE_SWAP: + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: OP_COMP_AND_SWAP=%lx," + "%lx rkey 0x%x va %#016Lx\n", + compare_add, swap, remote_iov->rmr_context, + remote_iov->virtual_address); + + wr.remote_ops.vaddr = remote_iov->virtual_address; + wr.remote_ops.rkey = remote_iov->rmr_context; + wr.remote_ops.atomic1 = compare_add; + wr.remote_ops.atomic2 = swap; + break; + + case WR_FETCH_ADD: + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: OP_FETCH_AND_ADD=%lx," + "%lx rkey 0x%x va %#016Lx\n", + compare_add, remote_iov->rmr_context, + remote_iov->virtual_address); + + wr.remote_ops.vaddr = remote_iov->virtual_address; + wr.remote_ops.rkey = remote_iov->rmr_context; + break; + + default: + break; + } + + /* set completion flags in work request */ + wr.send_opt |= (DAT_COMPLETION_SUPPRESS_FLAG & + completion_flags) ? 0 : IB_SEND_OPT_SIGNALED; + wr.send_opt |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & + completion_flags) ? IB_SEND_OPT_FENCE : 0; + wr.send_opt |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & + completion_flags) ? IB_SEND_OPT_SOLICITED : 0; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", + wr.wr_type, wr.send_opt, wr.ds_array, wr.num_ds); + + ib_status = ib_post_send(ep_ptr->qp_handle, &wr, &bad_wr); + + if (num_segs > DAPL_DEFAULT_DS_ENTRIES) + dapl_os_free(wr.ds_array, num_segs * sizeof(ib_local_ds_t)); + + if ( ib_status == IB_SUCCESS ) + return DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsPS: EP=%p post_send status = %s\n", + ep_ptr, ib_get_err_str(ib_status)); + /* + * Moving QP to error state; + */ + ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle); + ep_ptr->qp_state = IB_QPS_ERROR; + + return (dapl_ib_status_convert (ib_status)); +} + +#endif /* DAT_EXTENSIONS */ + +#endif /* _DAPL_IBAL_DTO_H */ diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_extensions.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_extensions.c new file mode 100644 index 00000000..d1351f5a --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_extensions.c @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_extensions.c + * + * PURPOSE: Extensions routines for OpenIB-windows IBAL provider + * + * $Id: $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_ibal_util.h" +#include "dapl_ep_util.h" +#include "dapl_cookie.h" +#include + + +DAT_RETURN +dapli_post_ext( IN DAT_EP_HANDLE ep_handle, + IN DAT_UINT64 cmp_add, + IN DAT_UINT64 swap, + IN DAT_UINT32 immed_data, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN int op_type, + IN DAT_COMPLETION_FLAGS flags ); + + +/* + * dapl_extensions + * + * Process extension requests + * + * Input: + * ext_type, + * ... + * + * Output: + * Depends.... + * + * Returns: + * DAT_SUCCESS + * DAT_NOT_IMPLEMENTED + * ..... + * + */ +DAT_RETURN +dapl_extensions(IN DAT_HANDLE dat_handle, + IN DAT_EXTENDED_OP ext_op, + IN va_list args) +{ + DAT_EP_HANDLE ep; + DAT_LMR_TRIPLET *lmr_p; + DAT_DTO_COOKIE cookie; + const DAT_RMR_TRIPLET *rmr_p; + DAT_UINT64 dat_uint64a, dat_uint64b; + DAT_UINT32 dat_uint32; + DAT_COUNT segments = 1; + DAT_COMPLETION_FLAGS comp_flags; + DAT_RETURN status = DAT_NOT_IMPLEMENTED; + + dapl_dbg_log(DAPL_DBG_TYPE_API, + "dapl_extensions(hdl %p operation %d, ...)\n", + dat_handle, ext_op); + + DAPL_CNTR(DCNT_EXTENSION); + + switch ((int)ext_op) + { + + case DAT_IB_RDMA_WRITE_IMMED_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " WRITE_IMMED_DATA extension call\n"); + + ep = dat_handle; /* ep_handle */ + segments = va_arg( args, DAT_COUNT); /* num segments */ + lmr_p = va_arg( args, DAT_LMR_TRIPLET*); + cookie = va_arg( args, DAT_DTO_COOKIE); + rmr_p = va_arg( args, const DAT_RMR_TRIPLET*); + dat_uint32 = va_arg( args, DAT_UINT32); /* immed data */ + comp_flags = va_arg( args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, 0, 0, dat_uint32, segments, lmr_p, + cookie, rmr_p, OP_RDMA_WRITE_IMM, + comp_flags ); + break; + + case DAT_IB_CMP_AND_SWAP_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " CMP_AND_SWAP extension call\n"); + + ep = dat_handle; /* ep_handle */ + dat_uint64a = va_arg( args, DAT_UINT64); /* cmp_value */ + dat_uint64b = va_arg( args, DAT_UINT64); /* swap_value */ + lmr_p = va_arg( args, DAT_LMR_TRIPLET*); + cookie = va_arg( args, DAT_DTO_COOKIE); + rmr_p = va_arg( args, const DAT_RMR_TRIPLET*); + comp_flags = va_arg( args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, dat_uint64a, dat_uint64b, + 0, segments, lmr_p, cookie, rmr_p, + OP_COMP_AND_SWAP, comp_flags ); + break; + + case DAT_IB_FETCH_AND_ADD_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " FETCH_AND_ADD extension call\n"); + + ep = dat_handle; /* ep_handle */ + dat_uint64a = va_arg( args, DAT_UINT64); /* add value */ + lmr_p = va_arg( args, DAT_LMR_TRIPLET*); + cookie = va_arg( args, DAT_DTO_COOKIE); + rmr_p = va_arg( args, const DAT_RMR_TRIPLET*); + comp_flags = va_arg( args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, dat_uint64a, 0, 0, segments, + lmr_p, cookie, rmr_p, + OP_FETCH_AND_ADD, comp_flags ); + + break; + + default: + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + "unsupported extension(%d)\n", (int)ext_op); + } + + return(status); +} + + +DAT_RETURN +dapli_post_ext( IN DAT_EP_HANDLE ep_handle, + IN DAT_UINT64 cmp_add, + IN DAT_UINT64 swap, + IN DAT_UINT32 immed_data, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN int op_type, + IN DAT_COMPLETION_FLAGS flags ) +{ + DAPL_EP *ep_ptr; + ib_qp_handle_t qp_ptr; + DAPL_COOKIE *cookie; + DAT_RETURN dat_status = DAT_SUCCESS; + + dapl_dbg_log(DAPL_DBG_TYPE_API, + " post_ext_op: ep %p cmp_val %d " + "swap_val %d cookie 0x%x, r_iov %p, flags 0x%x\n", + ep_handle, (unsigned)cmp_add, (unsigned)swap, + (unsigned)user_cookie.as_64, remote_iov, flags); + + if (DAPL_BAD_HANDLE(ep_handle, DAPL_MAGIC_EP)) + return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); + + if ((NULL == remote_iov) || (NULL == local_iov)) + return DAT_INVALID_PARAMETER; + + ep_ptr = (DAPL_EP *) ep_handle; + qp_ptr = ep_ptr->qp_handle; + + /* + * Synchronization ok since this buffer is only used for send + * requests, which aren't allowed to race with each other. + * only if completion is expected + */ + if (!(DAT_COMPLETION_SUPPRESS_FLAG & flags)) { + + dat_status = dapls_dto_cookie_alloc( + &ep_ptr->req_buffer, + DAPL_DTO_TYPE_EXTENSION, + user_cookie, + &cookie ); + + if ( dat_status != DAT_SUCCESS ) + goto bail; + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc(&ep_ptr->req_count); +dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> %s() EP %lx req_count %d\n",__FUNCTION__, + ep_ptr, dapl_os_atomic_read(&ep_ptr->req_count)); + + } + + /* + * Invoke provider specific routine to post DTO + */ + dat_status = dapls_ib_post_ext_send(ep_ptr, + op_type, + cookie, + segments, /* data segments */ + local_iov, + remote_iov, + immed_data, /* immed data */ + cmp_add, /* compare or add */ + swap, /* swap */ + flags); + + if (dat_status != DAT_SUCCESS) { + if ( cookie != NULL ) { + dapl_os_atomic_dec(&ep_ptr->req_count); + dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie); + } + } + +bail: + return dat_status; + +} + + +/* + * New provider routine to process extended DTO events + */ +void +dapls_cqe_to_event_extension(IN DAPL_EP *ep_ptr, + IN DAPL_COOKIE *cookie, + IN ib_work_completion_t *cqe_ptr, + IN DAT_EVENT *event_ptr) +{ + uint32_t ibtype; + DAT_DTO_COMPLETION_EVENT_DATA *dto = + &event_ptr->event_data.dto_completion_event_data; + DAT_IB_EXTENSION_EVENT_DATA *ext_data = + (DAT_IB_EXTENSION_EVENT_DATA *) + &event_ptr->event_extension_data[0]; + DAT_DTO_COMPLETION_STATUS dto_status; + + /* Get status from cqe */ + dto_status = dapls_ib_get_dto_status(cqe_ptr); + + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: dto_ptr %p ext_ptr %p status %d\n", + dto, ext_data, dto_status); + + event_ptr->event_number = DAT_IB_DTO_EVENT; + dto->ep_handle = cookie->ep; + dto->user_cookie = cookie->val.dto.cookie; + dto->operation = DAPL_GET_CQE_DTOS_OPTYPE(cqe_ptr); /* new for 2.0 */ + dto->status = ext_data->status = dto_status; + + if (dto_status != DAT_DTO_SUCCESS) + return; + + /* + * Get operation type from CQ work completion entry and + * if extented operation then set extended event data + */ + ibtype = DAPL_GET_CQE_OPTYPE(cqe_ptr); + + switch (ibtype) { + + case OP_RDMA_WRITE_IMM: + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: OP_RDMA_WRITE_IMMED\n"); + + /* type and outbound rdma write transfer size */ + dto->transfered_length = cookie->val.dto.size; + ext_data->type = DAT_IB_RDMA_WRITE_IMMED; + break; + + case OP_RECEIVE_IMM: + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: OP_RECEIVE_RDMA_IMMED\n"); + + /* immed recvd, type and inbound rdma write transfer size */ + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + ext_data->type = DAT_IB_RDMA_WRITE_IMMED_DATA; + ext_data->val.immed.data = DAPL_GET_CQE_IMMED_DATA(cqe_ptr); + break; + + case OP_COMP_AND_SWAP: + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: COMP_AND_SWAP_RESP\n"); + + /* original data is returned in LMR provided with post */ + ext_data->type = DAT_IB_CMP_AND_SWAP; + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + break; + + case OP_FETCH_AND_ADD: + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: FETCH_AND_ADD_RESP\n"); + + /* original data is returned in LMR provided with post */ + ext_data->type = DAT_IB_FETCH_AND_ADD; + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + break; + + default: + /* not extended operation */ + ext_data->status = DAT_IB_OP_ERR; + dto->status = DAT_DTO_ERR_TRANSPORT; + break; + } +} + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_kmod.h b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_kmod.h new file mode 100644 index 00000000..086ec7ae --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_kmod.h @@ -0,0 +1,91 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_kmod.h + * + * PURPOSE: Utility defs & routines for access to Intel IBAL APIs + * + * $Id: dapl_ibal_kmod.h 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#ifndef _DAPL_IBAL_KMOD_H_ +#define _DAPL_IBAL_KMOD_H_ + +#include +#include +#include + +#define MVDAPL_DEV_KEY 'm' +#define MVDAPL_GET_ANY_SVID _IO ( MVDAPL_DEV_KEY, psp_get_any_svid ) +#define MVDAPL_MRDB_RECORD_INSERT _IO ( MVDAPL_DEV_KEY, mrdb_record_insert ) +#define MVDAPL_MRDB_RECORD_DEC _IO ( MVDAPL_DEV_KEY, mrdb_record_dec ) +#define MVDAPL_MRDB_RECORD_QUERY _IO ( MVDAPL_DEV_KEY, mrdb_record_query ) +#define MVDAPL_MRDB_RECORD_UPDATE _IO ( MVDAPL_DEV_KEY, mrdb_record_update ) + +typedef enum +{ + psp_get_any_svid, + mrdb_record_insert, + mrdb_record_dec, + mrdb_record_query, + mrdb_record_update, + mvdapl_max_ops +} mvdapl_dev_ops_t; + +typedef struct _mvdapl_user_ctx +{ + cl_spinlock_t oust_mrdb_lock; + cl_qlist_t oust_mrdb_head; +} mvdapl_user_ctx_t; + + +typedef struct _mvdapl_ca_t +{ + cl_spinlock_t mrdb_lock; + cl_qlist_t mrdb_head; + boolean_t initialized; + cl_dev_handle_t mrdb_dev_handle; + ib_net64_t ca_guid; +} mvdapl_ca_t; + + +typedef struct _mvdapl_root +{ + ib_al_handle_t h_al; + intn_t guid_count; + mvdapl_ca_t *mvdapl_ca_tbl; + +} mvdapl_root_t; + +typedef struct _mrdb_record_t +{ + cl_list_item_t next; + ib_lmr_cookie_t key_cookie; + void *mr_handle; + int ib_shmid; + uint32_t ref_count; + boolean_t initialized; + cl_spinlock_t record_lock; +} mrdb_record_t; + + +typedef struct _oust_mrdb_rec +{ + cl_list_item_t next; + mrdb_record_t *p_record; + uint32_t ref_count; +} oust_mrdb_rec_t; + + +#endif /* _DAPL_IBAL_KMOD_H_ */ diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.c new file mode 100644 index 00000000..e19bb871 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.c @@ -0,0 +1,392 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_mrdb.c + * + * PURPOSE: Utility routines for access to IBAL APIs + * + * $Id: dapl_ibal_mrdb.c 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_ibal_kmod.h" +#include "dapl_ibal_mrdb.h" + +DAT_RETURN dapls_mrdb_init ( + IN DAPL_HCA *hca_ptr) +{ + cl_status_t cl_status; + char name[32]; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsMI"); + return DAT_INVALID_PARAMETER; + } + + sprintf (name, + "/dev/mvdapl%x", + (uint32_t) cl_ntoh64 (p_ca->p_ca_attr->ca_guid)); + + cl_status = cl_open_device ( (cl_dev_name_t) name, &p_ca->mlnx_device); + + if (cl_status != CL_SUCCESS) + { + /* dapl_dbg_log ( DAPL_DBG_TYPE_UTIL, + "--> DsMI: Init MRDB failed = 0x%x\n", cl_status); */ + p_ca->mlnx_device = 0; + } + + return DAT_SUCCESS; +} + + +DAT_RETURN dapls_mrdb_exit ( + IN DAPL_HCA *hca_ptr) +{ + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsME"); + return DAT_INVALID_PARAMETER; + } + + if (p_ca->mlnx_device) + { + cl_close_device (p_ca->mlnx_device); + } + + return DAT_SUCCESS; +} + + +DAT_RETURN dapls_mrdb_record_insert ( + IN DAPL_HCA *hca_ptr, + IN DAT_LMR_COOKIE shared_mem_id, + OUT int *p_ib_shmid) +{ + cl_status_t cl_status; + mrdb_rec_insert_ioctl_t ioctl_buf; + uintn_t bytes_ret; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsMRI"); + return DAT_INVALID_PARAMETER; + } + + bytes_ret = 0; + cl_memclr (&ioctl_buf, sizeof (ioctl_buf)); + cl_memcpy (ioctl_buf.shared_mem_id, shared_mem_id, IBAL_LMR_COOKIE_SIZE); + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRI: MVDAPL_MRDB_REC_INSERT mem_cookie %p\n", + shared_mem_id); +#if defined(DAPL_DBG) + { + int i; + char *c = (char *) shared_mem_id; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRI: mem_cookie: \n"); + + for ( i = 0; i < IBAL_LMR_COOKIE_SIZE ; i++) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "0x%x ", *(c+i)); + } + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "\n"); + + } +#endif /* DAPL_DBG */ + + cl_status = cl_ioctl_device ( p_ca->mlnx_device, + MVDAPL_MRDB_RECORD_INSERT, + &ioctl_buf, + sizeof (mrdb_rec_insert_ioctl_t), + &bytes_ret); + if ((cl_status != CL_SUCCESS) || + (ioctl_buf.status == IB_INSUFFICIENT_MEMORY)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsMRI: Failed to IOCTL record_insert 0x%x\n", cl_status); + return DAT_INSUFFICIENT_RESOURCES; + } + + *p_ib_shmid = (int) ioctl_buf.inout_f; + + if (ioctl_buf.status == IB_ERROR) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRI: There is a record with shmid 0x%x\n", + *p_ib_shmid); + return DAT_INVALID_STATE; + } + else + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRI: Insert new mrdb record with shmid 0x%x\n", + *p_ib_shmid); + } + + return DAT_SUCCESS; +} + +DAT_RETURN dapls_mrdb_record_dec ( + IN DAPL_HCA *hca_ptr, + IN DAT_LMR_COOKIE shared_mem_id) +{ + cl_status_t cl_status; + mrdb_rec_dec_ioctl_t ioctl_buf; + uintn_t bytes_ret; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsMRD"); + return DAT_INVALID_PARAMETER; + } + + bytes_ret = 0; + cl_memclr (&ioctl_buf, sizeof (ioctl_buf)); + cl_memcpy (ioctl_buf.shared_mem_id, shared_mem_id, IBAL_LMR_COOKIE_SIZE); + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRD: MVDAPL_MRDB_REC_DEC mem_cookie 0x%p\n", + shared_mem_id); +#if defined(DAPL_DBG) + { + int i; + char *c = (char *) shared_mem_id; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRD: mem_cookie: \n"); + + for ( i = 0; i < IBAL_LMR_COOKIE_SIZE ; i++) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "0x%x ", *(c+i)); + } + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "\n"); + + } +#endif /* DAPL_DBG */ + + cl_status = cl_ioctl_device ( p_ca->mlnx_device, + MVDAPL_MRDB_RECORD_DEC, + &ioctl_buf, + sizeof (mrdb_rec_dec_ioctl_t), + &bytes_ret); + if ((cl_status != CL_SUCCESS) || + (ioctl_buf.status != IB_SUCCESS)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsMRD: IOCTL failed 'cause there is no record %s\n", + ib_get_err_str(ioctl_buf.status)); + return DAT_INVALID_STATE; + } + + return DAT_SUCCESS; +} + +DAT_RETURN dapls_mrdb_record_update ( + IN DAPL_HCA *hca_ptr, + IN DAT_LMR_COOKIE shared_mem_id, + IN ib_mr_handle_t mr_handle) +{ + cl_status_t cl_status; + mrdb_rec_update_ioctl_t ioctl_buf; + uintn_t bytes_ret; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsMRU"); + return DAT_INVALID_PARAMETER; + } + + bytes_ret = 0; + cl_memclr (&ioctl_buf, sizeof (ioctl_buf)); + cl_memcpy (ioctl_buf.shared_mem_id, shared_mem_id, IBAL_LMR_COOKIE_SIZE); + ioctl_buf.mr_handle = mr_handle; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRU: MVDAPL_MRDB_REC_UPDATE mr_handle %p\n", mr_handle); +#if defined(DAPL_DBG) + { + int i; + char *c = (char *) shared_mem_id; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRU: mem_cookie: \n"); + + for ( i = 0; i < IBAL_LMR_COOKIE_SIZE ; i++) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "0x%x ", *(c+i)); + } + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "\n"); + + } +#endif /* DAPL_DBG */ + + cl_status = cl_ioctl_device ( p_ca->mlnx_device, + MVDAPL_MRDB_RECORD_UPDATE, + &ioctl_buf, + sizeof (mrdb_rec_update_ioctl_t), + &bytes_ret); + if ((cl_status != CL_SUCCESS) || + (ioctl_buf.status != IB_SUCCESS)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsMRU: IOCTL update_record failed %s\n", + ib_get_err_str(ioctl_buf.status)); + return DAT_INTERNAL_ERROR; + } + + return DAT_SUCCESS; +} + + +DAT_RETURN dapls_mrdb_record_query ( + IN DAPL_HCA *hca_ptr, + IN DAT_LMR_COOKIE shared_mem_id, + OUT int *p_ib_shmid, + OUT ib_mr_handle_t *p_mr_handle) +{ + cl_status_t cl_status; + mrdb_rec_query_ioctl_t ioctl_buf; + uintn_t bytes_ret; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsMRQ"); + return DAT_INVALID_PARAMETER; + } + + bytes_ret = 0; + cl_memclr (&ioctl_buf, sizeof (ioctl_buf)); + + cl_memcpy (ioctl_buf.shared_mem_id, shared_mem_id, IBAL_LMR_COOKIE_SIZE); + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRQ: MVDAPL_MRDB_REC_QUERY mem_cookie 0x%p\n", + shared_mem_id); + #if defined(DAPL_DBG) + { + int i; + char *c = (char *) shared_mem_id; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRQ: mem_cookie: \n"); + + for ( i = 0; i < IBAL_LMR_COOKIE_SIZE ; i++) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "0x%x ", *(c+i)); + } + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "\n"); + + } + #endif + + cl_status = cl_ioctl_device ( p_ca->mlnx_device, + MVDAPL_MRDB_RECORD_QUERY, + &ioctl_buf, + sizeof (mrdb_rec_query_ioctl_t), + &bytes_ret); + if ((cl_status != CL_SUCCESS) || + (ioctl_buf.status != IB_SUCCESS)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsMRQ: IOCTL query_record failed %s\n", + ib_get_err_str(ioctl_buf.status)); + return DAT_INTERNAL_ERROR; + } + + *p_mr_handle = ioctl_buf.mr_handle; + *p_ib_shmid = (int) ioctl_buf.inout_f; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsMRQ: MVDAPL_MRDB_REC_QUERY mr_handle 0x%p shmid 0x%x\n", + *p_mr_handle, *p_ib_shmid); + + return DAT_SUCCESS; +} + + +DAT_RETURN dapls_ib_get_any_svid ( + IN DAPL_HCA *hca_ptr, + OUT DAT_CONN_QUAL *p_svid) +{ + cl_status_t cl_status; + psp_get_any_svid_ioctl_t ioctl_buf; + uintn_t bytes_ret; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: p_ca is NULL\n","DsPGAS"); + return DAT_INVALID_PARAMETER; + } + + bytes_ret = 0; + cl_memclr (&ioctl_buf, sizeof (ioctl_buf)); + + cl_status = cl_ioctl_device ( p_ca->mlnx_device, + MVDAPL_GET_ANY_SVID, + &ioctl_buf, + sizeof (psp_get_any_svid_ioctl_t), + &bytes_ret); + if ((cl_status != CL_SUCCESS) || + (ioctl_buf.status != IB_SUCCESS)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsMRQ: IOCTL query_record failed %s\n", + ib_get_err_str(ioctl_buf.status)); + return DAT_INTERNAL_ERROR; + } + + *p_svid = (DAT_CONN_QUAL) ioctl_buf.inout_f; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsPGAS: new ServiceID 0x%x\n", + *p_svid); + + return DAT_SUCCESS; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.h b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.h new file mode 100644 index 00000000..39dde9ba --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_mrdb.h @@ -0,0 +1,52 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_mrdb.h + * + * PURPOSE: Utility defs & routines for access to Intel IBAL APIs + * + * $Id: dapl_ibal_mrdb.h 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#ifndef _DAPL_IBAL_MRDB_H_ +#define _DAPL_IBAL_MRDB_H_ + +#include +#include + +#define MVDAPL_BASE_SHMID 0xF00 +#define MVDAPL_BASE_SVID 0xF +#define MVDAPL_MAX_SHMID 0xFFFFFFFF +#define MVDAPL_MAX_SVID 0xEFFFFFFF + +#define IBAL_LMR_COOKIE_SIZE 40 +typedef char (* ib_lmr_cookie_t)[IBAL_LMR_COOKIE_SIZE]; + +typedef struct _mrdb_record_ioctl +{ + char *shared_mem_id[IBAL_LMR_COOKIE_SIZE]; + void *mr_handle; + ib_net64_t inout_f; + ib_api_status_t status; +} mrdb_record_ioctl_t; + +typedef mrdb_record_ioctl_t mrdb_rec_dec_ioctl_t; +typedef mrdb_record_ioctl_t mrdb_rec_insert_ioctl_t; +typedef mrdb_record_ioctl_t mrdb_rec_query_ioctl_t; +typedef mrdb_record_ioctl_t mrdb_rec_update_ioctl_t; +typedef mrdb_record_ioctl_t psp_get_any_svid_ioctl_t; + + +#endif /* _DAPL_IBAL_MRDB_H_ */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_name_service.h b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_name_service.h new file mode 100644 index 00000000..9ebb255a --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_name_service.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under either one of the following two licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * OR + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * Licensee has the right to choose either one of the above two licenses. + * + * Redistributions of source code must retain both the above copyright + * notice and either one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, either one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_name_service.h + * + * PURPOSE: Utility defs & routines supporting name services + * + * $Id: dapl_name_service.h 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * Prototypes for name service routines + */ + +//DAT_RETURN dapls_ns_init (void); + +#ifdef NO_NAME_SERVICE + +DAT_RETURN dapls_ns_lookup_address ( + IN DAPL_IA *ia_ptr, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + OUT GID *gid); + + +#else + +DAT_RETURN dapls_ns_create_gid_map(DAPL_HCA *hca_ptr); +DAT_RETURN dapls_ns_remove_gid_map(DAPL_HCA *hca_ptr); + +DAT_RETURN dapls_ns_map_gid ( + IN DAPL_HCA *hca_ptr, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + OUT GID *gid); + +DAT_RETURN dapls_ns_map_ipaddr ( + IN DAPL_HCA *hca_ptr, + IN GID gid, + OUT DAT_IA_ADDRESS_PTR remote_ia_address); + +#endif /* NO_NAME_SERVICE */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c new file mode 100644 index 00000000..acc3be40 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c @@ -0,0 +1,692 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_qp.c + * + * PURPOSE: IB QP routines for access to IBAL APIs + * + * $Id: dapl_ibal_qp.c 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_ibal_util.h" + +#define DAPL_IBAL_QKEY 0 +#define DAPL_IBAL_START_PSN 0 + +extern DAT_RETURN +dapls_ib_cq_late_alloc ( IN ib_pd_handle_t pd_handle, + IN DAPL_EVD *evd_ptr ); + +static void +dapli_ib_qp_async_error_cb( + IN ib_async_event_rec_t* p_err_rec ) +{ + DAPL_EP *ep_ptr = (DAPL_EP *)p_err_rec->context; + DAPL_EVD *evd_ptr; + DAPL_IA *ia_ptr; + dapl_ibal_ca_t *p_ca; + dapl_ibal_evd_cb_t *evd_cb; + + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiQpAEC QP error %d for qp context %p\n", + p_err_rec->code, p_err_rec->context); + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiQpAEC qp_handle %p qpn %u\n", + ((DAPL_EP *)p_err_rec->context)->qp_handle, + ((DAPL_EP *)p_err_rec->context)->qpn); + + /* + * Verify handles EP, EVD, and hca_handle + */ + if ( DAPL_BAD_HANDLE (ep_ptr, DAPL_MAGIC_EP ) || + DAPL_BAD_HANDLE (ep_ptr->param.connect_evd_handle, DAPL_MAGIC_EVD) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiQpAEC: invalid EP %p \n", ep_ptr); + return; + } + ia_ptr = ep_ptr->header.owner_ia; + evd_ptr = ia_ptr->async_error_evd; + + if (DAPL_BAD_HANDLE (evd_ptr, DAPL_MAGIC_EVD) || + ! (evd_ptr->evd_flags & DAT_EVD_ASYNC_FLAG)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiQpAEC: invalid EVD %p \n", evd_ptr); + return; + } + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiQpAEC: can't find %s HCA\n", + (ia_ptr->header.provider)->device_name); + return; + } + + /* find QP error callback using ia_ptr for context */ + evd_cb = dapli_find_evd_cb_by_context (ia_ptr, p_ca); + if ((evd_cb == NULL) || (evd_cb->pfn_async_qp_err_cb == NULL)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> DiQpAEC: no ERROR cb on %p found \n", p_ca); + return; + } + + dapl_os_lock (&ep_ptr->header.lock); + ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECT_PENDING; + dapl_os_unlock (&ep_ptr->header.lock); + + /* force disconnect, QP error state, to insure DTO's get flushed */ + dapls_ib_disconnect ( ep_ptr, DAT_CLOSE_ABRUPT_FLAG ); + + /* maps to dapl_evd_qp_async_error_callback(), context is EP */ + evd_cb->pfn_async_qp_err_cb( (ib_hca_handle_t)p_ca, + (ib_error_record_t*)&p_err_rec->code, ep_ptr); +} + +/* + * dapl_ib_qp_alloc + * + * Alloc a QP + * + * Input: + * *ia_ptr pointer to DAPL IA + * *ep_ptr pointer to DAPL EP + * *ep_ctx_ptr pointer to DAPL EP context + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_qp_alloc ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr, + IN DAPL_EP *ep_ctx_ptr) +{ + DAT_EP_ATTR *attr; + DAPL_EVD *recv_evd_ptr, *request_evd_ptr; + DAT_RETURN dat_status; + ib_api_status_t ib_status; + ib_qp_create_t qp_create; + ib_pd_handle_t ib_pd_handle; + ib_cq_handle_t cq_recv; + ib_cq_handle_t cq_send; + dapl_ibal_ca_t *p_ca; + dapl_ibal_port_t *p_active_port; + ib_qp_attr_t qp_attr; + + attr = &ep_ptr->param.ep_attr; + + dapl_os_assert ( ep_ptr->param.pz_handle != NULL ); + + ib_pd_handle = ((DAPL_PZ *)ep_ptr->param.pz_handle)->pd_handle; + dapl_os_assert(ib_pd_handle); + recv_evd_ptr = (DAPL_EVD *) ep_ptr->param.recv_evd_handle; + request_evd_ptr = (DAPL_EVD *) ep_ptr->param.request_evd_handle; + + cq_recv = IB_INVALID_HANDLE; + cq_send = IB_INVALID_HANDLE; + + dapl_os_assert ( recv_evd_ptr != DAT_HANDLE_NULL ); + { + cq_recv = (ib_cq_handle_t) recv_evd_ptr->ib_cq_handle; + + if ((cq_recv == IB_INVALID_HANDLE) && + ( 0 != (recv_evd_ptr->evd_flags & ~DAT_EVD_SOFTWARE_FLAG) )) + { + dat_status = dapls_ib_cq_late_alloc ( ib_pd_handle, recv_evd_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> %s: failed to create CQ\n","DsQA"); + return (dat_status); + } + + dat_status = dapls_set_cq_notify (ia_ptr, recv_evd_ptr); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> %s: failed to enable notify CQ\n","DsQA"); + return (dat_status); + } + + cq_recv = (ib_cq_handle_t) recv_evd_ptr->ib_cq_handle; + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsQA: alloc_recv_CQ = %p\n", cq_recv); + + } + } + + dapl_os_assert ( request_evd_ptr != DAT_HANDLE_NULL ); + { + cq_send = (ib_cq_handle_t) request_evd_ptr->ib_cq_handle; + + if ((cq_send == IB_INVALID_HANDLE) && + ( 0 != (request_evd_ptr->evd_flags & ~DAT_EVD_SOFTWARE_FLAG) )) + { + dat_status = dapls_ib_cq_late_alloc (ib_pd_handle, request_evd_ptr); + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> %s: failed to create CQ\n","DsQA"); + return (dat_status); + } + + dat_status = dapls_set_cq_notify (ia_ptr, request_evd_ptr); + + if (dat_status != DAT_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> %s: failed to enable notify CQ\n","DsQA"); + return (dat_status); + } + + cq_send = (ib_cq_handle_t) request_evd_ptr->ib_cq_handle; + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsQA: alloc_send_CQ = %p\n", cq_send); + } + } + + /* + * Get the CA structure + */ + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + + dapl_os_memzero (&qp_create, sizeof (qp_create)); + qp_create.qp_type = IB_QPT_RELIABLE_CONN; + qp_create.sq_depth = attr->max_request_dtos; + qp_create.rq_depth = attr->max_recv_dtos; + qp_create.sq_sge = attr->max_recv_iov; + qp_create.rq_sge = attr->max_request_iov; + qp_create.h_sq_cq = cq_send; + qp_create.h_rq_cq = cq_recv; + qp_create.sq_signaled = FALSE; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsQA: sqd,iov=%d,%d rqd,iov=%d,%d\n", + attr->max_request_dtos, attr->max_request_iov, + attr->max_recv_dtos, attr->max_recv_iov); + + ib_status = ib_create_qp ( + ib_pd_handle, + &qp_create, + (void *) ep_ctx_ptr /* context */, + dapli_ib_qp_async_error_cb, + &ep_ptr->qp_handle); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsQA: Create QP failed = %s\n", + ib_get_err_str(ib_status)); + return (DAT_INSUFFICIENT_RESOURCES); + } + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsQA: EP=%p, tEVD=%p, rEVD=%p QP=%p\n", + ep_ptr, ep_ptr->param.request_evd_handle, + ep_ptr->param.recv_evd_handle, + ep_ptr->qp_handle ); + + ep_ptr->qp_state = IB_QPS_RESET; + + p_active_port = dapli_ibal_get_port(p_ca,(uint8_t)ia_ptr->hca_ptr->port_num); + + if (NULL == p_active_port) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DsQA: Port %d is not available = %d\n", + ia_ptr->hca_ptr->port_num, __LINE__); + return (DAT_INVALID_STATE); + } + + ib_status = dapls_modify_qp_state_to_init ( ep_ptr->qp_handle, + &ep_ptr->param.ep_attr, + p_active_port ); + + if ( ib_status != IB_SUCCESS ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DsQA: Change QP state to INIT failed = %s\n", + ib_get_err_str(ib_status)); + return (DAT_INVALID_HANDLE); + } + ib_status = ib_query_qp ( ep_ptr->qp_handle, &qp_attr ); + + ep_ptr->qp_state = qp_attr.state; + ep_ptr->qpn = qp_attr.num; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsQAQA: EP:%p new_QP %p state %s\n", + ep_ptr, + ep_ptr->qp_handle, + ib_get_port_state_str(ep_ptr->qp_state)); + + return (DAT_SUCCESS); +} + + +/* + * dapl_ib_qp_free + * + * Free a QP + * + * Input: + * *ia_ptr pointer to IA structure + * *ep_ptr pointer to EP structure + * + * Output: + * none + * + * Returns: + * none + * + */ +DAT_RETURN +dapls_ib_qp_free ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr ) +{ + + ib_qp_handle_t qp_handle; + UNREFERENCED_PARAMETER(ia_ptr); + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsQF: free %p, state %s\n", + ep_ptr->qp_handle, + ib_get_port_state_str(ep_ptr->qp_state)); + + if (( ep_ptr->qp_handle != IB_INVALID_HANDLE ) && + ( ep_ptr->qp_state != DAPL_QP_STATE_UNATTACHED )) + { + qp_handle = ep_ptr->qp_handle; + ep_ptr->qp_handle = IB_INVALID_HANDLE; + ep_ptr->qp_state = DAPL_QP_STATE_UNATTACHED; + ib_destroy_qp ( qp_handle, NULL /* callback */); + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsQF: freed QP %p\n", qp_handle ); + } + + return DAT_SUCCESS; +} + + +/* + * dapl_ib_qp_modify + * + * Set the QP to the parameters specified in an EP_PARAM + * + * We can't be sure what state the QP is in so we first obtain the state + * from the driver. The EP_PARAM structure that is provided has been + * sanitized such that only non-zero values are valid. + * + * Input: + * *ia_ptr pointer to DAPL IA + * *ep_ptr pointer to DAPL EP + * *ep_attr pointer to DAT EP attribute + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_qp_modify ( + IN DAPL_IA *ia_ptr, + IN DAPL_EP *ep_ptr, + IN DAT_EP_ATTR *ep_attr ) +{ + ib_qp_attr_t qp_attr; + ib_api_status_t ib_status; + ib_qp_handle_t qp_handle; + ib_qp_state_t qp_state; + ib_qp_mod_t qp_mod; + ib_av_attr_t *p_av_attr; + ib_qp_opts_t *p_qp_opts; + uint32_t *p_sq_depth, *p_rq_depth; + DAT_BOOLEAN need_modify; + DAT_RETURN dat_status; + + qp_handle = ep_ptr->qp_handle; + need_modify = DAT_FALSE; + dat_status = DAT_SUCCESS; + if ( ia_ptr == NULL || ia_ptr->header.magic != DAPL_MAGIC_IA ) + { + dat_status = DAT_INVALID_HANDLE; + goto bail; + } + /* + * Query the QP to get the current state */ + ib_status = ib_query_qp ( qp_handle, &qp_attr ); + + if ( ib_status != IB_SUCCESS ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsIQM: Query QP failed = %s\n", ib_get_err_str(ib_status)); + dat_status = DAT_INTERNAL_ERROR; + goto bail; + } + + qp_state = qp_attr.state; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsIQM: modify qp state=%d\n",qp_state); + /* + * Check if we have the right qp_state or not + */ + if ( (qp_state != IB_QPS_RTR ) && + (qp_state != IB_QPS_RTS ) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsIQM: postpone to modify qp to EP values later\n"); + dat_status = DAT_SUCCESS; + goto bail; + } + + dapl_os_memzero (&qp_mod, sizeof (ib_qp_mod_t)); + + if (qp_state == IB_QPS_RTR) + { + p_av_attr = &qp_mod.state.rtr.primary_av; + p_qp_opts = &qp_mod.state.rtr.opts; + p_sq_depth = &qp_mod.state.rtr.sq_depth; + p_rq_depth = &qp_mod.state.rtr.rq_depth; + } + else + { + /* + * RTS does not have primary_av field + */ + p_av_attr = &qp_mod.state.rts.alternate_av; + p_qp_opts = &qp_mod.state.rts.opts; + p_sq_depth = &qp_mod.state.rts.sq_depth; + p_rq_depth = &qp_mod.state.rts.rq_depth; + } + + if ( (ep_attr->max_recv_dtos > 0) && + ((DAT_UINT32)ep_attr->max_recv_dtos != qp_attr.rq_depth) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsIQM: rq_depth modified (%d,%d)\n", + qp_attr.rq_depth, ep_attr->max_recv_dtos); + + *p_rq_depth = ep_attr->max_recv_dtos; + *p_qp_opts |= IB_MOD_QP_RQ_DEPTH; + need_modify = DAT_TRUE; + } + + if ( (ep_attr->max_request_dtos > 0) && + ((DAT_UINT32)ep_attr->max_request_dtos != qp_attr.sq_depth) ) + { + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsIQM: sq_depth modified (%d,%d)\n", + qp_attr.sq_depth, ep_attr->max_request_dtos); + + *p_sq_depth = ep_attr->max_request_dtos; + *p_qp_opts |= IB_MOD_QP_SQ_DEPTH; + need_modify = DAT_TRUE; + } + + qp_mod.req_state = qp_state; + + if ( need_modify == DAT_TRUE ) + { + ib_status = ib_modify_qp (qp_handle, &qp_mod); + + if ( ib_status != IB_SUCCESS) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: ib_status = %d\n", + "DsIQM", ib_status); + dat_status = DAT_INTERNAL_ERROR; + } + } + +bail: + + return dat_status; +} + + +ib_api_status_t +dapls_modify_qp_state_to_error ( ib_qp_handle_t qp_handle ) +{ + ib_qp_mod_t qp_mod; + ib_api_status_t ib_status; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsIQM_Err: QP state change\n"); + + dapl_os_memzero (&qp_mod, sizeof (ib_qp_mod_t)); + + qp_mod.req_state = IB_QPS_ERROR; + + ib_status = ib_modify_qp (qp_handle, &qp_mod); + + return (ib_status); +} + + +ib_api_status_t +dapls_modify_qp_state_to_reset ( ib_qp_handle_t qp_handle ) +{ + ib_qp_mod_t qp_mod; + ib_api_status_t ib_status; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsIQM_RESET: QP state change\n"); + + dapl_os_memzero (&qp_mod, sizeof (ib_qp_mod_t)); + + qp_mod.req_state = IB_QPS_RESET; + + ib_status = ib_modify_qp (qp_handle, &qp_mod); + + return (ib_status); +} + + +ib_api_status_t +dapls_modify_qp_state_to_init ( + IN ib_qp_handle_t qp_handle, + IN DAT_EP_ATTR *p_attr, + IN dapl_ibal_port_t *p_port ) +{ + ib_qp_mod_t qp_mod; + ib_api_status_t ib_status; + + dapl_os_memzero (&qp_mod, sizeof (ib_qp_mod_t)); + + qp_mod.req_state = IB_QPS_INIT; + qp_mod.state.init.primary_port = p_port->p_attr->port_num; + qp_mod.state.init.qkey = DAPL_IBAL_QKEY; + qp_mod.state.init.pkey_index = 0; + qp_mod.state.init.access_ctrl = + IB_AC_LOCAL_WRITE|IB_AC_RDMA_WRITE|IB_AC_MW_BIND; + if ((p_attr->max_rdma_read_in > 0) || + (p_attr->max_rdma_read_out > 0)) + { + qp_mod.state.init.access_ctrl |= IB_AC_RDMA_READ; + } + ib_status = ib_modify_qp (qp_handle, &qp_mod); + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsIQM_INIT: QP(%p) state change, %s\n", + qp_handle, ib_get_err_str(ib_status)); + + return (ib_status); +} + +ib_api_status_t +dapls_modify_qp_state_to_rtr ( + ib_qp_handle_t qp_handle, + ib_net32_t dest_qp, + ib_lid_t dest_lid, + dapl_ibal_port_t *p_port) +{ + ib_qp_mod_t qp_mod; + ib_api_status_t ib_status; + + dapl_os_memzero (&qp_mod, sizeof (ib_qp_mod_t)); + + qp_mod.req_state = IB_QPS_RTR; + qp_mod.state.rtr.rq_psn = DAPL_IBAL_START_PSN; + qp_mod.state.rtr.dest_qp = dest_qp; + qp_mod.state.rtr.resp_res = 7; + qp_mod.state.rtr.rnr_nak_timeout = 7; + qp_mod.state.rtr.primary_av.sl = 0; + qp_mod.state.rtr.primary_av.dlid = dest_lid; + qp_mod.state.rtr.primary_av.port_num = p_port->p_attr->port_num; + qp_mod.state.rtr.primary_av.grh_valid = 0; /* FALSE */ + qp_mod.state.rtr.primary_av.path_bits = 0; + qp_mod.state.rtr.primary_av.static_rate = IB_PATH_RECORD_RATE_10_GBS; + qp_mod.state.rtr.primary_av.conn.path_mtu = p_port->p_attr->mtu; + qp_mod.state.rtr.primary_av.conn.rnr_retry_cnt = 7; + qp_mod.state.rtr.primary_av.conn.local_ack_timeout = 7; + qp_mod.state.rtr.primary_av.conn.seq_err_retry_cnt = 7; + + qp_mod.state.rtr.opts = IB_MOD_QP_PRIMARY_AV | IB_MOD_QP_RESP_RES; + + ib_status = ib_modify_qp (qp_handle, &qp_mod); + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsIQM_RTR: QP(%p) state change %s\n", + qp_handle, ib_get_err_str(ib_status)); + + return (ib_status); +} + +ib_api_status_t +dapls_modify_qp_state_to_rts ( ib_qp_handle_t qp_handle ) +{ + ib_qp_mod_t qp_mod; + ib_api_status_t ib_status; + + dapl_os_memzero (&qp_mod, sizeof (ib_qp_mod_t)); + + qp_mod.req_state = IB_QPS_RTS; + qp_mod.state.rts.sq_psn = DAPL_IBAL_START_PSN; + qp_mod.state.rts.retry_cnt = 7; + qp_mod.state.rts.rnr_retry_cnt = 6; + qp_mod.state.rts.rnr_nak_timeout = 7; + qp_mod.state.rts.local_ack_timeout = 7; + qp_mod.state.rts.init_depth = 4; + + ib_status = ib_modify_qp (qp_handle, &qp_mod); + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsIQM_RTS: QP(%p) state change %s\n", + qp_handle, ib_get_err_str(ib_status)); + + return (ib_status); +} + + +/* + * dapls_ib_reinit_ep + * + * Move the QP to INIT state again. + * + * Input: + * ep_ptr DAPL_EP + * + * Output: + * none + * + * Returns: + * void + * + */ +void +dapls_ib_reinit_ep ( IN DAPL_EP *ep_ptr ) +{ + DAPL_IA *ia_ptr; + ib_api_status_t ib_status; + dapl_ibal_ca_t *p_ca; + dapl_ibal_port_t *p_active_port; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsIQM_REINIT: EP(%p) QP(%p) state change\n", + ep_ptr, ep_ptr->qp_handle ); + + if ( ep_ptr->param.ep_state != DAT_EP_STATE_DISCONNECTED ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsIRE: EP invalid state(%d)\n", + ep_ptr->param.ep_state); + return /*DAT_INVALID_STATE*/; + } + + ia_ptr = ep_ptr->header.owner_ia; + + /* Re-create QP if cleaned up, alloc will return init state */ + if ( ep_ptr->qp_handle == IB_INVALID_HANDLE ) + { + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsIRE: !EP(%p)->qp_handle, re-create QP\n",ep_ptr); + ib_status = dapls_ib_qp_alloc ( ia_ptr, ep_ptr, ep_ptr ); + if ( ib_status != IB_SUCCESS ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DsIRE: failed to move qp to RESET status = %s\n", + ib_get_err_str(ib_status)); + } + return /*ib_status*/; + } + + ib_status = dapls_modify_qp_state_to_reset ( ep_ptr->qp_handle); + + if ( ib_status != IB_SUCCESS ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DsIRE: failed to move qp to RESET status = %s\n", + ib_get_err_str(ib_status)); + return /*DAT_INTERNAL_ERROR*/; + } + + ep_ptr->qp_state = IB_QPS_RESET; + + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + p_active_port = dapli_ibal_get_port ( p_ca, + (uint8_t)ia_ptr->hca_ptr->port_num ); + if (NULL == p_active_port) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DsIRE: Port %d is not available = %d\n", + ia_ptr->hca_ptr->port_num, __LINE__); + return /*DAT_INTERNAL_ERROR*/; + } + + /* May fail if QP still RESET and in timewait, keep in reset state */ + ib_status = dapls_modify_qp_state_to_init ( ep_ptr->qp_handle, + &ep_ptr->param.ep_attr, + p_active_port); + if ( ib_status != IB_SUCCESS ) + { + ep_ptr->qp_state = IB_QPS_RESET; + + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DsIRE: failed to move qp to INIT status %s\n", + ib_get_err_str(ib_status)); + return /*DAT_INTERNAL_ERROR*/; + } + + ep_ptr->qp_state = IB_QPS_INIT; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.c new file mode 100644 index 00000000..182d6dbf --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.c @@ -0,0 +1,2257 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_util.c + * + * PURPOSE: Utility routines for access to IBAL APIs + * + * $Id: dapl_ibal_util.c 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" +#include "dapl_lmr_util.h" +#include "dapl_rmr_util.h" +#include "dapl_cookie.h" +#include "dapl_ring_buffer_util.h" + +#ifdef DAT_EXTENSIONS +#include +#endif + +#ifndef NO_NAME_SERVICE +#include "dapl_name_service.h" +#endif /* NO_NAME_SERVICE */ + +#define DAPL_IBAL_MAX_CA 4 +#define DAT_ADAPTER_NAME "InfiniHost (Tavor)" +#define DAT_VENDOR_NAME "Mellanox Technolgy Inc." + +/* + * Root data structure for DAPL_IIBA. + */ +dapl_ibal_root_t dapl_ibal_root; +DAPL_HCA_NAME dapl_ibal_hca_name_array [DAPL_IBAL_MAX_CA] = + {"IbalHca0", "IbalHca1", "IbalHca2", "IbalHca3"}; +ib_net64_t *gp_ibal_ca_guid_tbl = NULL; + +/* + * DAT spec does not tie max_mtu_size with IB MTU + * +static ib_net32_t dapl_ibal_mtu_table[6] = {0, 256, 512, 1024, 2048, 4096}; + */ + +int g_loopback_connection = 0; + + +static cl_status_t +dapli_init_root_ca_list( + IN dapl_ibal_root_t *root ) +{ + cl_status_t status; + + cl_qlist_init (&root->ca_head); + status = cl_spinlock_init (&root->ca_lock); + + if (status == CL_SUCCESS) + { + /* + * Get the time ready to go but don't start here + */ + root->shutdown = FALSE; + root->initialized = TRUE; + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DiIRCL: cl_spinlock_init returned %d\n", status ); + root->initialized = FALSE; + } + + root->h_al = NULL; + + return (status); +} + + +static cl_status_t +dapli_destroy_root_ca_list( + IN dapl_ibal_root_t *root ) +{ + + root->initialized = FALSE; + + /* + * At this point the lock should not be necessary + */ + if (!cl_is_qlist_empty (&root->ca_head) ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> Destroying nonempty ca list (%s)\n", "DiDRCL"); + } + cl_spinlock_destroy (&root->ca_lock); + + return CL_SUCCESS; +} + + +static void +dapli_shutdown_port_access( + IN dapl_ibal_ca_t *ca ) +{ + dapl_ibal_port_t *p_port; + + TAKE_LOCK( ca->port_lock ); + { + while ( ! cl_is_qlist_empty( &ca->port_head ) ) + { + p_port = (dapl_ibal_port_t *)cl_qlist_remove_head( &ca->port_head ); + RELEASE_LOCK( ca->port_lock ); + { + REMOVE_REFERENCE( &p_port->refs ); + REMOVE_REFERENCE( &p_port->ca->refs ); + + dapl_os_free (p_port, sizeof (dapl_ibal_port_t)); + } + TAKE_LOCK( ca->port_lock ); + } + } + RELEASE_LOCK( ca->port_lock ); +} + + +static void dapli_shutdown_ca_access (void) +{ + dapl_ibal_ca_t *ca; + + if ( dapl_ibal_root.initialized == FALSE ) + { + goto destroy_root; + } + + TAKE_LOCK (dapl_ibal_root.ca_lock); + { + while ( ! cl_is_qlist_empty (&dapl_ibal_root.ca_head) ) + { + ca = (dapl_ibal_ca_t *) + cl_qlist_remove_head (&dapl_ibal_root.ca_head); + + if (ca->p_ca_attr) + { + dapl_os_free (ca->p_ca_attr, sizeof (ib_ca_attr_t)); + } + + + RELEASE_LOCK (dapl_ibal_root.ca_lock); + { + dapli_shutdown_port_access (ca); + REMOVE_REFERENCE (&ca->refs); + } + TAKE_LOCK (dapl_ibal_root.ca_lock); + } + } + RELEASE_LOCK (dapl_ibal_root.ca_lock); + +destroy_root: + /* + * Destroy the root CA list and list lock + */ + dapli_destroy_root_ca_list (&dapl_ibal_root); + + /* + * Signal we're all done and wake any waiter + */ + dapl_ibal_root.shutdown = FALSE; +} + + +dapl_ibal_evd_cb_t * +dapli_find_evd_cb_by_context( + IN void *context, + IN dapl_ibal_ca_t *ca) +{ + dapl_ibal_evd_cb_t *evd_cb = NULL; + + TAKE_LOCK( ca->evd_cb_lock ); + + evd_cb = (dapl_ibal_evd_cb_t *) cl_qlist_head( &ca->evd_cb_head ); + while ( &evd_cb->next != cl_qlist_end( &ca->evd_cb_head ) ) + { + if ( context == evd_cb->context) + { + goto found; + } + + /* + * Try again + */ + evd_cb = (dapl_ibal_evd_cb_t *) cl_qlist_next( &evd_cb->next ); + } + /* + * No joy + */ + evd_cb = NULL; + +found: + + RELEASE_LOCK( ca->evd_cb_lock ); + + return ( evd_cb ); +} + + +static cl_status_t +dapli_init_ca_evd_cb_list( + IN dapl_ibal_ca_t *ca ) +{ + cl_status_t status; + + cl_qlist_init( &ca->evd_cb_head ); + status = cl_spinlock_init( &ca->evd_cb_lock ); + if ( status != CL_SUCCESS ) + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DiICECL: cl_spinlock_init returned %d\n", status); + return ( status ); +} + + +static cl_status_t +dapli_init_ca_port_list( + IN dapl_ibal_ca_t *ca ) +{ + cl_status_t status; + + cl_qlist_init( &ca->port_head ); + status = cl_spinlock_init( &ca->port_lock ); + if ( status != CL_SUCCESS ) + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DiICPL: cl_spinlock_init returned %d\n", status ); + return ( status ); +} + +dapl_ibal_port_t * +dapli_ibal_get_port ( + IN dapl_ibal_ca_t *p_ca, + IN uint8_t port_num) +{ + cl_list_item_t *p_active_port = NULL; + + TAKE_LOCK (p_ca->port_lock); + for ( p_active_port = cl_qlist_head( &p_ca->port_head ); + p_active_port != cl_qlist_end ( &p_ca->port_head); + p_active_port = cl_qlist_next ( p_active_port ) ) + { + if (((dapl_ibal_port_t *)p_active_port)->p_attr->port_num == port_num) + break; + } + RELEASE_LOCK (p_ca->port_lock); + + return (dapl_ibal_port_t *)p_active_port; +} + + +void +dapli_ibal_ca_async_error_callback( IN ib_async_event_rec_t *p_err_rec ) +{ + dapl_ibal_ca_t *p_ca = (dapl_ibal_ca_t*)((void *)p_err_rec->context); + dapl_ibal_evd_cb_t *evd_cb; + DAPL_IA *ia_ptr; + + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DiCaAEC: CA error %d for context %p\n", + p_err_rec->code, p_err_rec->context); + + if (p_ca == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DiCaAEC: invalid p_ca" + "(%p)in async event rec\n",p_ca); + return; + } + + ia_ptr = (DAPL_IA*)p_ca->ia_ptr; + if (ia_ptr == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCaAEC: invalid ia_ptr in %p ca \n", p_ca ); + return; + } + + if (ia_ptr->async_error_evd == NULL) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCqAEC: can't find async_error_evd on %s HCA\n", + (ia_ptr->header.provider)->device_name ); + return; + } + + /* find QP error callback using p_ca for context */ + evd_cb = dapli_find_evd_cb_by_context (ia_ptr->async_error_evd, p_ca); + if ((evd_cb == NULL) || (evd_cb->pfn_async_err_cb == NULL)) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> DiCaAEC: no ERROR cb on %p found \n", p_ca); + return; + } + + /* maps to dapl_evd_un_async_error_callback(), context is async_evd */ + evd_cb->pfn_async_err_cb( (ib_hca_handle_t)p_ca, + (ib_error_record_t*)&p_err_rec->code, + ia_ptr->async_error_evd ); + +} + + +static dapl_ibal_port_t * +dapli_alloc_port( + IN dapl_ibal_ca_t *ca, + IN ib_port_attr_t *ib_port ) +{ + dapl_ibal_port_t *p_port = NULL; + + if (ca->h_ca == NULL ) + { + return NULL; + } + + /* + * Allocate the port structure memory. This will also deal with the + * copying ib_port_attr_t including GID and P_Key tables + */ + p_port = dapl_os_alloc ( sizeof(dapl_ibal_port_t ) ); + + if ( p_port ) + { + dapl_os_memzero (p_port, sizeof(dapl_ibal_port_t ) ); + + /* + * We're good to go after initializing reference. + */ + INIT_REFERENCE( &p_port->refs, 1, p_port, NULL /* pfn_destructor */ ); + + p_port->p_attr = ib_port; + } + return ( p_port ); +} + +static void +dapli_add_active_port( + IN dapl_ibal_ca_t *ca ) +{ + dapl_ibal_port_t *p_port; + ib_port_attr_t *p_port_attr; + ib_ca_attr_t *p_ca_attr; + int i; + + p_ca_attr = ca->p_ca_attr; + + dapl_os_assert (p_ca_attr != NULL); + + for (i = 0; i < p_ca_attr->num_ports; i++) + { + p_port_attr = &p_ca_attr->p_port_attr[i]; + + { + p_port = dapli_alloc_port( ca, p_port_attr ); + if ( p_port ) + { + TAKE_REFERENCE (&ca->refs); + + /* + * Record / update attribues + */ + p_port->p_attr = p_port_attr; + + /* + * Remember the parant CA keeping the reference we took above + */ + p_port->ca = ca; + + /* + * We're good to go - Add the new port to the list on the CA + */ + LOCK_INSERT_TAIL( ca->port_lock, ca->port_head, p_port->next ); + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: Could not allocate dapl_ibal_port_t\n", + "DiAAP"); + } + } + dapl_dbg_log( DAPL_DBG_TYPE_UTIL, + "--> DiAAP: Port %d logical link %s lid = %#x\n", + p_port_attr->port_num, + ( p_port_attr->link_state != IB_LINK_ACTIVE + ? "DOWN": "UP" ), + CL_HTON16(p_port_attr->lid) ); + + } /* for loop */ +} + +static dapl_ibal_ca_t * +dapli_alloc_ca( + IN ib_al_handle_t h_al, + IN ib_net64_t ca_guid) +{ + dapl_ibal_ca_t *p_ca; + ib_api_status_t status; + uint32_t attr_size; + + /* + * Allocate the CA structure + */ + p_ca = dapl_os_alloc( sizeof(dapl_ibal_ca_t) ); + dapl_os_memzero (p_ca, sizeof(dapl_ibal_ca_t) ); + + if ( p_ca ) + { + /* + * Now we pass dapli_ibal_ca_async_error_callback as the + * async error callback + */ + status = ib_open_ca( h_al, + ca_guid, + dapli_ibal_ca_async_error_callback, + p_ca, + &p_ca->h_ca ); + if ( status != IB_SUCCESS ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DiAC: ib_open_ca returned %s\n", + ib_get_err_str(status)); + dapl_os_free (p_ca, sizeof (dapl_ibal_ca_t)); + return (NULL); + } + + /* + * Get port list lock and list head initialized + */ + if (( dapli_init_ca_port_list( p_ca ) != CL_SUCCESS ) || + ( dapli_init_ca_evd_cb_list( p_ca ) != CL_SUCCESS )) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: dapli_init_ca_port_list returned failed\n", + "DiAC"); + goto close_and_free_ca; + } + + attr_size = 0; + status = ib_query_ca (p_ca->h_ca, NULL, &attr_size); + if (status != IB_INSUFFICIENT_MEMORY) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DiAC: ib_query_ca returned failed status = %d\n", + status); + goto close_and_free_ca; + } + + p_ca->p_ca_attr = dapl_os_alloc ((int)attr_size); + if (p_ca->p_ca_attr == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: dapli_alloc_ca failed to alloc memory\n", + "DiAC"); + goto close_and_free_ca; + } + + status = ib_query_ca ( + p_ca->h_ca, + p_ca->p_ca_attr, + &attr_size); + if (status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> ib_query_ca returned failed status = %d\n", + status); + dapl_os_free (p_ca->p_ca_attr, (int)attr_size); + goto close_and_free_ca; + } + + p_ca->ca_attr_size = attr_size; + + INIT_REFERENCE( &p_ca->refs, 1, p_ca, NULL /* pfn_destructor */ ); + + dapli_add_active_port (p_ca); + + /* + * We're good to go + */ + return ( p_ca ); + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: Error allocating CA structure\n","DiAC"); + return ( NULL ); + } + +close_and_free_ca: + /* + * Close the CA. + */ + (void) ib_close_ca ( p_ca->h_ca, NULL /* callback */); + dapl_os_free (p_ca, sizeof (dapl_ibal_ca_t)); + + /* + * If we get here, there was an initialization failure + */ + return ( NULL ); +} + + +static dapl_ibal_ca_t * +dapli_add_ca ( + IN ib_al_handle_t h_al, + IN ib_net64_t ca_guid ) +{ + dapl_ibal_ca_t *p_ca; + + /* + * Allocate a CA structure + */ + p_ca = dapli_alloc_ca( h_al, ca_guid ); + if ( p_ca ) + { + /* + * Add the new CA to the list + */ + LOCK_INSERT_TAIL( dapl_ibal_root.ca_lock, + dapl_ibal_root.ca_head, p_ca->next ); + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: Could not allocate dapl_ibal_ca_t " + " for CA guid " F64x "\n","DiAA",ca_guid); + } + + return ( p_ca ); +} + + +int32_t +dapls_ib_init (void) +{ + ib_api_status_t status; + + /* + * Initialize the root structure + */ + if ( dapli_init_root_ca_list (&dapl_ibal_root) == CL_SUCCESS ) + { + /* + * Register with the access layer + */ + status = ib_open_al (&dapl_ibal_root.h_al); + + if (status == IB_SUCCESS) + { + intn_t guid_count; + + status = ib_get_ca_guids ( dapl_ibal_root.h_al, + NULL, + &(size_t)guid_count ); + if (status != IB_INSUFFICIENT_MEMORY) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: ib_get_ca_guids failed = %d\n", + __FUNCTION__,status); + return -1; + } + + if (guid_count == 0) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: found NO HCA in the system\n", + __FUNCTION__); + return -1; + } + + if (guid_count > DAPL_IBAL_MAX_CA) + { + guid_count = DAPL_IBAL_MAX_CA; + } + + gp_ibal_ca_guid_tbl = (ib_net64_t*) + dapl_os_alloc ( (int)(guid_count * + sizeof (ib_net64_t)) ); + + if (gp_ibal_ca_guid_tbl == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> %s() can not alloc " + "gp_ibal_ca_guid_tbl\n", __FUNCTION__); + + return -1; + } + + status = ib_get_ca_guids ( dapl_ibal_root.h_al, + gp_ibal_ca_guid_tbl, + &(size_t)guid_count ); + + + if ( status != IB_SUCCESS ) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: ib_get_ca_guids failed '%s'\n", + __FUNCTION__, ib_get_err_str(status) ); + return -1; + } + + dapl_dbg_log ( DAPL_DBG_TYPE_UTIL, + "--> %s: Success open AL & found %d HCA avail,\n", + __FUNCTION__, guid_count); + return 0; + } + else + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: ib_open_al() failed '%s'\n", + __FUNCTION__, ib_get_err_str(status) ); + /* + * Undo CA list + */ + dapli_destroy_root_ca_list (&dapl_ibal_root); + } + } + return -1; +} + + +int32_t dapls_ib_release (void) +{ + dapl_ibal_root.shutdown = TRUE; + + dapli_shutdown_ca_access(); + + /* + * If shutdown not complete, wait for it + */ + if (dapl_ibal_root.shutdown) + { + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsIR: timeout waiting for completion\n"); + } + + if ( dapl_ibal_root.h_al != NULL ) + { + (void) ib_close_al (dapl_ibal_root.h_al); + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> DsIR: ib_close_al() returns\n"); + dapl_ibal_root.h_al = NULL; + } +#ifdef DBG + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> %s: Exit\n",__FUNCTION__); +#endif + + return 0; +} + + +/* + * dapls_ib_enum_hcas + * + * Enumerate all HCAs on the system + * + * Input: + * none + * + * Output: + * hca_names Array of hca names + * total_hca_count + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_enum_hcas ( + IN const char *vendor, + OUT DAPL_HCA_NAME **hca_names, + OUT DAT_COUNT *total_hca_count ) +{ + intn_t guid_count; + ib_api_status_t ib_status; + UNREFERENCED_PARAMETER(vendor); + + ib_status = ib_get_ca_guids (dapl_ibal_root.h_al, NULL, &(size_t)guid_count); + if (ib_status != IB_INSUFFICIENT_MEMORY) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIEH: ib_get_ca_guids failed '%s'\n", + ib_get_err_str(ib_status) ); + return dapl_ib_status_convert (ib_status); + } + + if (guid_count == 0) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: ib_get_ca_guids no HCA in the system\n", + "DsIEH"); + return (DAT_PROVIDER_NOT_FOUND); + } + + if (guid_count > DAPL_IBAL_MAX_CA) + { + guid_count = DAPL_IBAL_MAX_CA; + } + + gp_ibal_ca_guid_tbl = (ib_net64_t *)dapl_os_alloc ((int)(guid_count * sizeof (ib_net64_t)) ); + + if (gp_ibal_ca_guid_tbl == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: can not alloc resources @line%d\n", "DsIEH", + __LINE__); + return (DAT_INSUFFICIENT_RESOURCES); + } + + ib_status = ib_get_ca_guids ( dapl_ibal_root.h_al, + gp_ibal_ca_guid_tbl, + &(size_t)guid_count); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIEH: ib_get_ca_guids failed status = %s\n", + ib_get_err_str(ib_status) ); + return dapl_ib_status_convert (ib_status); + } + + *hca_names = (DAPL_HCA_NAME*) + dapl_os_alloc ((int)(guid_count * sizeof (DAPL_HCA_NAME))); + + if (*hca_names == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: can not alloc resources @line%d\n", + "DsIEH", __LINE__); + return (DAT_INSUFFICIENT_RESOURCES); + } + + dapl_os_memcpy (*hca_names, + dapl_ibal_hca_name_array, + (int)(guid_count * sizeof (DAPL_HCA_NAME)) ); + + *total_hca_count = (DAT_COUNT)guid_count; + + { + int i; + + for (i = 0; i < guid_count; i++) + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> DsIEH: %d) hca_names = %s\n", + i, dapl_ibal_hca_name_array[i]); + } + + return (DAT_SUCCESS); +} + + + +IB_HCA_NAME +dapl_ib_convert_name( + IN char *name) +{ + int i; + + if (gp_ibal_ca_guid_tbl == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DICN: found no HCA with name %s\n", name ); + return 0; + } + + for (i = 0; i < DAPL_IBAL_MAX_CA; i++) + { + if (strcmp (name, dapl_ibal_hca_name_array[i]) == 0) + { + break; + } + } + + if (i >= DAPL_IBAL_MAX_CA) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DICN: can't find any HCA with name %s\n", name); + return 0; + } + + return (gp_ibal_ca_guid_tbl[i]); +} + + +/* + * dapls_ib_open_hca + * + * Open HCA + * + * Input: + * *hca_name pointer to provider device name + * *ib_hca_handle_p pointer to provide HCA handle + * + * Output: + * none + * + * Return: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN dapls_ib_open_hca ( IN char *hca_name, + IN DAPL_HCA *p_hca ) +{ + dapl_ibal_ca_t *p_ca; + IB_HCA_NAME ca_guid; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL," open_hca: %s - %p\n", hca_name, p_hca); + + if (gp_ibal_ca_guid_tbl == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsIOH: found no HCA with ca_guid" + F64x "\n", hca_name); + return (DAT_PROVIDER_NOT_FOUND); + } + + ca_guid = dapl_ib_convert_name(hca_name); + + p_ca = dapli_add_ca (dapl_ibal_root.h_al, ca_guid); + + if (p_ca == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIOH: can not create ca for '%s' guid " F64x "\n", + hca_name, ca_guid); + return (DAT_INSUFFICIENT_RESOURCES); + } + + p_hca->ib_hca_handle = (ib_hca_handle_t) p_ca; + p_hca->ib_trans.ib_hca_handle = p_hca->ib_hca_handle; + p_hca->ib_trans.d_hca = p_hca; // back-link + + /* initialize hca wait object for uAT event */ + dapl_os_wait_object_init(&p_hca->ib_trans.wait_object); + + return (DAT_SUCCESS); +} + + +/* + * dapls_ib_close_hca + * + * Open HCA + * + * Input: + * ib_hca_handle provide HCA handle + * + * Output: + * none + * + * Return: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN dapls_ib_close_hca ( IN DAPL_HCA *p_hca ) +{ + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) p_hca->ib_hca_handle; + + /* + * Remove it from the list + */ + TAKE_LOCK (dapl_ibal_root.ca_lock); + { + cl_qlist_remove_item (&dapl_ibal_root.ca_head, &p_ca->next); + } + RELEASE_LOCK (dapl_ibal_root.ca_lock); + + dapli_shutdown_port_access (p_ca); + + /* + * Remove the constructor reference + */ + REMOVE_REFERENCE (&p_ca->refs); + + cl_spinlock_destroy (&p_ca->port_lock); + cl_spinlock_destroy (&p_ca->evd_cb_lock); + + if (p_ca->p_ca_attr) + dapl_os_free (p_ca->p_ca_attr, sizeof (ib_ca_attr_t)); + + (void) ib_close_ca (p_ca->h_ca, NULL /* close_callback */); + + p_hca->ib_hca_handle = IB_INVALID_HANDLE; + dapl_os_free (p_ca, sizeof (dapl_ibal_ca_t)); + + return (DAT_SUCCESS); +} + + + +/* + * dapl_ib_pd_alloc + * + * Alloc a PD + * + * Input: + * ia_handle IA handle + * PZ_ptr pointer to PZEVD struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_pd_alloc ( + IN DAPL_IA *ia, + IN DAPL_PZ *pz) +{ + ib_api_status_t ib_status; + dapl_ibal_ca_t *p_ca; + + p_ca = (dapl_ibal_ca_t *) ia->hca_ptr->ib_hca_handle; + + ib_status = ib_alloc_pd ( + p_ca->h_ca, + IB_PDT_NORMAL, + ia, + &pz->pd_handle ); + + return dapl_ib_status_convert (ib_status); +} + + +/* + * dapl_ib_pd_free + * + * Free a PD + * + * Input: + * PZ_ptr pointer to PZ struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_pd_free ( + IN DAPL_PZ *pz) +{ + ib_api_status_t ib_status; + + ib_status = ib_dealloc_pd (pz->pd_handle, /* destroy_callback */ NULL); + + pz->pd_handle = IB_INVALID_HANDLE; + + return dapl_ib_status_convert (ib_status); +} + + +/* + * dapl_ib_mr_register + * + * Register a virtual memory region + * + * Input: + * ia_handle IA handle + * lmr pointer to dapl_lmr struct + * virt_addr virtual address of beginning of mem region + * length length of memory region + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mr_register ( + IN DAPL_IA *ia, + IN DAPL_LMR *lmr, + IN DAT_PVOID virt_addr, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type) +{ + ib_api_status_t ib_status; + ib_mr_handle_t mr_handle; + ib_mr_create_t mr_create; + uint32_t l_key, r_key; + + if ( ia == NULL || ia->header.magic != DAPL_MAGIC_IA ) + { + return DAT_INVALID_HANDLE; + } + + /* IBAL does not support */ + if (va_type == DAT_VA_TYPE_ZB) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + "--> va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n"); + return DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + } + + mr_create.vaddr = (void *) virt_addr; + mr_create.length = (size_t)length; + mr_create.access_ctrl = dapl_lmr_convert_privileges (privileges); + mr_create.access_ctrl |= IB_AC_MW_BIND; + + if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) + { + ib_status = ib_reg_shmid ( + ((DAPL_PZ *)lmr->param.pz_handle)->pd_handle, + (const uint8_t*)lmr->shmid, + &mr_create, + (uint64_t *)&virt_addr, + &l_key, + &r_key, + &mr_handle); + } + else + { + ib_status = ib_reg_mem ( ((DAPL_PZ *)lmr->param.pz_handle)->pd_handle, + &mr_create, + &l_key, + &r_key, + &mr_handle ); + } + + if (ib_status != IB_SUCCESS) + { + return (dapl_ib_status_convert (ib_status)); + } + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> DsIMR: lmr (%p) lkey = 0x%x " + "r_key= %#x mr_handle %p vaddr 0x%LX len 0x%LX\n", + lmr, l_key, r_key, mr_handle, virt_addr, length); + + lmr->param.lmr_context = l_key; + lmr->param.rmr_context = r_key; + lmr->param.registered_size = length; + lmr->param.registered_address = (DAT_VADDR)virt_addr; + lmr->mr_handle = mr_handle; + + return (DAT_SUCCESS); +} + + +/* + * dapl_ib_mr_deregister + * + * Free a memory region + * + * Input: + * lmr pointer to dapl_lmr struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mr_deregister ( + IN DAPL_LMR *lmr) +{ + ib_api_status_t ib_status; + + ib_status = ib_dereg_mr (lmr->mr_handle); + + if (ib_status != IB_SUCCESS) + { + return dapl_ib_status_convert (ib_status); + } + + lmr->param.lmr_context = 0; + lmr->mr_handle = IB_INVALID_HANDLE; + + return (DAT_SUCCESS); +} + + +/* + * dapl_ib_mr_register_shared + * + * Register a virtual memory region + * + * Input: + * ia_handle IA handle + * lmr pointer to dapl_lmr struct + * virt_addr virtual address of beginning of mem region + * length length of memory region + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mr_register_shared ( + IN DAPL_IA *ia, + IN DAPL_LMR *lmr, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type ) +{ + DAT_VADDR virt_addr; + ib_mr_handle_t mr_handle; + ib_api_status_t ib_status; + ib_mr_handle_t new_mr_handle; + ib_access_t access_ctrl; + uint32_t l_key, r_key; + ib_mr_create_t mr_create; + if ( ia == NULL || ia->header.magic != DAPL_MAGIC_IA ) + { + return DAT_INVALID_HANDLE; + } + + /* IBAL does not support?? */ + if (va_type == DAT_VA_TYPE_ZB) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n"); + return DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + } + + virt_addr = dapl_mr_get_address (lmr->param.region_desc, + lmr->param.mem_type); + + access_ctrl = dapl_lmr_convert_privileges (privileges); + access_ctrl |= IB_AC_MW_BIND; + + mr_create.vaddr = (void *) virt_addr; + mr_create.access_ctrl = access_ctrl; + mr_handle = (ib_mr_handle_t) lmr->mr_handle; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsIMRS: orig mr_handle %p vaddr %p\n", + mr_handle, virt_addr); + + if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) + { + ib_status = ib_reg_shmid ( ((DAPL_PZ *)lmr->param.pz_handle)->pd_handle, + (const uint8_t*)lmr->shmid, + &mr_create, + &virt_addr, + &l_key, + &r_key, + &new_mr_handle ); + } + else + { + + ib_status = ib_reg_shared ( mr_handle, + ((DAPL_PZ *)lmr->param.pz_handle)->pd_handle, + access_ctrl, + /* in/out */(DAT_UINT64 *)&virt_addr, + &l_key, + &r_key, + &new_mr_handle ); + } + + if (ib_status != IB_SUCCESS) + { + return dapl_ib_status_convert (ib_status); + } + /* + * FIXME - Vu + * What if virt_addr as an OUTPUT having the actual virtual address + * assigned to the register region + */ + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> DsIMRS: lmr (%p) lkey = 0x%x " + "new mr_handle %p vaddr %p\n", + lmr, l_key, new_mr_handle, virt_addr); + + lmr->param.lmr_context = l_key; + lmr->param.rmr_context = r_key; + lmr->param.registered_address = (DAT_VADDR) (uintptr_t) virt_addr; + lmr->mr_handle = new_mr_handle; + + return (DAT_SUCCESS); +} + + +/* + * dapls_ib_mw_alloc + * + * Bind a protection domain to a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mw_alloc ( IN DAPL_RMR *rmr ) +{ + ib_api_status_t ib_status; + uint32_t r_key; + ib_mw_handle_t mw_handle; + + ib_status = ib_create_mw ( + ((DAPL_PZ *)rmr->param.pz_handle)->pd_handle, + &r_key, + &mw_handle); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIMA: create MW failed = %s\n", + ib_get_err_str(ib_status) ); + return dapl_ib_status_convert (ib_status); + } + + rmr->mw_handle = mw_handle; + rmr->param.rmr_context = (DAT_RMR_CONTEXT) r_key; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsIMA: mw_handle %p r_key = 0x%x\n", + mw_handle, r_key); + + return (DAT_SUCCESS); +} + + +/* + * dapls_ib_mw_free + * + * Release bindings of a protection domain to a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mw_free ( + IN DAPL_RMR *rmr) +{ + ib_api_status_t ib_status; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsIMF: mw_handle %p\n", rmr->mw_handle); + + ib_status = ib_destroy_mw (rmr->mw_handle); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsIMF: Free MW failed = %s\n", + ib_get_err_str(ib_status)); + return dapl_ib_status_convert (ib_status); + } + + rmr->param.rmr_context = 0; + rmr->mw_handle = IB_INVALID_HANDLE; + + return (DAT_SUCCESS); +} + +/* + * dapls_ib_mw_bind + * + * Bind a protection domain to a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mw_bind ( + IN DAPL_RMR *rmr, + IN DAPL_LMR *lmr, + IN DAPL_EP *ep, + IN DAPL_COOKIE *cookie, + IN DAT_VADDR virtual_address, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN ib_bool_t is_signaled) +{ + ib_api_status_t ib_status; + ib_bind_wr_t bind_wr_prop; + uint32_t new_rkey; + + bind_wr_prop.local_ds.vaddr = virtual_address; + bind_wr_prop.local_ds.length = (uint32_t)length; + bind_wr_prop.local_ds.lkey = lmr->param.lmr_context; + bind_wr_prop.current_rkey = rmr->param.rmr_context; + bind_wr_prop.access_ctrl = dapl_rmr_convert_privileges (mem_priv); + bind_wr_prop.send_opt = (is_signaled == TRUE) ? + IB_SEND_OPT_SIGNALED : 0; + bind_wr_prop.wr_id = (uint64_t) ((uintptr_t) cookie); + bind_wr_prop.h_mr = lmr->mr_handle; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> DsIMB: mr_handle %p, mw_handle %p " + "vaddr %#I64x length %#I64x\n", + lmr->mr_handle, rmr->mw_handle, virtual_address, length); + + ib_status = ib_bind_mw ( + rmr->mw_handle, + ep->qp_handle, + &bind_wr_prop, + &new_rkey); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsIMB: Bind MW failed = %s\n", + ib_get_err_str(ib_status)); + return (dapl_ib_status_convert (ib_status)); + } + + rmr->param.rmr_context = (DAT_RMR_CONTEXT) new_rkey; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "--> DsIMB: new_rkey = 0x%x\n",new_rkey); + + return (DAT_SUCCESS); +} + +/* + * dapls_ib_mw_unbind + * + * Unbind a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mw_unbind ( + IN DAPL_RMR *rmr, + IN DAPL_EP *ep, + IN DAPL_COOKIE *cookie, + IN ib_bool_t is_signaled) +{ + ib_api_status_t ib_status; + ib_bind_wr_t bind_wr_prop; + uint32_t new_rkey; + + bind_wr_prop.local_ds.vaddr = 0; + bind_wr_prop.local_ds.length = 0; + bind_wr_prop.local_ds.lkey = 0; + bind_wr_prop.access_ctrl = 0; + bind_wr_prop.send_opt = (is_signaled == TRUE) ? + IB_SEND_OPT_SIGNALED : 0; + bind_wr_prop.wr_id = (uint64_t) ((uintptr_t) cookie); + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsIMU: mw_handle = %p\n", rmr->mw_handle); + + ib_status = ib_bind_mw ( + rmr->mw_handle, + ep->qp_handle, + &bind_wr_prop, + &new_rkey); + + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsIMU: Unbind MW failed = %s\n", + ib_get_err_str(ib_status)); + return (dapl_ib_status_convert (ib_status)); + } + + rmr->param.rmr_context = (DAT_RMR_CONTEXT) new_rkey; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + "--> DsIMU: unbind new_rkey = 0x%x\n", new_rkey); + + return (DAT_SUCCESS); +} + + +/* + * dapls_ib_setup_async_callback + * + * Set up an asynchronous callbacks of various kinds + * + * Input: + * ia_handle IA handle + * handler_type type of handler to set up + * callback_handle handle param for completion callbacks + * callback callback routine pointer + * context argument for callback routine + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_setup_async_callback ( + IN DAPL_IA *ia_ptr, + IN DAPL_ASYNC_HANDLER_TYPE handler_type, + IN DAPL_EVD *evd_ptr, + IN ib_async_handler_t callback, + IN void *context ) +{ + dapl_ibal_ca_t *p_ca; + dapl_ibal_evd_cb_t *evd_cb; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, + " setup_async_cb: ia %p type %d hdl %p cb %p ctx %p\n", + ia_ptr, handler_type, evd_ptr, callback, context); + + p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> DsISAC: can't find %s HCA\n", + (ia_ptr->header.provider)->device_name); + return (DAT_INVALID_HANDLE); + } + + if (handler_type != DAPL_ASYNC_CQ_COMPLETION) + { + evd_cb = dapli_find_evd_cb_by_context (context, p_ca); + + if (evd_cb == NULL) + { + /* + * No record for this evd. We allocate one + */ + evd_cb = dapl_os_alloc (sizeof (dapl_ibal_evd_cb_t)); + dapl_os_memzero (evd_cb, sizeof(dapl_ibal_evd_cb_t)); + + if (evd_cb == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> %s: can't alloc res\n","DsISAC"); + return (DAT_INSUFFICIENT_RESOURCES); + } + + evd_cb->context = context; + + /* + * Add the new EVD CB to the list + */ + LOCK_INSERT_TAIL( p_ca->evd_cb_lock, + p_ca->evd_cb_head, + evd_cb->next ); + } + + switch (handler_type) + { + case DAPL_ASYNC_UNAFILIATED: + evd_cb->pfn_async_err_cb = callback; + break; + case DAPL_ASYNC_CQ_ERROR: + evd_cb->pfn_async_cq_err_cb = callback; + break; + case DAPL_ASYNC_QP_ERROR: + evd_cb->pfn_async_qp_err_cb = callback; + break; + default: + break; + } + + } + + return DAT_SUCCESS; +} + + +/* + * dapls_ib_query_hca + * + * Query the hca attribute + * + * Input: + * hca_handl hca handle + * ep_attr attribute of the ep + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ + +DAT_RETURN dapls_ib_query_hca ( + IN DAPL_HCA *hca_ptr, + OUT DAT_IA_ATTR *ia_attr, + OUT DAT_EP_ATTR *ep_attr, + OUT DAT_SOCK_ADDR6 *ip_addr) +{ + ib_ca_attr_t *p_hca_attr; + dapl_ibal_ca_t *p_ca; + ib_api_status_t ib_status; + ib_hca_port_t port_num; + GID gid; + DAT_SOCK_ADDR6 *p_sock_addr; + DAT_RETURN dat_status = DAT_SUCCESS; + port_num = hca_ptr->port_num; + + p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle; + + if (p_ca == NULL) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> %s: invalid handle %p", + "DsIQH", hca_ptr); + return (DAT_INVALID_HANDLE); + } + + ib_status = ib_query_ca ( + p_ca->h_ca, + p_ca->p_ca_attr, + &p_ca->ca_attr_size); + if (ib_status != IB_SUCCESS) + { + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + "--> DsIQH: ib_query_ca returned failed status = %s\n", + ib_get_err_str(ib_status)); + return (dapl_ib_status_convert (ib_status)); + } + + p_hca_attr = p_ca->p_ca_attr; + + if (ip_addr != NULL) + { + p_sock_addr = dapl_os_alloc(sizeof(DAT_SOCK_ADDR6)); + if ( !p_sock_addr ) + { + dat_status = DAT_INSUFFICIENT_RESOURCES; + dapl_dbg_log ( DAPL_DBG_TYPE_ERR, + " Query Hca alloc Err: status %d\n", + dat_status); + return dat_status; + } + dapl_os_memzero(p_sock_addr, sizeof(DAT_SOCK_ADDR6)); + + gid.gid_prefix = p_hca_attr->p_port_attr[port_num-1].p_gid_table->unicast.prefix; + gid.guid = p_hca_attr->p_port_attr[port_num-1].p_gid_table->unicast.interface_id; + + dat_status = dapls_ns_map_ipaddr( hca_ptr, + gid, + (DAT_IA_ADDRESS_PTR)p_sock_addr); + + if ( dat_status != DAT_SUCCESS ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + " SA Query for local IP failed= %d\n", dat_status ); + /* what to do next ? */ + } + else + { + dapl_dbg_log (DAPL_DBG_TYPE_CM, "SA query GID for IP: "); + dapl_dbg_log ( DAPL_DBG_TYPE_CM, "%0d:%d:%d:%d\n", + (uint8_t)((DAT_IA_ADDRESS_PTR )p_sock_addr)->sa_data[2]&0xff, + (uint8_t)((DAT_IA_ADDRESS_PTR )p_sock_addr)->sa_data[3]&0xff, + (uint8_t)((DAT_IA_ADDRESS_PTR )p_sock_addr)->sa_data[4]&0xff, + (uint8_t)((DAT_IA_ADDRESS_PTR )p_sock_addr)->sa_data[5]&0xff); + } + + hca_ptr->hca_address = *p_sock_addr; + + /* if structure address not from our hca_ptr */ + if ( ip_addr != &hca_ptr->hca_address ) + { + *ip_addr = *p_sock_addr; + } + + dapl_os_free (p_sock_addr, sizeof(DAT_SOCK_ADDR6)); + + } /* ip_addr != NULL */ + + if ( ia_attr != NULL ) + { + dapl_os_memzero( ia_attr->adapter_name, + (int)sizeof(ia_attr->adapter_name )); + dapl_os_memcpy(ia_attr->adapter_name, + DAT_ADAPTER_NAME, + min ( (int)dapl_os_strlen(DAT_ADAPTER_NAME), + (int)(DAT_NAME_MAX_LENGTH)-1 ) ); + + dapl_os_memzero ( ia_attr->vendor_name, + (int)sizeof(ia_attr->vendor_name) ); + dapl_os_memcpy ( ia_attr->vendor_name, + DAT_VENDOR_NAME, + min ( (int)dapl_os_strlen(DAT_VENDOR_NAME), + (int)(DAT_NAME_MAX_LENGTH)-1 ) ); + + /* FIXME : Vu + * this value should be revisited + * It can be set by DAT consumers + */ + ia_attr->ia_address_ptr = (DAT_PVOID)&hca_ptr->hca_address; + ia_attr->hardware_version_major = p_hca_attr->dev_id; + ia_attr->hardware_version_minor = p_hca_attr->revision; + ia_attr->max_eps = p_hca_attr->max_qps; + ia_attr->max_dto_per_ep = p_hca_attr->max_wrs; + ia_attr->max_rdma_read_per_ep = p_hca_attr->max_qp_resp_res; + ia_attr->max_evds = p_hca_attr->max_cqs; + ia_attr->max_evd_qlen = p_hca_attr->max_cqes; + ia_attr->max_iov_segments_per_dto = p_hca_attr->max_sges; + ia_attr->max_lmrs = p_hca_attr->init_regions; + ia_attr->max_lmr_block_size = p_hca_attr->init_region_size; + ia_attr->max_rmrs = p_hca_attr->init_windows; + ia_attr->max_lmr_virtual_address = p_hca_attr->max_addr_handles; + ia_attr->max_rmr_target_address = p_hca_attr->max_addr_handles; + ia_attr->max_pzs = p_hca_attr->max_pds; + /* + * DAT spec does not tie max_mtu_size with IB MTU + * + ia_attr->max_mtu_size = + dapl_ibal_mtu_table[p_hca_attr->p_port_attr->mtu]; + */ + ia_attr->max_mtu_size = + p_hca_attr->p_port_attr->max_msg_size; + ia_attr->max_rdma_size = + p_hca_attr->p_port_attr->max_msg_size; + ia_attr->num_transport_attr = 0; + ia_attr->transport_attr = NULL; + ia_attr->num_vendor_attr = 0; + ia_attr->vendor_attr = NULL; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " --> DsIMU_qHCA: (ver=%x) ep %d ep_q %d evd %d evd_q %d\n", + ia_attr->hardware_version_major, + ia_attr->max_eps, ia_attr->max_dto_per_ep, + ia_attr->max_evds, ia_attr->max_evd_qlen ); + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " --> DsIMU_qHCA: mtu %llu rdma %llu iov %d lmr %d rmr %d" + " rdma_io %d\n", + ia_attr->max_mtu_size, ia_attr->max_rdma_size, + ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs, + ia_attr->max_rmrs, ia_attr->max_rdma_read_per_ep ); + } + + if ( ep_attr != NULL ) + { + /* + * DAT spec does not tie max_mtu_size with IB MTU + * + ep_attr->max_mtu_size = + dapl_ibal_mtu_table[p_hca_attr->p_port_attr->mtu]; + */ + ep_attr->max_mtu_size = p_hca_attr->p_port_attr->max_msg_size; + ep_attr->max_rdma_size = p_hca_attr->p_port_attr->max_msg_size; + ep_attr->max_recv_dtos = p_hca_attr->max_wrs; + ep_attr->max_request_dtos = p_hca_attr->max_wrs; + ep_attr->max_recv_iov = p_hca_attr->max_sges; + ep_attr->max_request_iov = p_hca_attr->max_sges; + ep_attr->max_rdma_read_in = p_hca_attr->max_qp_resp_res; + ep_attr->max_rdma_read_out= p_hca_attr->max_qp_resp_res; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " --> DsIMU_qHCA: msg %llu dto %d iov %d rdma i%d,o%d\n", + ep_attr->max_mtu_size, + ep_attr->max_recv_dtos, ep_attr->max_recv_iov, + ep_attr->max_rdma_read_in, ep_attr->max_rdma_read_out); + } + return DAT_SUCCESS; +} + + +DAT_RETURN +dapls_ib_completion_poll ( IN DAPL_HCA *p_hca, + IN DAPL_EVD *p_evd, + IN ib_work_completion_t *cqe_ptr ) +{ + ib_api_status_t ib_status; + ib_work_completion_t *cqe_filled; + + /* + * FIXME - Vu + * Now we only poll for one cqe. We can poll for more than + * one completions later for better. However, this requires + * to change the logic in dapl_evd_dto_callback function + * to process more than one completion. + */ + cqe_ptr->p_next = NULL; + cqe_filled = NULL; + + if ( !p_hca->ib_hca_handle ) + { + return DAT_INVALID_HANDLE; + } + + ib_status = ib_poll_cq (p_evd->ib_cq_handle, &cqe_ptr, &cqe_filled); + + if ( ib_status == IB_INVALID_CQ_HANDLE ) + { + ib_status = IB_NOT_FOUND; + } + + return dapl_ib_status_convert (ib_status); +} + + +DAT_RETURN +dapls_ib_completion_notify ( IN ib_hca_handle_t hca_handle, + IN DAPL_EVD *p_evd, + IN ib_notification_type_t type ) +{ + ib_api_status_t ib_status; + DAT_BOOLEAN solic_notify; + + if ( !hca_handle ) + { + return DAT_INVALID_HANDLE; + } + solic_notify = (type == IB_NOTIFY_ON_SOLIC_COMP) ? DAT_TRUE : DAT_FALSE; + ib_status = ib_rearm_cq ( p_evd->ib_cq_handle, solic_notify ); + + return dapl_ib_status_convert (ib_status); +} + + + +DAT_RETURN +dapls_ib_wait_object_create ( + IN cl_waitobj_handle_t* p_cq_wait_obj_handle) +{ + cl_status_t cl_status; + + cl_status = cl_waitobj_create (FALSE /* auto_reset */, p_cq_wait_obj_handle); + + if (cl_status == CL_SUCCESS) + return DAT_SUCCESS; + + return DAT_INTERNAL_ERROR; +} + + +DAT_RETURN +dapls_ib_wait_object_destroy ( + IN cl_waitobj_handle_t cq_wait_obj_handle) +{ + cl_status_t cl_status; + + cl_status = cl_waitobj_destroy (cq_wait_obj_handle); + + if (cl_status == CL_SUCCESS) + return DAT_SUCCESS; + + return DAT_INTERNAL_ERROR; +} + + +DAT_RETURN +dapls_ib_wait_object_wakeup ( + IN cl_waitobj_handle_t cq_wait_obj_handle) +{ + cl_status_t cl_status; + + cl_status = cl_waitobj_signal (cq_wait_obj_handle); + + if (cl_status == CL_SUCCESS) + return DAT_SUCCESS; + + return DAT_INTERNAL_ERROR; +} + + +DAT_RETURN +dapls_ib_wait_object_wait ( + IN cl_waitobj_handle_t cq_wait_obj_handle, + IN uint32_t timeout) +{ + cl_status_t cl_status; + + cl_status = cl_waitobj_wait_on (cq_wait_obj_handle, timeout, TRUE ); + + switch (cl_status) + { + case CL_SUCCESS: + return DAT_SUCCESS; + case CL_TIMEOUT: + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> wait_object_wait: cl_timeout: %d\n", timeout); + return DAT_TIMEOUT_EXPIRED; + case CL_NOT_DONE: + return DAT_SUCCESS; + default: + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "--> wait_object_wait: cl_error: %d\n", cl_status); + return DAT_INTERNAL_ERROR; + } +} + + +/* + * dapls_ib_get_async_event + * + * Translate an asynchronous event type to the DAT event. + * Note that different providers have different sets of errors. + * + * Input: + * cause_ptr provider event cause + * + * Output: + * async_event DAT mapping of error + * + * Returns: + * DAT_SUCCESS + * DAT_NOT_IMPLEMENTED Caller is not interested this event + */ + +DAT_RETURN dapls_ib_get_async_event( + IN ib_async_event_rec_t *cause_ptr, + OUT DAT_EVENT_NUMBER *async_event) +{ + ib_async_event_t event_id; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + event_id = cause_ptr->code; + + dapl_dbg_log (DAPL_DBG_TYPE_WARN, + "--> DsAE: event_id = %d%d\n", event_id); + + switch (event_id ) + { + case IB_AE_SQ_ERROR: + case IB_AE_SQ_DRAINED: + case IB_AE_RQ_ERROR: + { + *async_event = DAT_ASYNC_ERROR_EP_BROKEN; + break; + } + + + + /* INTERNAL errors */ + case IB_AE_QP_FATAL: + case IB_AE_CQ_ERROR: + case IB_AE_LOCAL_FATAL: + case IB_AE_WQ_REQ_ERROR: + case IB_AE_WQ_ACCESS_ERROR: + { + *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR; + break; + } + + /* CATASTROPHIC errors */ + case IB_AE_FLOW_CTRL_ERROR: + case IB_AE_BUF_OVERRUN: + { + *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC; + break; + } + default: + { + /* + * Errors we are not interested in reporting: + * IB_AE_QP_APM + * IB_AE_PKEY_TRAP + * IB_AE_QKEY_TRAP + * IB_AE_MKEY_TRAP + * IB_AE_PORT_TRAP + * IB_AE_QP_APM_ERROR + * IB_AE_PORT_ACTIVE + * ... + */ + dat_status = DAT_NOT_IMPLEMENTED; + } + + } + + return dat_status; +} + +/* + * dapls_ib_get_dto_status + * + * Return the DAT status of a DTO operation + * + * Input: + * cqe_ptr pointer to completion queue entry + * + * Output: + * none + * + * Returns: + * Value from ib_status_map table above + */ + +DAT_DTO_COMPLETION_STATUS +dapls_ib_get_dto_status( + IN ib_work_completion_t *cqe_ptr) +{ + ib_uint32_t ib_status; + + ib_status = DAPL_GET_CQE_STATUS (cqe_ptr); + + switch (ib_status) + { + case IB_COMP_ST_SUCCESS : + return DAT_DTO_SUCCESS; + case IB_COMP_ST_LOCAL_LEN_ERR: + return DAT_DTO_ERR_LOCAL_LENGTH; + case IB_COMP_ST_LOCAL_OP_ERR: + return DAT_DTO_ERR_LOCAL_EP; + case IB_COMP_ST_LOCAL_PROTECT_ERR: + return DAT_DTO_ERR_LOCAL_PROTECTION; + case IB_COMP_ST_WR_FLUSHED_ERR: + return DAT_DTO_ERR_FLUSHED; + case IB_COMP_ST_MW_BIND_ERR: + return DAT_RMR_OPERATION_FAILED; + case IB_COMP_ST_REM_ACC_ERR: + return DAT_DTO_ERR_REMOTE_ACCESS; + case IB_COMP_ST_REM_OP_ERR: + return DAT_DTO_ERR_REMOTE_RESPONDER; + case IB_COMP_ST_RNR_COUNTER: + return DAT_DTO_ERR_RECEIVER_NOT_READY; + case IB_COMP_ST_TRANSP_COUNTER: + return DAT_DTO_ERR_TRANSPORT; + case IB_COMP_ST_REM_REQ_ERR: + return DAT_DTO_ERR_REMOTE_RESPONDER; + case IB_COMP_ST_BAD_RESPONSE_ERR: + return DAT_DTO_ERR_BAD_RESPONSE; + case IB_COMP_ST_EE_STATE_ERR: + case IB_COMP_ST_EE_CTX_NO_ERR: + return DAT_DTO_ERR_TRANSPORT; + default: + return DAT_DTO_FAILURE; + } +} + + +/* + * Map all IBAPI DTO completion codes to the DAT equivelent. + * + * dapls_ib_get_dat_event + * + * Return a DAT connection event given a provider CM event. + * + * N.B. Some architectures combine async and CM events into a + * generic async event. In that case, dapls_ib_get_dat_event() + * and dapls_ib_get_async_event() should be entry points that + * call into a common routine. + * + * Input: + * ib_cm_event event provided to the dapl callback routine + * active switch indicating active or passive connection + * + * Output: + * none + * + * Returns: + * DAT_EVENT_NUMBER of translated provider value + */ + +DAT_EVENT_NUMBER +dapls_ib_get_dat_event ( + IN const ib_cm_events_t ib_cm_event, + IN DAT_BOOLEAN active) +{ + DAT_EVENT_NUMBER dat_event_num = 0; + UNREFERENCED_PARAMETER (active); + + switch ( ib_cm_event) + { + case IB_CME_CONNECTED: + dat_event_num = DAT_CONNECTION_EVENT_ESTABLISHED; + break; + case IB_CME_DISCONNECTED: + dat_event_num = DAT_CONNECTION_EVENT_DISCONNECTED; + break; + case IB_CME_DISCONNECTED_ON_LINK_DOWN: + dat_event_num = DAT_CONNECTION_EVENT_DISCONNECTED; + break; + case IB_CME_CONNECTION_REQUEST_PENDING: + dat_event_num = DAT_CONNECTION_REQUEST_EVENT; + break; + case IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA: + dat_event_num = DAT_CONNECTION_REQUEST_EVENT; + break; + case IB_CME_DESTINATION_REJECT: + dat_event_num = DAT_CONNECTION_EVENT_NON_PEER_REJECTED; + break; + case IB_CME_DESTINATION_REJECT_PRIVATE_DATA: + dat_event_num = DAT_CONNECTION_EVENT_PEER_REJECTED; + break; + case IB_CME_DESTINATION_UNREACHABLE: + dat_event_num = DAT_CONNECTION_EVENT_UNREACHABLE; + break; + case IB_CME_TOO_MANY_CONNECTION_REQUESTS: + dat_event_num = DAT_CONNECTION_EVENT_NON_PEER_REJECTED; + break; + case IB_CME_LOCAL_FAILURE: + dat_event_num = DAT_CONNECTION_EVENT_BROKEN; + break; + case IB_CME_REPLY_RECEIVED: + case IB_CME_REPLY_RECEIVED_PRIVATE_DATA: + default: + break; + } +#if 0 + dapl_dbg_log (DAPL_DBG_TYPE_CM, + " dapls_ib_get_dat_event: event translation: (%s) " + "ib_event 0x%x dat_event 0x%x\n", + active ? "active" : "passive", + ib_cm_event, + dat_event_num); +#endif + return dat_event_num; +} + + +/* + * dapls_ib_get_dat_event + * + * Return a DAT connection event given a provider CM event. + * + * N.B. Some architectures combine async and CM events into a + * generic async event. In that case, dapls_ib_get_cm_event() + * and dapls_ib_get_async_event() should be entry points that + * call into a common routine. + * + * WARNING: In this implementation, there are multiple CM + * events that map to a single DAT event. Be very careful + * with provider routines that depend on this reverse mapping, + * they may have to accomodate more CM events than they + * 'naturally' would. + * + * Input: + * dat_event_num DAT event we need an equivelent CM event for + * + * Output: + * none + * + * Returns: + * ib_cm_event of translated DAPL value + */ +ib_cm_events_t +dapls_ib_get_cm_event ( + IN DAT_EVENT_NUMBER dat_event_num) +{ + ib_cm_events_t ib_cm_event = 0; + + switch (dat_event_num) + { + case DAT_CONNECTION_EVENT_ESTABLISHED: + ib_cm_event = IB_CME_CONNECTED; + break; + case DAT_CONNECTION_EVENT_DISCONNECTED: + ib_cm_event = IB_CME_DISCONNECTED; + break; + case DAT_CONNECTION_REQUEST_EVENT: + ib_cm_event = IB_CME_CONNECTION_REQUEST_PENDING; + break; + case DAT_CONNECTION_EVENT_NON_PEER_REJECTED: + ib_cm_event = IB_CME_DESTINATION_REJECT; + break; + case DAT_CONNECTION_EVENT_PEER_REJECTED: + ib_cm_event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA; + break; + case DAT_CONNECTION_EVENT_UNREACHABLE: + ib_cm_event = IB_CME_DESTINATION_UNREACHABLE; + break; + case DAT_CONNECTION_EVENT_BROKEN: + ib_cm_event = IB_CME_LOCAL_FAILURE; + break; + default: + break; + } + + return ib_cm_event; +} + + + +/* + * dapls_set_provider_specific_attr + * + * Input: + * attr_ptr Pointer provider specific attributes + * + * Output: + * none + * + * Returns: + * void + */ + +#ifdef DAT_EXTENSIONS +static DAT_NAMED_ATTR ib_attrs[] = { + { + "DAT_EXTENSION_INTERFACE", "TRUE" + }, + { + DAT_IB_ATTR_FETCH_AND_ADD, "TRUE" + }, + { + DAT_IB_ATTR_CMP_AND_SWAP, "TRUE" + }, + { + DAT_IB_ATTR_IMMED_DATA, "TRUE" + }, +}; +#define SPEC_ATTR_SIZE( x ) (sizeof( x ) / sizeof( DAT_NAMED_ATTR)) +#else +static DAT_NAMED_ATTR *ib_attrs = NULL; +#define SPEC_ATTR_SIZE( x ) 0 +#endif + +void dapls_query_provider_specific_attr( IN DAT_PROVIDER_ATTR *attr_ptr ) +{ + attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs); + attr_ptr->provider_specific_attr = ib_attrs; +} + + +DAT_RETURN dapls_ns_map_gid ( + IN DAPL_HCA *hca_ptr, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + OUT GID *gid) +{ + return (dapls_ib_ns_map_gid (hca_ptr, remote_ia_address, gid)); +} + +DAT_RETURN dapls_ns_map_ipaddr ( + IN DAPL_HCA *hca_ptr, + IN GID gid, + OUT DAT_IA_ADDRESS_PTR remote_ia_address) +{ + return (dapls_ib_ns_map_ipaddr (hca_ptr, gid, remote_ia_address)); +} + + +#ifdef NOT_USED +/* + * dapls_ib_post_recv - defered.until QP ! in init state. + * + * Provider specific Post RECV function + */ + +DAT_RETURN +dapls_ib_post_recv_defered ( + IN DAPL_EP *ep_ptr, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov) +{ + ib_api_status_t ib_status; + ib_recv_wr_t *recv_wr, *rwr; + ib_local_ds_t *ds_array_p; + DAT_COUNT i, total_len; + + if (ep_ptr->qp_state != IB_QPS_INIT) + { + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPR: BAD QP state(%s), not init? " + "EP %p QP %p cookie %p, num_seg %d\n", + ib_get_port_state_str(ep_ptr->qp_state), ep_ptr, + ep_ptr->qp_handle, cookie, num_segments); + return (DAT_INSUFFICIENT_RESOURCES); + } + + recv_wr = dapl_os_alloc (sizeof(ib_recv_wr_t) + + (num_segments*sizeof(ib_local_ds_t))); + if (NULL == recv_wr) + { + return (DAT_INSUFFICIENT_RESOURCES); + } + + dapl_os_memzero(recv_wr, sizeof(ib_recv_wr_t)); + recv_wr->wr_id = (DAT_UINT64) cookie; + recv_wr->num_ds = num_segments; + + ds_array_p = (ib_local_ds_t*)(recv_wr+1); + + recv_wr->ds_array = ds_array_p; + + //total_len = 0; + + for (total_len = i = 0; i < num_segments; i++, ds_array_p++) + { + ds_array_p->length = (uint32_t)local_iov[i].segment_length; + ds_array_p->lkey = local_iov[i].lmr_context; + ds_array_p->vaddr = local_iov[i].virtual_address; + total_len += ds_array_p->length; + } + + if (cookie != NULL) + { + cookie->val.dto.size = total_len; + + dapl_dbg_log (DAPL_DBG_TYPE_EP, + "--> DsPR: EP = %p QP = %p cookie= %p, num_seg= %d\n", + ep_ptr, ep_ptr->qp_handle, cookie, num_segments); + } + + recv_wr->p_next = NULL; + + /* find last defered recv work request, link new on the end */ + rwr=ep_ptr->cm_post; + if (rwr == NULL) + { + ep_ptr->cm_post = (void*)recv_wr; + i = 1; + } + else + { + for(i=2; rwr->p_next; rwr=rwr->p_next) i++; + rwr->p_next = recv_wr; + } + + dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPR: %s() EP %p QP %p cookie %p " + "num_seg %d Tdefered %d\n", + __FUNCTION__, ep_ptr, ep_ptr->qp_handle, cookie, num_segments, + i); + + return DAT_SUCCESS; +} +#endif + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.h b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.h new file mode 100644 index 00000000..02a84d52 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_util.h @@ -0,0 +1,497 @@ + +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "Common Public + * License" a copy of which is in the file LICENSE.txt in the root + * directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/cpl.php. + * + */ + +/********************************************************************** + * + * MODULE: dapl_ibal_util.h + * + * PURPOSE: Utility defs & routines for access to openib-windows IBAL APIs + * + * $Id: dapl_ibal_util.h 33 2005-07-11 19:51:17Z ftillier $ + * + **********************************************************************/ + +#ifndef _DAPL_IBAL_UTIL_H_ +#define _DAPL_IBAL_UTIL_H_ + +#include +#include +#include +#include + +#ifdef DAT_EXTENSIONS +#include +#endif + +STATIC _INLINE_ DAT_RETURN +dapl_ib_status_convert ( + IN int32_t ib_status); + +/* + * Typedefs to map IBAL types to more generic 'ib' types + */ +typedef ib_cm_handle_t *dp_ib_cm_handle_t; + +typedef ib_net64_t IB_HCA_NAME; +typedef ib_listen_handle_t ib_cm_srvc_handle_t; +typedef ib_ca_handle_t ib_hca_handle_t; +typedef DAT_PVOID ib_cqd_handle_t; +typedef ib_async_event_rec_t ib_error_record_t; +typedef ib_wr_type_t ib_send_op_type_t; +typedef ib_wc_t ib_work_completion_t; +typedef uint32_t ib_hca_port_t; +typedef uint32_t ib_uint32_t; +typedef uint32_t ib_comp_handle_t; +typedef ib_local_ds_t ib_data_segment_t; + +typedef unsigned __int3264 cl_dev_handle_t; + +typedef void (*ib_async_handler_t)( + IN ib_hca_handle_t ib_hca_handle, + IN ib_error_record_t *err_code, + IN void *context); + +typedef ib_net64_t ib_guid_t; +typedef ib_net16_t ib_lid_t; +typedef boolean_t ib_bool_t; + +typedef struct _GID +{ + uint64_t gid_prefix; + uint64_t guid; +} GID; + +typedef enum +{ + IB_CME_CONNECTED, + IB_CME_DISCONNECTED, + IB_CME_DISCONNECTED_ON_LINK_DOWN, + IB_CME_CONNECTION_REQUEST_PENDING, + IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, + IB_CME_DESTINATION_REJECT, + IB_CME_DESTINATION_REJECT_PRIVATE_DATA, + IB_CME_DESTINATION_UNREACHABLE, + IB_CME_TOO_MANY_CONNECTION_REQUESTS, + IB_CME_LOCAL_FAILURE, + IB_CME_REPLY_RECEIVED, + IB_CME_REPLY_RECEIVED_PRIVATE_DATA, + IB_CM_LOCAL_FAILURE +} ib_cm_events_t; + + +typedef enum +{ + IB_NOTIFY_ON_NEXT_COMP, + IB_NOTIFY_ON_SOLIC_COMP +} ib_notification_type_t; + +typedef struct _ib_hca_name +{ + DAT_NAME_PTR hca_name[DAT_NAME_MAX_LENGTH]; +} ib_hca_name_t; + + +#define IB_INVALID_HANDLE NULL +#define true TRUE +#define false FALSE + +#define IB_MAX_REQ_PDATA_SIZE 92 +#define IB_MAX_REP_PDATA_SIZE 196 +#define IB_MAX_REJ_PDATA_SIZE 148 +#define IB_MAX_DREQ_PDATA_SIZE 220 +#define IB_MAX_DREP_PDATA_SIZE 224 + +typedef void +(*dapl_ibal_pfn_destructor_t)( + IN void* context ); + +typedef struct _dapl_ibal_refs +{ + atomic32_t count; // number of references + void* context; // context for destructor + dapl_ibal_pfn_destructor_t destructor; // called when reference goes to zero + +} dapl_ibal_refs_t; + + +typedef struct _dapl_ibal_root +{ + ib_al_handle_t h_al; // handle to Access Layer + cl_spinlock_t ca_lock; // CA list lock + cl_qlist_t ca_head; // list head of CAs + boolean_t shutdown; // when true, driver is shutting down + boolean_t initialized; // when true, lib is initialized + +} dapl_ibal_root_t; + + +typedef struct _dapl_ibal_ca +{ + cl_list_item_t next; // peer CA list + ib_ca_handle_t h_ca; // handle to open CA + ib_ca_attr_t *p_ca_attr; // CA attributes + uint32_t ca_attr_size;// size of ca attribute + dapl_ibal_refs_t refs; // reference counting + cl_spinlock_t port_lock; // port list lock + cl_qlist_t port_head; // port list head for this CA + cl_spinlock_t evd_cb_lock; // EVD async error cb list lock + cl_qlist_t evd_cb_head; // EVD async error cb list head for this CA + cl_dev_handle_t mlnx_device; + DAT_PVOID *ia_ptr; // hook for CA async callbacks +} dapl_ibal_ca_t; + + +typedef struct _dapl_ibal_port +{ + cl_list_item_t next; // peer CA list + dapl_ibal_ca_t *ca; // pointer to parent CA + ib_port_attr_t *p_attr; // port attributes + dapl_ibal_refs_t refs; // reference counting +} dapl_ibal_port_t; + +typedef struct _dapl_ibal_evd_cb +{ + cl_list_item_t next; // peer CA list + ib_async_handler_t pfn_async_err_cb; + ib_async_handler_t pfn_async_qp_err_cb; + ib_async_handler_t pfn_async_cq_err_cb; + void *context; +} dapl_ibal_evd_cb_t; + +/* + * Definitions to map DTO OPs + */ +#define OP_RDMA_READ WR_RDMA_READ +#define OP_RDMA_WRITE WR_RDMA_WRITE +#define OP_SEND WR_SEND +#define OP_COMP_AND_SWAP WR_COMPARE_SWAP +#define OP_FETCH_AND_ADD WR_FETCH_ADD + +#define OP_RECEIVE 6 /* no-equip */ +#define OP_BIND_MW 7 /* no-equip */ +#define OP_RDMA_WRITE_IMM 8 /* no-equip */ +#define OP_RECEIVE_IMM 9 /* no-equip */ + +/* + * Definitions to map QP state + */ +#define IB_QP_STATE_RESET IB_QPS_RESET +#define IB_QP_STATE_INIT IB_QPS_INIT +#define IB_QP_STATE_RTR IB_QPS_RTR +#define IB_QP_STATE_RTS IB_QPS_RTS +#define IB_QP_STATE_SQE IB_QPS_SQERR +#define IB_QP_STATE_SQD IB_QPS_SQD +#define IB_QP_STATE_ERROR IB_QPS_ERROR + +/* + * Definitions to map Memory OPs + */ +#define IB_ACCESS_LOCAL_WRITE IB_AC_LOCAL_WRITE +#define IB_ACCESS_REMOTE_READ IB_AC_RDMA_READ +#define IB_ACCESS_REMOTE_WRITE IB_AC_RDMA_WRITE + +/* + * CQE status + */ +enum _dapl_comp_status +{ + IB_COMP_ST_SUCCESS = IB_WCS_SUCCESS, + IB_COMP_ST_LOCAL_LEN_ERR = IB_WCS_LOCAL_LEN_ERR, + IB_COMP_ST_LOCAL_OP_ERR = IB_WCS_LOCAL_OP_ERR, + IB_COMP_ST_LOCAL_PROTECT_ERR = IB_WCS_LOCAL_PROTECTION_ERR, + IB_COMP_ST_WR_FLUSHED_ERR = IB_WCS_WR_FLUSHED_ERR, + IB_COMP_ST_MW_BIND_ERR = IB_WCS_MEM_WINDOW_BIND_ERR, + IB_COMP_ST_REM_ACC_ERR = IB_WCS_REM_ACCESS_ERR, + IB_COMP_ST_REM_OP_ERR = IB_WCS_REM_OP_ERR, + IB_COMP_ST_RNR_COUNTER = IB_WCS_RNR_RETRY_ERR, + IB_COMP_ST_TRANSP_COUNTER = IB_WCS_TIMEOUT_RETRY_ERR, + IB_COMP_ST_REM_REQ_ERR = IB_WCS_REM_INVALID_REQ_ERR, + IB_COMP_ST_BAD_RESPONSE_ERR = IB_WCS_UNMATCHED_RESPONSE, + IB_COMP_ST_EE_STATE_ERR, + IB_COMP_ST_EE_CTX_NO_ERR +}; + + +/* + * Macro to check the state of an EP/QP + */ +#define DAPLIB_NEEDS_INIT(ep) ((ep)->qp_state == IB_QPS_ERROR) + + +/* + * Resolve IBAL return codes to their DAPL equivelent. + * Do not return invalid Handles, the user is not able + * to deal with them. + */ +STATIC _INLINE_ DAT_RETURN +dapl_ib_status_convert ( + IN int32_t ib_status) +{ + switch ( ib_status ) + { + case IB_SUCCESS: + { + return DAT_SUCCESS; + } + case IB_INSUFFICIENT_RESOURCES: + case IB_INSUFFICIENT_MEMORY: + case IB_RESOURCE_BUSY: + { + return DAT_INSUFFICIENT_RESOURCES; + } + case IB_INVALID_CA_HANDLE: + case IB_INVALID_CQ_HANDLE: + case IB_INVALID_QP_HANDLE: + case IB_INVALID_PD_HANDLE: + case IB_INVALID_MR_HANDLE: + case IB_INVALID_MW_HANDLE: + case IB_INVALID_AL_HANDLE: + case IB_INVALID_AV_HANDLE: + { + return DAT_INVALID_HANDLE; + } + case IB_INVALID_PKEY: + { + return DAT_PROTECTION_VIOLATION; + } + case IB_INVALID_LKEY: + case IB_INVALID_RKEY: + case IB_INVALID_PERMISSION: + { + return DAT_PRIVILEGES_VIOLATION; + } + case IB_INVALID_MAX_WRS: + case IB_INVALID_MAX_SGE: + case IB_INVALID_CQ_SIZE: + case IB_INVALID_SETTING: + case IB_INVALID_SERVICE_TYPE: + case IB_INVALID_GID: + case IB_INVALID_LID: + case IB_INVALID_GUID: + case IB_INVALID_PARAMETER: + { + return DAT_INVALID_PARAMETER; + } + case IB_INVALID_QP_STATE: + case IB_INVALID_APM_STATE: + case IB_INVALID_PORT_STATE: + case IB_INVALID_STATE: + { + return DAT_INVALID_STATE; + } + case IB_NOT_FOUND: + { + return DAT_QUEUE_EMPTY; + } + case IB_OVERFLOW: + { + return DAT_QUEUE_FULL; + } + case IB_UNSUPPORTED: + { + return DAT_NOT_IMPLEMENTED; + } + case IB_TIMEOUT: + { + return DAT_TIMEOUT_EXPIRED; + } + case IB_CANCELED: + { + return DAT_ABORT; + } + default: + { + return DAT_INTERNAL_ERROR; + } + } +} + +#define TAKE_LOCK( lock ) \ + cl_spinlock_acquire( &(lock) ) + +#define RELEASE_LOCK( lock ) \ + cl_spinlock_release( &(lock) ) + +#define LOCK_INSERT_HEAD( lock, head, item ) \ +{ \ + TAKE_LOCK( lock ); \ + cl_qlist_insert_head( &head, (cl_list_item_t*)(&item) ); \ + RELEASE_LOCK( lock ); \ +} + +#define LOCK_INSERT_TAIL( lock, tail, item ) \ +{ \ + TAKE_LOCK( lock ); \ + cl_qlist_insert_tail( &tail, (cl_list_item_t*)(&item) ); \ + RELEASE_LOCK( lock ); \ +} + +#define INIT_REFERENCE( p_ref, n, con, destruct ) \ +{ \ + (p_ref)->count = n; \ + (p_ref)->context = con; \ + (p_ref)->destructor = destruct; \ +} + +#define TAKE_REFERENCE( p_ref ) \ + cl_atomic_inc( &(p_ref)->count ) + +#define REMOVE_REFERENCE( p_ref ) \ +{ \ + if ( cl_atomic_dec( &(p_ref)->count ) == 0 ) \ + if ( (p_ref)->destructor ) \ + (p_ref)->destructor( (p_ref)->context ); \ +} + +/* + * dapl_llist_entry in dapl.h but dapl.h depends on provider + * typedef's in this file first. move dapl_llist_entry out of dapl.h + */ +struct ib_llist_entry +{ + struct dapl_llist_entry *flink; + struct dapl_llist_entry *blink; + void *data; + struct dapl_llist_entry *list_head; +}; + + +typedef struct _ib_hca_transport +{ + struct ib_llist_entry entry; + int destroy; + struct dapl_hca *d_hca; + DAPL_OS_WAIT_OBJECT wait_object; + +// struct ibv_device *ib_dev; +// struct ibv_context *ib_ctx; +// struct ib_cm_device *ib_cm; +// struct ibv_comp_channel *ib_cq; +// ib_cq_handle_t ib_cq_empty; +// int max_inline_send; +// union ibv_gid gid; +// ib_async_handler_t async_unafiliated; +// void *async_un_ctx; +// ib_async_cq_handler_t async_cq_error; +// ib_async_dto_handler_t async_cq; +// ib_async_qp_handler_t async_qp_error; + + /* from 1.1 winDAPL - Values specific to IB OS API */ + //IB_HCA_NAME name; + ib_hca_handle_t ib_hca_handle; + DAPL_ATOMIC handle_ref_count; /* count of ia_opens on handle */ + ib_hca_port_t port_num; /* number of physical port */ + ib_uint32_t partition_max; + ib_guid_t node_GUID; + ib_lid_t lid; + ib_cqd_handle_t ib_cqd_handle; /* cq domain handle */ + ib_cq_handle_t null_ib_cq_handle; /* CQ handle with 0 entries */ + + /* Name service support */ + void *name_service_handle; /* handle to name service */ + +} ib_hca_transport_t; + +/* provider specfic fields for shared memory support */ +typedef uint32_t ib_shm_transport_t; + +/* + * Prototype + */ + +extern ib_api_status_t +dapls_modify_qp_state_to_error ( + ib_qp_handle_t qp_handle ); + +extern ib_api_status_t +dapls_modify_qp_state_to_reset ( + ib_qp_handle_t); + +extern ib_api_status_t +dapls_modify_qp_state_to_init ( + ib_qp_handle_t, DAT_EP_ATTR *, dapl_ibal_port_t *); + +extern ib_api_status_t +dapls_modify_qp_state_to_rtr ( + ib_qp_handle_t, ib_net32_t, ib_lid_t, dapl_ibal_port_t *); + +extern ib_api_status_t +dapls_modify_qp_state_to_rts ( + ib_qp_handle_t); + +extern void +dapli_ibal_ca_async_error_callback( + IN ib_async_event_rec_t* p_err_rec ); + +extern dapl_ibal_port_t * +dapli_ibal_get_port ( + IN dapl_ibal_ca_t *p_ca, + IN uint8_t port_num); + +extern int32_t dapls_ib_init (void); +extern int32_t dapls_ib_release (void); + +extern dapl_ibal_evd_cb_t * +dapli_find_evd_cb_by_context( + IN void *context, + IN dapl_ibal_ca_t *ca); + +extern IB_HCA_NAME +dapl_ib_convert_name( + IN char *name); + + +STATIC _INLINE_ int32_t +dapl_lmr_convert_privileges ( IN DAT_MEM_PRIV_FLAGS privileges ) +{ + int32_t value = 0; + + /* + * if (DAT_MEM_PRIV_LOCAL_READ_FLAG & privileges) + * do nothing + */ + + if (DAT_MEM_PRIV_LOCAL_WRITE_FLAG & privileges) + { + value |= IB_ACCESS_LOCAL_WRITE; + } + + if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) + { + value |= IB_ACCESS_REMOTE_READ; + } + + if (DAT_MEM_PRIV_REMOTE_WRITE_FLAG & privileges) + { + value |= IB_ACCESS_REMOTE_WRITE; + } + return value; +} + + +STATIC _INLINE_ int32_t +dapl_rmr_convert_privileges ( IN DAT_MEM_PRIV_FLAGS privileges ) +{ + int32_t value = 0; + + if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) + { + value |= IB_ACCESS_REMOTE_READ; + } + + if (DAT_MEM_PRIV_REMOTE_WRITE_FLAG & privileges) + { + value |= IB_ACCESS_REMOTE_WRITE; + } + + return value; +} +#endif /* _DAPL_IBAL_UTIL_H_ */ diff --git a/trunk/ulp/dapl2/dapl/include/dapl.h b/trunk/ulp/dapl2/dapl/include/dapl.h new file mode 100644 index 00000000..eb1fecff --- /dev/null +++ b/trunk/ulp/dapl2/dapl/include/dapl.h @@ -0,0 +1,1267 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl.h + * + * PURPOSE: defines common data structures for the DAPL reference implemenation + * + * Description: This file describes the working data structures used within + * DAPL RI. + * + * + * $Id: dapl.h 1317 2005-04-25 17:29:42Z jlentini $ + **********************************************************************/ + +#ifndef _DAPL_H_ +#define _DAPL_H_ + +#if defined(__KERNEL__) +#include +#else +#include +#endif /* defined(__KERNEL__) */ +#include +#include "dapl_osd.h" +#include "dapl_debug.h" + +#ifdef IBAPI +#include "dapl_ibapi_util.h" +#elif VAPI +#include "dapl_vapi_util.h" +#elif __OPENIB__ +#include "dapl_openib_util.h" +#include "dapl_openib_cm.h" +#elif DUMMY +#include "dapl_dummy_util.h" +#elif OPENIB +#include "dapl_ib_util.h" +#else /* windows - IBAL */ +#include "dapl_ibal_util.h" +#endif + +/********************************************************************* + * * + * Enumerations * + * * + *********************************************************************/ + +typedef enum dapl_magic +{ + /* magic number values for verification & debug */ + DAPL_MAGIC_IA = 0xCafeF00d, + DAPL_MAGIC_EVD = 0xFeedFace, + DAPL_MAGIC_EP = 0xDeadBabe, + DAPL_MAGIC_LMR = 0xBeefCafe, + DAPL_MAGIC_RMR = 0xABadCafe, + DAPL_MAGIC_PZ = 0xDeafBeef, + DAPL_MAGIC_PSP = 0xBeadeD0c, + DAPL_MAGIC_RSP = 0xFab4Feed, + DAPL_MAGIC_SRQ = 0xC001Babe, + DAPL_MAGIC_CR = 0xBe12Cee1, + DAPL_MAGIC_CR_DESTROYED = 0xB12bDead, + DAPL_MAGIC_CNO = 0xDeadF00d, + DAPL_MAGIC_INVALID = 0xFFFFFFFF +} DAPL_MAGIC; + +typedef enum dapl_evd_state +{ + DAPL_EVD_STATE_TERMINAL, + DAPL_EVD_STATE_INITIAL, + DAPL_EVD_STATE_OPEN, + DAPL_EVD_STATE_WAITED, + DAPL_EVD_STATE_DEAD = 0xDEAD +} DAPL_EVD_STATE; + +typedef enum dapl_evd_completion +{ + DAPL_EVD_STATE_INIT, + DAPL_EVD_STATE_SOLICITED_WAIT, + DAPL_EVD_STATE_THRESHOLD, + DAPL_EVD_STATE_UNSIGNALLED +} DAPL_EVD_COMPLETION; + +typedef enum dapl_cno_state +{ + DAPL_CNO_STATE_UNTRIGGERED, + DAPL_CNO_STATE_TRIGGERED, + DAPL_CNO_STATE_DEAD = 0xDeadFeed, +} DAPL_CNO_STATE; + +typedef enum dapl_qp_state +{ + DAPL_QP_STATE_UNCONNECTED, + DAPL_QP_STATE_RESERVED, + DAPL_QP_STATE_PASSIVE_CONNECTION_PENDING, + DAPL_QP_STATE_ACTIVE_CONNECTION_PENDING, + DAPL_QP_STATE_TENTATIVE_CONNECTION_PENDING, + DAPL_QP_STATE_CONNECTED, + DAPL_QP_STATE_DISCONNECT_PENDING, + DAPL_QP_STATE_ERROR, + DAPL_QP_STATE_NOT_REUSABLE, + DAPL_QP_STATE_FREE +} DAPL_QP_STATE; + + +/********************************************************************* + * * + * Constants * + * * + *********************************************************************/ + +/* + * number of HCAs allowed + */ +#define DAPL_MAX_HCA_COUNT 4 + +/* + * Configures the RMR bind evd restriction + */ + +#define DAPL_RMR_BIND_EVD_RESTRICTION DAT_RMR_EVD_SAME_AS_REQUEST_EVD + +/* + * special qp_state indicating the EP does not have a QP attached yet + */ +#define DAPL_QP_STATE_UNATTACHED 0xFFF0 + +#define DAPL_MAX_PRIVATE_DATA_SIZE 256 + +/********************************************************************* + * * + * Macros * + * * + *********************************************************************/ + +#if defined (sun) || defined(__sun) || defined(_sun_) || defined (__solaris__) +#define DAPL_BAD_PTR(a) ((unsigned long)(a) & 3) +#elif defined(__linux__) +#define DAPL_BAD_PTR(a) ((unsigned long)(a) & 3) +#elif defined(_WIN64) +#define DAPL_BAD_PTR(a) ((unsigned long)((DAT_UINT64)(a)) & 3) +#elif defined(_WIN32) +#define DAPL_BAD_PTR(a) ((unsigned long)(a) & 3) +#endif + +/* + * Simple macro to verify a handle is bad. Conditions: + * - pointer is NULL + * - pointer is not word aligned + * - pointer's magic number is wrong + */ + +#define DAPL_BAD_HANDLE(h, magicNum) ( \ + ((h) == NULL) || \ + DAPL_BAD_PTR(h) || \ + (((DAPL_HEADER *)(h))->magic != (magicNum))) + +#define DAPL_MIN(a, b) ((a < b) ? (a) : (b)) +#define DAPL_MAX(a, b) ((a > b) ? (a) : (b)) + +#if NDEBUG > 0 +#define DEBUG_IS_BAD_HANDLE(h, magicNum) (DAPL_BAD_HANDLE(h, magicNum)) +#else +#define DEBUG_IS_BAD_HANDLE(h, magicNum) (0) +#endif + +#define DAT_ERROR(Type, SubType) ((DAT_RETURN)(DAT_CLASS_ERROR | Type | SubType)) + +/********************************************************************* + * * + * Typedefs * + * * + *********************************************************************/ + +typedef struct dapl_llist_entry DAPL_LLIST_ENTRY; +typedef DAPL_LLIST_ENTRY * DAPL_LLIST_HEAD; +typedef struct dapl_ring_buffer DAPL_RING_BUFFER; +typedef struct dapl_cookie_buffer DAPL_COOKIE_BUFFER; + +typedef struct dapl_hash_table DAPL_HASH_TABLE; +typedef struct dapl_hash_table *DAPL_HASH_TABLEP; +typedef DAT_UINT64 DAPL_HASH_KEY; +typedef void * DAPL_HASH_DATA; + +typedef struct dapl_hca DAPL_HCA; + +typedef struct dapl_header DAPL_HEADER; + +typedef struct dapl_ia DAPL_IA; +typedef struct dapl_cno DAPL_CNO; +typedef struct dapl_evd DAPL_EVD; +typedef struct dapl_ep DAPL_EP; +typedef struct dapl_srq DAPL_SRQ; +typedef struct dapl_pz DAPL_PZ; +typedef struct dapl_lmr DAPL_LMR; +typedef struct dapl_rmr DAPL_RMR; +typedef struct dapl_sp DAPL_SP; +typedef struct dapl_cr DAPL_CR; + +typedef struct dapl_cookie DAPL_COOKIE; +typedef struct dapl_dto_cookie DAPL_DTO_COOKIE; +typedef struct dapl_rmr_cookie DAPL_RMR_COOKIE; + +typedef struct dapl_private DAPL_PRIVATE; + +typedef void (*DAPL_CONNECTION_STATE_HANDLER) ( + IN DAPL_EP *, + IN ib_cm_events_t, + IN const void *, + OUT DAT_EVENT *); + + +/********************************************************************* + * * + * Structures * + * * + *********************************************************************/ + +struct dapl_llist_entry +{ + struct dapl_llist_entry *flink; + struct dapl_llist_entry *blink; + void *data; + DAPL_LLIST_HEAD *list_head; /* for consistency checking */ +}; + +struct dapl_ring_buffer +{ + void **base; /* base of element array */ + DAT_COUNT lim; /* mask, number of entries - 1 */ + DAPL_ATOMIC head; /* head pointer index */ + DAPL_ATOMIC tail; /* tail pointer index */ +}; + +struct dapl_cookie_buffer +{ + DAPL_COOKIE *pool; + DAT_COUNT pool_size; + DAPL_ATOMIC head; + DAPL_ATOMIC tail; +}; + +struct dapl_hca +{ + DAPL_OS_LOCK lock; + DAPL_LLIST_HEAD ia_list_head; /* list of all open IAs */ + DAPL_ATOMIC handle_ref_count; /* count of ia_opens on handle */ + DAPL_EVD *async_evd; + DAPL_EVD *async_error_evd; + DAT_SOCK_ADDR6 hca_address; /* local address of HCA*/ + char *name; /* provider name */ + ib_hca_handle_t ib_hca_handle; + unsigned long port_num; /* number of physical port */ + ib_hca_transport_t ib_trans; /* Values specific transport API */ + /* Memory Subsystem Support */ + DAPL_HASH_TABLE *lmr_hash_table; + /* Limits & useful HCA attributes */ + DAT_IA_ATTR ia_attr; +}; + +/* DAPL Objects always have the following header */ +struct dapl_header +{ + DAT_PROVIDER *provider; /* required by DAT - must be first */ + DAPL_MAGIC magic; /* magic number for verification */ + DAT_HANDLE_TYPE handle_type; /* struct type */ + DAPL_IA *owner_ia; /* ia which owns this stuct */ + DAPL_LLIST_ENTRY ia_list_entry; /* link entry on ia struct */ + DAT_CONTEXT user_context; /* user context - opaque to DAPL */ + DAPL_OS_LOCK lock; /* lock - in header for easier macros */ +}; + +/* DAPL_IA maps to DAT_IA_HANDLE */ +struct dapl_ia +{ + DAPL_HEADER header; + DAPL_HCA *hca_ptr; + DAPL_EVD *async_error_evd; + DAT_BOOLEAN cleanup_async_error_evd; + + DAPL_LLIST_ENTRY hca_ia_list_entry; /* HCAs list of IAs */ + DAPL_LLIST_HEAD ep_list_head; /* EP queue */ + DAPL_LLIST_HEAD lmr_list_head; /* LMR queue */ + DAPL_LLIST_HEAD rmr_list_head; /* RMR queue */ + DAPL_LLIST_HEAD pz_list_head; /* PZ queue */ + DAPL_LLIST_HEAD evd_list_head; /* EVD queue */ + DAPL_LLIST_HEAD cno_list_head; /* CNO queue */ + DAPL_LLIST_HEAD psp_list_head; /* PSP queue */ + DAPL_LLIST_HEAD rsp_list_head; /* RSP queue */ + DAPL_LLIST_HEAD srq_list_head; /* SRQ queue */ +}; + +/* DAPL_CNO maps to DAT_CNO_HANDLE */ +struct dapl_cno +{ + DAPL_HEADER header; + + /* A CNO cannot be freed while it is referenced elsewhere. */ + DAPL_ATOMIC cno_ref_count; + DAPL_CNO_STATE cno_state; + + DAT_COUNT cno_waiters; + DAPL_EVD *cno_evd_triggered; +#if defined(__KERNEL__) + DAT_UPCALL_OBJECT cno_upcall; + DAT_UPCALL_POLICY cno_upcall_policy; +#else + DAT_OS_WAIT_PROXY_AGENT cno_wait_agent; +#endif /* defined(__KERNEL__) */ + + DAPL_OS_WAIT_OBJECT cno_wait_object; +}; + +/* DAPL_EVD maps to DAT_EVD_HANDLE */ +struct dapl_evd +{ + DAPL_HEADER header; + + DAPL_EVD_STATE evd_state; + DAT_EVD_FLAGS evd_flags; + DAT_BOOLEAN evd_enabled; /* For attached CNO. */ + DAT_BOOLEAN evd_waitable; /* EVD state. */ + + /* Derived from evd_flags; see dapls_evd_internal_create. */ + DAT_BOOLEAN evd_producer_locking_needed; + + /* Every EVD has a CQ unless it is a SOFTWARE_EVENT only EVD */ + ib_cq_handle_t ib_cq_handle; + + /* Mellanox Specific completion handle for registration/de-registration */ + ib_comp_handle_t ib_comp_handle; + + /* An Event Dispatcher cannot be freed while + * it is referenced elsewhere. + */ + DAPL_ATOMIC evd_ref_count; + + /* Set if there has been a catastrophic overflow */ + DAT_BOOLEAN catastrophic_overflow; + + /* the actual events */ + DAT_COUNT qlen; + DAT_EVENT *events; + DAPL_RING_BUFFER free_event_queue; + DAPL_RING_BUFFER pending_event_queue; + + /* CQ Completions are not placed into 'deferred_events' + ** rather they are simply left on the Completion Queue + ** and the fact that there was a notification is flagged. + */ + DAT_BOOLEAN cq_notified; + DAPL_OS_TICKS cq_notified_when; + + DAT_COUNT cno_active_count; + DAPL_CNO *cno_ptr; + + DAPL_OS_WAIT_OBJECT wait_object; + +#ifdef CQ_WAIT_OBJECT + /* Some providers support a direct CQ wait object */ + ib_wait_obj_handle_t cq_wait_obj_handle; +#endif + + DAT_COUNT threshold; + DAPL_EVD_COMPLETION completion_type; +}; + +/* DAPL_PRIVATE used to pass private data in a connection */ +struct dapl_private +{ +#ifdef IBHOSTS_NAMING + DAT_SOCK_ADDR6 hca_address; /* local address of HCA*/ +#endif /* IBHOSTS_NAMING */ + unsigned char private_data[DAPL_MAX_PRIVATE_DATA_SIZE]; +}; + +/* uDAPL timer entry, used to queue timeouts */ +struct dapl_timer_entry +{ + DAPL_LLIST_ENTRY list_entry; /* link entry on ia struct */ + DAPL_OS_TIMEVAL expires; + void (*function) (uintptr_t); + void *data; +}; + +#ifdef DAPL_DBG_IO_TRC + +#define DBG_IO_TRC_QLEN 32 /* length of trace buffer */ +#define DBG_IO_TRC_IOV 3 /* iov elements we keep track of */ + +struct io_buf_track +{ + Ib_send_op_type op_type; + DAPL_COOKIE *cookie; + DAT_LMR_TRIPLET iov[DBG_IO_TRC_IOV]; + DAT_RMR_TRIPLET remote_iov; + unsigned int done; /* count to track completion ordering */ + int status; + void *wqe; +}; + +#endif /* DAPL_DBG_IO_TRC */ + +/* DAPL_EP maps to DAT_EP_HANDLE */ +struct dapl_ep +{ + DAPL_HEADER header; + /* What the DAT Consumer asked for */ + DAT_EP_PARAM param; + + /* The RC Queue Pair (IBM OS API) */ + ib_qp_handle_t qp_handle; + unsigned int qpn; /* qp number */ + ib_qp_state_t qp_state; + + /* communications manager handle (IBM OS API) */ + dp_ib_cm_handle_t cm_handle; + + /* store the remote IA address here, reference from the param + * struct which only has a pointer, no storage + */ + DAT_SOCK_ADDR6 remote_ia_address; + + /* For passive connections we maintain a back pointer to the CR */ + void * cr_ptr; + + /* pointer to connection timer, if set */ + struct dapl_timer_entry *cxn_timer; + + /* private data container */ + DAPL_PRIVATE private; + + /* DTO data */ + DAPL_ATOMIC req_count; + DAPL_ATOMIC recv_count; + + DAPL_COOKIE_BUFFER req_buffer; + DAPL_COOKIE_BUFFER recv_buffer; + + ib_data_segment_t *recv_iov; + DAT_COUNT recv_iov_num; + + ib_data_segment_t *send_iov; + DAT_COUNT send_iov_num; +#ifdef DAPL_DBG_IO_TRC + int ibt_dumped; + struct io_buf_track *ibt_base; + DAPL_RING_BUFFER ibt_queue; +#endif /* DAPL_DBG_IO_TRC */ +#if defined(_WIN32) || defined(_WIN64) + DAT_BOOLEAN recv_discreq; + DAT_BOOLEAN sent_discreq; + dp_ib_cm_handle_t ibal_cm_handle; +#endif +}; + +/* DAPL_SRQ maps to DAT_SRQ_HANDLE */ +struct dapl_srq +{ + DAPL_HEADER header; + DAT_SRQ_PARAM param; + DAPL_ATOMIC srq_ref_count; + DAPL_COOKIE_BUFFER recv_buffer; + DAPL_ATOMIC recv_count; +}; + +/* DAPL_PZ maps to DAT_PZ_HANDLE */ +struct dapl_pz +{ + DAPL_HEADER header; + ib_pd_handle_t pd_handle; + DAPL_ATOMIC pz_ref_count; +}; + +/* DAPL_LMR maps to DAT_LMR_HANDLE */ +struct dapl_lmr +{ + DAPL_HEADER header; + DAT_LMR_PARAM param; + ib_mr_handle_t mr_handle; + DAPL_ATOMIC lmr_ref_count; +#if !defined(__KDAPL__) + char shmid[DAT_LMR_COOKIE_SIZE]; /* shared memory ID */ + ib_shm_transport_t ib_trans; /* provider specific data */ +#endif /* !__KDAPL__ */ +}; + +/* DAPL_RMR maps to DAT_RMR_HANDLE */ +struct dapl_rmr +{ + DAPL_HEADER header; + DAT_RMR_PARAM param; + DAPL_EP *ep; + DAPL_PZ *pz; + DAPL_LMR *lmr; + ib_mw_handle_t mw_handle; +}; + +/* SP types, indicating the state and queue */ +typedef enum dapl_sp_state +{ + DAPL_SP_STATE_FREE, + DAPL_SP_STATE_PSP_LISTENING, + DAPL_SP_STATE_PSP_PENDING, + DAPL_SP_STATE_RSP_LISTENING, + DAPL_SP_STATE_RSP_PENDING +} DAPL_SP_STATE; + +/* DAPL_SP maps to DAT_PSP_HANDLE and DAT_RSP_HANDLE */ +struct dapl_sp +{ + DAPL_HEADER header; + DAPL_SP_STATE state; /* type and queue of the SP */ + + /* PSP/RSP PARAM fields */ + DAT_CONN_QUAL conn_qual; + DAT_EVD_HANDLE evd_handle; + DAT_PSP_FLAGS psp_flags; + DAT_EP_HANDLE ep_handle; + + /* maintenence fields */ + DAT_BOOLEAN listening; /* PSP is registered & active */ + ib_cm_srvc_handle_t cm_srvc_handle; /* Used by Mellanox CM */ + DAPL_LLIST_HEAD cr_list_head; /* CR pending queue */ + DAT_COUNT cr_list_count; /* count of CRs on queue */ +#if _VENDOR_IBAL_ + DAPL_OS_WAIT_OBJECT wait_object; /* cancel & destroy. */ +#endif +}; + +/* DAPL_CR maps to DAT_CR_HANDLE */ +struct dapl_cr +{ + DAPL_HEADER header; + + /* for convenience the data is kept as a DAT_CR_PARAM. + * however, the "local_endpoint" field is always NULL + * so this wastes a pointer. This is probably ok to + * simplify code, espedially dat_cr_query. + */ + DAT_CR_PARAM param; + /* IB specific fields */ + dp_ib_cm_handle_t ib_cm_handle; + + DAT_SOCK_ADDR6 remote_ia_address; + /* Assuming that the maximum private data size is small. + * If it gets large, use of a pointer may be appropriate. + */ + unsigned char private_data[DAPL_MAX_PRIVATE_DATA_SIZE]; + /* + * Need to be able to associate the CR back to the PSP for + * dapl_cr_reject. + */ + DAPL_SP *sp_ptr; +}; + +typedef enum dapl_dto_type +{ + DAPL_DTO_TYPE_SEND, + DAPL_DTO_TYPE_RECV, + DAPL_DTO_TYPE_RDMA_WRITE, + DAPL_DTO_TYPE_RDMA_READ, +#ifdef DAT_EXTENSIONS + DAPL_DTO_TYPE_EXTENSION, +#endif +} DAPL_DTO_TYPE; + +typedef enum dapl_cookie_type +{ + DAPL_COOKIE_TYPE_NULL, + DAPL_COOKIE_TYPE_DTO, + DAPL_COOKIE_TYPE_RMR, +} DAPL_COOKIE_TYPE; + +/* DAPL_DTO_COOKIE used as context for DTO WQEs */ +struct dapl_dto_cookie +{ + DAPL_DTO_TYPE type; + DAT_DTO_COOKIE cookie; + DAT_COUNT size; /* used for SEND and RDMA write */ +}; + +/* DAPL_RMR_COOKIE used as context for bind WQEs */ +struct dapl_rmr_cookie +{ + DAPL_RMR *rmr; + DAT_RMR_COOKIE cookie; +}; + +/* DAPL_COOKIE used as context for WQEs */ +struct dapl_cookie +{ + DAPL_COOKIE_TYPE type; /* Must be first, to define struct. */ + DAPL_EP *ep; + DAT_COUNT index; + union + { + DAPL_DTO_COOKIE dto; + DAPL_RMR_COOKIE rmr; + } val; +}; + +/* + * Private Data operations. Used to obtain the size of the private + * data from the provider layer. + */ +typedef enum dapl_private_data_op +{ + DAPL_PDATA_CONN_REQ = 0, /* connect request */ + DAPL_PDATA_CONN_REP = 1, /* connect reply */ + DAPL_PDATA_CONN_REJ = 2, /* connect reject */ + DAPL_PDATA_CONN_DREQ = 3, /* disconnect request */ + DAPL_PDATA_CONN_DREP = 4, /* disconnect reply */ +} DAPL_PDATA_OP; + + +/* + * Generic HCA name field + */ +#define DAPL_HCA_NAME_MAX_LEN 260 +typedef char DAPL_HCA_NAME[DAPL_HCA_NAME_MAX_LEN+1]; + +#ifdef IBHOSTS_NAMING + +/* + * Simple mapping table to match IP addresses to GIDs. Loaded + * by dapl_init. + */ +typedef struct _dapl_gid_map_table +{ + uint32_t ip_address; + ib_gid_t gid; +} DAPL_GID_MAP; + +#endif /* IBHOSTS_NAMING */ + +/* + * IBTA defined reason for reject message: See IBTA 1.1 specification, + * 12.6.7.2 REJECTION REASON section. + */ +#define IB_CM_REJ_REASON_CONSUMER_REJ 0x001C + + +#if defined(DAPL_DBG_IO_TRC) +/********************************************************************* + * * + * Debug I/O tracing support prototypes * + * * + *********************************************************************/ +/* + * I/O tracing support + */ +void dapls_io_trc_alloc ( + DAPL_EP *ep_ptr); + +void dapls_io_trc_update_completion ( + DAPL_EP *ep_ptr, + DAPL_COOKIE *cookie, + ib_uint32_t ib_status ); + +void dapls_io_trc_dump ( + DAPL_EP *ep_ptr, + ib_work_completion_t *cqe_ptr, + ib_uint32_t ib_status); + +#else /* DAPL_DBG_IO_TRC */ + +#define dapls_io_trc_alloc(a) +#define dapls_io_trc_update_completion(a, b, c) +#define dapls_io_trc_dump(a, b, c) + +#endif /* DAPL_DBG_IO_TRC */ + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +/* + * DAT Mandated functions + */ + +extern DAT_RETURN DAT_API dapl_ia_open ( + IN const DAT_NAME_PTR, /* name */ + IN DAT_COUNT, /* asynch_evd_qlen */ + INOUT DAT_EVD_HANDLE *, /* asynch_evd_handle */ + OUT DAT_IA_HANDLE *); /* ia_handle */ + +extern DAT_RETURN DAT_API dapl_ia_close ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CLOSE_FLAGS ); /* ia_flags */ + + +extern DAT_RETURN DAT_API dapl_ia_query ( + IN DAT_IA_HANDLE, /* ia handle */ + OUT DAT_EVD_HANDLE *, /* async_evd_handle */ + IN DAT_IA_ATTR_MASK, /* ia_params_mask */ + OUT DAT_IA_ATTR *, /* ia_params */ + IN DAT_PROVIDER_ATTR_MASK, /* provider_params_mask */ + OUT DAT_PROVIDER_ATTR * ); /* provider_params */ + + +/* helper functions */ + +extern DAT_RETURN DAT_API dapl_set_consumer_context ( + IN DAT_HANDLE, /* dat handle */ + IN DAT_CONTEXT); /* context */ + +extern DAT_RETURN DAT_API dapl_get_consumer_context ( + IN DAT_HANDLE, /* dat handle */ + OUT DAT_CONTEXT * ); /* context */ + +extern DAT_RETURN DAT_API dapl_get_handle_type ( + IN DAT_HANDLE, + OUT DAT_HANDLE_TYPE * ); + +/* CNO functions */ + +#if !defined(__KERNEL__) +extern DAT_RETURN DAT_API dapl_cno_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_OS_WAIT_PROXY_AGENT, /* agent */ + OUT DAT_CNO_HANDLE *); /* cno_handle */ + +extern DAT_RETURN DAT_API dapl_cno_modify_agent ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_OS_WAIT_PROXY_AGENT); /* agent */ + +extern DAT_RETURN DAT_API dapl_cno_query ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_CNO_PARAM_MASK, /* cno_param_mask */ + OUT DAT_CNO_PARAM * ); /* cno_param */ + +extern DAT_RETURN DAT_API dapl_cno_free ( + IN DAT_CNO_HANDLE); /* cno_handle */ + +extern DAT_RETURN DAT_API dapl_cno_wait ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_TIMEOUT, /* timeout */ + OUT DAT_EVD_HANDLE *); /* evd_handle */ + +extern DAT_RETURN DAT_API dapl_cno_fd_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_FD *, /* file_descriptor */ + OUT DAT_CNO_HANDLE *); /* cno_handle */ + +extern DAT_RETURN DAT_API dapl_cno_trigger ( + IN DAT_CNO_HANDLE, /* cno_handle */ + OUT DAT_EVD_HANDLE *); /* evd_handle */ + +#endif /* !defined(__KERNEL__) */ + +/* CR Functions */ + +extern DAT_RETURN DAT_API dapl_cr_query ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_CR_PARAM_MASK, /* cr_args_mask */ + OUT DAT_CR_PARAM * ); /* cwr_args */ + +extern DAT_RETURN DAT_API dapl_cr_accept ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +extern DAT_RETURN DAT_API dapl_cr_reject ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +extern DAT_RETURN DAT_API dapl_cr_handoff ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_CONN_QUAL); /* handoff */ + +/* EVD Functions */ + +#if defined(__KERNEL__) +extern DAT_RETURN dapl_ia_memtype_hint ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_VLEN, /* length */ + IN DAT_MEM_OPT, /* mem_optimization */ + OUT DAT_VLEN *, /* suggested_length */ + OUT DAT_VADDR *); /* suggested_alignment */ + +extern DAT_RETURN dapl_evd_kcreate ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COUNT, /* evd_min_qlen */ + IN DAT_UPCALL_POLICY, /* upcall_policy */ + IN const DAT_UPCALL_OBJECT *, /* upcall */ + IN DAT_EVD_FLAGS, /* evd_flags */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +extern DAT_RETURN dapl_evd_kquery ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_EVD_PARAM_MASK, /* evd_args_mask */ + OUT DAT_EVD_PARAM * ); /* evd_args */ + +#else +extern DAT_RETURN DAT_API dapl_evd_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COUNT, /* evd_min_qlen */ + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_EVD_FLAGS, /* evd_flags */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +extern DAT_RETURN DAT_API dapl_evd_query ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_EVD_PARAM_MASK, /* evd_args_mask */ + OUT DAT_EVD_PARAM * ); /* evd_args */ +#endif /* defined(__KERNEL__) */ + +#if defined(__KERNEL__) +extern DAT_RETURN dapl_evd_modify_upcall ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_UPCALL_POLICY, /* upcall_policy */ + IN const DAT_UPCALL_OBJECT * ); /* upcall */ + +#else + +extern DAT_RETURN DAT_API dapl_evd_modify_cno ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_CNO_HANDLE); /* cno_handle */ +#endif + +extern DAT_RETURN DAT_API dapl_evd_enable ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +extern DAT_RETURN DAT_API dapl_evd_disable ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +#if !defined(__KERNEL__) +extern DAT_RETURN DAT_API dapl_evd_wait ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* threshold */ + OUT DAT_EVENT *, /* event */ + OUT DAT_COUNT *); /* nmore */ +#endif /* !defined(__KERNEL__) */ + +extern DAT_RETURN DAT_API dapl_evd_resize ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_COUNT ); /* evd_qlen */ + +extern DAT_RETURN DAT_API dapl_evd_post_se ( + DAT_EVD_HANDLE, /* evd_handle */ + const DAT_EVENT * ); /* event */ + +extern DAT_RETURN DAT_API dapl_evd_dequeue ( + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_EVENT * ); /* event */ + +extern DAT_RETURN DAT_API dapl_evd_free ( + IN DAT_EVD_HANDLE ); + +extern DAT_RETURN DAT_API +dapl_evd_set_unwaitable ( + IN DAT_EVD_HANDLE evd_handle ); + +extern DAT_RETURN DAT_API +dapl_evd_clear_unwaitable ( + IN DAT_EVD_HANDLE evd_handle ); + +/* EP functions */ + +extern DAT_RETURN DAT_API dapl_ep_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_EVD_HANDLE, /* in_dto_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* out_dto_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* connect_evd_handle */ + IN const DAT_EP_ATTR *, /* ep_parameters */ + OUT DAT_EP_HANDLE * ); /* ep_handle */ + +extern DAT_RETURN DAT_API dapl_ep_query ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_PARAM_MASK, /* ep_args_mask */ + OUT DAT_EP_PARAM * ); /* ep_args */ + +extern DAT_RETURN DAT_API dapl_ep_modify ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_PARAM_MASK, /* ep_args_mask */ + IN const DAT_EP_PARAM * ); /* ep_args */ + +extern DAT_RETURN DAT_API dapl_ep_connect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR, /* remote_ia_address */ + IN DAT_CONN_QUAL, /* remote_conn_qual */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID, /* private_data */ + IN DAT_QOS, /* quality_of_service */ + IN DAT_CONNECT_FLAGS ); /* connect_flags */ + +extern DAT_RETURN DAT_API dapl_ep_common_connect ( + IN DAT_EP_HANDLE ep, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR remote_addr, /* remote_ia_address */ + IN DAT_TIMEOUT timeout, /* timeout */ + IN DAT_COUNT pdata_size, /* private_data_size */ + IN const DAT_PVOID pdata ); /* private_data */ + +extern DAT_RETURN DAT_API dapl_ep_dup_connect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_HANDLE, /* ep_dup_handle */ + IN DAT_TIMEOUT, /* timeout*/ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID, /* private_data */ + IN DAT_QOS); /* quality_of_service */ + +extern DAT_RETURN DAT_API dapl_ep_disconnect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_CLOSE_FLAGS ); /* close_flags */ + +extern DAT_RETURN DAT_API dapl_ep_post_send ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dapl_ep_post_recv ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dapl_ep_post_rdma_read ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *, /* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dapl_ep_post_rdma_read_to_rmr ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN const DAT_RMR_TRIPLET *,/* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS); /* completion_flags */ + +extern DAT_RETURN DAT_API dapl_ep_post_rdma_write ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *, /* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dapl_ep_post_send_with_invalidate ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + IN DAT_BOOLEAN, /* invalidate_flag */ + IN DAT_RMR_CONTEXT); /* RMR context */ + +extern DAT_RETURN DAT_API dapl_ep_get_status ( + IN DAT_EP_HANDLE, /* ep_handle */ + OUT DAT_EP_STATE *, /* ep_state */ + OUT DAT_BOOLEAN *, /* in_dto_idle */ + OUT DAT_BOOLEAN * ); /* out_dto_idle */ + +extern DAT_RETURN DAT_API dapl_ep_free ( + IN DAT_EP_HANDLE); /* ep_handle */ + +extern DAT_RETURN DAT_API dapl_ep_reset ( + IN DAT_EP_HANDLE); /* ep_handle */ + +extern DAT_RETURN DAT_API dapl_ep_create_with_srq ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_EVD_HANDLE, /* recv_evd_handle */ + IN DAT_EVD_HANDLE, /* request_evd_handle */ + IN DAT_EVD_HANDLE, /* connect_evd_handle */ + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN const DAT_EP_ATTR *, /* ep_attributes */ + OUT DAT_EP_HANDLE *); /* ep_handle */ + +extern DAT_RETURN DAT_API dapl_ep_recv_query ( + IN DAT_EP_HANDLE, /* ep_handle */ + OUT DAT_COUNT *, /* nbufs_allocated */ + OUT DAT_COUNT *); /* bufs_alloc_span */ + +extern DAT_RETURN DAT_API dapl_ep_set_watermark ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* soft_high_watermark */ + IN DAT_COUNT); /* hard_high_watermark */ + +/* LMR functions */ + +#if defined(__KERNEL__) +extern DAT_RETURN dapl_lmr_kcreate ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* length */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_MEM_PRIV_FLAGS, /* privileges */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_MEM_OPT, /* optimization */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT *, /* lmr_context */ + OUT DAT_RMR_CONTEXT *, /* rmr_context */ + OUT DAT_VLEN *, /* registered_length */ + OUT DAT_VADDR * ); /* registered_address */ +#else +extern DAT_RETURN DAT_API dapl_lmr_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* length */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_MEM_PRIV_FLAGS, /* privileges */ + IN DAT_VA_TYPE, /* va_type */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT *, /* lmr_context */ + OUT DAT_RMR_CONTEXT *, /* rmr_context */ + OUT DAT_VLEN *, /* registered_length */ + OUT DAT_VADDR * ); /* registered_address */ +#endif /* defined(__KERNEL__) */ + +extern DAT_RETURN DAT_API dapl_lmr_query ( + IN DAT_LMR_HANDLE, + IN DAT_LMR_PARAM_MASK, + OUT DAT_LMR_PARAM *); + +extern DAT_RETURN DAT_API dapl_lmr_free ( + IN DAT_LMR_HANDLE); + +extern DAT_RETURN DAT_API dapl_lmr_sync_rdma_read ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_LMR_TRIPLET *, /* local_segments */ + IN DAT_VLEN); /* num_segments */ + +extern DAT_RETURN DAT_API dapl_lmr_sync_rdma_write ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_LMR_TRIPLET *, /* local_segments */ + IN DAT_VLEN); /* num_segments */ + +/* RMR Functions */ + +extern DAT_RETURN DAT_API dapl_rmr_create ( + IN DAT_PZ_HANDLE, /* pz_handle */ + OUT DAT_RMR_HANDLE *); /* rmr_handle */ + +extern DAT_RETURN DAT_API dapl_rmr_create_for_ep ( + IN DAT_PZ_HANDLE pz_handle, /* pz_handle */ + OUT DAT_RMR_HANDLE *rmr_handle); /* rmr_handle */ + +extern DAT_RETURN DAT_API dapl_rmr_query ( + IN DAT_RMR_HANDLE, /* rmr_handle */ + IN DAT_RMR_PARAM_MASK, /* rmr_args_mask */ + OUT DAT_RMR_PARAM *); /* rmr_args */ + +extern DAT_RETURN DAT_API dapl_rmr_bind ( + IN DAT_RMR_HANDLE, /* rmr_handle */ + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN const DAT_LMR_TRIPLET *, /* lmr_triplet */ + IN DAT_MEM_PRIV_FLAGS, /* mem_priv */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_RMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + INOUT DAT_RMR_CONTEXT * ); /* context */ + +extern DAT_RETURN DAT_API dapl_rmr_free ( + IN DAT_RMR_HANDLE); + +/* PSP Functions */ + +extern DAT_RETURN DAT_API dapl_psp_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CONN_QUAL, /* conn_qual */ + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_PSP_FLAGS, /* psp_flags */ + OUT DAT_PSP_HANDLE * ); /* psp_handle */ + +extern DAT_RETURN DAT_API dapl_psp_create_any ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_CONN_QUAL *, /* conn_qual */ + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_PSP_FLAGS, /* psp_flags */ + OUT DAT_PSP_HANDLE *); /* psp_handle */ + +extern DAT_RETURN DAT_API dapl_psp_query ( + IN DAT_PSP_HANDLE, + IN DAT_PSP_PARAM_MASK, + OUT DAT_PSP_PARAM * ); + +extern DAT_RETURN DAT_API dapl_psp_free ( + IN DAT_PSP_HANDLE ); /* psp_handle */ + +/* RSP Functions */ + +extern DAT_RETURN DAT_API dapl_rsp_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CONN_QUAL, /* conn_qual */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_RSP_HANDLE * ); /* rsp_handle */ + +extern DAT_RETURN DAT_API dapl_rsp_query ( + IN DAT_RSP_HANDLE, + IN DAT_RSP_PARAM_MASK, + OUT DAT_RSP_PARAM * ); + +extern DAT_RETURN DAT_API dapl_rsp_free ( + IN DAT_RSP_HANDLE ); /* rsp_handle */ + +/* PZ Functions */ + +extern DAT_RETURN DAT_API dapl_pz_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_PZ_HANDLE * ); /* pz_handle */ + +extern DAT_RETURN DAT_API dapl_pz_query ( + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_PZ_PARAM_MASK, /* pz_args_mask */ + OUT DAT_PZ_PARAM * ); /* pz_args */ + +extern DAT_RETURN DAT_API dapl_pz_free ( + IN DAT_PZ_HANDLE ); /* pz_handle */ + +/* SRQ functions */ + +extern DAT_RETURN DAT_API dapl_srq_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_SRQ_ATTR *, /* srq_attr */ + OUT DAT_SRQ_HANDLE *); /* srq_handle */ + +extern DAT_RETURN DAT_API dapl_srq_free ( + IN DAT_SRQ_HANDLE); /* srq_handle */ + +extern DAT_RETURN DAT_API dapl_srq_post_recv ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE); /* user_cookie */ + +extern DAT_RETURN DAT_API dapl_srq_query ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_SRQ_PARAM_MASK, /* srq_param_mask */ + OUT DAT_SRQ_PARAM *); /* srq_param */ + +extern DAT_RETURN DAT_API dapl_srq_resize ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT); /* srq_max_recv_dto */ + +extern DAT_RETURN DAT_API dapl_srq_set_lw ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT); /* low_watermark */ + +/* CSP functions */ +extern DAT_RETURN DAT_API dapl_csp_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COMM *, /* communicator */ + IN DAT_IA_ADDRESS_PTR, /* address */ + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_CSP_HANDLE *); /* csp_handle */ + +extern DAT_RETURN DAT_API dapl_csp_query ( + IN DAT_CSP_HANDLE, /* csp_handle */ + IN DAT_CSP_PARAM_MASK, /* csp_param_mask */ + OUT DAT_CSP_PARAM *); /* csp_param */ + +extern DAT_RETURN DAT_API dapl_csp_free ( + IN DAT_CSP_HANDLE); /* csp_handle */ + +/* HA functions */ +DAT_RETURN DAT_API dapl_ia_ha ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_NAME_PTR, /* provider */ + OUT DAT_BOOLEAN *); /* answer */ + +#ifdef DAT_EXTENSIONS +#include +extern DAT_RETURN DAT_API dapl_extensions ( + IN DAT_HANDLE, /* handle */ + IN DAT_EXTENDED_OP, /* extended op */ + IN va_list); /* argument list */ +#endif + +/* + * DAPL internal utility function prototpyes + */ + +extern void dapl_llist_init_head ( + DAPL_LLIST_HEAD * head); + +extern void dapl_llist_init_entry ( + DAPL_LLIST_ENTRY * entry); + +extern DAT_BOOLEAN dapl_llist_is_empty ( + DAPL_LLIST_HEAD * head); + +extern void dapl_llist_add_head ( + DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry, + void * data); + +extern void dapl_llist_add_tail ( + DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry, + void * data); + +extern void dapl_llist_add_entry ( + DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry, + DAPL_LLIST_ENTRY * new_entry, + void * data); + +extern void * dapl_llist_remove_head ( + DAPL_LLIST_HEAD * head); + +extern void * dapl_llist_remove_tail ( + DAPL_LLIST_HEAD * head); + +extern void * dapl_llist_remove_entry ( + DAPL_LLIST_HEAD * head, + DAPL_LLIST_ENTRY * entry); + +extern void * dapl_llist_peek_head ( + DAPL_LLIST_HEAD * head); + +extern void * dapl_llist_next_entry ( + IN DAPL_LLIST_HEAD *head, + IN DAPL_LLIST_ENTRY *cur_ent); + +extern void dapl_llist_debug_print_list ( + DAPL_LLIST_HEAD * head); + + +#endif diff --git a/trunk/ulp/dapl2/dapl/include/dapl_debug.h b/trunk/ulp/dapl2/dapl/include/dapl_debug.h new file mode 100644 index 00000000..57bab60f --- /dev/null +++ b/trunk/ulp/dapl2/dapl/include/dapl_debug.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_debug.h + * + * PURPOSE: defines common deuggging flags & data for the DAPL reference + * implemenation + * + * Description: + * + * + * $Id:$ + **********************************************************************/ + +#ifndef _DAPL_DEBUG_H_ +#define _DAPL_DEBUG_H_ + +/* + * Debug level switches + * + * Use these bits to enable various tracing/debug options. Each bit + * represents debugging in a particular subsystem or area of the code. + * + * The ERR bit should always be on unless someone disables it for a + * reason: The ERR flag is used sparingly and will print useful + * information if it fires. + */ +typedef enum +{ + DAPL_DBG_TYPE_ERR = 0x0001, + DAPL_DBG_TYPE_WARN = 0x0002, + DAPL_DBG_TYPE_EVD = 0x0004, + DAPL_DBG_TYPE_CM = 0x0008, + DAPL_DBG_TYPE_EP = 0x0010, + DAPL_DBG_TYPE_UTIL = 0x0020, + DAPL_DBG_TYPE_CALLBACK = 0x0040, + DAPL_DBG_TYPE_DTO_COMP_ERR = 0x0080, + DAPL_DBG_TYPE_API = 0x0100, + DAPL_DBG_TYPE_RTN = 0x0200, + DAPL_DBG_TYPE_EXCEPTION = 0x0400, + DAPL_DBG_TYPE_SRQ = 0x0800, + DAPL_DBG_TYPE_CNTR = 0x1000 + +} DAPL_DBG_TYPE; + +typedef enum +{ + DAPL_DBG_DEST_STDOUT = 0x0001, + DAPL_DBG_DEST_SYSLOG = 0x0002, +} DAPL_DBG_DEST; + + +#if defined(DAPL_DBG) + +extern DAPL_DBG_TYPE g_dapl_dbg_type; +extern DAPL_DBG_DEST g_dapl_dbg_dest; + +#define dapl_dbg_log g_dapl_dbg_type==0 ? (void) 1 : dapl_internal_dbg_log +extern void dapl_internal_dbg_log ( DAPL_DBG_TYPE type, const char *fmt, ...); + +#else /* !DAPL_DBG */ + +#if defined(_WIN32) || defined(_WIN64) +/* sigh - no support for (...) in macros. Compiler should optimize this away */ +static __inline void dapl_dbg_log ( DAPL_DBG_TYPE type, const char *fmt, ...) +{ +} +#else +#define dapl_dbg_log(...) +#endif + +#endif /* !DAPL_DBG */ + +/* + * Counters + */ +#define DCNT_EP_CREATE 0 +#define DCNT_EP_FREE 1 +#define DCNT_EP_CONNECT 2 +#define DCNT_EP_DISCONNECT 3 +#define DCNT_POST_SEND 4 +#define DCNT_POST_RECV 5 +#define DCNT_POST_RDMA_WRITE 6 +#define DCNT_POST_RDMA_READ 7 +#define DCNT_EVD_CREATE 8 +#define DCNT_EVD_FREE 9 +#define DCNT_EVD_WAIT 10 +#define DCNT_EVD_WAIT_BLOCKED 11 +#define DCNT_EVD_WAIT_CMP_NTFY 12 +#define DCNT_EVD_DTO_CALLBACK 13 +#define DCNT_EVD_CONN_CALLBACK 14 +#define DCNT_EVD_DEQUEUE 15 +#define DCNT_EVD_DEQUEUE_POLL 16 +#define DCNT_EVD_DEQUEUE_FOUND 17 +#define DCNT_EVD_DEQUEUE_NOT_FOUND 18 +#define DCNT_TIMER_SET 19 +#define DCNT_TIMER_CANCEL 20 +#define DCNT_NUM_COUNTERS 21 +#define DCNT_ALL_COUNTERS DCNT_NUM_COUNTERS + +#if defined(DAPL_COUNTERS) + +extern void dapl_dump_cntr( int cntr ); +extern int dapl_dbg_counters[]; + +#define DAPL_CNTR(cntr) dapl_os_atomic_inc (&dapl_dbg_counters[cntr]); +#define DAPL_DUMP_CNTR(cntr) dapl_dump_cntr( cntr ); +#define DAPL_COUNTERS_INIT() + +#else + +#define DAPL_CNTR(cntr) +#define DAPL_DUMP_CNTR(cntr) +#define DAPL_COUNTERS_INIT() + +#endif /* DAPL_COUNTERS */ + + +#endif /* _DAPL_DEBUG_H_ */ diff --git a/trunk/ulp/dapl2/dapl/include/dapl_ipoib_names.h b/trunk/ulp/dapl2/dapl/include/dapl_ipoib_names.h new file mode 100644 index 00000000..a1a094c1 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/include/dapl_ipoib_names.h @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: ipoib_naming.h + * + * PURPOSE: Defines flags and prototypes for IPoIB API + * + * Description: + * This defines a simple naming interface for discovering + * the IP addresses available to a provider, then a set + * of query mechanisms useful to map an IP address to + * a provider specific address; a GID in InfiniBand. + * + * NOTE: As implementations mature this may not be necessary. + * + * $Id:$ + **********************************************************************/ + +#ifndef _IPOIB_NAMING_H_ +#define _IPOIB_NAMING_H_ + +typedef enum _ipoib_port_num { + HCA_PORT_1= 1, + HCA_PORT_2, + HCA_PORT_ANY +} IPOIB_PORT_NUM; + +typedef struct if_query_info +{ + uint64_t guid; + uint32_t port_num; + uint32_t state; +}IF_QUERY_INFO; + +/*********************************************************************** + * ipoib_enum_if() + * + * PURPOSE + * Returns count of IP interfaces. + * + * ARGUMENTS + * hca_index: index of HCA in the provider library. In general + * terms, the index represents the HCA number, e.g. + * 1 == First HCA, 2 == Second HCA, etc. + * + * port: an enum of + * HCA_PORT_0 + * HCA_PORT_1 + * HCA_PORT_ANY + * HCA_PORT_ANY enum value returns all IP instances assigned to the HCA. + * + * RETURNS + * count of IP interfaces supported on physical port + * + ***********************************************************************/ +int +ipoib_enum_if( + IN uint32_t hca_index, + IN IPOIB_PORT_NUM port); + + +/*********************************************************************** + * ipoib_get_if() + * + * PURPOSE + * Returns array of IP Addresses of all instances. Port parameter may + * restrict instances of interest. + * + * ARGUMENTS + * hca_index: index of HCA in the provider library. + * + * port: IPOIB_PORT_NUM as described above + * + * ip_addr_list: pointer to user-allocated space in which an array of + * IP addresses found for this hca and port will be returned + * + * ip_addr_count: number of returned addresses + * + * RETURNS + * 0 for SUCCESS + * !0 for failure + * + ***********************************************************************/ +int +ipoib_get_if( + IN uint32_t hca_index, + IN IPOIB_PORT_NUM port, + OUT struct sockaddr **ip_addr_list, + OUT int *ip_addr_count); + +/*********************************************************************** + * + * PURPOSE + * Returns a handle to this interface, to be used for subsequent + * operations + * + * ARGUMENTS + * ip_address: input IP address + * + * ipoib_handle: handle to be used in subsequent operations. + * + * RETURNS + * 0 for SUCCESS + * !0 for failure + * + ***********************************************************************/ +int +ipoib_open_if( + IN struct sockaddr *ip_address, + OUT void *ipoib_handle); + +/*********************************************************************** + * ipoib_query_if() + * + * PURPOSE + * if_query_if returns information on local ipoib_handle such as GID, + * Port number, IPoIB state, anything interesting + * + * ARGUMENTS + * ipoib_handle: handle for instance + * + * if_qry_info: info struct. Looks like: + * + * RETURNS + * 0 for SUCCESS + * !0 for failure + * + ***********************************************************************/ +int +ipoib_query_if( + IN void *ipoib_handle, + OUT IF_QUERY_INFO *if_qry_info); + +/*********************************************************************** + * + * + * PURPOSE + * Obtain a GID from an IP Address. Used by the active side of + * a connection. + * + * The behavior of this routine is specified to provide control + * over the underlying implementation. + * Returns immediately if the remote information is available. If + * callback_routine_ptr is NULL then it will block until information is + * available or known to be unavailable. If callback_routine_ptr is + * specified then it will be invoked when remote information is + * available or known to be unavailable. Remote_Addr_info contains + * remote GID information. + * + * ARGUMENTS + * ipoib_handle: handle for instance + * + * remote_ip_address: IP address of remote instance + * + * callback_routine_ptr: routine to invoke for asynch callback. If + * NULL ipoib_getaddrinfo() will block. + * + * context: argument to pass to asynch callback_routine. + * + * Remote_Addr_info: Remote GID + * + * RETURNS + * 0 for SUCCESS + * !0 for failure + * + ***********************************************************************/ +int +ipoib_getaddrinfo( + IN void *ipoib_handle, + IN struct sockaddr *remote_ip_address, + IN void *callback_routine_ptr, + IN void *context, + OUT void *Remote_Addr_info ); + +/*********************************************************************** + * + * + * PURPOSE + * Obtain an IP Address from a GID. Used by the passive side of a + * connection. + * + * The behavior of this routine is specified to provide control over + * the underlying implementation. Returns immediately if the remote + * information is available. If callback_routine_ptr is NULL then it + * will block until information is available or known to be + * unavailable. If callback_routine_ptr is specified then it will be + * invoked when remote information is available or known to be + * unavailable. + * + * ARGUMENTS + * ipoib_handle: handle for instance + * + * remote_gidAddr: Remote GID. It is not defined on how the application + * will obtain this GID from the connection manager. + * + * callback_routine_ptr: + * routine to invoke for async callback. If NULL + * ipoib_getgidinfo() will block. + * + * context: argument to pass to asynch callback_routine. + * + * remote_ip_address: + * IP address of remote instance + * + * RETURNS + * 0 for SUCCESS + * !0 for failure + * + ***********************************************************************/ +int +ipoib_getgidinfo( + IN void *ipoib_handle, + IN GID *remote_gid, + IN void *callback_routine_ptr, + IN void *context, + OUT struct sockaddr *remote_ip_address); + +/*********************************************************************** + * + * PURPOSE + * Release handle. + * + * ARGUMENTS + * ipoib_handle: handle for instance + * + * RETURNS + * 0 for SUCCESS + * !0 for failure + * + ***********************************************************************/ +int +ipoib_close( + IN void *ipoib_handle); + + +#endif /* _IPOIB_NAMING_H_ */ diff --git a/trunk/ulp/dapl2/dapl/include/dapl_vendor.h b/trunk/ulp/dapl2/dapl/include/dapl_vendor.h new file mode 100644 index 00000000..951e2faf --- /dev/null +++ b/trunk/ulp/dapl2/dapl/include/dapl_vendor.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_vendor.h + * + * PURPOSE: + * Vendor provides values for their implementation. Most of + * these values are returned in the DAT_IA_ATTR parameter of + * dat_ia_query() + * + * $Id:$ + **********************************************************************/ + +/********************************************************************** + * DAT_IA_ATTR attributes + * + * These values are used in the provider support routine + * dapls_ib_query_hca (). Many of the values there are HW + * specific, the the vendor should look to make sure they are + * appropriate for their implementation. Specifically, + * vendors are encouraged to update transport and vendor + * attributes: the reference implementation sets these to NULL. + */ + +/* + * Product name of the adapter. + * Returned in DAT_IA_ATTR.adapter_name + */ +#define VN_ADAPTER_NAME "Generic InfiniBand HCA" + + +/* + * Vendor name + * Returned in DAT_IA_ATTR.vendor_name + */ +#define VN_VENDOR_NAME "DAPL Reference Implementation" + + +/********************************************************************** + * PROVIDER Attributes + * + * These values are used in ./common/dapl_ia_query.c, in dapl_ia_query (). + * The values below are the most common for vendors to change, but + * there are several other values that may be updated once the + * implementation becomes mature. + * + */ + +/* + * Provider Versions + * Returned in DAT_PROVIDER_ATTR.provider_version_major and + * DAT_PROVIDER_ATTR.provider_version_minor + */ + +#define VN_PROVIDER_MAJOR 1 +#define VN_PROVIDER_MINOR 0 + +/* + * Provider support for memory types. The reference implementation + * always supports DAT_MEM_TYPE_VIRTUAL and DAT_MEM_TYPE_LMR, so + * the vendor must indicate if they support DAT_MEM_TYPE_SHARED_VIRTUAL. + * Set this value to '1' if DAT_MEM_TYPE_SHARED_VIRTUAL is supported. + * + * Returned in DAT_PROVIDER_ATTR.lmr_mem_types_supported + */ + +#define VN_MEM_SHARED_VIRTUAL_SUPPORT 1 + + +/********************************************************************** + * + * This value will be assigned to dev_name_prefix in ./udapl/dapl_init.c. + * + * DAT is designed to support multiple DAPL instances simultaneously, + * with different dapl libraries originating from different providers. + * There is always the possibility of name conflicts, so a dat name + * prefix is provided to make a vendor's adapter name unique. This is + * especially true of the IBM Access API, which returns adapter + * names that are simply ordinal numbers (e.g. 0, 1, 2). If + * a vendor doesn't need or want a prefix, it should be left + * as a NULL (use ""). + * + * Values that might be used: + * #define VN_PREFIX "ia" (generic prefix) + * #define VN_PREFIX "jni" (JNI: OS Acces API) + * #define VN_PREFIX "ibm" (IBM: OS Acces API) + * #define VN_PREFIX "" (Mellanox: VAPI) + * #define VN_PREFIX "" (Intel: IB Common API) + */ +#define VN_PREFIX "ia" diff --git a/trunk/ulp/dapl2/dapl/udapl/Makefile.linux b/trunk/ulp/dapl2/dapl/udapl/Makefile.linux new file mode 100644 index 00000000..94569c68 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/Makefile.linux @@ -0,0 +1,444 @@ +# +# Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. +# +# This Software is licensed under one of the following licenses: +# +# 1) under the terms of the "Common Public License 1.0" a copy of which is +# in the file LICENSE.txt in the root directory. The license is also +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/cpl.php. +# +# 2) under the terms of the "The BSD License" a copy of which is in the file +# LICENSE2.txt in the root directory. The license is also available from +# the Open Source Initiative, see +# http://www.opensource.org/licenses/bsd-license.php. +# +# 3) under the terms of the "GNU General Public License (GPL) Version 2" a +# copy of which is in the file LICENSE3.txt in the root directory. The +# license is also available from the Open Source Initiative, see +# http://www.opensource.org/licenses/gpl-license.php. +# +# Licensee has the right to choose one of the above licenses. +# +# Redistributions of source code must retain the above copyright +# notice and one of the license notices. +# +# Redistributions in binary form must reproduce both the above copyright +# notice, one of the license notices in the documentation +# and/or other materials provided with the distribution. +# + +#********************************************************************** +# +# MODULE: Makefile +# +# PURPOSE: Makefile for dapl reference provider +# +# $Id: Makefile 1367 2005-12-05 16:48:37Z jlentini $ +#*********************************************************************/ + +TOPDIR = $(shell /bin/pwd) + +SRC_PATH = $(TOPDIR) +OBJ_PATH = $(TOPDIR)/Obj +TARGET_PATH = $(TOPDIR)/Target + +TARGET = $(TARGET_PATH)/libdapl.so +STATIC = $(TARGET_PATH)/libdapl.a +COMMON = $(TOPDIR)/../common +LINUX = $(TOPDIR)/../udapl/linux +CC = gcc + +MACH=$(shell uname -m) +OSRELEASE=$(shell expr `uname -r | cut -f1 -d.` \* 65536 + `uname -r | cut -f2 -d.`) + +# +# Set up the default provider +# +ifndef $VERBS +VERBS=openib_cma +endif + +# +# Set an OS Vendor +# +# OS_VENDOR = REDHAT_EL4 +# OS_VENDOR = SuSE +# + +# +# CFLAGS definition +# +# The makefile will build for multiple providers, but each provider +# must have a set of compiler definitions that include, but are not +# limited to: +# +# PROVIDER Directory of provider sources +# CFLAGS Compile time flags for build +# + +CFLAGS = -O2 $(CPPFLAGS) -DOS_VERSION=$(OSRELEASE) -DDAPL_DBG + +ifdef OS_VENDOR +CFLAGS += -D$(OS_VENDOR) +endif + +# +# dummy provider +# +ifeq ($(VERBS),dummy) +PROVIDER = $(TOPDIR)/../dummy +CFLAGS += -DDAPL_ATS -DDUMMY +endif + +# +# JNI Provider +# +ifeq ($(VERBS),jni) +PROVIDER = $(TOPDIR)/../ibapi +CFLAGS += -DDAPL_ATS -DIBAPI # -DCM_BUSTED +CFLAGS += -I../include/ib/IBM +CFLAGS += -I../include/ib/IBM/us +endif + +# +# Mellanox Provider +# +ifeq ($(VERBS),mellanox) +DAPL_IBLIB_DIR = /usr/mellanox/lib +PROVIDER = $(TOPDIR)/../vapi +CFLAGS += -DVAPI -DSMR_BUSTED # -DCM_BUSTED # -DMW_BUSTED +#CFLAGS += -DIBHOSTS_NAMING +#CFLAGS += -I/usr/mellanox/include +CFLAGS += -I../include/ib/MELLANOX +endif + +# +# OpenIB Gen one provider +# +ifeq ($(VERBS),openib_gen_one) +OPENIB_UNAME=$(shell uname -r) + +OPENIB_USER_DIR ?= /usr/local/ibgd +OPENIB_KERNEL_DIR ?= /lib/modules/$(OPENIB_UNAME)/build/drivers/infiniband +OPENIB_THCA_DIR ?= $(OPENIB_USER_DIR)/driver/infinihost + +PROVIDER = $(TOPDIR)/../openib_gen_one +CFLAGS += -D__OPENIB__ -DSMR_BUSTED # -DMW_BUSTED +CFLAGS += -DDAPL_ATS +CFLAGS += -I$(OPENIB_KERNEL_DIR)/include +CFLAGS += -I$(OPENIB_KERNEL_DIR)/ulp/dapl_srv +CFLAGS += -I$(OPENIB_USER_DIR)/driver/infinihost/include +endif + +# +# OpenIB provider +# +ifeq ($(VERBS),openib) +PROVIDER = $(TOPDIR)/../openib +CFLAGS += -DOPENIB +CFLAGS += -DCQ_WAIT_OBJECT +CFLAGS += -I/usr/local/include/infiniband +endif + +# +# OpenIB provider with Socket CM +# +ifeq ($(VERBS),openib_scm) +PROVIDER = $(TOPDIR)/../openib_scm +CFLAGS += -DOPENIB +CFLAGS += -DCQ_WAIT_OBJECT +CFLAGS += -I/usr/local/include/infiniband +endif + +# +# OpenIB provider with IB CMA +# +ifeq ($(VERBS),openib_cma) +PROVIDER = $(TOPDIR)/../openib_cma +CFLAGS += -DOPENIB +CFLAGS += -DCQ_WAIT_OBJECT +CFLAGS += -DDAT_EXTENSIONS +CFLAGS += -I/usr/local/include/infiniband +endif + +# +# If an implementation supports CM and DTO completions on the same EVD +# then DAPL_MERGE_CM_DTO should be set +# CFLAGS += -DDAPL_MERGE_CM_DTO=1 + +# +# If an implementation supports Shared Memory, enable +# VN_MEM_SHARED_VIRTUAL_SUPPORT +# CFLAGS += -DVN_MEM_SHARED_VIRTUAL_SUPPORT=1 + +CFLAGS += -I. +CFLAGS += -I.. +CFLAGS += -I../../dat/include +CFLAGS += -I../include + +CFLAGS += -I$(PROVIDER) +CFLAGS += -I../udapl/linux +CFLAGS += -I../common +CFLAGS += -Wall +CFLAGS += -Wstrict-prototypes +CFLAGS += -Wmissing-prototypes +CFLAGS += -Wmissing-declarations +CFLAGS += --no-strict-aliasing +CFLAGS += -Werror +CFLAGS += -g3 +CFLAGS += -fPIC + +ifdef GPROF +CFLAGS += -pg +endif + +LD = ld + +# +# LDFLAGS definition +# +LDFLAGS = -shared +LDFLAGS += -lpthread +LDFLAGS += -init dapl_init +LDFLAGS += -fini dapl_fini +ifeq ($(OS_VENDOR),SuSE) +LDFLAGS += -lgcc_s +endif + + +AR = ar +# +# ARFLAGS definition +# +ARFLAGS = r + +# +# To build with JNI verbs: make VERBS=jni [default] +# To build with Mellanox verbs: make VERBS=mellanox +# To build with XYZ verbs: make VERBS=XYZ +# +# Verb libraries should be in /usr/lib or vendor specified directories +# + +# Allow specification in the environment of a location for the +# verbs library, to allow running DAPL without installing +# the verbs library in /usr/lib. For development. +ifneq (${DAPL_IBLIB_DIR},) +LDFLAGS += -L${DAPL_IBLIB_DIR} -R${DAPL_IBLIB_DIR} +endif + +ifeq ($(VERBS),dummy) +PROVIDER_SRCS = dapl_dummy_util.c +endif + +ifeq ($(VERBS),ibm) +CFLAGS += -D_IBM -DIBM +LDFLAGS += -lIBusd +LDFLAGS += -lIBlueHCAd +LDFLAGS += -ldl +PROVIDER_SRCS = dapl_ibapi_util.c dapl_ibapi_qp.c dapl_ibapi_cm.c +endif + +ifeq ($(VERBS),jni) +CFLAGS += -DJNI +LDFLAGS += -lJniTavorVerbs +LDFLAGS += -lpthread +LDFLAGS += -rpath /opt/JNIhca/lib/ -L /opt/JNIhca/lib/ +PROVIDER_SRCS = dapl_ibapi_util.c dapl_ibapi_qp.c dapl_ibapi_cm.c +endif + +ifeq ($(VERBS),mellanox) +CFLAGS += -DMTL_MODULE=M_dapl -DMAX_TRACE=8 -DMAX_DEBUG=8 -DMAX_ERROR=8 +PROVIDER_SRCS = dapl_vapi_util.c dapl_vapi_qp.c dapl_vapi_cm.c +LDFLAGS += -lvapi +LDFLAGS += -lmpga +LDFLAGS += -lmtl_common +LDFLAGS += -lcm +endif + +ifeq ($(VERBS),openib_gen_one) +CFLAGS += -DMTL_MODULE=M_dapl -DMAX_TRACE=8 -DMAX_DEBUG=8 -DMAX_ERROR=8 +LDFLAGS += -lvapi +LDFLAGS += -lmpga +LDFLAGS += -lmtl_common +LDFLAGS += -lcm +LDFLAGS += -L$(OPENIB_USER_DIR)/lib -L$(OPENIB_THCA_DIR)/lib +PROVIDER_SRCS = dapl_openib.c dapl_openib_qp.c +PROVIDER_SRCS += dapl_openib_util.c dapl_openib_cm.c dapl_openib_cm_util_user.c +endif + +ifeq ($(VERBS),openib) +LDFLAGS += -libverbs -libcm -libat +LDFLAGS += -rpath /usr/local/lib -L /usr/local/lib +PROVIDER_SRCS = dapl_ib_util.c dapl_ib_cq.c dapl_ib_qp.c +PROVIDER_SRCS += dapl_ib_cm.c dapl_ib_mem.c +endif + +ifeq ($(VERBS),openib_scm) +LDFLAGS += -libverbs +LDFLAGS += -rpath /usr/local/lib -L /usr/local/lib +PROVIDER_SRCS = dapl_ib_util.c dapl_ib_cq.c dapl_ib_qp.c \ + dapl_ib_cm.c dapl_ib_mem.c +endif + +ifeq ($(VERBS),openib_cma) +LDFLAGS += -libverbs -lrdmacm +LDFLAGS += -rpath /usr/local/lib -L /usr/local/lib +PROVIDER_SRCS = dapl_ib_util.c dapl_ib_cq.c dapl_ib_qp.c \ + dapl_ib_cm.c dapl_ib_mem.c +endif + +UDAPL_SRCS = dapl_init.c \ + dapl_evd_create.c \ + dapl_evd_query.c \ + dapl_cno_create.c \ + dapl_cno_modify_agent.c \ + dapl_cno_free.c \ + dapl_cno_wait.c \ + dapl_cno_query.c \ + dapl_lmr_create.c \ + dapl_evd_wait.c \ + dapl_evd_disable.c \ + dapl_evd_enable.c \ + dapl_evd_modify_cno.c \ + dapl_evd_set_unwaitable.c \ + dapl_evd_clear_unwaitable.c + +COMMON_SRCS = dapl_cookie.c \ + dapl_cr_accept.c \ + dapl_cr_query.c \ + dapl_cr_reject.c \ + dapl_cr_util.c \ + dapl_cr_callback.c \ + dapl_cr_handoff.c \ + dapl_ep_connect.c \ + dapl_ep_create.c \ + dapl_ep_disconnect.c \ + dapl_ep_dup_connect.c \ + dapl_ep_free.c \ + dapl_ep_reset.c \ + dapl_ep_get_status.c \ + dapl_ep_modify.c \ + dapl_ep_post_rdma_read.c \ + dapl_ep_post_rdma_write.c \ + dapl_ep_post_recv.c \ + dapl_ep_post_send.c \ + dapl_ep_query.c \ + dapl_ep_util.c \ + dapl_evd_dequeue.c \ + dapl_evd_free.c \ + dapl_evd_post_se.c \ + dapl_evd_resize.c \ + dapl_evd_util.c \ + dapl_evd_cq_async_error_callb.c \ + dapl_evd_qp_async_error_callb.c \ + dapl_evd_un_async_error_callb.c \ + dapl_evd_connection_callb.c \ + dapl_evd_dto_callb.c \ + dapl_get_consumer_context.c \ + dapl_get_handle_type.c \ + dapl_hash.c \ + dapl_hca_util.c \ + dapl_ia_close.c \ + dapl_ia_open.c \ + dapl_ia_query.c \ + dapl_ia_util.c \ + dapl_llist.c \ + dapl_lmr_free.c \ + dapl_lmr_query.c \ + dapl_lmr_util.c \ + dapl_lmr_sync_rdma_read.c \ + dapl_lmr_sync_rdma_write.c \ + dapl_mr_util.c \ + dapl_provider.c \ + dapl_sp_util.c \ + dapl_psp_create.c \ + dapl_psp_create_any.c \ + dapl_psp_free.c \ + dapl_psp_query.c \ + dapl_pz_create.c \ + dapl_pz_free.c \ + dapl_pz_query.c \ + dapl_pz_util.c \ + dapl_rmr_create.c \ + dapl_rmr_free.c \ + dapl_rmr_bind.c \ + dapl_rmr_query.c \ + dapl_rmr_util.c \ + dapl_rsp_create.c \ + dapl_rsp_free.c \ + dapl_rsp_query.c \ + dapl_cno_util.c \ + dapl_set_consumer_context.c \ + dapl_ring_buffer_util.c \ + dapl_name_service.c \ + dapl_timer_util.c \ + dapl_ep_create_with_srq.c \ + dapl_ep_recv_query.c \ + dapl_ep_set_watermark.c \ + dapl_srq_create.c \ + dapl_srq_free.c \ + dapl_srq_query.c \ + dapl_srq_resize.c \ + dapl_srq_post_recv.c \ + dapl_srq_set_lw.c \ + dapl_srq_util.c \ + dapl_debug.c + +LINUX_SRCS = dapl_osd.c + +SRCS = $(UDAPL_SRCS) $(COMMON_SRCS) $(LINUX_SRCS) $(PROVIDER_SRCS) + +UDAPL_OBJS = $(UDAPL_SRCS:%.c=$(OBJ_PATH)/%.o) +COMMON_OBJS = $(COMMON_SRCS:%.c=$(OBJ_PATH)/%.o) +LINUX_OBJS = $(LINUX_SRCS:%.c=$(OBJ_PATH)/%.o) +PROVIDER_OBJS = $(PROVIDER_SRCS:%.c=$(OBJ_PATH)/%.o) + +OBJS = $(UDAPL_OBJS) $(COMMON_OBJS) $(LINUX_OBJS) $(PROVIDER_OBJS) + + +all: mkdirs $(TARGET) $(STATIC) MAKE_DONE + +mkdirs: + @[ -d $(TARGET_PATH) ] || /bin/mkdir -p $(TARGET_PATH) + @[ -d $(OBJ_PATH) ] || /bin/mkdir -p $(OBJ_PATH) + +$(UDAPL_OBJS): $(OBJ_PATH)/%.o : %.c + @echo Compiling $< + $(CC) $(CFLAGS) -c $< -o $@ + +$(COMMON_OBJS): $(OBJ_PATH)/%.o : $(COMMON)/%.c + @echo Compiling $< + $(CC) $(CFLAGS) -c $< -o $@ + +$(LINUX_OBJS): $(OBJ_PATH)/%.o : $(LINUX)/%.c + @echo Compiling $< + $(CC) $(CFLAGS) -c $< -o $@ + +$(PROVIDER_OBJS): $(OBJ_PATH)/%.o : $(PROVIDER)/%.c + @echo Compiling $< + $(CC) $(CFLAGS) -c $< -o $@ + +$(TARGET): $(OBJS) + @echo Linking $(TARGET) + $(LD) $(LDFLAGS) $^ -o $(TARGET) + +$(STATIC): $(OBJS) + @echo Archiving $(STATIC) + @$(AR) $(ARFLAGS) $(STATIC) $^ + +MAKE_DONE: + @echo "--- Make done ----" + +clean: + rm -f $(OBJS) + rm -f $(TARGET) $(STATIC) + +tidy: + rm -f *~ + rm -f ../common/*~ + rm -f ../include/*~ + rm -f ../vapi/*~ + rm -f ../ibapi/*~ + rm -f linux/*~ diff --git a/trunk/ulp/dapl2/dapl/udapl/SOURCES b/trunk/ulp/dapl2/dapl/udapl/SOURCES new file mode 100644 index 00000000..7f426abc --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/SOURCES @@ -0,0 +1,59 @@ +!if $(FREEBUILD) +TARGETNAME=dapl2 +!else +TARGETNAME=dapl2d +!endif +TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=DYNLINK +DLLENTRY=_DllMainCRTStartup +DLLDEF=$O\udapl_exports.def +USE_CRTDLL=1 + +# pickup local files, then via udapl_sources.c get common files + +SOURCES=udapl.rc \ + dapl_cno_create.c \ + dapl_cno_free.c \ + dapl_cno_modify_agent.c \ + dapl_cno_query.c \ + dapl_cno_wait.c \ + dapl_evd_clear_unwaitable.c \ + dapl_evd_create.c \ + dapl_evd_disable.c \ + dapl_evd_enable.c \ + dapl_evd_modify_cno.c \ + dapl_evd_query.c \ + dapl_evd_set_unwaitable.c \ + dapl_evd_wait.c \ + dapl_init.c \ + dapl_lmr_create.c \ + udapl_sources.c + +INCLUDES=..\include;..\common;windows;..\ibal;..\..\dat\include;\ + ..\..\..\..\inc;..\..\..\..\inc\user; + +DAPL_OPTS= -DEXPORT_DAPL_SYMBOLS -D_VENDOR_IBAL_ -DDAPL_MERGE_CM_DTO + +#DAPL_OPTS= $(DAPL_OPTS) -DDAT_EXTENSIONS + +USER_C_FLAGS=$(USER_C_FLAGS) $(DAPL_OPTS) +!if !$(FREEBUILD) +USER_C_FLAGS=$(USER_C_FLAGS) -DDAPL_DBG +!endif + +TARGETLIBS= \ + $(SDK_LIB_PATH)\kernel32.lib \ + $(SDK_LIB_PATH)\ws2_32.lib \ +!if $(FREEBUILD) + $(TARGETPATH)\*\dat2.lib \ + $(TARGETPATH)\*\complib.lib \ + $(TARGETPATH)\*\ibal.lib +!else + $(TARGETPATH)\*\dat2d.lib \ + $(TARGETPATH)\*\complibd.lib \ + $(TARGETPATH)\*\ibald.lib +!endif + +# FIX ME ASAP +#MSC_WARNING_LEVEL= /W3 +MSC_WARNING_LEVEL= /W1 /wd4113 diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_cno_create.c b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_create.c new file mode 100644 index 00000000..bc58feca --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_create.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cno_create.c + * + * PURPOSE: Consumer Notification Object creation + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.2.1 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_cno_util.h" +#include "dapl_ia_util.h" + +/* + * dapl_cno_create + * + * DAPL Requirements Version xxx, 6.3.4.1 + * + * Create a consumer notification object instance + * + * Input: + * ia_handle + * wait_agent + * cno_handle + * + * Output: + * cno_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API dapl_cno_create( + IN DAT_IA_HANDLE ia_handle, /* ia_handle */ + IN DAT_OS_WAIT_PROXY_AGENT wait_agent, /* agent */ + OUT DAT_CNO_HANDLE *cno_handle) /* cno_handle */ + +{ + DAPL_IA *ia_ptr; + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + + ia_ptr = (DAPL_IA *)ia_handle; + cno_ptr = NULL; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (ia_handle, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + + cno_ptr = dapl_cno_alloc(ia_ptr, wait_agent); + + if (!cno_ptr) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + cno_ptr->cno_state = DAPL_CNO_STATE_UNTRIGGERED; + + dapl_ia_link_cno(ia_ptr, cno_ptr); + + *cno_handle = cno_ptr; + + bail: + if (dat_status != DAT_SUCCESS && cno_ptr != NULL) + { + dapl_cno_dealloc(cno_ptr); + } + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_cno_free.c b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_free.c new file mode 100644 index 00000000..f84f3330 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_free.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cno_free.c + * + * PURPOSE: Consumer Notification Object destruction + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.2.2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_ia_util.h" +#include "dapl_cno_util.h" + +/* + * dapl_cno_free + * + * DAPL Requirements Version xxx, 6.3.2.2 + * + * Destroy a consumer notification object instance + * + * Input: + * cno_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + */ +DAT_RETURN DAT_API dapl_cno_free( + IN DAT_CNO_HANDLE cno_handle) /* cno_handle */ + +{ + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + cno_ptr = (DAPL_CNO *)cno_handle; + + if (DAPL_BAD_HANDLE (cno_handle, DAPL_MAGIC_CNO)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CNO); + goto bail; + } + + if (dapl_os_atomic_read(&cno_ptr->cno_ref_count) != 0 + || cno_ptr->cno_waiters != 0) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_CNO_IN_USE); + goto bail; + } + + dapl_ia_unlink_cno (cno_ptr->header.owner_ia, cno_ptr); + dapl_cno_dealloc (cno_ptr); + +bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_cno_modify_agent.c b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_modify_agent.c new file mode 100644 index 00000000..277de66d --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_modify_agent.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cno_modify_agent.c + * + * PURPOSE: Modify the wait proxy agent associted with the CNO + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.2.4 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_cno_modify_agent + * + * DAPL Requirements Version xxx, 6.3.2.4 + * + * Modify the wait proxy agent associted with the CNO + * + * Input: + * cno_handle + * prx_agent + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API dapl_cno_modify_agent( + IN DAT_CNO_HANDLE cno_handle, /* cno_handle */ + IN DAT_OS_WAIT_PROXY_AGENT prx_agent ) /* agent */ + +{ + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + if ( DAPL_BAD_HANDLE (cno_handle, DAPL_MAGIC_CNO) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CNO); + goto bail; + } + + cno_ptr = (DAPL_CNO *) cno_handle; + dapl_os_lock(&cno_ptr->header.lock); + cno_ptr->cno_wait_agent = prx_agent; + dapl_os_unlock(&cno_ptr->header.lock); + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_cno_query.c b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_query.c new file mode 100644 index 00000000..47984834 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_query.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cno_query.c + * + * PURPOSE: Return the consumer parameters of the CNO + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.2.5 + * + * $Id: dapl_cno_query.c 1301 2005-03-24 05:58:55Z jlentini $ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_cno_query + * + * DAPL Requirements Version xxx, 6.3.2.5 + * + * Return the consumer parameters of the CNO + * + * Input: + * cno_handle + * cno_param_mask + * cno_param + * + * Output: + * cno_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API dapl_cno_query( + IN DAT_CNO_HANDLE cno_handle, /* cno_handle */ + IN DAT_CNO_PARAM_MASK cno_param_mask, /* cno_param_mask */ + OUT DAT_CNO_PARAM *cno_param ) /* cno_param */ + + +{ + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (cno_handle, DAPL_MAGIC_CNO) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CNO); + goto bail; + } + + if ( NULL == cno_param ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + cno_ptr = (DAPL_CNO *) cno_handle; + cno_param->ia_handle = cno_ptr->header.owner_ia; + cno_param->proxy_type = DAT_PROXY_TYPE_AGENT; + cno_param->proxy.agent = cno_ptr->cno_wait_agent; + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_cno_wait.c b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_wait.c new file mode 100644 index 00000000..bfcdbfd1 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_cno_wait.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_cno_wait.c + * + * PURPOSE: Wait for a consumer notification event + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.2.3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_cno_wait + * + * DAPL Requirements Version xxx, 6.3.2.3 + * + * Wait for a consumer notification event + * + * Input: + * cno_handle + * timeout + * evd_handle + * + * Output: + * evd_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * DAT_QUEUE_EMPTY + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API dapl_cno_wait( + IN DAT_CNO_HANDLE cno_handle, /* cno_handle */ + IN DAT_TIMEOUT timeout, /* agent */ + OUT DAT_EVD_HANDLE *evd_handle) /* ia_handle */ + +{ + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + + if ( DAPL_BAD_HANDLE (cno_handle, DAPL_MAGIC_CNO) ) + { + dat_status = DAT_INVALID_HANDLE | DAT_INVALID_HANDLE_CNO; + goto bail; + } + + dat_status = DAT_SUCCESS; + + cno_ptr = (DAPL_CNO *) cno_handle; + + if ( cno_ptr->cno_state == DAPL_CNO_STATE_DEAD ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_CNO_DEAD); + goto bail; + } + + dapl_os_lock ( &cno_ptr->header.lock ); + while (cno_ptr->cno_state == DAPL_CNO_STATE_UNTRIGGERED + && DAT_GET_TYPE(dat_status) != DAT_TIMEOUT_EXPIRED) + { + cno_ptr->cno_waiters++; + dapl_os_unlock ( &cno_ptr->header.lock ); + dat_status = dapl_os_wait_object_wait (&cno_ptr->cno_wait_object, + timeout); + dapl_os_lock ( &cno_ptr->header.lock ); + cno_ptr->cno_waiters--; + } + + if ( cno_ptr->cno_state == DAPL_CNO_STATE_DEAD ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_CNO_DEAD); + } + else if ( dat_status == DAT_SUCCESS ) + { + /* + * After the first triggering, this will be a valid handle. + * If we're racing with wakeups of other CNO waiters, + * that's ok. + */ + dapl_os_assert ( cno_ptr->cno_state == DAPL_CNO_STATE_TRIGGERED ); + cno_ptr->cno_state = DAPL_CNO_STATE_UNTRIGGERED; + *evd_handle = cno_ptr->cno_evd_triggered; + } + else if ( DAT_GET_TYPE(dat_status) == DAT_TIMEOUT_EXPIRED ) + { + cno_ptr->cno_state = DAPL_CNO_STATE_UNTRIGGERED; + *evd_handle = NULL; + dat_status = DAT_QUEUE_EMPTY; + } + else + { + /* + * The only other reason we could have made it out of + * the loop is an interrupted system call. + */ + dapl_os_assert(DAT_GET_TYPE(dat_status) == DAT_INTERRUPTED_CALL); + } + dapl_os_unlock ( &cno_ptr->header.lock ); + + bail: + return dat_status; +} + diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_clear_unwaitable.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_clear_unwaitable.c new file mode 100644 index 00000000..c779604b --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_clear_unwaitable.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_clear_unwaitable.c + * + * PURPOSE: EVENT management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.4.8 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_evd_clear_unwaitable + * + * DAPL Requirements Version 1.1, 6.3.4.8 + * + * Transition the Event Dispatcher into a waitable state + * + * Input: + * evd_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ +DAT_RETURN DAT_API +dapl_evd_clear_unwaitable ( + IN DAT_EVD_HANDLE evd_handle ) +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD) ) + + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + dapl_os_lock ( &evd_ptr->header.lock ); + evd_ptr->evd_waitable = DAT_TRUE; + dapl_os_unlock ( &evd_ptr->header.lock ); + + dat_status = DAT_SUCCESS; + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_create.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_create.c new file mode 100644 index 00000000..ee305442 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_create.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_create.c + * + * PURPOSE: EVENT management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" + +/* + * dapl_evd_create + * + * DAPL Requirements Version xxx, 6.3.2.1 + * + * Create and instance of Event Dispatcher. + * + * Input: + * ia_handle + * cno_handle + * evd_min_qlen + * evd_flags + * + * Output: + * evd_handle + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + */ + +/* ** REVISIT ** + * + * Selecting the cqe handing domain must still be done. + * We *probably* want one per hca, but we could have one + * per provider or one per consumer. + */ +/* Note that if there already is a cq, it is not deleted + * even if it is not required. However, it will not be armed. + */ + +DAT_RETURN DAT_API dapl_evd_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_COUNT evd_min_qlen, + IN DAT_CNO_HANDLE cno_handle, + IN DAT_EVD_FLAGS evd_flags, + OUT DAT_EVD_HANDLE *evd_handle ) +{ + DAPL_IA *ia_ptr; + DAPL_EVD *evd_ptr; + DAPL_CNO *cno_ptr; + DAT_RETURN dat_status; + DAT_PROVIDER_ATTR provider_attr; + int i; + int j; + int flag_mask[6]; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_evd_create (%p, %d, %p, 0x%x, %p)\n", + ia_handle, + evd_min_qlen, + cno_handle, + evd_flags, + evd_handle); + DAPL_CNTR(DCNT_EVD_CREATE); + + ia_ptr = (DAPL_IA *)ia_handle; + cno_ptr = (DAPL_CNO *)cno_handle; + evd_ptr = NULL; + *evd_handle = NULL; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (ia_handle, DAPL_MAGIC_IA)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_IA); + goto bail; + } + + if ( evd_min_qlen <= 0 ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + if ( evd_min_qlen > ia_ptr->hca_ptr->ia_attr.max_evd_qlen ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_TEVD); + goto bail; + } + + if (cno_handle != DAT_HANDLE_NULL + && DAPL_BAD_HANDLE(cno_handle, DAPL_MAGIC_CNO)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,DAT_INVALID_HANDLE_CNO); + goto bail; + } + + /* + * Check the merging attributes to ensure the combination of + * flags requested is supported. + */ + dapl_ia_query (ia_handle, NULL, + 0, NULL, + DAT_PROVIDER_FIELD_ALL, &provider_attr); + + /* Set up an array of flags to compare against; the EVD bits are + * a sparse array that need to be mapped to the merging flags + */ + flag_mask[0] = DAT_EVD_SOFTWARE_FLAG; + flag_mask[1] = DAT_EVD_CR_FLAG; + flag_mask[2] = DAT_EVD_DTO_FLAG; + flag_mask[3] = DAT_EVD_CONNECTION_FLAG; + flag_mask[4] = DAT_EVD_RMR_BIND_FLAG; + flag_mask[5] = DAT_EVD_ASYNC_FLAG; + + for (i = 0; i < 6; i++) + { + if (flag_mask[i] & evd_flags) + { + for (j = 0; j < 6; j++) + { + if (flag_mask[j] & evd_flags) + { + if (provider_attr.evd_stream_merging_supported[i][j] == DAT_FALSE) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4); + goto bail; + } + } + } /* end for j */ + } + } /* end for i */ + + dat_status = dapls_evd_internal_create (ia_ptr, + cno_ptr, + evd_min_qlen, + evd_flags, + &evd_ptr); + if (dat_status != DAT_SUCCESS) + { + goto bail; + } + + evd_ptr->evd_state = DAPL_EVD_STATE_OPEN; + + *evd_handle = (DAT_EVD_HANDLE) evd_ptr; + +bail: + if (dat_status != DAT_SUCCESS) + { + if (evd_ptr) + { + dapl_evd_free (evd_ptr); + } + } + + dapl_dbg_log (DAPL_DBG_TYPE_RTN, + "dapl_evd_create () returns 0x%x\n", + dat_status); + + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_disable.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_disable.c new file mode 100644 index 00000000..f4fb863b --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_disable.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_disable.c + * + * PURPOSE: EVENT management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_evd_disable + * + * DAPL Requirements Version xxx, 6.3.2.5 + * + * Modify the size of the event queue of an Event Dispatcher + * + * Input: + * evd_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ + +DAT_RETURN DAT_API dapl_evd_disable ( + IN DAT_EVD_HANDLE evd_handle) +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD) ) + + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + + evd_ptr->evd_enabled = DAT_FALSE; + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_enable.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_enable.c new file mode 100644 index 00000000..aef2f240 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_enable.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_enable.c + * + * PURPOSE: EVENT management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_enable + * + * DAPL Requirements Version xxx, 6.3.2.5 + * + * Modify the size of the event queue of an Event Dispatcher + * + * Input: + * evd_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ + +DAT_RETURN DAT_API dapl_evd_enable ( + IN DAT_EVD_HANDLE evd_handle) +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD) ) + + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + + evd_ptr->evd_enabled = DAT_TRUE; + + /* We need to enable the callback handler if there is a CNO. */ + if (evd_ptr->cno_ptr != NULL && + evd_ptr->ib_cq_handle != IB_INVALID_HANDLE ) + { + dat_status = dapls_ib_completion_notify ( + evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle, + evd_ptr, + IB_NOTIFY_ON_NEXT_COMP); + + /* FIXME report error */ + dapl_os_assert(dat_status == DAT_SUCCESS); + } + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_modify_cno.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_modify_cno.c new file mode 100644 index 00000000..64515c08 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_modify_cno.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_modify_cno.c + * + * PURPOSE: Event Management + * + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_modify_cno + * + * DAPL Requirements Version xxx, 6.3.2.4 + * + * Modify the CNO associated with the EVD + * + * Input: + * evd_handle + * cno_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCSSS + * DAT_INVALID_HANDLE + */ + +DAT_RETURN DAT_API dapl_evd_modify_cno ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_CNO_HANDLE cno_handle) + + +{ + DAPL_EVD *evd_ptr; + DAPL_CNO *cno_ptr; + DAPL_CNO *old_cno_ptr; + DAT_RETURN dat_status; + + evd_ptr = (DAPL_EVD *)evd_handle; + cno_ptr = (DAPL_CNO *)cno_handle; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, 0); + goto bail; + } + if (cno_handle != NULL && + DAPL_BAD_HANDLE (cno_handle, DAPL_MAGIC_CNO)) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CNO); + goto bail; + } + dapl_os_lock (&evd_ptr->header.lock); + old_cno_ptr = evd_ptr->cno_ptr; + evd_ptr->cno_ptr = cno_ptr; + dapl_os_unlock (&evd_ptr->header.lock); + if (cno_ptr) + { + dapl_os_atomic_inc ( & (cno_ptr->cno_ref_count) ); + } + if (old_cno_ptr) + { + dapl_os_atomic_dec ( & (old_cno_ptr->cno_ref_count) ); + } + + /* We need to enable the callback handler if the EVD is enabled. */ + if (evd_ptr->evd_enabled && + cno_handle != DAT_HANDLE_NULL && + evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) + { + dat_status = dapls_ib_completion_notify ( + evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle, + evd_ptr, + IB_NOTIFY_ON_NEXT_COMP); + + /* FIXME report error */ + dapl_os_assert (dat_status == DAT_SUCCESS); + } + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_query.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_query.c new file mode 100644 index 00000000..59dffdd7 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_query.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_query.c + * + * PURPOSE: Event management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" + +/* + * dapl_evd_query + * + * DAPL Requirements Version xxx, 6.3.2.3 + * + * Provides the consumer with arguments of the Event Dispatcher. + * + * Input: + * evd_handle + * evd_mask + * + * Output: + * evd_param + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + */ +DAT_RETURN DAT_API +dapl_evd_query ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_EVD_PARAM_MASK evd_param_mask, + OUT DAT_EVD_PARAM *evd_param ) +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + if ( NULL == evd_param ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + + /* Note: the spec. allows for events to be directed to a NULL EVD */ + /* with handle of type DAT_HANDLE_NULL. See 6.3.1 */ + if ( DAT_HANDLE_NULL == evd_handle ) + { + dapl_os_memzero (evd_param, sizeof (DAT_EVD_PARAM)); + } + else + { + if ( DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + + evd_ptr = (DAPL_EVD *) evd_handle; + + /* + * We may be racing against the thread safe modify + * calls here (dat_evd_{enable,disable,{set,clear}_unwaitable}). + * They are thread safe, so our reads need to be atomic with + * regard to those calls. The below is ok (a single bit + * read counts as atomic; if it's in transition you'll get one + * of the correct values) but we'll need to be careful + * about reading the state variable atomically when we add + * in waitable/unwaitable. + */ + evd_param->evd_state = + (evd_ptr->evd_enabled ? DAT_EVD_STATE_ENABLED : DAT_EVD_STATE_DISABLED); + evd_param->evd_state |= + (evd_ptr->evd_waitable ? DAT_EVD_STATE_WAITABLE : DAT_EVD_STATE_UNWAITABLE); + evd_param->ia_handle = evd_ptr->header.owner_ia; + evd_param->evd_qlen = evd_ptr->qlen; + evd_param->cno_handle = (DAT_CNO_HANDLE) evd_ptr->cno_ptr; + evd_param->evd_flags = evd_ptr->evd_flags; + } + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_set_unwaitable.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_set_unwaitable.c new file mode 100644 index 00000000..4bc74147 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_set_unwaitable.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_set_unwaitable.c + * + * PURPOSE: EVENT management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 3.4.7 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_set_unwaitable + * + * DAPL Requirements Version 1.1, 6.3.4.7 + * + * Transition the Event Dispatcher into an unwaitable state + * + * Input: + * evd_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ +DAT_RETURN DAT_API +dapl_evd_set_unwaitable ( + IN DAT_EVD_HANDLE evd_handle ) +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if ( DAPL_BAD_HANDLE (evd_handle, DAPL_MAGIC_EVD) ) + + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + dapl_os_lock ( &evd_ptr->header.lock ); + evd_ptr->evd_waitable = DAT_FALSE; + dapl_os_unlock ( &evd_ptr->header.lock ); + + /* + * If this evd is waiting, wake it up. There is an obvious race + * condition here where we may wakeup the waiter before it goes to + * sleep; but the wait_object allows this and will do the right + * thing. + */ + if (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED) + { +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + dapls_ib_wait_object_wakeup (evd_ptr->cq_wait_obj_handle); + else +#endif + dapl_os_wait_object_wakeup (&evd_ptr->wait_object); + } +bail: + return dat_status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_evd_wait.c b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_wait.c new file mode 100644 index 00000000..c0f1b8ca --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_evd_wait.c @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_evd_wait.c + * + * PURPOSE: EVENT management + * + * Description: Interfaces in this file are completely defined in + * the uDAPL 1.1 API specification + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include "dapl_evd_util.h" +#include "dapl_ring_buffer_util.h" +#include "dapl_adapter_util.h" + +/* + * dapl_evd_wait + * + * UDAPL Requirements Version xxx, + * + * Wait, up to specified timeout, for notification event on EVD. + * Then return first available event. + * + * Input: + * evd_handle + * timeout + * + * Output: + * event + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * DAT_INVALID_STATE + */ + +DAT_RETURN DAT_API dapl_evd_wait ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_TIMEOUT time_out, + IN DAT_COUNT threshold, + OUT DAT_EVENT *event, + OUT DAT_COUNT *nmore) + +{ + DAPL_EVD *evd_ptr; + DAT_RETURN dat_status; + DAT_EVENT *local_event; + DAT_BOOLEAN notify_requested = DAT_FALSE; + DAT_BOOLEAN waitable; + DAPL_EVD_STATE evd_state; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_evd_wait (%p, %d, %d, %p, %p)\n", + evd_handle, + time_out, + threshold, + event, + nmore); + DAPL_CNTR(DCNT_EVD_WAIT); + + evd_ptr = (DAPL_EVD *)evd_handle; + dat_status = DAT_SUCCESS; + + if (DAPL_BAD_HANDLE (evd_ptr, DAPL_MAGIC_EVD)) + { + /* + * We return directly rather than bailing because + * bailing attempts to update the evd, and we don't have + * one. + */ + dat_status = DAT_ERROR (DAT_INVALID_HANDLE,0); + goto bail; + } + if (!event) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4); + goto bail; + } + if (!nmore) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG5); + goto bail; + } + if (threshold <= 0 || + (threshold > 1 && evd_ptr->completion_type != DAPL_EVD_STATE_THRESHOLD) || + threshold > evd_ptr->qlen) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG3); + goto bail; + } + if ( evd_ptr->catastrophic_overflow ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,0); + goto bail; + } + + dapl_dbg_log (DAPL_DBG_TYPE_EVD, + "dapl_evd_wait: EVD %p, CQ %p\n", + evd_ptr, + (void *)evd_ptr->ib_cq_handle); + + /* + * Make sure there are no other waiters and the evd is active. + * Currently this means only the OPEN state is allowed. + * Do this atomically. We need to take a lock to synchronize + * with dapl_evd_dequeue(), but the atomic transition allows + * non-locking synchronization with dapl_evd_query() and + * dapl_evd_{query,enable,disable,{set,clear}_unwaitable}. + */ + + dapl_os_lock ( &evd_ptr->header.lock ); + waitable = evd_ptr->evd_waitable; + + dapl_os_assert ( sizeof(DAT_COUNT) == sizeof(DAPL_EVD_STATE) ); + evd_state = dapl_os_atomic_assign ( (DAPL_ATOMIC *)&evd_ptr->evd_state, + (DAT_COUNT) DAPL_EVD_STATE_OPEN, + (DAT_COUNT) DAPL_EVD_STATE_WAITED ); + dapl_os_unlock ( &evd_ptr->header.lock ); + + if ( evd_state != DAPL_EVD_STATE_OPEN ) + { + /* Bogus state, bail out */ + dat_status = DAT_ERROR (DAT_INVALID_STATE,0); + goto bail; + } + + if (!waitable) + { + /* This EVD is not waitable, reset the state and bail */ + (void) dapl_os_atomic_assign ((DAPL_ATOMIC *)&evd_ptr->evd_state, + (DAT_COUNT) DAPL_EVD_STATE_WAITED, + evd_state); + dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_EVD_UNWAITABLE); + goto bail; + } + + /* + * We now own the EVD, even though we don't have the lock anymore, + * because we're in the WAITED state. + */ + + evd_ptr->threshold = threshold; + + for (;;) + { + /* + * Ideally we'd just check the number of entries on the CQ, but + * we don't have a way to do that. Because we have to set *nmore + * at some point in this routine, we'll need to do this copy + * sometime even if threshold == 1. + * + * For connection evd or async evd, the function checks and + * return right away if the ib_cq_handle associate with these evd + * equal to IB_INVALID_HANDLE + */ + dapls_evd_copy_cq(evd_ptr); + + if (dapls_rbuf_count(&evd_ptr->pending_event_queue) >= threshold) + { + break; + } + + /* + * Do not enable the completion notification if this evd is not + * a DTO_EVD or RMR_BIND_EVD + */ + if ( (!notify_requested) && + ((evd_ptr->evd_flags & DAT_EVD_DTO_FLAG) || + (evd_ptr->evd_flags & DAT_EVD_RMR_BIND_FLAG)) ) + { + dat_status = dapls_ib_completion_notify ( + evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle, + evd_ptr, + (evd_ptr->completion_type == DAPL_EVD_STATE_SOLICITED_WAIT) ? + IB_NOTIFY_ON_SOLIC_COMP : IB_NOTIFY_ON_NEXT_COMP ); + + DAPL_CNTR(DCNT_EVD_WAIT_CMP_NTFY); + /* FIXME report error */ + dapl_os_assert(dat_status == DAT_SUCCESS); + + notify_requested = DAT_TRUE; + + /* Try again. */ + continue; + } + + + /* + * Unused by poster; it has no way to tell how many + * items are on the queue without copying them over to the + * EVD queue, and we're the only ones allowed to dequeue + * from the CQ for synchronization/locking reasons. + */ + evd_ptr->threshold = threshold; + + DAPL_CNTR(DCNT_EVD_WAIT_BLOCKED); + +#ifdef CQ_WAIT_OBJECT + if (evd_ptr->cq_wait_obj_handle) + dat_status = dapls_ib_wait_object_wait ( + evd_ptr->cq_wait_obj_handle, time_out ); + else +#endif + dat_status = dapl_os_wait_object_wait ( + &evd_ptr->wait_object, time_out ); + /* + * FIXME: if the thread loops around and waits again + * the time_out value needs to be updated. + */ + + notify_requested = DAT_FALSE; /* We've used it up. */ + + /* See if we were awakened by evd_set_unwaitable */ + if ( !evd_ptr->evd_waitable ) + { + dat_status = DAT_ERROR (DAT_INVALID_STATE,0); + } + + if (dat_status != DAT_SUCCESS) + { + /* + * If the status is DAT_TIMEOUT, we'll break out of the + * loop, *not* dequeue an event (because dat_status + * != DAT_SUCCESS), set *nmore (as we should for timeout) + * and return DAT_TIMEOUT. + */ + break; + } + } + + evd_ptr->evd_state = DAPL_EVD_STATE_OPEN; + + if (dat_status == DAT_SUCCESS) + { + local_event = dapls_rbuf_remove(&evd_ptr->pending_event_queue); + *event = *local_event; + dapls_rbuf_add(&evd_ptr->free_event_queue, local_event); + } + + /* + * Valid if dat_status == DAT_SUCCESS || dat_status == DAT_TIMEOUT + * Undefined otherwise, so ok to set it. + */ + *nmore = dapls_rbuf_count(&evd_ptr->pending_event_queue); + + bail: + if ( dat_status ) { + dapl_dbg_log (DAPL_DBG_TYPE_RTN, "dapl_evd_wait () returns 0x%x\n", + dat_status); + } + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_init.c b/trunk/ulp/dapl2/dapl/udapl/dapl_init.c new file mode 100644 index 00000000..c0c2681d --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_init.c @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_init.c + * + * PURPOSE: Interface Adapter management + * Description: Interfaces in this file are completely described in + * the DAPL 1.1 API, Chapter 6, section 2 + * + * $Id:$ + **********************************************************************/ + +#include "dapl.h" +#include /* Provider API function prototypes */ +#include "dapl_hca_util.h" +#include "dapl_init.h" +#include "dapl_provider.h" +#include "dapl_mr_util.h" +#include "dapl_osd.h" +#include "dapl_adapter_util.h" +#include "dapl_name_service.h" +#include "dapl_timer_util.h" +#include "dapl_vendor.h" + + +/* + * dapl_init + * + * initialize this provider + * includes initialization of all global variables + * as well as registering all supported IAs with the dat registry + * + * This function needs to be called once when the provider is loaded. + * + * Input: + * none + * + * Output: + * none + * + * Return Values: + */ +void dapl_init ( void ) +{ + DAT_RETURN dat_status; + +#if defined(DAPL_DBG) + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: (dapl_init)\n"); + + /* set up debug type */ + g_dapl_dbg_type = dapl_os_get_env_val ( "DAPL_DBG_TYPE", + DAPL_DBG_TYPE_ERR | DAPL_DBG_TYPE_WARN); + /* set up debug destination */ + g_dapl_dbg_dest = dapl_os_get_env_val ( "DAPL_DBG_DEST", + DAPL_DBG_DEST_STDOUT ); +#endif /* DAPL_DBG */ + + /* See if the user is on a loopback setup */ + g_dapl_loopback_connection = dapl_os_get_env_bool ( "DAPL_LOOPBACK" ); + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: %s Setting Loopback\n", + g_dapl_loopback_connection ? "" : "NOT"); + + /* initialize verbs library */ + dapls_ib_init (); + + /* initialize the timer */ + dapls_timer_init(); + + /* Set up name services */ + dat_status = dapls_ns_init (); + if (DAT_SUCCESS != dat_status) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dapls_ns_init failed %d\n", dat_status); + goto bail; + } + + /* initialize the provider list */ + dat_status = dapl_provider_list_create (); + + if (DAT_SUCCESS != dat_status) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dapl_provider_list_create failed %d\n", dat_status); + goto bail; + } + + DAPL_COUNTERS_INIT(); + + return; + +bail: + dapl_dbg_log (DAPL_DBG_TYPE_ERR, "ERROR: dapl_init failed\n"); + return; +} + + +/* + * dapl_fini + * + * finalize this provider + * includes freeing of all global variables + * as well as deregistering all supported IAs from the dat registry + * + * This function needs to be called once when the provider is loaded. + * + * Input: + * none + * + * Output: + * none + * + * Return Values: + */ +void dapl_fini ( void ) +{ + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: ENTER (dapl_fini)\n"); + + dat_status = dapl_provider_list_destroy (); + if (DAT_SUCCESS != dat_status) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dapl_provider_list_destroy failed %d\n", dat_status); + } + + dapls_ib_release (); + + dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: Exit (dapl_fini)\n"); + + return; +} + + +/* + * + * This function is called by the registry to initialize a provider + * + * The instance data string is expected to have the following form: + * + * + * + */ +void DAT_API +DAT_PROVIDER_INIT_FUNC_NAME ( + IN const DAT_PROVIDER_INFO *provider_info, + IN const char *instance_data ) +{ + DAT_PROVIDER *provider; + DAPL_HCA *hca_ptr; + DAT_RETURN dat_status; + char * data; + char * name; + char * port; + unsigned int len; + unsigned int i; + + data = NULL; + provider = NULL; + hca_ptr = NULL; + +#if defined(_WIN32) || defined(_WIN64) + /* initialize DAPL library here as when called from DLL context in DLLmain() + * the IB (ibal) call hangs. + */ + dapl_init(); +#endif + + dat_status = dapl_provider_list_insert(provider_info->ia_name, &provider); + if ( DAT_SUCCESS != dat_status ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dat_provider_list_insert failed: %x\n", dat_status); + goto bail; + } + + data = dapl_os_strdup(instance_data); + if ( NULL == data ) + { + goto bail; + } + + len = dapl_os_strlen(data); + + for ( i = 0; i < len; i++ ) + { + if ( ' ' == data[i] ) + { + data[i] = '\0'; + break; + } + } + + /* if the instance data did not have a valid format */ + if ( i == len ) + { + goto bail; + } + + name = data; + port = data + (i + 1); + + hca_ptr = dapl_hca_alloc (name, port); + if ( NULL == hca_ptr ) + { + goto bail; + } + + provider->extension = hca_ptr; + dat_status = dat_registry_add_provider (provider, provider_info); + if ( DAT_SUCCESS != dat_status ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dat_registry_add_provider failed: %x\n", dat_status); + goto bail; + } + + bail: + if ( NULL != data ) + { + dapl_os_free(data, len + 1); + } + + if ( DAT_SUCCESS != dat_status ) + { + if ( NULL != provider ) + { + (void) dapl_provider_list_remove(provider_info->ia_name); + } + + if ( NULL != hca_ptr ) + { + dapl_hca_free (hca_ptr); + } + } +} + + +/* + * + * This function is called by the registry to de-initialize a provider + * + */ +void DAT_API +DAT_PROVIDER_FINI_FUNC_NAME ( + IN const DAT_PROVIDER_INFO *provider_info ) +{ + DAT_PROVIDER *provider; + DAT_RETURN dat_status; + + dat_status = dapl_provider_list_search(provider_info->ia_name, &provider); + if ( DAT_SUCCESS != dat_status ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dat_registry_add_provider failed: %x\n", dat_status); + return; + } + + dat_status = dat_registry_remove_provider (provider, provider_info); + if ( DAT_SUCCESS != dat_status ) + { + dapl_dbg_log (DAPL_DBG_TYPE_ERR, + "dat_registry_add_provider failed: %x\n", dat_status); + } + + /* + * free HCA memory + */ + dapl_hca_free (provider->extension); + + (void) dapl_provider_list_remove(provider_info->ia_name); + +#if defined(_WIN32) || defined(_WIN64) + /* cleanup DAPL library - relocated here from OSD DLL context as the IBAL + * calls hung in the DLL context? + */ + dapl_fini(); +#endif +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/udapl/dapl_lmr_create.c b/trunk/ulp/dapl2/dapl/udapl/dapl_lmr_create.c new file mode 100644 index 00000000..b3f8c6c1 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/dapl_lmr_create.c @@ -0,0 +1,539 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_lmr_create.c + * + * PURPOSE: Memory management + * + * $Id:$ + **********************************************************************/ + +#include "dapl_lmr_util.h" +#include "dapl_adapter_util.h" +#include "dapl_vendor.h" + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +STATIC _INLINE_ DAT_RETURN +dapli_lmr_create_virtual ( + IN DAPL_IA *ia, + IN DAT_PVOID virt_addr, + IN DAT_VLEN length, + IN DAPL_PZ *pz, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ); + +STATIC _INLINE_ DAT_RETURN +dapli_lmr_create_lmr ( + IN DAPL_IA *ia, + IN DAPL_LMR *original_lmr, + IN DAPL_PZ *pz, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ); + + +STATIC _INLINE_ DAT_RETURN +dapli_lmr_create_shared ( + IN DAPL_IA *ia, + IN DAT_REGION_DESCRIPTION reg_desc, + IN DAT_VLEN length, + IN DAPL_PZ *pz, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ); + +/********************************************************************* + * * + * Function Definitions * + * * + *********************************************************************/ + +DAT_RETURN +dapli_lmr_create_virtual ( + IN DAPL_IA *ia, + IN DAT_PVOID virt_addr, + IN DAT_VLEN length, + IN DAPL_PZ *pz, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ) +{ + DAPL_LMR *lmr; + DAT_REGION_DESCRIPTION reg_desc; + DAT_RETURN dat_status; + + reg_desc.for_va = virt_addr; + dat_status = DAT_SUCCESS; + + lmr = dapl_lmr_alloc (ia, + DAT_MEM_TYPE_VIRTUAL, + reg_desc, + length, + (DAT_PZ_HANDLE) pz, + privileges); + + if ( NULL == lmr ) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_status = dapls_ib_mr_register (ia, + lmr, + virt_addr, + length, + privileges, + va_type); + + if (DAT_SUCCESS != dat_status) + { + dapl_lmr_dealloc (lmr); + goto bail; + } + + /* if the LMR context is already in the hash table */ + dat_status = dapls_hash_search (ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, + NULL); + if (dat_status == DAT_SUCCESS) + { + (void)dapls_ib_mr_deregister (lmr); + dapl_lmr_dealloc (lmr); + + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_LMR_IN_USE); + goto bail; + } + + dat_status = dapls_hash_insert (ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, + lmr); + if (dat_status != DAT_SUCCESS) + { + (void)dapls_ib_mr_deregister (lmr); + dapl_lmr_dealloc (lmr); + + /* The value returned by dapls_hash_insert(.) is not */ + /* returned to the consumer because the spec. requires */ + /* that dat_lmr_create(.) return only certain values. */ + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_atomic_inc (&pz->pz_ref_count); + *lmr_handle = (DAT_LMR_HANDLE) lmr; + + if (NULL != lmr_context) + { + *lmr_context = lmr->param.lmr_context; + } + if (NULL != rmr_context) + { + *rmr_context = lmr->param.rmr_context; + } + if (NULL != registered_length) + { + *registered_length = lmr->param.registered_size; + } + if (NULL != registered_address) + { + *registered_address = lmr->param.registered_address; + } + + bail: + return dat_status; +} + + +DAT_RETURN +dapli_lmr_create_lmr ( + IN DAPL_IA *ia, + IN DAPL_LMR *original_lmr, + IN DAPL_PZ *pz, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ) +{ + DAPL_LMR *lmr; + DAT_REGION_DESCRIPTION reg_desc; + DAT_RETURN dat_status; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_lmr_create_lmr (%p, %p, %p, %x, %x, %p, %p, %p, %p)\n", + ia, + original_lmr, + pz, privileges, va_type, + lmr_handle, + lmr_context, + registered_length, + registered_address); + + dat_status = dapls_hash_search (ia->hca_ptr->lmr_hash_table, + original_lmr->param.lmr_context, + (DAPL_HASH_DATA *) &lmr); + if ( dat_status != DAT_SUCCESS ) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); + goto bail; + } + + reg_desc.for_lmr_handle = (DAT_LMR_HANDLE) original_lmr; + + lmr = dapl_lmr_alloc (ia, + DAT_MEM_TYPE_LMR, + reg_desc, + original_lmr->param.length, + (DAT_PZ_HANDLE) pz, + privileges); + + if (NULL == lmr) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_status = dapls_ib_mr_register_shared (ia, + lmr, + privileges, + va_type); + + if (DAT_SUCCESS != dat_status) + { + dapl_lmr_dealloc (lmr); + goto bail; + } + + /* if the LMR context is already in the hash table */ + dat_status = dapls_hash_search (ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, + NULL); + if (dat_status == DAT_SUCCESS) + { + dapls_ib_mr_deregister (lmr); + dapl_lmr_dealloc (lmr); + + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_LMR_IN_USE); + goto bail; + } + + dat_status = dapls_hash_insert (ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, + lmr); + if (dat_status != DAT_SUCCESS) + { + dapls_ib_mr_deregister (lmr); + dapl_lmr_dealloc (lmr); + + /* The value returned by dapls_hash_insert(.) is not */ + /* returned to the consumer because the spec. requires */ + /* that dat_lmr_create(.) return only certain values. */ + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_atomic_inc (&pz->pz_ref_count); + *lmr_handle = (DAT_LMR_HANDLE) lmr; + + if (NULL != lmr_context) + { + *lmr_context = lmr->param.lmr_context; + } + if (NULL != rmr_context) + { + *rmr_context = lmr->param.rmr_context; + } + if (NULL != registered_length) + { + *registered_length = lmr->param.registered_size; + } + if (NULL != registered_address) + { + *registered_address = lmr->param.registered_address; + } + + bail: + return dat_status; +} + + + +DAT_RETURN +dapli_lmr_create_shared ( + IN DAPL_IA *ia, + IN DAT_REGION_DESCRIPTION reg_desc, + IN DAT_VLEN length, + IN DAPL_PZ *pz, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ) +{ + DAPL_LMR *lmr; + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + + dapl_dbg_log (DAPL_DBG_TYPE_API, + "dapl_lmr_create_shared_virtual (ia=%p len=%x pz=%p priv=%x)\n", + ia, + length, + pz, privileges); + + lmr = dapl_lmr_alloc (ia, + DAT_MEM_TYPE_LMR, + reg_desc, + length, /* length is meaningless */ + (DAT_PZ_HANDLE) pz, + privileges); + + if (NULL == lmr) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + /* + * Added for the shared memory support - - - + * Save the region description. We need to copy the shared + * memory id because the region_desc only contains a pointer + * to it. + */ + dapl_os_memcpy(&lmr->shmid, + reg_desc.for_shared_memory.shared_memory_id, + sizeof(lmr->shmid)); + lmr->param.region_desc = reg_desc; + lmr->param.length = length; + lmr->param.mem_type = DAT_MEM_TYPE_SHARED_VIRTUAL; + lmr->param.region_desc.for_shared_memory.shared_memory_id = &lmr->shmid; + + dat_status = dapls_ib_mr_register_shared(ia, lmr, privileges, va_type); + if (dat_status != DAT_SUCCESS) + { + dapl_lmr_dealloc (lmr); + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY_REGION); + goto bail; + } + + /* if the LMR context is already in the hash table */ + dat_status = dapls_hash_search(ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, + NULL); + if (DAT_SUCCESS == dat_status) + { + (void)dapls_ib_mr_deregister (lmr); + dapl_lmr_dealloc (lmr); + + dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_LMR_IN_USE); + goto bail; + } + + dat_status = dapls_hash_insert(ia->hca_ptr->lmr_hash_table, + lmr->param.lmr_context, + lmr); + if (dat_status != DAT_SUCCESS) + { + (void)dapls_ib_mr_deregister (lmr); + dapl_lmr_dealloc (lmr); + + /* The value returned by dapls_hash_insert(.) is not */ + /* returned to the consumer because the spec. requires */ + /* that dat_lmr_create(.) return only certain values. */ + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + dapl_os_atomic_inc (&pz->pz_ref_count); + *lmr_handle = (DAT_LMR_HANDLE) lmr; + + if (NULL != lmr_context) + { + *lmr_context = (DAT_LMR_CONTEXT) lmr->param.lmr_context; + } + if (NULL != rmr_context) + { + *rmr_context = (DAT_LMR_CONTEXT) lmr->param.rmr_context; + } + if (NULL != registered_length) + { + *registered_length = lmr->param.length; + } + if (NULL != registered_address) + { + *registered_address = (DAT_VADDR) (uintptr_t) + lmr->param.region_desc.for_shared_memory.virtual_address; + } + bail: + + return dat_status; +} + + +/* + * dapl_lmr_create + * + * Register a memory region with an Interface Adaptor. + * + * Input: + * ia_handle + * mem_type + * region_description + * length + * pz_handle + * privileges + * + * Output: + * lmr_handle + * lmr_context + * registered_length + * registered_address + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * DAT_INVALID_HANDLE + * DAT_INVALID_STATE + * DAT_MODEL_NOT_SUPPORTED + * + */ +DAT_RETURN DAT_API +dapl_lmr_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_MEM_TYPE mem_type, + IN DAT_REGION_DESCRIPTION region_description, + IN DAT_VLEN length, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ) +{ + DAPL_IA *ia; + DAPL_PZ *pz; + DAT_RETURN dat_status; + + if ( DAPL_BAD_HANDLE (ia_handle, DAPL_MAGIC_IA) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + if ( DAPL_BAD_HANDLE (pz_handle, DAPL_MAGIC_PZ) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + goto bail; + } + if (NULL == lmr_handle) + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG7); + goto bail; + } + + ia = (DAPL_IA *) ia_handle; + pz = (DAPL_PZ *) pz_handle; + + switch (mem_type) + { + case DAT_MEM_TYPE_VIRTUAL: + { + dat_status = dapli_lmr_create_virtual ( + ia, region_description.for_va, length, pz, privileges, va_type, + lmr_handle, lmr_context, rmr_context, registered_length, + registered_address); + break; + } + case DAT_MEM_TYPE_LMR: + { + DAPL_LMR *lmr; + + if ( DAPL_BAD_HANDLE (region_description.for_lmr_handle, DAPL_MAGIC_LMR) ) + { + dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_LMR); + goto bail; + } + + lmr = (DAPL_LMR *) region_description.for_lmr_handle; + + dat_status = dapli_lmr_create_lmr ( + ia, lmr, pz, privileges, va_type, lmr_handle, + lmr_context, rmr_context, registered_length, registered_address); + break; + } + case DAT_MEM_TYPE_SHARED_VIRTUAL: + { +#if (VN_MEM_SHARED_VIRTUAL_SUPPORT > 0) + dat_status = dapli_lmr_create_shared ( + ia, region_description, length, pz, privileges, va_type, + lmr_handle, lmr_context, rmr_context, registered_length, + registered_address); +#else + dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); +#endif + break; + } + default: + { + dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + break; + } + } + +bail: + return dat_status; +} diff --git a/trunk/ulp/dapl2/dapl/udapl/libdaplcma.map b/trunk/ulp/dapl2/dapl/udapl/libdaplcma.map new file mode 100644 index 00000000..de06305f --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/libdaplcma.map @@ -0,0 +1,7 @@ +DAPL_CMA_2.0 { + global: + dat_provider_fini; + dat_provider_init; + dapl_extensions; + local: *; +}; diff --git a/trunk/ulp/dapl2/dapl/udapl/libdaplscm.map b/trunk/ulp/dapl2/dapl/udapl/libdaplscm.map new file mode 100644 index 00000000..b5dec4ef --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/libdaplscm.map @@ -0,0 +1,6 @@ +DAPL_SCM_1.2 { + global: + dat_provider_fini; + dat_provider_init; + local: *; +}; diff --git a/trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.c b/trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.c new file mode 100644 index 00000000..0ef66cd1 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.c @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_osd.c + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent functions with a canonical DAPL + * interface. Designed to be portable and hide OS specific quirks + * of common functions. + * + * + * $Id:$ + **********************************************************************/ + +#include "dapl_osd.h" +#include "dapl.h" +#include "dapl_hca_util.h" +#include "dapl_ia_util.h" +#include "dapl_rmr_util.h" +#include "dapl_lmr_util.h" +#include "dapl_pz_util.h" +#include "dapl_ep_util.h" +#include "dapl_cr_util.h" +#include "dapl_evd_util.h" +#include "dapl_sp_util.h" +#include "dapl_adapter_util.h" +#include "dapl_provider.h" +#include "dapl_hash.h" +#include "dapl_timer_util.h" +#include "dapl_debug.h" + +#include +#include /* needed for getenv() */ +#include /* needed for pthread_atfork() */ +#include /* needed for thread setup */ + +static void dapls_osd_fork_cleanup (void); + +/* + * dapl_osd_init + * + * Do Linux initialization: + * - Set up fork handler to clean up DAPL resources in the child + * process after a fork(). + * + * Input: + * none + * + * Returns: + * DAT_SUCCESS + */ +void +dapl_os_init ( ) +{ + int status; + + /* + * Set up fork control + */ + status = pthread_atfork ( NULL, NULL, dapls_osd_fork_cleanup ); + if ( status != 0 ) + { + dapl_os_printf ("WARNING: pthread_atfork %d\n", status); + } +} + + +/* + * dapl_os_get_time + * + * Return 64 bit value of current time in microseconds. + * + * Input: + * loc User location to place current time + * + * Returns: + * DAT_SUCCESS + */ + +DAT_RETURN +dapl_os_get_time ( + OUT DAPL_OS_TIMEVAL * loc) +{ + struct timeval tv; + struct timezone tz; + + + gettimeofday (&tv, &tz); + *loc = ((DAT_UINT64) (tv.tv_sec) * 1000000L) + (DAT_UINT64) tv.tv_usec; + + return DAT_SUCCESS; +} + + +/* + * dapl_os_get__env_bool + * + * Return boolean value of passed in environment variable: 1 if present, + * 0 if not + * + * Input: + * + * + * Returns: + * TRUE or FALSE + */ +int +dapl_os_get_env_bool ( + char *env_str ) +{ + char *env_var; + + env_var = getenv (env_str); + if (env_var != NULL) + { + return 1; + } + + return 0; +} + + +/* + * dapl_os_get_env_val + * + * Update val to value of passed in environment variable if present + * + * Input: + * env_str + * def_val default value if environment variable does not exist + * + * Returns: + * TRUE or FALSE + */ +int +dapl_os_get_env_val ( + char *env_str, + int def_val ) +{ + char *env_var; + + env_var = getenv (env_str); + if ( env_var != NULL ) + { + def_val = strtol (env_var, NULL, 0); + } + + return def_val; +} + + +/* + * Wait object routines + */ + +/* + * dapl_os_wait_object_init + * + * Initialize a wait object + * + * Input: + * wait_obj + * + * Returns: + * DAT_SUCCESS + * DAT_INTERNAL_ERROR + */ +DAT_RETURN +dapl_os_wait_object_init ( + IN DAPL_OS_WAIT_OBJECT *wait_obj) +{ + wait_obj->signaled = DAT_FALSE; + if ( 0 != pthread_cond_init ( &wait_obj->cv, NULL ) ) + { + return DAT_ERROR (DAT_INTERNAL_ERROR,0); + } + + /* Always returns 0. */ + pthread_mutex_init ( &wait_obj->lock, NULL ); + + return DAT_SUCCESS; +} + + +/* Wait on the supplied wait object, up to the specified time_out. + * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. + * Timeout should be specified in micro seconds. + * + * Functional returns: + * DAT_SUCCESS -- another thread invoked dapl_os_wait object_wakeup + * DAT_INVALID_STATE -- someone else is already waiting in this wait + * object. + * only one waiter is allowed at a time. + * DAT_ABORT -- another thread invoked dapl_os_wait_object_destroy + * DAT_TIMEOUT -- the specified time limit was reached. + */ + +DAT_RETURN +dapl_os_wait_object_wait ( + IN DAPL_OS_WAIT_OBJECT *wait_obj, + IN DAT_TIMEOUT timeout_val) +{ + DAT_RETURN dat_status; + int pthread_status; + struct timespec future; + + dat_status = DAT_SUCCESS; + pthread_status = 0; + + if ( timeout_val != DAT_TIMEOUT_INFINITE ) + { + struct timeval now; + struct timezone tz; + unsigned int microsecs; + + gettimeofday (&now, &tz); +#define USEC_PER_SEC 1000000 + microsecs = now.tv_usec + timeout_val; + now.tv_sec = now.tv_sec + microsecs/USEC_PER_SEC; + now.tv_usec = microsecs % USEC_PER_SEC; + + /* Convert timeval to timespec */ + future.tv_sec = now.tv_sec; + future.tv_nsec = now.tv_usec * 1000; + + pthread_mutex_lock (&wait_obj->lock); + while ( wait_obj->signaled == DAT_FALSE && pthread_status == 0) + { + pthread_status = pthread_cond_timedwait ( + &wait_obj->cv , &wait_obj->lock , &future ); + + /* + * No need to reset &future if we go around the loop; + * It's an absolute time. + */ + } + /* Reset the signaled status if we were woken up. */ + if (pthread_status == 0) + { + wait_obj->signaled = DAT_FALSE; + } + pthread_mutex_unlock (&wait_obj->lock); + } + else + { + pthread_mutex_lock (&wait_obj->lock); + while ( wait_obj->signaled == DAT_FALSE && pthread_status == 0) + { + pthread_status = pthread_cond_wait ( + &wait_obj->cv , &wait_obj->lock ); + } + /* Reset the signaled status if we were woken up. */ + if (pthread_status == 0) + { + wait_obj->signaled = DAT_FALSE; + } + pthread_mutex_unlock (&wait_obj->lock); + } + + if (ETIMEDOUT == pthread_status) + { + dat_status = DAT_ERROR (DAT_TIMEOUT_EXPIRED, 0); + } + else if ( EINTR == pthread_status) + { + dat_status = DAT_ERROR (DAT_INTERRUPTED_CALL, 0); + } + else if ( 0 != pthread_status) + { + dat_status = DAT_ERROR (DAT_INTERNAL_ERROR, 0); + } + + return dat_status; +} + + +/* + * dapl_os_wait_object_wakeup + * + * Wakeup a thread waiting on a wait object + * + * Input: + * wait_obj + * + * Returns: + * DAT_SUCCESS + * DAT_INTERNAL_ERROR + */ +DAT_RETURN +dapl_os_wait_object_wakeup ( + IN DAPL_OS_WAIT_OBJECT *wait_obj) +{ + pthread_mutex_lock ( &wait_obj->lock ); + wait_obj->signaled = DAT_TRUE; + pthread_mutex_unlock ( &wait_obj->lock ); + if ( 0 != pthread_cond_signal ( &wait_obj->cv ) ) + { + return DAT_ERROR (DAT_INTERNAL_ERROR,0); + } + + return DAT_SUCCESS; +} + + +/* + * dapl_os_wait_object_destroy + * + * Destroy a wait object + * + * Input: + * wait_obj + * + * Returns: + * DAT_SUCCESS + * DAT_INTERNAL_ERROR + */ +DAT_RETURN +dapl_os_wait_object_destroy ( + IN DAPL_OS_WAIT_OBJECT *wait_obj) +{ + if ( 0 != pthread_cond_destroy ( &wait_obj->cv ) ) + { + return DAT_ERROR (DAT_INTERNAL_ERROR,0); + } + if ( 0 != pthread_mutex_destroy ( &wait_obj->lock ) ) + { + return DAT_ERROR (DAT_INTERNAL_ERROR,0); + } + + return DAT_SUCCESS; +} + + +/* + * dapls_osd_fork_cleanup + * + * Update val to value of passed in environment variable if present + * + * Input: + * env_str + * val Updated if environment variable exists + * + * Returns: + * TRUE or FALSE + */ +void dapls_osd_fork_cleanup (void) +{ + DAPL_PROVIDER_LIST_NODE *cur_node; + DAPL_HCA *hca_ptr; + DAPL_IA *ia_ptr; + DAPL_LMR *lmr_ptr; + DAPL_RMR *rmr_ptr; + DAPL_PZ *pz_ptr; + DAPL_CR *cr_ptr; + DAPL_EP *ep_ptr; + DAPL_EVD *evd_ptr; + DAT_EP_PARAM *param; + DAPL_SP *sp_ptr; + + while ( NULL != g_dapl_provider_list.head ) + { + cur_node = g_dapl_provider_list.head; + g_dapl_provider_list.head = cur_node->next; + + hca_ptr = (DAPL_HCA *) cur_node->data.extension; + + /* + * Walk the list of IA ptrs & clean up. This is purposely + * a destructive list walk, we really don't want to preserve + * any of it. + */ + while (!dapl_llist_is_empty ( &hca_ptr->ia_list_head ) ) + { + ia_ptr = (DAPL_IA *) + dapl_llist_peek_head ( &hca_ptr->ia_list_head ); + + /* + * The rest of the cleanup code is similar to dapl_ia_close, + * the big difference is that we don't release IB resources, + * only memory; the underlying IB subsystem doesn't deal + * with fork at all, so leave IB handles alone. + */ + while (!dapl_llist_is_empty ( &ia_ptr->rmr_list_head ) ) + { + rmr_ptr = (DAPL_RMR *) + dapl_llist_peek_head (&ia_ptr->rmr_list_head); + if ( rmr_ptr->param.lmr_triplet.virtual_address != 0 ) + { + dapl_os_atomic_dec (&rmr_ptr->lmr->lmr_ref_count); + rmr_ptr->param.lmr_triplet.virtual_address = 0; + } + dapl_os_atomic_dec ( &rmr_ptr->pz->pz_ref_count ); + dapl_ia_unlink_rmr ( rmr_ptr->header.owner_ia, rmr_ptr ); + dapl_rmr_dealloc ( rmr_ptr ); + } + + while (!dapl_llist_is_empty ( &ia_ptr->rsp_list_head )) + { + sp_ptr = (DAPL_SP *) + dapl_llist_peek_head ( &ia_ptr->rsp_list_head ); + dapl_os_atomic_dec (& ((DAPL_EVD *)sp_ptr->evd_handle)->evd_ref_count); + dapls_ia_unlink_sp ( ia_ptr, sp_ptr ); + dapls_sp_free_sp ( sp_ptr ); + } + + while (!dapl_llist_is_empty ( &ia_ptr->ep_list_head ) ) + { + ep_ptr = (DAPL_EP *) + dapl_llist_peek_head ( &ia_ptr->ep_list_head ); + param = &ep_ptr->param; + if ( param->pz_handle != NULL ) + { + dapl_os_atomic_dec ( & ((DAPL_PZ *)param->pz_handle)->pz_ref_count ); + } + if ( param->recv_evd_handle != NULL ) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)param->recv_evd_handle)->evd_ref_count); + } + if ( param->request_evd_handle ) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)param->request_evd_handle)->evd_ref_count); + } + if ( param->connect_evd_handle != NULL ) + { + dapl_os_atomic_dec (& ((DAPL_EVD *)param->connect_evd_handle)->evd_ref_count); + } + + /* ...and free the resource */ + dapl_ia_unlink_ep ( ia_ptr, ep_ptr ); + dapl_ep_dealloc ( ep_ptr ); + } + + while ( !dapl_llist_is_empty (&ia_ptr->lmr_list_head) ) + { + lmr_ptr = (DAPL_LMR *) + dapl_llist_peek_head ( &ia_ptr->lmr_list_head ); + + (void) dapls_hash_remove ( lmr_ptr->header.owner_ia->hca_ptr->lmr_hash_table, + lmr_ptr->param.lmr_context, + NULL ); + + pz_ptr = (DAPL_PZ *) lmr_ptr->param.pz_handle; + dapl_os_atomic_dec ( &pz_ptr->pz_ref_count ); + dapl_ia_unlink_lmr ( lmr_ptr->header.owner_ia, lmr_ptr ); + dapl_lmr_dealloc ( lmr_ptr ); + } + + while ( !dapl_llist_is_empty ( &ia_ptr->psp_list_head ) ) + { + sp_ptr = (DAPL_SP *) + dapl_llist_peek_head ( &ia_ptr->psp_list_head ); + while ( !dapl_llist_is_empty (&sp_ptr->cr_list_head) ) + { + cr_ptr = (DAPL_CR *) + dapl_llist_peek_head ( &sp_ptr->cr_list_head ); + dapl_sp_remove_cr (sp_ptr, cr_ptr); + dapls_cr_free (cr_ptr); + } + + dapls_ia_unlink_sp ( ia_ptr, sp_ptr ); + dapl_os_atomic_dec (& ((DAPL_EVD *)sp_ptr->evd_handle)->evd_ref_count); + dapls_sp_free_sp ( sp_ptr ); + } + + while (!dapl_llist_is_empty ( &ia_ptr->pz_list_head ) ) + { + pz_ptr = (DAPL_PZ *) + dapl_llist_peek_head ( &ia_ptr->pz_list_head ); + dapl_ia_unlink_pz (pz_ptr->header.owner_ia, pz_ptr); + dapl_pz_dealloc ( pz_ptr ); + } + + while (!dapl_llist_is_empty (&ia_ptr->evd_list_head)) + { + evd_ptr = (DAPL_EVD *) + dapl_llist_peek_head ( &ia_ptr->evd_list_head ); + dapl_ia_unlink_evd ( evd_ptr->header.owner_ia, evd_ptr ); + /* reset the cq_handle to avoid having it removed */ + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; + dapls_evd_dealloc ( evd_ptr ); + } + + dapl_hca_unlink_ia ( ia_ptr->hca_ptr, ia_ptr ); + /* asycn error evd was taken care of above, reset the pointer */ + ia_ptr->async_error_evd = NULL; + dapls_ia_free ( ia_ptr ); + } /* end while ( ia_ptr != NULL ) */ + + + dapl_os_free (cur_node, sizeof (DAPL_PROVIDER_LIST_NODE)); + } /* end while (NULL != g_dapl_provider_list.head) */ +} + + +/* + * Structure to contain all elements of a thread in order to enable a + * local routine to intercept and do some necessary initialization. + */ +struct thread_draft +{ + void (*func) (void *); /* start routine */ + void *data; /* argument to start routine */ +}; + +void dapli_thread_init ( struct thread_draft *thread_draft); + +/* + * dapls_os_thread_create + * + * Create a thread for dapl + * + * Input: + * func function to invoke thread + * f_arg argument to pass to function + * + * Output + * thread_id handle for thread + * + * Returns: + * DAT_SUCCESS + */ +DAT_RETURN +dapl_os_thread_create ( + IN void (*func) (void *), + IN void *data, + OUT DAPL_OS_THREAD *thread_id ) +{ + pthread_attr_t thread_attr; + struct thread_draft *thread_draft; + int status; + + /* + * Get default set of thread attributes + */ + status = pthread_attr_init ( &thread_attr ); + if ( status != 0 ) + { + return DAT_ERROR (DAT_INTERNAL_ERROR,0); + } + + /* Create dapl threads as detached from this process */ + status = pthread_attr_setdetachstate ( &thread_attr, + PTHREAD_CREATE_DETACHED ); + if ( status != 0 ) + { + return DAT_ERROR (DAT_INTERNAL_ERROR,0); + } + + thread_draft = dapl_os_alloc (sizeof ( struct thread_draft)); + + thread_draft->func = func; + thread_draft->data = data; + + /* Create the thread. Observe that we first invoke a local + * routine to set up OS parameters, before invoking the user + * specified routine. + */ + status = pthread_create ( thread_id, + &thread_attr, + (void * (*) (void *))dapli_thread_init, + (void *)thread_draft ); + + /* clean up resources */ + (void) pthread_attr_destroy ( &thread_attr ); + + if ( status != 0 ) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,0); + } + + return DAT_SUCCESS; +} + +/* + * dapli_thread_init + * + * Need to mask all signals from this thread in order to be a good + * citizen. Signals will arrive randomly and will be processed by + * whatever thread is running unless they are specifically blocked; and + * this should be a user thread, not a dapl thread + */ + +void +dapli_thread_init ( + struct thread_draft *thread_draft) +{ + sigset_t sigset; + void (*func) (void *); + void *data; + + sigfillset (&sigset); + pthread_sigmask ( SIG_BLOCK, &sigset, NULL); + + func = thread_draft->func; + data = thread_draft->data; + dapl_os_free (thread_draft, sizeof ( struct thread_draft )); + + (*func) (data); +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.h b/trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.h new file mode 100644 index 00000000..98ccd093 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/linux/dapl_osd.h @@ -0,0 +1,542 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_osd.h + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent data structures & functions with + * a canonical DAPL interface. Designed to be portable + * and hide OS specific quirks of common functions. + * + * $Id:$ + **********************************************************************/ + +#ifndef _DAPL_OSD_H_ +#define _DAPL_OSD_H_ + +/* + * This file is defined for Linux systems only, including it on any + * other build will cause an error + */ +#ifndef __linux__ +#error UNDEFINED OS TYPE +#endif /* __linux__ */ + +#if !defined (__i386__) && !defined (__ia64__) && !defined(__x86_64__) && !defined(__PPC64__) +#error UNDEFINED ARCH +#endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for printf */ +#include +#include +#include /* for getaddrinfo */ + +#include "dapl_debug.h" + +/* + * Include files for setting up a network name + */ +#include +#include +#include +#include + +#if !defined(REDHAT_EL5) && (defined(__ia64__) || defined(__PPC64__)) +#include +#endif +#if defined(__PPC64__) +#include +#endif + +/* Useful debug definitions */ +#ifndef STATIC +#define STATIC static +#endif /* STATIC */ +#ifndef _INLINE_ +#define _INLINE_ __inline__ +#endif /* _INLINE_ */ + +#define LINUX_VERSION(a,b) (((a) << 16) + (b)) + +void dapl_os_init ( void ); /* initialization function */ + +#define dapl_os_panic(...) \ + do { \ + fprintf(stderr, "PANIC in %s:%i:%s\n", __FILE__, __LINE__, __func__); \ + fprintf(stderr, __VA_ARGS__); \ + exit(1); \ + } while(0) + +#define dapl_ip_addr6(sockaddr) (((struct sockaddr_in6 *)sockaddr)->sin6_addr.s6_addr32) + +/* + * Atomic operations + */ + +typedef volatile DAT_COUNT DAPL_ATOMIC; + +/* atomic function prototypes */ +STATIC _INLINE_ void /* DAT_COUNT */ +dapl_os_atomic_inc ( + INOUT DAPL_ATOMIC *v); + +STATIC _INLINE_ void /* DAT_COUNT */ +dapl_os_atomic_dec ( + INOUT DAPL_ATOMIC *v); + +STATIC _INLINE_ DAT_COUNT +dapl_os_atomic_assign ( + INOUT DAPL_ATOMIC *v, + IN DAT_COUNT match_value, + IN DAT_COUNT new_value ); + +#define dapl_os_atomic_read(v) (*v) +#define dapl_os_atomic_set(v,i) (*v = i) + +int dapl_os_get_env_bool ( + char *env_str ); + +int dapl_os_get_env_val ( + char *env_str, + int def_val ); + + + +/* atomic functions */ + +/* dapl_os_atomic_inc + * + * get the current value of '*v', and then increment it. + * + * This is equivalent to an IB atomic fetch and add of 1, + * except that a DAT_COUNT might be 32 bits, rather than 64 + * and it occurs in local memory. + */ + +STATIC _INLINE_ void +dapl_os_atomic_inc ( + INOUT DAPL_ATOMIC *v) +{ +#ifdef __ia64__ + DAT_COUNT old_value; +#if defined(REDHAT_EL5) + old_value = __sync_fetch_and_add(v, 1); +#elif !defined(REDHAT_EL4) && (OS_RELEASE >= LINUX_VERSION(2,6)) + IA64_FETCHADD(old_value,v,1,4,rel); +#else + IA64_FETCHADD(old_value,v,1,4); +#endif +#elif defined(__PPC64__) + atomic_inc((atomic_t *) v); +#else /* !__ia64__ */ + __asm__ __volatile__ ( + "lock;" "incl %0" + :"=m" (*v) + :"m" (*v)); +#endif + return; +} + + +/* dapl_os_atomic_dec + * + * decrement the current value of '*v'. No return value is required. + */ + +STATIC _INLINE_ void +dapl_os_atomic_dec ( + INOUT DAPL_ATOMIC *v) +{ +#ifdef __ia64__ + DAT_COUNT old_value; +#if defined(REDHAT_EL5) + old_value = __sync_fetch_and_sub(v, 1); +#elif !defined(REDHAT_EL4) && (OS_RELEASE >= LINUX_VERSION(2,6)) + IA64_FETCHADD(old_value,v,-1,4,rel); +#else + IA64_FETCHADD(old_value,v,-1,4); +#endif +#elif defined (__PPC64__) + atomic_dec((atomic_t *)v); + +#else /* !__ia64__ */ + __asm__ __volatile__ ( + "lock;" "decl %0" + :"=m" (*v) + :"m" (*v)); +#endif + return; +} + + +/* dapl_os_atomic_assign + * + * assign 'new_value' to '*v' if the current value + * matches the provided 'match_value'. + * + * Make no assignment if there is no match. + * + * Return the current value in any case. + * + * This matches the IBTA atomic operation compare & swap + * except that it is for local memory and a DAT_COUNT may + * be only 32 bits, rather than 64. + */ + +STATIC _INLINE_ DAT_COUNT +dapl_os_atomic_assign ( + INOUT DAPL_ATOMIC *v, + IN DAT_COUNT match_value, + IN DAT_COUNT new_value ) +{ + DAT_COUNT current_value; + + /* + * Use the Pentium compare and exchange instruction + */ + +#ifdef __ia64__ +#if defined(REDHAT_EL5) + current_value = __sync_val_compare_and_swap(v,match_value,new_value); +#elif defined(REDHAT_EL4) + current_value = ia64_cmpxchg("acq",v,match_value,new_value,4); +#else + current_value = ia64_cmpxchg(acq,v,match_value,new_value,4); +#endif /* __ia64__ */ +#elif defined(__PPC64__) + __asm__ __volatile__ ( +" lwsync\n\ +1: lwarx %0,0,%2 # __cmpxchg_u32\n\ + cmpw 0,%0,%3\n\ + bne- 2f\n\ + stwcx. %4,0,%2\n\ + bne- 1b\n\ + isync\n\ +2:" + : "=&r" (current_value), "=m" (*v) + : "r" (v), "r" (match_value), "r" (new_value), "m" (*v) + : "cc", "memory"); +#else + __asm__ __volatile__ ( + "lock; cmpxchgl %1, %2" + : "=a" (current_value) + : "q" (new_value), "m" (*v), "0" (match_value) + : "memory"); +#endif + return current_value; +} + +/* + * Thread Functions + */ +typedef pthread_t DAPL_OS_THREAD; + +DAT_RETURN +dapl_os_thread_create ( + IN void (*func) (void *), + IN void *data, + OUT DAPL_OS_THREAD *thread_id ); + + +/* + * Lock Functions + */ + +typedef pthread_mutex_t DAPL_OS_LOCK; + +/* function prototypes */ +STATIC _INLINE_ DAT_RETURN +dapl_os_lock_init ( + IN DAPL_OS_LOCK *m); + +STATIC _INLINE_ DAT_RETURN +dapl_os_lock ( + IN DAPL_OS_LOCK *m); + +STATIC _INLINE_ DAT_RETURN +dapl_os_unlock ( + IN DAPL_OS_LOCK *m); + +STATIC _INLINE_ DAT_RETURN +dapl_os_lock_destroy ( + IN DAPL_OS_LOCK *m); + +/* lock functions */ +STATIC _INLINE_ DAT_RETURN +dapl_os_lock_init ( + IN DAPL_OS_LOCK *m) +{ + /* pthread_mutex_init always returns 0 */ + pthread_mutex_init (m, NULL); + + return DAT_SUCCESS; +} + +STATIC _INLINE_ DAT_RETURN +dapl_os_lock ( + IN DAPL_OS_LOCK *m) +{ + if (0 == pthread_mutex_lock (m)) + { + return DAT_SUCCESS; + } + else + { + return DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } +} + +STATIC _INLINE_ DAT_RETURN +dapl_os_unlock ( + IN DAPL_OS_LOCK *m) +{ + if (0 == pthread_mutex_unlock (m)) + { + return DAT_SUCCESS; + } + else + { + return DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } +} + +STATIC _INLINE_ DAT_RETURN +dapl_os_lock_destroy ( + IN DAPL_OS_LOCK *m) +{ + if (0 == pthread_mutex_destroy (m)) + { + return DAT_SUCCESS; + } + else + { + return DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } +} + + +/* + * Wait Objects + */ + +/* + * The wait object invariant: Presuming a call to dapl_os_wait_object_wait + * occurs at some point, there will be at least one wakeup after each call + * to dapl_os_wait_object_signal. I.e. Signals are not ignored, though + * they may be coallesced. + */ + +typedef struct +{ + DAT_BOOLEAN signaled; + pthread_cond_t cv; + pthread_mutex_t lock; +} DAPL_OS_WAIT_OBJECT; + +/* function prototypes */ +DAT_RETURN +dapl_os_wait_object_init ( + IN DAPL_OS_WAIT_OBJECT *wait_obj); + +DAT_RETURN +dapl_os_wait_object_wait ( + IN DAPL_OS_WAIT_OBJECT *wait_obj, + IN DAT_TIMEOUT timeout_val); + +DAT_RETURN +dapl_os_wait_object_wakeup ( + IN DAPL_OS_WAIT_OBJECT *wait_obj); + +DAT_RETURN +dapl_os_wait_object_destroy ( + IN DAPL_OS_WAIT_OBJECT *wait_obj); + +/* + * Memory Functions + */ + +/* function prototypes */ +STATIC _INLINE_ void *dapl_os_alloc (int size); + +STATIC _INLINE_ void *dapl_os_realloc (void *ptr, int size); + +STATIC _INLINE_ void dapl_os_free (void *ptr, int size); + +STATIC _INLINE_ void * dapl_os_memzero (void *loc, int size); + +STATIC _INLINE_ void * dapl_os_memcpy (void *dest, const void *src, int len); + +STATIC _INLINE_ int dapl_os_memcmp (const void *mem1, const void *mem2, int len); + +/* memory functions */ + + +STATIC _INLINE_ void *dapl_os_alloc (int size) +{ + return malloc (size); +} + +STATIC _INLINE_ void *dapl_os_realloc (void *ptr, int size) +{ + return realloc(ptr, size); +} + +STATIC _INLINE_ void dapl_os_free (void *ptr, int size) +{ + free (ptr); +} + +STATIC _INLINE_ void * dapl_os_memzero (void *loc, int size) +{ + return memset (loc, 0, size); +} + +STATIC _INLINE_ void * dapl_os_memcpy (void *dest, const void *src, int len) +{ + return memcpy (dest, src, len); +} + +STATIC _INLINE_ int dapl_os_memcmp (const void *mem1, const void *mem2, int len) +{ + return memcmp (mem1, mem2, len); +} + +/* + * Memory coherency functions + * For i386 Linux, there are no coherency issues so we just return success. + */ +STATIC _INLINE_ DAT_RETURN +dapl_os_sync_rdma_read ( + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + return DAT_SUCCESS; +} + +STATIC _INLINE_ DAT_RETURN +dapl_os_sync_rdma_write ( + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + return DAT_SUCCESS; +} + + +/* + * String Functions + */ + +STATIC _INLINE_ unsigned int dapl_os_strlen(const char *str) +{ + return strlen(str); +} + +STATIC _INLINE_ char * dapl_os_strdup(const char *str) +{ + return strdup(str); +} + + +/* + * Timer Functions + */ + +typedef DAT_UINT64 DAPL_OS_TIMEVAL; +typedef struct dapl_timer_entry DAPL_OS_TIMER; +typedef unsigned long long int DAPL_OS_TICKS; + +/* function prototypes */ +DAT_RETURN dapl_os_get_time (DAPL_OS_TIMEVAL *); + + +/* + * + * Name Service Helper functions + * + */ +#if defined(IBHOSTS_NAMING) || defined(CM_BUSTED) +#define dapls_osd_getaddrinfo(name, addr_ptr) getaddrinfo(name,NULL,NULL,addr_ptr) +#define dapls_osd_freeaddrinfo(addr) freeaddrinfo (addr) + +#endif /* IBHOSTS_NAMING */ + +/* + * *printf format helpers. We use the C string constant concatenation + * ability to define 64 bit formats, which unfortunatly are non standard + * in the C compiler world. E.g. %llx for gcc, %I64x for Windows + */ +#define F64d "%lld" +#define F64u "%llu" +#define F64x "%llx" +#define F64X "%llX" + + +/* + * Conversion Functions + */ + +STATIC _INLINE_ long int +dapl_os_strtol(const char *nptr, char **endptr, int base) +{ + return strtol(nptr, endptr, base); +} + + +/* + * Helper Functions + */ + + +#define dapl_os_assert(expression) assert(expression) +#define dapl_os_printf(...) printf(__VA_ARGS__) +#define dapl_os_vprintf(fmt,args) vprintf(fmt,args) +#define dapl_os_syslog(fmt,args) vsyslog (LOG_USER | LOG_DEBUG,fmt,args) + + + +#endif /* _DAPL_OSD_H_ */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/udapl/makefile b/trunk/ulp/dapl2/dapl/udapl/makefile new file mode 100644 index 00000000..a0c06273 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\inc\openib.def diff --git a/trunk/ulp/dapl2/dapl/udapl/udapl.rc b/trunk/ulp/dapl2/dapl/udapl/udapl.rc new file mode 100644 index 00000000..dfcde32e --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/udapl.rc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id$ + */ + + +#include + +#define VER_FILETYPE VFT_DLL +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#if DBG +#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0 (Debug)" +#define VER_INTERNALNAME_STR "dapl2d.dll" +#define VER_ORIGINALFILENAME_STR "dapl2d.dll" +#else +#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0" +#define VER_INTERNALNAME_STR "dapl2.dll" +#define VER_ORIGINALFILENAME_STR "dapl2.dll" +#endif + +#include diff --git a/trunk/ulp/dapl2/dapl/udapl/udapl_exports.src b/trunk/ulp/dapl2/dapl/udapl/udapl_exports.src new file mode 100644 index 00000000..ea167531 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/udapl_exports.src @@ -0,0 +1,14 @@ +#if DBG +LIBRARY dapl2d.dll +#else +LIBRARY dapl2.dll +#endif + + +EXPORTS +dat_provider_init +dat_provider_fini +#ifdef DAT_EXTENSIONS +dapl_extensions +#endif + diff --git a/trunk/ulp/dapl2/dapl/udapl/udapl_sources.c b/trunk/ulp/dapl2/dapl/udapl/udapl_sources.c new file mode 100644 index 00000000..345ab674 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/udapl_sources.c @@ -0,0 +1,100 @@ +/* + * Include all files that are not in children directories. + */ +#include "dapl_cno_util.c" +#include "dapl_cookie.c" +#include "dapl_cr_accept.c" +#include "dapl_cr_callback.c" +#include "dapl_cr_handoff.c" +#include "dapl_cr_query.c" +#include "dapl_cr_reject.c" +#include "dapl_cr_util.c" +#include "dapl_csp.c" +#include "dapl_debug.c" +#include "dapl_ep_connect.c" +#include "dapl_ep_create.c" +#include "dapl_ep_disconnect.c" +#include "dapl_ep_dup_connect.c" +#include "dapl_ep_free.c" +#include "dapl_ep_get_status.c" +#include "dapl_ep_modify.c" +#include "dapl_ep_post_rdma_read.c" +#include "dapl_ep_post_rdma_read_to_rmr.c" +#include "dapl_ep_post_rdma_write.c" +#include "dapl_ep_post_recv.c" +#include "dapl_ep_post_send.c" +#include "dapl_ep_post_send_invalidate.c" +#include "dapl_ep_query.c" +#include "dapl_ep_recv_query.c" +#include "dapl_ep_reset.c" +#include "dapl_ep_set_watermark.c" +#include "dapl_ep_util.c" +#include "dapl_evd_connection_callb.c" +#include "dapl_evd_cq_async_error_callb.c" +#include "dapl_evd_dequeue.c" +#include "dapl_evd_dto_callb.c" +#include "dapl_evd_free.c" +#include "dapl_evd_post_se.c" +#include "dapl_evd_qp_async_error_callb.c" +#include "dapl_evd_resize.c" +#include "dapl_evd_un_async_error_callb.c" +#include "dapl_evd_util.c" +#include "dapl_get_consumer_context.c" +#include "dapl_get_handle_type.c" +#include "dapl_hash.c" +#include "dapl_hca_util.c" +#include "dapl_ia_close.c" +#include "dapl_ia_ha.c" +#include "dapl_ia_open.c" +#include "dapl_ia_query.c" +#include "dapl_ia_util.c" +#include "dapl_llist.c" +#include "dapl_lmr_free.c" +#include "dapl_lmr_query.c" +#include "dapl_lmr_sync_rdma_read.c" +#include "dapl_lmr_sync_rdma_write.c" +#include "dapl_lmr_util.c" +#include "dapl_mr_util.c" +#include "dapl_name_service.c" +#include "dapl_provider.c" +#include "dapl_psp_create.c" +#include "dapl_psp_create_any.c" +#include "dapl_psp_free.c" +#include "dapl_psp_query.c" +#include "dapl_pz_create.c" +#include "dapl_pz_free.c" +#include "dapl_pz_query.c" +#include "dapl_pz_util.c" +#include "dapl_ring_buffer_util.c" +#include "dapl_rmr_bind.c" +#include "dapl_rmr_create.c" +#include "dapl_rmr_free.c" +#include "dapl_rmr_query.c" +#include "dapl_rmr_util.c" +#include "dapl_rsp_create.c" +#include "dapl_rsp_free.c" +#include "dapl_rsp_query.c" +#include "dapl_set_consumer_context.c" +#include "dapl_sp_util.c" +#include "dapl_ep_create_with_srq.c" +#include "dapl_srq_create.c" +#include "dapl_srq_free.c" +#include "dapl_srq_post_recv.c" +#include "dapl_srq_query.c" +#include "dapl_srq_resize.c" +#include "dapl_srq_set_lw.c" +#include "dapl_srq_util.c" +#include "dapl_timer_util.c" +#include "..\ibal\dapl_ibal_cm.c" +#include "..\ibal\dapl_ibal_qp.c" +#include "..\ibal\dapl_ibal_cq.c" +#include "..\ibal\dapl_ibal_util.c" + +//#include "..\ibal\dapl_ibal_mrdb.c" + +#ifdef DAT_EXTENSIONS +#include "..\ibal\dapl_ibal_extensions.c" +#endif + +#include "windows\dapl_osd.c" + diff --git a/trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.c b/trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.c new file mode 100644 index 00000000..0551ecb0 --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.c @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under either one of the following two licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * OR + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * Licensee has the right to choose either one of the above two licenses. + * + * Redistributions of source code must retain both the above copyright + * notice and either one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, either one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_osd.c + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent functions with a canonical DAPL + * interface. Designed to be portable and hide OS specific quirks + * of common functions. + * + * + * $Id: dapl_osd.c 33 2005-07-11 19:51:17Z ftillier $ + **********************************************************************/ + +/* + * MUST have the Microsoft Platform SDK installed for Windows to build + * and work properly + */ +#include "dapl.h" +#include "dapl_init.h" +#include "dapl_osd.h" +#include +#include /* needed for getenv() */ + + + +/* + * DllMain + * + * Primary Windows entry point + * + * Input: + * hDllHandle handle to DLL module + * fdwReason reason for calling function + * lpReserved reserved + * + * Returns: + * DAT_SUCCESS + */ + +BOOL WINAPI +DllMain ( + IN HINSTANCE hDllHandle, + IN DWORD fdwReason, + IN LPVOID lpReserved ) +{ + UNREFERENCED_PARAMETER(lpReserved); + + switch( fdwReason ) + { + case DLL_PROCESS_ATTACH: + /* + * We don't attach/detach threads that need any sort + * of initialization, so disable this ability to optimize + * the working set size of the DLL. Also allows us to + * remove two case statemens: + * DLL_THREAD_DETACH and DLL_THREAD_ATTACH + */ + if ( (DisableThreadLibraryCalls( hDllHandle )) != 0) + { +#if 0 + /* + * Relocated to [dapl_init.c] ..._PROVIDER_INIT() as when called + * from here calls to dapl_init/dapls_ib_init/ib_open_al() hang + * in the DLL call context. + */ + dapl_init (); +#endif + } + else + { + DWORD err = GetLastError(); + dapl_os_printf("DAPL Init Failed with code %u\n", err); + } + break; + + case DLL_PROCESS_DETACH: + /* + * Do library cleanup + */ +#if 0 + /* + * Relocated to [dapl_init.c] ..._PROVIDER_FINI() as the call to + * dapl_fini/dapls_ib_release/ib_close_al() hangs in this DLL call + * context. + */ + dapl_fini (); +#endif + break; + } + return TRUE; +} + + +#ifdef NOT_USED +/* + * dapl_osd_init + * + * Do Windows specific initialization: + * - nothing at this time + * + * Input: + * none + * + * Returns: + * none + */ +void +dapl_osd_init ( ) +{ + return; +} +#endif + +/* + * dapl_os_get_time + * + * Return 64 bit value of current time in microseconds. + * + * Input: + * loc User location to place current time + * + * Returns: + * DAT_SUCCESS + */ + +DAT_RETURN +dapl_os_get_time ( + OUT DAPL_OS_TIMEVAL * loc) +{ + struct _timeb tb; + + _ftime ( &tb ); + + *loc = ((DAT_UINT64) (tb.time * 1000000L) + (DAT_UINT64) tb.millitm * 1000); + + return DAT_SUCCESS; +} + + +/* + * dapl_os_get_bool_env + * + * Return boolean value of passed in environment variable: 1 if present, + * 0 if not + * + * Input: + * + * + * Returns: + * TRUE or FALSE + */ +int +dapl_os_get_env_bool ( + char *env_str ) +{ + char *env_var; + + env_var = getenv (env_str); + if (env_var != NULL) + { + return 1; + } + return 0; +} + + +/* + * dapl_os_get_val_env + * + * Update val to value of passed in environment variable if present + * + * Input: + * env_str + * def_val default value if environment variable does not exist + * + * Returns: + * TRUE or FALSE + */ +int +dapl_os_get_env_val ( + char *env_str, + int def_val ) +{ + char *env_var; + + env_var = getenv (env_str); + if ( env_var != NULL ) + { + def_val = strtol (env_var, NULL, 0); + } + + return def_val; +} + + +/* + * dapls_os_thread_create + * + * Create a thread for dapl + * + * Input: + * func function to invoke thread + * data argument to pass to function + * + * Output + * thread_id handle for thread + * + * Returns: + * DAT_SUCCESS + */ +DAT_RETURN +dapl_os_thread_create ( + IN void (*func) (void *), + IN void *data, + OUT DAPL_OS_THREAD *thread_id ) +{ + + *thread_id = CreateThread( + NULL, /* &thread security attrs */ + 8 * 1024, /* initial thread stack size */ + (LPTHREAD_START_ROUTINE)func, /* &thread function */ + data, /* argument for new thread */ + 0, /* creation flags */ + NULL); /* thread ID (ignore) */ + + if ( *thread_id == NULL ) + { + return DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, 0); + } + + return DAT_SUCCESS; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.h b/trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.h new file mode 100644 index 00000000..f290ddda --- /dev/null +++ b/trunk/ulp/dapl2/dapl/udapl/windows/dapl_osd.h @@ -0,0 +1,536 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under either one of the following two licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * OR + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * Licensee has the right to choose either one of the above two licenses. + * + * Redistributions of source code must retain both the above copyright + * notice and either one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, either one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dapl_osd.h + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent data structures & functions with + * a canonical DAPL interface. Designed to be portable + * and hide OS specific quirks of common functions. + * + * $Id: dapl_osd.h 33 2005-07-11 19:51:17Z ftillier $ + **********************************************************************/ + +#ifndef _DAPL_OSD_H_ +#define _DAPL_OSD_H_ + +/* + * This file is defined for Windows systems only, including it on any + * other build will cause an error + */ +#if !defined(_WIN32) && !defined(_WIN64) +#error UNDEFINED OS TYPE +#endif /* WIN32 */ + +#include +#include +#pragma warning ( push, 3 ) +#include +#include +#include +#include +#include +#include +#pragma warning ( pop ) + +#include "dapl_debug.h" + +/* Export Header */ +#ifdef EXPORT_DAPL_SYMBOLS /* 1 when building DAPL DLL, 0 for clients */ +#define DAPL_EXPORT __declspec( dllexport ) +#else +#define DAPL_EXPORT __declspec( dllimport ) +#endif + +/* Useful debug definitions */ +#ifndef STATIC +#define STATIC static +#endif /* STATIC */ + +#ifndef _INLINE_ +#define _INLINE_ __inline +#endif /* _INLINE_ */ + +#define dapl_os_panic(str) \ + { \ + fprintf(stderr, "PANIC in %s:%i:%s\n", __FILE__, __LINE__); \ + fprintf(stderr, str); \ + exit(1); \ + } + +/* + * Atomic operations + */ + +typedef volatile DAT_COUNT DAPL_ATOMIC; + +/* atomic function prototypes */ +STATIC __inline DAT_COUNT +dapl_os_atomic_inc ( + INOUT DAPL_ATOMIC *v); + +STATIC __inline DAT_COUNT +dapl_os_atomic_dec ( + INOUT DAPL_ATOMIC *v); + +STATIC __inline DAT_COUNT +dapl_os_atomic_assign ( + INOUT DAPL_ATOMIC *v, + IN DAT_COUNT match_value, + IN DAT_COUNT new_value ); + +#define dapl_os_atomic_read(v) (*v) +#define dapl_os_atomic_set(v,i) (*v = i) + +int dapl_os_get_env_bool ( + char *env_str ); + +int dapl_os_get_env_val ( + char *env_str, + int def_val ); + + +/* atomic functions */ + +/* dapl_os_atomic_inc + * + * get the current value of '*v', and then increment it. + * + * This is equivalent to an IB atomic fetch and add of 1, + * except that a DAT_COUNT might be 32 bits, rather than 64 + * and it occurs in local memory. + */ + +STATIC __inline DAT_COUNT +dapl_os_atomic_inc ( + INOUT DAPL_ATOMIC *v) +{ + return InterlockedIncrement( v ); +} + + +/* dapl_os_atomic_dec + * + * decrement the current value of '*v'. No return value is required. + */ + +STATIC __inline DAT_COUNT +dapl_os_atomic_dec ( + INOUT DAPL_ATOMIC *v) +{ + return InterlockedDecrement( v ); +} + + +/* dapl_os_atomic_assign + * + * assign 'new_value' to '*v' if the current value + * matches the provided 'match_value'. + * + * Make no assignment if there is no match. + * + * Return the current value in any case. + * + * This matches the IBTA atomic operation compare & swap + * except that it is for local memory and a DAT_COUNT may + * be only 32 bits, rather than 64. + */ + +STATIC __inline DAT_COUNT +dapl_os_atomic_assign ( + INOUT DAPL_ATOMIC *v, + IN DAT_COUNT match_value, + IN DAT_COUNT new_value ) +{ + return InterlockedCompareExchange((LPLONG)v, + new_value, + match_value); +} + + +/* + * Thread Functions + */ +typedef HANDLE DAPL_OS_THREAD; + +DAT_RETURN +dapl_os_thread_create ( + IN void (*func) (void *), + IN void *data, + OUT DAPL_OS_THREAD *thread_id ); + + +/* + * Lock Functions + */ +typedef HANDLE DAPL_OS_LOCK; + +/* function prototypes */ +STATIC __inline DAT_RETURN +dapl_os_lock_init ( + IN DAPL_OS_LOCK *m); + +STATIC __inline DAT_RETURN +dapl_os_lock ( + IN DAPL_OS_LOCK *m); + +STATIC __inline DAT_RETURN +dapl_os_unlock ( + IN DAPL_OS_LOCK *m); + +STATIC __inline DAT_RETURN +dapl_os_lock_destroy ( + IN DAPL_OS_LOCK *m); + +/* lock functions */ +STATIC __inline DAT_RETURN +dapl_os_lock_init ( + IN DAPL_OS_LOCK *m) +{ + *m = CreateMutex (0, FALSE, 0); + + return *m ? DAT_SUCCESS : (DAT_CLASS_ERROR | DAT_INSUFFICIENT_RESOURCES); +} + +STATIC __inline DAT_RETURN +dapl_os_lock ( + IN DAPL_OS_LOCK *m) +{ + WaitForSingleObject (*m, INFINITE); + + return DAT_SUCCESS; +} + +STATIC __inline DAT_RETURN +dapl_os_unlock ( + IN DAPL_OS_LOCK *m) +{ + ReleaseMutex (*m); + + return DAT_SUCCESS; +} + +STATIC __inline DAT_RETURN +dapl_os_lock_destroy ( + IN DAPL_OS_LOCK *m) +{ + CloseHandle (*m); + + return DAT_SUCCESS; +} + + +/* + * Wait Objects + */ + +/* + * The wait object invariant: Presuming a call to dapl_os_wait_object_wait + * occurs at some point, there will be at least one wakeup after each call + * to dapl_os_wait_object_signal. I.e. Signals are not ignored, though + * they may be coallesced. + */ + +/* wait_object functions */ + +typedef HANDLE DAPL_OS_WAIT_OBJECT; + +/* Initialize a wait object to an empty state + */ + +STATIC __inline DAT_RETURN +dapl_os_wait_object_init ( + IN DAPL_OS_WAIT_OBJECT *wait_obj) +{ + *wait_obj = CreateEvent(NULL,FALSE,FALSE,NULL); + + if ( *wait_obj == NULL ) + { + return DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return DAT_SUCCESS; +} + + +/* Wait on the supplied wait object, up to the specified time_out, + * and reacquiring it after the wait ends. + * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. + * Timeout should be specified in micro seconds. + * + * Functional returns: + * DAT_SUCCESS -- another thread invoked dapl_os_wait object_wakeup + * DAT_INVALID_STATE -- someone else is already waiting in this wait + * object. + * only one waiter is allowed at a time. + * DAT_ABORT -- another thread invoked dapl_os_wait_object_destroy + * DAT_TIMEOUT -- the specified time limit was reached. + */ + +STATIC __inline DAT_RETURN +dapl_os_wait_object_wait ( + IN DAPL_OS_WAIT_OBJECT *wait_obj, + IN DAT_TIMEOUT timeout_val) +{ + DAT_RETURN status; + DWORD op_status; + + status = DAT_SUCCESS; + + if ( DAT_TIMEOUT_INFINITE == timeout_val ) + { + op_status = WaitForSingleObject(*wait_obj, INFINITE); + } + else + { + /* convert to milliseconds */ + op_status = WaitForSingleObject(*wait_obj, timeout_val/1000); + } + + if (op_status == WAIT_TIMEOUT) + { + status = DAT_CLASS_ERROR | DAT_TIMEOUT_EXPIRED; + } + else if ( op_status == WAIT_FAILED) + { + status = DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return status; +} + +STATIC __inline DAT_RETURN +dapl_os_wait_object_wakeup ( + IN DAPL_OS_WAIT_OBJECT *wait_obj) +{ + DWORD op_status; + + op_status = SetEvent(*wait_obj); + if ( op_status == 0 ) + { + return DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return DAT_SUCCESS; +} + +STATIC __inline DAT_RETURN +dapl_os_wait_object_destroy ( + IN DAPL_OS_WAIT_OBJECT *wait_obj) +{ + DWORD op_status; + DAT_RETURN status = DAT_SUCCESS; + + op_status = CloseHandle(*wait_obj); + + if ( op_status == 0 ) + { + status = DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return status; +} + + +/* + * Memory Functions + */ + +/* function prototypes */ +STATIC __inline void *dapl_os_alloc (int size); + +STATIC __inline void *dapl_os_realloc (void *ptr, int size); + +STATIC __inline void dapl_os_free (void *ptr, int size); + +STATIC __inline void * dapl_os_memzero (void *loc, int size); + +STATIC __inline void * dapl_os_memcpy (void *dest, const void *src, int len); + +STATIC __inline int dapl_os_memcmp (const void *mem1, const void *mem2, int len); + +/* + * Memory coherency functions + * For i386/x86_64 Windows, there are no coherency issues - just return success. + */ +STATIC __inline DAT_RETURN +dapl_os_sync_rdma_read ( + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + return DAT_SUCCESS; +} + +STATIC __inline DAT_RETURN +dapl_os_sync_rdma_write ( + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + return DAT_SUCCESS; +} + + +/* memory functions */ + + +STATIC __inline void *dapl_os_alloc (int size) +{ + return malloc (size); +} + +STATIC __inline void *dapl_os_realloc (void *ptr, int size) +{ + return realloc(ptr, size); +} + +STATIC __inline void dapl_os_free (void *ptr, int size) +{ + size = size; + free (ptr); + ptr = NULL; +} + +STATIC __inline void * dapl_os_memzero (void *loc, int size) +{ + return memset (loc, 0, size); +} + +STATIC __inline void * dapl_os_memcpy (void *dest, const void *src, int len) +{ + return memcpy (dest, src, len); +} + +STATIC __inline int dapl_os_memcmp (const void *mem1, const void *mem2, int len) +{ + return memcmp (mem1, mem2, len); +} + + +STATIC __inline unsigned int dapl_os_strlen(const char *str) +{ + return ((unsigned int)strlen(str)); +} + +STATIC __inline char * dapl_os_strdup(const char *str) +{ + return _strdup(str); +} + + +/* + * Timer Functions + */ + +typedef DAT_UINT64 DAPL_OS_TIMEVAL; +typedef struct dapl_timer_entry DAPL_OS_TIMER; +typedef unsigned long DAPL_OS_TICKS; + +/* function prototypes */ + +/* + * Sleep for the number of micro seconds specified by the invoking + * function + */ +STATIC __inline void dapl_os_sleep_usec (int sleep_time) +{ + Sleep(sleep_time/1000); +} + +STATIC __inline DAPL_OS_TICKS dapl_os_get_ticks (void); + +STATIC __inline int dapl_os_ticks_to_seconds (DAPL_OS_TICKS ticks); + +DAT_RETURN dapl_os_get_time (DAPL_OS_TIMEVAL *); +/* timer functions */ + +STATIC __inline DAPL_OS_TICKS dapl_os_get_ticks (void) +{ + return GetTickCount (); +} + +STATIC __inline int dapl_os_ticks_to_seconds (DAPL_OS_TICKS ticks) +{ + ticks = ticks; + /* NOT YET IMPLEMENTED IN USER-SPACE */ + return 0; +} + + +/* + * + * Name Service Helper functions + * + */ +#ifdef IBHOSTS_NAMING +#define dapls_osd_getaddrinfo(name, addr_ptr) getaddrinfo(name,NULL,NULL,addr_ptr) +#define dapls_osd_freeaddrinfo(addr) freeaddrinfo (addr) + +#endif /* IBHOSTS_NAMING */ + +/* + * *printf format helpers. We use the C string constant concatenation + * ability to define 64 bit formats, which unfortunatly are non standard + * in the C compiler world. E.g. %llx for gcc, %I64x for Windows + */ +#define F64d "%I64d" +#define F64u "%I64u" +#define F64x "%I64x" +#define F64X "%I64X" + +/* + * Conversion Functions + */ + +STATIC __inline long int +dapl_os_strtol(const char *nptr, char **endptr, int base) +{ + return strtol(nptr, endptr, base); +} + + +/* + * Debug Helper Functions + */ + +#define dapl_os_assert(expression) CL_ASSERT(expression) + +#define dapl_os_printf printf +#define dapl_os_vprintf(fmt,args) vprintf(fmt,args) +#define dapl_os_syslog(fmt,args) /* XXX Need log routine call */ + + +#endif /* _DAPL_OSD_H_ */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dat/common/dat_api.c b/trunk/ulp/dapl2/dat/common/dat_api.c new file mode 100644 index 00000000..142715bd --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_api.c @@ -0,0 +1,1301 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/********************************************************************** + * + * MODULE: dat_api.c + * + * PURPOSE: DAT Provider and Consumer registry functions. + * Also provide small integers for IA_HANDLES + * + * $Id: dat_api.c 1326 2005-05-20 22:25:31Z jlentini $ + **********************************************************************/ + +#include "dat_osd.h" +#include "dat_init.h" +#include + +/* + * structure to deal with IA handles + */ +typedef struct +{ + DAT_OS_LOCK handle_lock; + unsigned long handle_max; + void **handle_array; +} DAT_HANDLE_VEC; + +/* + * Number of IA Handle entries allocated at a time. + */ +#define DAT_HANDLE_ENTRY_STEP 64 + +static DAT_HANDLE_VEC g_hv; + + +/*********************************************************************** + * Function: dats_handle_init + * + * Initialize a handle_vector. Happens when the module initializes + ***********************************************************************/ +DAT_RETURN +dats_handle_vector_init ( void ) +{ + DAT_RETURN dat_status; + unsigned long i; + + dat_status = DAT_SUCCESS; + + g_hv.handle_max = DAT_HANDLE_ENTRY_STEP; + + dat_status = dat_os_lock_init (&g_hv.handle_lock); + if ( DAT_SUCCESS != dat_status ) + { + return dat_status; + } + + g_hv.handle_array = dat_os_alloc (sizeof(void *) * DAT_HANDLE_ENTRY_STEP); + if (g_hv.handle_array == NULL) + { + dat_status = DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto bail; + } + + for (i = 0; i < g_hv.handle_max; i++) + { + g_hv.handle_array[i] = NULL; + } + + bail: + return dat_status; +} + + +/*********************************************************************** + * Function: dats_set_ia_handle + * + * Install an ia_handle into a handle vector and return a small + * integer. + ***********************************************************************/ +DAT_IA_HANDLE +dats_set_ia_handle ( + IN DAT_IA_HANDLE ia_handle ) +{ + unsigned long i; + void **h; + + dat_os_lock (&g_hv.handle_lock); + + /* + * Don't give out handle zero since that is DAT_HANDLE_NULL! + */ + for (i = 1; i < g_hv.handle_max; i++) + { + if (g_hv.handle_array[i] == NULL) + { + g_hv.handle_array[i] = ia_handle; + + dat_os_unlock (&g_hv.handle_lock); + + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "dat_set_handle %p to %d\n", ia_handle, i); + return UL_TO_DAT_IA_HANDLE(i); + } + } + + /* Didn't find an entry, grow the list & try again */ + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "Growing the handle array from %d to %d\n", + g_hv.handle_max, g_hv.handle_max + DAT_HANDLE_ENTRY_STEP); + + /* reallocate the handle array */ + h = dat_os_alloc (sizeof(void *) * (g_hv.handle_max + DAT_HANDLE_ENTRY_STEP)); + if (h == NULL) + { + dat_os_unlock (&g_hv.handle_lock); + return UL_TO_DAT_IA_HANDLE(-1); + } + /* copy old data to new area & free old memory*/ + memcpy((void *)h, (void *)g_hv.handle_array, sizeof(void *) * g_hv.handle_max); + dat_os_free (g_hv.handle_array, sizeof(void *) * g_hv.handle_max); + g_hv.handle_array = h; + + /* Initialize the new entries */ + for (i = g_hv.handle_max; i < g_hv.handle_max + DAT_HANDLE_ENTRY_STEP; i++) + { + g_hv.handle_array[i] = NULL; + } + i = g_hv.handle_max; + g_hv.handle_array[i] = ia_handle; + g_hv.handle_max += DAT_HANDLE_ENTRY_STEP; + dat_os_unlock (&g_hv.handle_lock); + + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "dat_set_handle x %p to %d\n", ia_handle, i); + + return UL_TO_DAT_IA_HANDLE(i); +} + +/*********************************************************************** + * Function: dats_get_ia_handle( + * + * Get a handle from a handle vector and return it in an OUT parameter + ***********************************************************************/ +DAT_RETURN +dats_get_ia_handle( + IN DAT_IA_HANDLE handle, + OUT DAT_IA_HANDLE *ia_handle_p ) +{ + DAT_RETURN dat_status; + + if (DAT_IA_HANDLE_TO_UL(handle) > g_hv.handle_max) + { + dat_status = DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + *ia_handle_p = g_hv.handle_array[DAT_IA_HANDLE_TO_UL(handle)]; + + if (*ia_handle_p == NULL) + { + dat_status = DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + dat_status = DAT_SUCCESS; + + bail: + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "dat_get_ia_handle from %d to %p\n", + handle, *ia_handle_p); + + return dat_status; +} + + +/*********************************************************************** + * Function: dats_is_ia_handle + * + * Unlike other handles, a DAT_IA_HANDLE is a small integer. Therefore, + * we must be able to distinguish between a DAT_IA_HANDLE and other + * types of handles (which are actually pointers). + * + * The current implementation assumes that any value for which an IA + * handle exists is a DAT_IA_HANDLE. Unfortunately this will result in + * false positives. In particular it may identify a NULL pointer as IA + * handle 0. An implmenetation that does not have this deficiency would + * be preferable. + * + ***********************************************************************/ +DAT_RETURN +dats_is_ia_handle ( + IN DAT_HANDLE dat_handle) +{ + unsigned long handle = DAT_IA_HANDLE_TO_UL((DAT_IA_HANDLE)dat_handle); + + if (g_hv.handle_max < handle ) + { + return DAT_FALSE; + } + else if ( NULL == g_hv.handle_array[handle] ) + { + return DAT_FALSE; + } + else + { + return DAT_TRUE; + } +} + + +/*********************************************************************** + * Function: dats_free_ia_handle + * + * Free a handle in a handle vector + ***********************************************************************/ +DAT_RETURN +dats_free_ia_handle ( + IN DAT_IA_HANDLE handle) +{ + DAT_RETURN dat_status; + + if (DAT_IA_HANDLE_TO_UL(handle) > g_hv.handle_max) + { + dat_status = DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + goto bail; + } + g_hv.handle_array[DAT_IA_HANDLE_TO_UL(handle)] = NULL; + dat_status = DAT_SUCCESS; + + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "dats_free_ia_handle %d\n", handle); + + bail: + return dat_status; +} + +/********************************************************************** + * API definitions for common API entry points + **********************************************************************/ +DAT_RETURN DAT_API dat_ia_query ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_EVD_HANDLE *async_evd_handle, + IN DAT_IA_ATTR_MASK ia_attr_mask, + OUT DAT_IA_ATTR *ia_attr, + IN DAT_PROVIDER_ATTR_MASK provider_attr_mask, + OUT DAT_PROVIDER_ATTR *provider_attr) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_IA_QUERY (dapl_ia_handle, + async_evd_handle, + ia_attr_mask, + ia_attr, + provider_attr_mask, + provider_attr); + } + + return dat_status; +} + +DAT_RETURN DAT_API dat_set_consumer_context ( + IN DAT_HANDLE dat_handle, + IN DAT_CONTEXT context) +{ + if ( dats_is_ia_handle (dat_handle) ) + { + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle((DAT_IA_HANDLE)dat_handle, + &dapl_ia_handle); + + /* failure to map the handle is unlikely but possible */ + /* in a mult-threaded environment */ + if ( DAT_SUCCESS == dat_status ) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE1); + } + + dat_handle = dapl_ia_handle; + } + + return DAT_SET_CONSUMER_CONTEXT (dat_handle, + context); +} + + +DAT_RETURN DAT_API dat_get_consumer_context ( + IN DAT_HANDLE dat_handle, + OUT DAT_CONTEXT *context) +{ + if ( dats_is_ia_handle (dat_handle) ) + { + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle((DAT_IA_HANDLE)dat_handle, + &dapl_ia_handle); + + /* failure to map the handle is unlikely but possible */ + /* in a mult-threaded environment */ + if ( DAT_SUCCESS == dat_status ) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE1); + } + + dat_handle = dapl_ia_handle; + } + + return DAT_GET_CONSUMER_CONTEXT (dat_handle, + context); +} + + +DAT_RETURN DAT_API dat_get_handle_type ( + IN DAT_HANDLE dat_handle, + OUT DAT_HANDLE_TYPE *type) +{ + if ( dats_is_ia_handle (dat_handle) ) + { + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle((DAT_IA_HANDLE)dat_handle, + &dapl_ia_handle); + + /* failure to map the handle is unlikely but possible */ + /* in a mult-threaded environment */ + if ( DAT_SUCCESS == dat_status ) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE1); + } + + dat_handle = dapl_ia_handle; + } + + return DAT_GET_HANDLE_TYPE (dat_handle, type); +} + + +DAT_RETURN DAT_API dat_cr_query ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_CR_PARAM_MASK cr_param_mask, + OUT DAT_CR_PARAM *cr_param) +{ + if (cr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CR); + } + return DAT_CR_QUERY (cr_handle, + cr_param_mask, + cr_param); +} + + +DAT_RETURN DAT_API dat_cr_accept ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data) +{ + if (cr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CR); + } + return DAT_CR_ACCEPT (cr_handle, + ep_handle, + private_data_size, + private_data); +} + + +DAT_RETURN DAT_API dat_cr_reject ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data) +{ + if (cr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CR); + } + return DAT_CR_REJECT (cr_handle, private_data_size, private_data); +} + + +DAT_RETURN DAT_API dat_evd_resize ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_COUNT evd_min_qlen) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_RESIZE (evd_handle, + evd_min_qlen); +} + + +DAT_RETURN DAT_API dat_evd_post_se ( + IN DAT_EVD_HANDLE evd_handle, + IN const DAT_EVENT *event) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_POST_SE (evd_handle, + event); +} + + +DAT_RETURN DAT_API dat_evd_dequeue ( + IN DAT_EVD_HANDLE evd_handle, + OUT DAT_EVENT *event) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_DEQUEUE (evd_handle, + event); +} + + +DAT_RETURN DAT_API dat_evd_free ( + IN DAT_EVD_HANDLE evd_handle) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_FREE (evd_handle); +} + +DAT_RETURN DAT_API dat_evd_query ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_EVD_PARAM_MASK evd_param_mask, + OUT DAT_EVD_PARAM *evd_param) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_QUERY (evd_handle, + evd_param_mask, + evd_param); +} + + +DAT_RETURN DAT_API dat_ep_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_EVD_HANDLE recv_completion_evd_handle, + IN DAT_EVD_HANDLE request_completion_evd_handle, + IN DAT_EVD_HANDLE connect_evd_handle, + IN const DAT_EP_ATTR *ep_attributes, + OUT DAT_EP_HANDLE *ep_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_EP_CREATE (dapl_ia_handle, + pz_handle, + recv_completion_evd_handle, + request_completion_evd_handle, + connect_evd_handle, + ep_attributes, + ep_handle); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_ep_query ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_PARAM_MASK ep_param_mask, + OUT DAT_EP_PARAM *ep_param) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_QUERY (ep_handle, + ep_param_mask, + ep_param); +} + + +DAT_RETURN DAT_API dat_ep_modify ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_PARAM_MASK ep_param_mask, + IN const DAT_EP_PARAM *ep_param) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_MODIFY (ep_handle, + ep_param_mask, + ep_param); +} + +DAT_RETURN DAT_API dat_ep_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + IN DAT_CONN_QUAL remote_conn_qual, + IN DAT_TIMEOUT timeout, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data, + IN DAT_QOS quality_of_service, + IN DAT_CONNECT_FLAGS connect_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_CONNECT (ep_handle, + remote_ia_address, + remote_conn_qual, + timeout, + private_data_size, + private_data, + quality_of_service, + connect_flags); +} + +DAT_RETURN DAT_API dat_ep_common_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + IN DAT_TIMEOUT timeout, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_COMMON_CONNECT (ep_handle, + remote_ia_address, + timeout, + private_data_size, + private_data); +} + +DAT_RETURN DAT_API dat_ep_dup_connect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_EP_HANDLE ep_dup_handle, + IN DAT_TIMEOUT timeout, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data, + IN DAT_QOS quality_of_service) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_DUP_CONNECT (ep_handle, + ep_dup_handle, + timeout, + private_data_size, + private_data, + quality_of_service); +} + + +DAT_RETURN DAT_API dat_ep_disconnect ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_CLOSE_FLAGS close_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_DISCONNECT (ep_handle, + close_flags); +} + +DAT_RETURN DAT_API dat_ep_post_send ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_POST_SEND (ep_handle, + num_segments, + local_iov, + user_cookie, + completion_flags); +} + +DAT_RETURN DAT_API dat_ep_post_send_with_invalidate ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags, + IN DAT_BOOLEAN invalidate_flag, + IN DAT_RMR_CONTEXT rmr_context) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_POST_SEND_WITH_INVALIDATE (ep_handle, + num_segments, + local_iov, + user_cookie, + completion_flags, + invalidate_flag, + rmr_context); +} + +DAT_RETURN DAT_API dat_ep_post_recv ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_POST_RECV (ep_handle, + num_segments, + local_iov, + user_cookie, + completion_flags); +} + + +DAT_RETURN DAT_API dat_ep_post_rdma_read ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_POST_RDMA_READ (ep_handle, + num_segments, + local_iov, + user_cookie, + remote_iov, + completion_flags); +} + + +DAT_RETURN DAT_API dat_ep_post_rdma_read_to_rmr ( + IN DAT_EP_HANDLE ep_handle, + IN const DAT_RMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_POST_RDMA_READ_TO_RMR (ep_handle, + local_iov, + user_cookie, + remote_iov, + completion_flags); +} + + +DAT_RETURN DAT_API dat_ep_post_rdma_write ( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET *local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_POST_RDMA_WRITE (ep_handle, + num_segments, + local_iov, + user_cookie, + remote_iov, + completion_flags); +} + + +DAT_RETURN DAT_API dat_ep_get_status ( + IN DAT_EP_HANDLE ep_handle, + OUT DAT_EP_STATE *ep_state, + OUT DAT_BOOLEAN *recv_idle, + OUT DAT_BOOLEAN *request_idle) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_GET_STATUS (ep_handle, + ep_state, + recv_idle, + request_idle); +} + + +DAT_RETURN DAT_API dat_ep_free ( + IN DAT_EP_HANDLE ep_handle) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_FREE (ep_handle); +} + + +DAT_RETURN DAT_API dat_ep_reset ( + IN DAT_EP_HANDLE ep_handle) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_RESET (ep_handle); +} + + +DAT_RETURN DAT_API dat_lmr_free ( + IN DAT_LMR_HANDLE lmr_handle) +{ + if (lmr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_LMR); + } + return DAT_LMR_FREE (lmr_handle); +} + + +DAT_RETURN DAT_API dat_rmr_create ( + IN DAT_PZ_HANDLE pz_handle, + OUT DAT_RMR_HANDLE *rmr_handle) +{ + if (pz_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + } + return DAT_RMR_CREATE (pz_handle, + rmr_handle); +} + + +DAT_RETURN DAT_API dat_rmr_create_for_ep ( + IN DAT_PZ_HANDLE pz_handle, + OUT DAT_RMR_HANDLE *rmr_handle) +{ + if (pz_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + } + return DAT_RMR_CREATE_FOR_EP (pz_handle, + rmr_handle); +} +DAT_RETURN DAT_API dat_rmr_query ( + IN DAT_RMR_HANDLE rmr_handle, + IN DAT_RMR_PARAM_MASK rmr_param_mask, + OUT DAT_RMR_PARAM *rmr_param) +{ + if (rmr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RMR); + } + return DAT_RMR_QUERY (rmr_handle, + rmr_param_mask, + rmr_param); +} + + +DAT_RETURN DAT_API dat_rmr_bind ( + IN DAT_RMR_HANDLE rmr_handle, + IN DAT_LMR_HANDLE lmr_handle, + IN const DAT_LMR_TRIPLET *lmr_triplet, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN DAT_VA_TYPE va_type, + IN DAT_EP_HANDLE ep_handle, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags, + OUT DAT_RMR_CONTEXT *context) +{ + if (rmr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RMR); + } + return DAT_RMR_BIND (rmr_handle, + lmr_handle, + lmr_triplet, + mem_priv, + va_type, + ep_handle, + user_cookie, + completion_flags, + context); +} + + +DAT_RETURN DAT_API dat_rmr_free ( + IN DAT_RMR_HANDLE rmr_handle) +{ + if (rmr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RMR); + } + return DAT_RMR_FREE (rmr_handle); +} + +DAT_RETURN DAT_API dat_lmr_sync_rdma_read( + IN DAT_IA_HANDLE ia_handle, + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_LMR_SYNC_RDMA_READ (dapl_ia_handle, + local_segments, + num_segments); + + } + + return dat_status; +} + +DAT_RETURN DAT_API dat_lmr_sync_rdma_write( + IN DAT_IA_HANDLE ia_handle, + IN const DAT_LMR_TRIPLET *local_segments, + IN DAT_VLEN num_segments) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_LMR_SYNC_RDMA_WRITE (dapl_ia_handle, + local_segments, + num_segments); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_psp_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_CONN_QUAL conn_qual, + IN DAT_EVD_HANDLE evd_handle, + IN DAT_PSP_FLAGS psp_flags, + OUT DAT_PSP_HANDLE *psp_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_PSP_CREATE (dapl_ia_handle, + conn_qual, + evd_handle, + psp_flags, + psp_handle); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_psp_create_any ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_CONN_QUAL *conn_qual, + IN DAT_EVD_HANDLE evd_handle, + IN DAT_PSP_FLAGS psp_flags, + OUT DAT_PSP_HANDLE *psp_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_PSP_CREATE_ANY (dapl_ia_handle, + conn_qual, + evd_handle, + psp_flags, + psp_handle); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_psp_query ( + IN DAT_PSP_HANDLE psp_handle, + IN DAT_PSP_PARAM_MASK psp_param_mask, + OUT DAT_PSP_PARAM *psp_param) +{ + if (psp_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PSP); + } + return DAT_PSP_QUERY (psp_handle, + psp_param_mask, + psp_param); +} + + +DAT_RETURN DAT_API dat_psp_free ( + IN DAT_PSP_HANDLE psp_handle) +{ + if (psp_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PSP); + } + return DAT_PSP_FREE (psp_handle); +} + +DAT_RETURN DAT_API dat_csp_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_COMM *comm, + IN DAT_IA_ADDRESS_PTR address, + IN DAT_EVD_HANDLE evd_handle, + OUT DAT_CSP_HANDLE *csp_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_CSP_CREATE (dapl_ia_handle, + comm, + address, + evd_handle, + csp_handle); + } + return dat_status; +} + +DAT_RETURN DAT_API dat_csp_query ( + IN DAT_CSP_HANDLE csp_handle, + IN DAT_CSP_PARAM_MASK csp_param_mask, + OUT DAT_CSP_PARAM *csp_param) +{ + if (csp_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CSP); + } + return DAT_CSP_QUERY (csp_handle, + csp_param_mask, + csp_param); +} + +DAT_RETURN DAT_API dat_csp_free ( + IN DAT_CSP_HANDLE csp_handle) +{ + if (csp_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CSP); + } + return DAT_CSP_FREE (csp_handle); +} + + +DAT_RETURN DAT_API dat_rsp_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_CONN_QUAL conn_qual, + IN DAT_EP_HANDLE ep_handle, + IN DAT_EVD_HANDLE evd_handle, + OUT DAT_RSP_HANDLE *rsp_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_RSP_CREATE (dapl_ia_handle, + conn_qual, + ep_handle, + evd_handle, + rsp_handle); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_rsp_query ( + IN DAT_RSP_HANDLE rsp_handle, + IN DAT_RSP_PARAM_MASK rsp_param_mask, + OUT DAT_RSP_PARAM *rsp_param) +{ + if (rsp_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RSP); + } + return DAT_RSP_QUERY (rsp_handle, + rsp_param_mask, + rsp_param); +} + + +DAT_RETURN DAT_API dat_rsp_free ( + IN DAT_RSP_HANDLE rsp_handle) +{ + if (rsp_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RSP); + } + return DAT_RSP_FREE (rsp_handle); +} + + +DAT_RETURN DAT_API dat_pz_create ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_PZ_HANDLE *pz_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_PZ_CREATE (dapl_ia_handle, + pz_handle); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_pz_query ( + IN DAT_PZ_HANDLE pz_handle, + IN DAT_PZ_PARAM_MASK pz_param_mask, + OUT DAT_PZ_PARAM *pz_param) +{ + if (pz_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + } + return DAT_PZ_QUERY (pz_handle, + pz_param_mask, + pz_param); +} + + +DAT_RETURN DAT_API dat_pz_free ( + IN DAT_PZ_HANDLE pz_handle) +{ + if (pz_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ); + } + return DAT_PZ_FREE (pz_handle); +} + +DAT_RETURN DAT_API dat_ep_create_with_srq( + IN DAT_IA_HANDLE ia_handle, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_EVD_HANDLE recv_evd_handle, + IN DAT_EVD_HANDLE request_evd_handle, + IN DAT_EVD_HANDLE connect_evd_handle, + IN DAT_SRQ_HANDLE srq_handle, + IN const DAT_EP_ATTR *ep_attributes, + OUT DAT_EP_HANDLE *ep_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_EP_CREATE_WITH_SRQ (dapl_ia_handle, + pz_handle, + recv_evd_handle, + request_evd_handle, + connect_evd_handle, + srq_handle, + ep_attributes, + ep_handle); + } + + return dat_status; +} + +DAT_RETURN DAT_API dat_ep_recv_query( + IN DAT_EP_HANDLE ep_handle, + OUT DAT_COUNT * nbufs_allocated, + OUT DAT_COUNT * bufs_alloc_span) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_RECV_QUERY (ep_handle, + nbufs_allocated, + bufs_alloc_span); +} + +DAT_RETURN DAT_API dat_ep_set_watermark( + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT soft_high_watermark, + IN DAT_COUNT hard_high_watermark) +{ + if (ep_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); + } + return DAT_EP_SET_WATERMARK (ep_handle, + soft_high_watermark, + hard_high_watermark); +} + +/* SRQ functions */ + +DAT_RETURN DAT_API dat_srq_create( + IN DAT_IA_HANDLE ia_handle, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_SRQ_ATTR *srq_attr, + OUT DAT_SRQ_HANDLE *srq_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_SRQ_CREATE(dapl_ia_handle, + pz_handle, + srq_attr, + srq_handle); + } + + return dat_status; +} + +DAT_RETURN DAT_API dat_srq_free( + IN DAT_SRQ_HANDLE srq_handle) +{ + return DAT_SRQ_FREE (srq_handle); +} + +DAT_RETURN DAT_API dat_srq_post_recv( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_COUNT num_segments, + IN DAT_LMR_TRIPLET * local_iov, + IN DAT_DTO_COOKIE user_cookie) +{ + if (srq_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + } + return DAT_SRQ_POST_RECV (srq_handle, + num_segments, + local_iov, + user_cookie); +} + +DAT_RETURN DAT_API dat_srq_query( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_SRQ_PARAM_MASK srq_param_mask, + OUT DAT_SRQ_PARAM * srq_param) +{ + if (srq_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + } + return DAT_SRQ_QUERY (srq_handle, + srq_param_mask, + srq_param); +} + +DAT_RETURN DAT_API dat_srq_resize( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_COUNT srq_max_recv_dto) +{ + if (srq_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + } + return DAT_SRQ_RESIZE (srq_handle, + srq_max_recv_dto); +} + +DAT_RETURN DAT_API dat_srq_set_lw( + IN DAT_SRQ_HANDLE srq_handle, + IN DAT_COUNT low_watermark) +{ + if (srq_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ); + } + return DAT_SRQ_SET_LW (srq_handle, + low_watermark); +} + +#ifdef DAT_EXTENSIONS + +extern int g_dat_extensions; + +DAT_RETURN DAT_API dat_extension_op( + IN DAT_HANDLE handle, + IN DAT_EXTENDED_OP ext_op, + IN ... ) + +{ + DAT_RETURN status; + DAT_IA_HANDLE dapl_handle; + va_list args; + + /* If not IA handle then just passthrough */ + if (dats_get_ia_handle(handle, &dapl_handle) != DAT_SUCCESS) + { + dapl_handle = handle; + } + + /* verify provider extension support */ + if (!g_dat_extensions) + { + return DAT_ERROR(DAT_NOT_IMPLEMENTED, 0); + } + + /* extension will validate the handle based on op */ + va_start(args, ext_op); + status = DAT_HANDLE_EXTENDEDOP(dapl_handle, ext_op, args); + va_end(args); + + return status; +} +#endif + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dat/common/dat_dictionary.c b/trunk/ulp/dapl2/dat/common/dat_dictionary.c new file mode 100644 index 00000000..933cb40f --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_dictionary.c @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_dictionary.c + * + * PURPOSE: dictionary data structure + * + * $Id: dat_dictionary.c,v 1.17 2005/05/20 22:24:01 jlentini Exp $ + **********************************************************************/ + + +#include "dat_dictionary.h" + + +/********************************************************************* + * * + * Structures * + * * + *********************************************************************/ + +typedef struct DAT_DICTIONARY_NODE +{ + DAT_PROVIDER_INFO key; + DAT_DICTIONARY_DATA data; + struct DAT_DICTIONARY_NODE *prev; + struct DAT_DICTIONARY_NODE *next; +} DAT_DICTIONARY_NODE; + + +struct DAT_DICTIONARY +{ + DAT_DICTIONARY_NODE *head; + DAT_DICTIONARY_NODE *tail; + DAT_COUNT size; +}; + +/********************************************************************* + * * + * Function Declarations * + * * + *********************************************************************/ + +static DAT_RETURN +dat_dictionary_key_dup ( + const DAT_PROVIDER_INFO *old_key, + DAT_PROVIDER_INFO *new_key ); + +static DAT_BOOLEAN +dat_dictionary_key_is_equal ( + const DAT_PROVIDER_INFO *key_a, + const DAT_PROVIDER_INFO *key_b ); + + +/********************************************************************* + * * + * External Functions * + * * + *********************************************************************/ + + +/*********************************************************************** + * Function: dat_dictionary_create + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_create ( + OUT DAT_DICTIONARY **pp_dictionary) +{ + DAT_DICTIONARY *p_dictionary; + DAT_RETURN status; + + dat_os_assert ( NULL != pp_dictionary); + + status = DAT_SUCCESS; + + /* create the dictionary */ + p_dictionary = dat_os_alloc (sizeof (DAT_DICTIONARY)); + if (NULL == p_dictionary) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_memset (p_dictionary, '\0', sizeof (DAT_DICTIONARY)); + + /* create the head node */ + p_dictionary->head = dat_os_alloc (sizeof (DAT_DICTIONARY_NODE)); + if (NULL == p_dictionary->head) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_memset (p_dictionary->head, '\0', sizeof (DAT_DICTIONARY_NODE)); + + /* create the tail node */ + p_dictionary->tail = dat_os_alloc (sizeof (DAT_DICTIONARY_NODE)); + if (NULL == p_dictionary->tail) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_memset (p_dictionary->tail, '\0', sizeof (DAT_DICTIONARY_NODE)); + + p_dictionary->head->next = p_dictionary->tail; + p_dictionary->tail->prev = p_dictionary->head; + + *pp_dictionary = p_dictionary; + +bail: + if ( DAT_SUCCESS != status ) + { + if ( NULL != p_dictionary ) + { + if ( NULL != p_dictionary->head ) + { + dat_os_free (p_dictionary->head, sizeof (DAT_DICTIONARY_NODE)); + } + + if ( NULL != p_dictionary->tail ) + { + dat_os_free (p_dictionary->tail, sizeof (DAT_DICTIONARY_NODE)); + } + + dat_os_free (p_dictionary, sizeof (DAT_DICTIONARY)); + } + } + + return status; +} + + +/*********************************************************************** + * Function: dat_dictionary_destroy + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_destroy ( + IN DAT_DICTIONARY *p_dictionary) +{ + DAT_DICTIONARY_NODE *cur_node; + + dat_os_assert (NULL != p_dictionary); + + while (NULL != p_dictionary->head) + { + cur_node = p_dictionary->head; + p_dictionary->head = cur_node->next; + + dat_os_free (cur_node, sizeof (DAT_DICTIONARY_NODE)); + } + + dat_os_free (p_dictionary, sizeof (DAT_DICTIONARY)); + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_dictionary_size + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_size ( + IN DAT_DICTIONARY *p_dictionary, + OUT DAT_COUNT *p_size) +{ + dat_os_assert (NULL != p_dictionary); + dat_os_assert (NULL != p_size); + + *p_size = p_dictionary->size; + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_dictionary_entry_create + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_entry_create ( + OUT DAT_DICTIONARY_ENTRY *p_entry) +{ + DAT_DICTIONARY_NODE *node; + DAT_RETURN dat_status; + + dat_os_assert (NULL != p_entry); + + dat_status = DAT_SUCCESS; + + node = dat_os_alloc (sizeof (DAT_DICTIONARY_NODE)); + if (NULL == node) + { + dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + *p_entry = node; + +bail: + return dat_status; +} + + +/*********************************************************************** + * Function: dat_dictionary_entry_destroy + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_entry_destroy ( + OUT DAT_DICTIONARY_ENTRY entry) +{ + dat_os_free (entry, sizeof (DAT_DICTIONARY_NODE)); + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_dictionary_insert + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_insert ( + IN DAT_DICTIONARY *p_dictionary, + IN DAT_DICTIONARY_ENTRY entry, + IN const DAT_PROVIDER_INFO *key, + IN DAT_DICTIONARY_DATA data) +{ + DAT_RETURN dat_status; + DAT_DICTIONARY_NODE *cur_node, *prev_node, *next_node; + + dat_os_assert (NULL != p_dictionary); + dat_os_assert (NULL != entry); + + cur_node = entry; + + if ( DAT_SUCCESS == dat_dictionary_search (p_dictionary, key, NULL) ) + { + dat_status = DAT_ERROR (DAT_PROVIDER_ALREADY_REGISTERED, 0); + goto bail; + } + + dat_status = dat_dictionary_key_dup ( key, &cur_node->key ); + if ( DAT_SUCCESS != dat_status ) + { + goto bail; + } + + /* insert node at end of list to preserve registration order*/ + prev_node = p_dictionary->tail->prev; + next_node = p_dictionary->tail; + + cur_node->data = data; + cur_node->next = next_node; + cur_node->prev = prev_node; + + prev_node->next = cur_node; + next_node->prev = cur_node; + + p_dictionary->size++; + +bail: + return dat_status; +} + + +/*********************************************************************** + * Function: dat_dictionary_search + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_search ( + IN DAT_DICTIONARY *p_dictionary, + IN const DAT_PROVIDER_INFO *key, + OUT DAT_DICTIONARY_DATA *p_data) +{ + DAT_DICTIONARY_NODE *cur_node; + DAT_RETURN status; + + dat_os_assert (NULL != p_dictionary); + + status = DAT_ERROR (DAT_PROVIDER_NOT_FOUND, DAT_NAME_NOT_REGISTERED); + + for (cur_node = p_dictionary->head->next; + p_dictionary->tail != cur_node; + cur_node = cur_node->next) + { + if ( DAT_TRUE == dat_dictionary_key_is_equal (&cur_node->key, key) ) + { + if ( NULL != p_data ) + { + *p_data = cur_node->data; + } + + status = DAT_SUCCESS; + goto bail; + } + } + +bail: + return status; +} + + +/*********************************************************************** + * Function: dat_dictionary_enumerate + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_enumerate ( + IN DAT_DICTIONARY *p_dictionary, + IN DAT_DICTIONARY_DATA array[], + IN DAT_COUNT array_size) +{ + DAT_DICTIONARY_NODE *cur_node; + DAT_COUNT i; + DAT_RETURN status; + + dat_os_assert (NULL != p_dictionary); + dat_os_assert (NULL != array); + + status = DAT_SUCCESS; + + if ( array_size < p_dictionary->size ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, 0); + goto bail; + } + + for (cur_node = p_dictionary->head->next, i = 0; + p_dictionary->tail != cur_node; + cur_node = cur_node->next, i++) + { + array[i] = cur_node->data; + } + +bail: + return status; +} + + +/*********************************************************************** + * Function: dat_dictionary_remove + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_remove ( + IN DAT_DICTIONARY *p_dictionary, + IN DAT_DICTIONARY_ENTRY *p_entry, + IN const DAT_PROVIDER_INFO *key, + OUT DAT_DICTIONARY_DATA *p_data) +{ + DAT_DICTIONARY_NODE *cur_node, *prev_node, *next_node; + DAT_RETURN status; + + dat_os_assert (NULL != p_dictionary); + dat_os_assert (NULL != p_entry); + + status = DAT_ERROR (DAT_PROVIDER_NOT_FOUND, DAT_NAME_NOT_REGISTERED); + + for (cur_node = p_dictionary->head->next; + p_dictionary->tail != cur_node; + cur_node = cur_node->next) + { + if ( DAT_TRUE == dat_dictionary_key_is_equal (&cur_node->key, key) ) + { + if ( NULL != p_data ) + { + *p_data = cur_node->data; + } + + prev_node = cur_node->prev; + next_node = cur_node->next; + + prev_node->next = next_node; + next_node->prev = prev_node; + + *p_entry = cur_node; + + p_dictionary->size--; + + status = DAT_SUCCESS; + goto bail; + } + } + +bail: + return status; +} + + +/********************************************************************* + * * + * Internal Function Definitions * + * * + *********************************************************************/ + + +/*********************************************************************** + * Function: dat_dictionary_key_create + ***********************************************************************/ + +DAT_RETURN +dat_dictionary_key_dup ( + const DAT_PROVIDER_INFO *old_key, + DAT_PROVIDER_INFO *new_key ) +{ + dat_os_assert (NULL != old_key); + dat_os_assert (NULL != new_key); + + dat_os_strncpy (new_key->ia_name, old_key->ia_name, DAT_NAME_MAX_LENGTH); + new_key->dapl_version_major = old_key->dapl_version_major; + new_key->dapl_version_minor = old_key->dapl_version_minor; + new_key->is_thread_safe = old_key->is_thread_safe; + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_dictionary_key_is_equal + ***********************************************************************/ + +DAT_BOOLEAN +dat_dictionary_key_is_equal ( + const DAT_PROVIDER_INFO *key_a, + const DAT_PROVIDER_INFO *key_b ) +{ + if ( ( dat_os_strlen (key_a->ia_name) == dat_os_strlen (key_b->ia_name) ) && + ( !dat_os_strncmp (key_a->ia_name, key_b->ia_name, dat_os_strlen (key_a->ia_name)) ) && + ( key_a->dapl_version_major == key_b->dapl_version_major ) && + ( key_a->dapl_version_minor == key_b->dapl_version_minor ) && + ( key_a->is_thread_safe == key_b->is_thread_safe ) ) + { + return DAT_TRUE; + } + else + { + return DAT_FALSE; + } +} diff --git a/trunk/ulp/dapl2/dat/common/dat_dictionary.h b/trunk/ulp/dapl2/dat/common/dat_dictionary.h new file mode 100644 index 00000000..6f9a49f2 --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_dictionary.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_dictionary.h + * + * PURPOSE: dictionary data structure + * + * $Id: dat_dictionary.h,v 1.10 2005/03/24 05:58:27 jlentini Exp $ + **********************************************************************/ + +#ifndef _DAT_DICTIONARY_H_ +#define _DAT_DICTIONARY_H_ + + +#include "dat_osd.h" + + +/********************************************************************* + * * + * Typedefs * + * * + *********************************************************************/ + +typedef struct DAT_DICTIONARY DAT_DICTIONARY; +typedef void * DAT_DICTIONARY_DATA; +typedef void * DAT_DICTIONARY_ENTRY; + + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +extern DAT_RETURN +dat_dictionary_create ( + OUT DAT_DICTIONARY **pp_dictionary); + +extern DAT_RETURN +dat_dictionary_destroy ( + IN DAT_DICTIONARY *p_dictionary); + +extern DAT_RETURN +dat_dictionary_size ( + IN DAT_DICTIONARY *p_dictionary, + OUT DAT_COUNT *p_size); + +extern DAT_RETURN +dat_dictionary_entry_create ( + OUT DAT_DICTIONARY_ENTRY *p_entry); + +extern DAT_RETURN +dat_dictionary_entry_destroy ( + IN DAT_DICTIONARY_ENTRY entry); + +extern DAT_RETURN +dat_dictionary_insert ( + IN DAT_DICTIONARY *p_dictionary, + IN DAT_DICTIONARY_ENTRY entry, + IN const DAT_PROVIDER_INFO *key, + IN DAT_DICTIONARY_DATA data); + +extern DAT_RETURN +dat_dictionary_search ( + IN DAT_DICTIONARY *p_dictionary, + IN const DAT_PROVIDER_INFO *key, + OUT DAT_DICTIONARY_DATA *p_data); + +extern DAT_RETURN +dat_dictionary_enumerate ( + IN DAT_DICTIONARY *p_dictionary, + IN DAT_DICTIONARY_DATA array[], + IN DAT_COUNT array_size); + + +extern DAT_RETURN +dat_dictionary_remove ( + IN DAT_DICTIONARY *p_dictionary, + IN DAT_DICTIONARY_ENTRY *p_entry, + IN const DAT_PROVIDER_INFO *key, + OUT DAT_DICTIONARY_DATA *p_data); + +#endif diff --git a/trunk/ulp/dapl2/dat/common/dat_dr.c b/trunk/ulp/dapl2/dat/common/dat_dr.c new file mode 100644 index 00000000..4a86f19b --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_dr.c @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_dr.c + * + * PURPOSE: dynamic registry implementation + * + * $Id: dat_dr.c,v 1.17 2005/03/24 05:58:27 jlentini Exp $ + **********************************************************************/ + + +#include "dat_dr.h" + +#include "dat_dictionary.h" + + +/********************************************************************* + * * + * Global Variables * + * * + *********************************************************************/ + +static DAT_OS_LOCK g_dr_lock; +static DAT_DICTIONARY *g_dr_dictionary = NULL; + + +/********************************************************************* + * * + * External Functions * + * * + *********************************************************************/ + + +//*********************************************************************** +// Function: dat_dr_init +//*********************************************************************** + +DAT_RETURN +dat_dr_init ( void ) +{ + DAT_RETURN status; + + status = dat_os_lock_init (&g_dr_lock); + if ( DAT_SUCCESS != status ) + { + return status; + } + + status = dat_dictionary_create (&g_dr_dictionary); + if ( DAT_SUCCESS != status ) + { + return status; + } + + return DAT_SUCCESS; +} + + +//*********************************************************************** +// Function: dat_dr_fini +//*********************************************************************** + +DAT_RETURN +dat_dr_fini ( void ) +{ + DAT_RETURN status; + + status = dat_os_lock_destroy (&g_dr_lock); + if ( DAT_SUCCESS != status ) + { + return status; + } + + status = dat_dictionary_destroy (g_dr_dictionary); + if ( DAT_SUCCESS != status ) + { + return status; + } + + return DAT_SUCCESS; +} + + +//*********************************************************************** +// Function: dat_dr_insert +//*********************************************************************** + +DAT_RETURN +dat_dr_insert ( + IN const DAT_PROVIDER_INFO *info, + IN DAT_DR_ENTRY *entry ) +{ + DAT_RETURN status; + DAT_DICTIONARY_ENTRY dict_entry; + DAT_DR_ENTRY *data; + + data = dat_os_alloc (sizeof (DAT_DR_ENTRY)); + if ( NULL == data ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + *data = *entry; + + dict_entry = NULL; + status = dat_dictionary_entry_create (&dict_entry); + if ( DAT_SUCCESS != status ) + { + goto bail; + } + + dat_os_lock (&g_dr_lock); + + status = dat_dictionary_insert (g_dr_dictionary, + dict_entry, + info, + (DAT_DICTIONARY_DATA *) data); + + dat_os_unlock (&g_dr_lock); + +bail: + if ( DAT_SUCCESS != status ) + { + if ( NULL != data ) + { + dat_os_free (data, sizeof (DAT_DR_ENTRY)); + } + + + if ( NULL != dict_entry ) + { + (void) dat_dictionary_entry_destroy (dict_entry); + } + } + + return status; +} + + +//*********************************************************************** +// Function: dat_dr_remove +//*********************************************************************** + +DAT_RETURN +dat_dr_remove ( + IN const DAT_PROVIDER_INFO *info ) +{ + DAT_DR_ENTRY *data; + DAT_DICTIONARY_ENTRY dict_entry; + DAT_RETURN status; + + dict_entry = NULL; + dat_os_lock (&g_dr_lock); + + status = dat_dictionary_search ( g_dr_dictionary, + info, + (DAT_DICTIONARY_DATA *) &data); + + if ( DAT_SUCCESS != status ) + { + /* return status from dat_dictionary_search() */ + goto bail; + } + + if ( 0 != data->ref_count ) + { + status = DAT_ERROR (DAT_PROVIDER_IN_USE, 0); + goto bail; + } + + status = dat_dictionary_remove ( g_dr_dictionary, + &dict_entry, + info, + (DAT_DICTIONARY_DATA *) &data); + if ( DAT_SUCCESS != status ) + { + /* return status from dat_dictionary_remove() */ + goto bail; + } + + dat_os_free (data, sizeof (DAT_DR_ENTRY)); + +bail: + dat_os_unlock (&g_dr_lock); + + if ( NULL != dict_entry ) + { + (void) dat_dictionary_entry_destroy (dict_entry); + } + + return status; +} + + +//*********************************************************************** +// Function: dat_dr_provider_open +//*********************************************************************** + +DAT_RETURN +dat_dr_provider_open ( + IN const DAT_PROVIDER_INFO *info, + OUT DAT_IA_OPEN_FUNC *p_ia_open_func ) +{ + DAT_RETURN status; + DAT_DR_ENTRY *data; + + dat_os_lock (&g_dr_lock); + + status = dat_dictionary_search ( g_dr_dictionary, + info, + (DAT_DICTIONARY_DATA *) &data); + + dat_os_unlock (&g_dr_lock); + + if ( DAT_SUCCESS == status ) + { + data->ref_count++; + *p_ia_open_func = data->ia_open_func; + } + + return status; +} + + +//*********************************************************************** +// Function: dat_dr_provider_close +//*********************************************************************** + +DAT_RETURN +dat_dr_provider_close ( + IN const DAT_PROVIDER_INFO *info ) +{ + DAT_RETURN status; + DAT_DR_ENTRY *data; + + dat_os_lock (&g_dr_lock); + + status = dat_dictionary_search ( g_dr_dictionary, + info, + (DAT_DICTIONARY_DATA *) &data); + + dat_os_unlock (&g_dr_lock); + + if ( DAT_SUCCESS == status ) + { + data->ref_count--; + } + + return status; +} + + +//*********************************************************************** +// Function: dat_dr_size +//*********************************************************************** + +DAT_RETURN +dat_dr_size ( + OUT DAT_COUNT *size ) +{ + return dat_dictionary_size (g_dr_dictionary, size); +} + + +//*********************************************************************** +// Function: dat_dr_list +//*********************************************************************** + +DAT_RETURN +dat_dr_list ( + IN DAT_COUNT max_to_return, + OUT DAT_COUNT *entries_returned, + OUT DAT_PROVIDER_INFO * (dat_provider_list[]) ) +{ + DAT_DR_ENTRY **array; + DAT_COUNT array_size; + DAT_COUNT i; + DAT_RETURN status; + + array = NULL; + status = DAT_SUCCESS; + + /* The dictionary size may increase between the call to */ + /* dat_dictionary_size() and dat_dictionary_enumerate(). */ + /* Therefore we loop until a successful enumeration is made. */ + *entries_returned = 0; + for (;;) + { + status = dat_dictionary_size (g_dr_dictionary, &array_size); + if ( status != DAT_SUCCESS ) + { + goto bail; + } + + if (array_size == 0) + { + status = DAT_SUCCESS; + goto bail; + } + + array = dat_os_alloc (array_size * sizeof (DAT_DR_ENTRY *)); + if ( array == NULL ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_lock (&g_dr_lock); + + status = dat_dictionary_enumerate (g_dr_dictionary, + (DAT_DICTIONARY_DATA *) array, + array_size); + + dat_os_unlock (&g_dr_lock); + + if ( DAT_SUCCESS == status ) + { + break; + } + else + { + dat_os_free (array, array_size * sizeof (DAT_DR_ENTRY *)); + array = NULL; + continue; + } + } + + for ( i = 0; (i < max_to_return) && (i < array_size); i++) + { + if ( NULL == dat_provider_list[i] ) + { + status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + *dat_provider_list[i] = array[i]->info; + } + + *entries_returned = i; + +bail: + if ( NULL != array ) + { + dat_os_free (array, array_size * sizeof (DAT_DR_ENTRY *)); + } + + return status; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dat/common/dat_dr.h b/trunk/ulp/dapl2/dat/common/dat_dr.h new file mode 100644 index 00000000..e9361a62 --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_dr.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_dr.h + * + * PURPOSE: dynamic registry interface declarations + * + * $Id: dat_dr.h,v 1.12 2005/03/24 05:58:27 jlentini Exp $ + **********************************************************************/ + +#ifndef __DAT_DR_H__ +#define __DAT_DR_H__ + + +#include "dat_osd.h" +#include /* Provider API function prototypes */ + + +/********************************************************************* + * * + * Strucutres * + * * + *********************************************************************/ + +typedef struct +{ + DAT_COUNT ref_count; + DAT_IA_OPEN_FUNC ia_open_func; + DAT_PROVIDER_INFO info; +} DAT_DR_ENTRY; + + +/********************************************************************* + * * + * Function Declarations * + * * + *********************************************************************/ + +extern DAT_RETURN +dat_dr_init ( void ); + +extern DAT_RETURN +dat_dr_fini ( void ); + +extern DAT_RETURN +dat_dr_insert ( + IN const DAT_PROVIDER_INFO *info, + IN DAT_DR_ENTRY *entry ); + +extern DAT_RETURN +dat_dr_remove ( + IN const DAT_PROVIDER_INFO *info ); + + +extern DAT_RETURN +dat_dr_provider_open ( + IN const DAT_PROVIDER_INFO *info, + OUT DAT_IA_OPEN_FUNC *p_ia_open_func ); + +extern DAT_RETURN +dat_dr_provider_close ( + IN const DAT_PROVIDER_INFO *info); + +extern DAT_RETURN +dat_dr_size ( + OUT DAT_COUNT *size); + +extern DAT_RETURN +dat_dr_list ( + IN DAT_COUNT max_to_return, + OUT DAT_COUNT *entries_returned, + OUT DAT_PROVIDER_INFO * (dat_provider_list[]) ); + +#endif diff --git a/trunk/ulp/dapl2/dat/common/dat_init.c b/trunk/ulp/dapl2/dat/common/dat_init.c new file mode 100644 index 00000000..4bdaf0dc --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_init.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_init.c + * + * PURPOSE: DAT registry implementation for uDAPL + * Description: init and fini functions for DAT module. + * + * $Id: dat_init.c,v 1.18 2005/03/24 05:58:27 jlentini Exp $ + **********************************************************************/ + +#include "dat_init.h" + +#include "dat_dr.h" +#include "dat_osd.h" + +#ifndef DAT_NO_STATIC_REGISTRY +#include "dat_sr.h" +#endif + + +/********************************************************************* + * * + * Global Variables * + * * + *********************************************************************/ + +/* + * Ideally, the following two rules could be enforced: + * + * - The DAT Registry's initialization function is executed before that + * of any DAT Providers and hence all calls into the registry occur + * after the registry module is initialized. + * + * - The DAT Registry's deinitialization function is executed after that + * of any DAT Providers and hence all calls into the registry occur + * before the registry module is deinitialized. + * + * However, on many platforms few guarantees are provided regarding the + * order in which module initialization and deinitialization functions + * are invoked. + * + * To understand why these rules are difficult to enforce using only + * features common to all platforms, consider the Linux platform. The order + * in which Linux shared libraries are loaded into a process's address space + * is undefined. When a DAT consumer explicitly links to DAT provider + * libraries, the order in which library initialization and deinitialization + * functions are invoked becomes important. In this scenario, a DAPL provider + * may call dat_registry_add_provider() before the registry has been + * initialized. + * + * We assume that modules are loaded with a single thread. Given + * this assumption, we can use a simple state variable to determine + * the state of the DAT registry. + */ + +static DAT_MODULE_STATE g_module_state = DAT_MODULE_STATE_UNINITIALIZED; + + +//*********************************************************************** +// Function: dat_module_get_state +//*********************************************************************** + +DAT_MODULE_STATE +dat_module_get_state ( void ) +{ + return g_module_state; +} + + +//*********************************************************************** +// Function: dat_init +//*********************************************************************** + +void +dat_init ( void ) +{ + if ( DAT_MODULE_STATE_UNINITIALIZED == g_module_state ) + { + /* + * update the module state flag immediately in case there + * is a recursive call to dat_init(). + */ + g_module_state = DAT_MODULE_STATE_INITIALIZING; + + dat_os_dbg_init (); + + dats_handle_vector_init (); + + dat_os_dbg_print (DAT_OS_DBG_TYPE_GENERIC, + "DAT Registry: Started (dat_init)\n"); + +#ifndef DAT_NO_STATIC_REGISTRY + dat_sr_init (); +#endif + dat_dr_init (); + + g_module_state = DAT_MODULE_STATE_INITIALIZED; + } +} + + +//*********************************************************************** +// Function: dat_fini +//*********************************************************************** + +void +dat_fini ( void ) +{ + if ( DAT_MODULE_STATE_INITIALIZED == g_module_state ) + { + g_module_state = DAT_MODULE_STATE_DEINITIALIZING; + + dat_dr_fini (); +#ifndef DAT_NO_STATIC_REGISTRY + dat_sr_fini (); +#endif + + dat_os_dbg_print (DAT_OS_DBG_TYPE_GENERIC, "DAT Registry: Stopped (dat_fini)\n"); + + g_module_state = DAT_MODULE_STATE_DEINITIALIZED; + } +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dat/common/dat_init.h b/trunk/ulp/dapl2/dat/common/dat_init.h new file mode 100644 index 00000000..c3b17866 --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_init.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_init.h + * + * PURPOSE: DAT registry global data + * + * $Id: dat_init.h,v 1.16 2005/03/24 05:58:27 jlentini Exp $ + **********************************************************************/ + +#ifndef _DAT_INIT_H_ +#define _DAT_INIT_H_ + +#include "dat_osd.h" + +/********************************************************************* + * * + * Enumerations * + * * + *********************************************************************/ + +typedef enum +{ + DAT_MODULE_STATE_UNINITIALIZED, + DAT_MODULE_STATE_INITIALIZING, + DAT_MODULE_STATE_INITIALIZED, + DAT_MODULE_STATE_DEINITIALIZING, + DAT_MODULE_STATE_DEINITIALIZED +} DAT_MODULE_STATE; + +/********************************************************************* + * * + * Function Prototypes * + * * + *********************************************************************/ + +DAT_MODULE_STATE +dat_module_get_state ( void ) ; + +#if defined(_MSC_VER) || defined(_WIN64) || defined(_WIN32) +/* NT. MSC compiler, Win32/64 platform */ +void +dat_init ( void ); + +void +dat_fini ( void ); + +#else /* GNU C */ + +void +dat_init ( void ) __attribute__ ((constructor)); + +void +dat_fini ( void ) __attribute__ ((destructor)); +#endif + +extern DAT_RETURN +dats_handle_vector_init ( void ); + +extern DAT_IA_HANDLE +dats_set_ia_handle ( + IN DAT_IA_HANDLE ia_handle); + +extern DAT_RETURN +dats_get_ia_handle( + IN DAT_IA_HANDLE handle, + OUT DAT_IA_HANDLE *ia_handle_p); + +extern DAT_BOOLEAN +dats_is_ia_handle ( + IN DAT_HANDLE dat_handle); + +extern DAT_RETURN +dats_free_ia_handle( + IN DAT_IA_HANDLE handle); + +#endif diff --git a/trunk/ulp/dapl2/dat/common/dat_sr.c b/trunk/ulp/dapl2/dat/common/dat_sr.c new file mode 100644 index 00000000..1608cf7c --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_sr.c @@ -0,0 +1,492 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_sr.c + * + * PURPOSE: static registry implementation + * + * $Id: dat_sr.c,v 1.17 2005/03/24 05:58:27 jlentini Exp $ + **********************************************************************/ + + +#include "dat_sr.h" + +#include "dat_dictionary.h" +#include "udat_sr_parser.h" + + +/********************************************************************* + * * + * Global Variables * + * * + *********************************************************************/ + +static DAT_OS_LOCK g_sr_lock; +static DAT_DICTIONARY *g_sr_dictionary = NULL; + + +/********************************************************************* + * * + * External Functions * + * * + *********************************************************************/ + + +//*********************************************************************** +// Function: dat_sr_init +//*********************************************************************** + +DAT_RETURN +dat_sr_init ( void ) +{ + DAT_RETURN status; + + status = dat_os_lock_init (&g_sr_lock); + if ( DAT_SUCCESS != status ) + { + return status; + } + + status = dat_dictionary_create (&g_sr_dictionary); + if ( DAT_SUCCESS != status ) + { + return status; + } + + /* + * Since DAT allows providers to be loaded by either the static + * registry or explicitly through OS dependent methods, do not + * return an error if no providers are loaded via the static registry. + */ + + (void) dat_sr_load (); + + return DAT_SUCCESS; +} + + +//*********************************************************************** +// Function: dat_sr_fini +//*********************************************************************** + +extern DAT_RETURN +dat_sr_fini ( void ) +{ + DAT_RETURN status; + + status = dat_os_lock_destroy (&g_sr_lock); + if ( DAT_SUCCESS != status ) + { + return status; + } + + status = dat_dictionary_destroy (g_sr_dictionary); + if ( DAT_SUCCESS != status ) + { + return status; + } + + return DAT_SUCCESS; +} + + +//*********************************************************************** +// Function: dat_sr_insert +//*********************************************************************** + +extern DAT_RETURN +dat_sr_insert ( + IN const DAT_PROVIDER_INFO *info, + IN DAT_SR_ENTRY *entry ) +{ + DAT_RETURN status; + DAT_SR_ENTRY *data, *prev_data; + DAT_OS_SIZE lib_path_size; + DAT_OS_SIZE lib_path_len; + DAT_OS_SIZE ia_params_size; + DAT_OS_SIZE ia_params_len; + DAT_DICTIONARY_ENTRY dict_entry; + + if ( NULL == (data = dat_os_alloc (sizeof (DAT_SR_ENTRY))) ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + lib_path_len = strlen (entry->lib_path); + lib_path_size = (lib_path_len + 1) * sizeof (char); + + if ( NULL == (data->lib_path = dat_os_alloc (lib_path_size)) ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_strncpy (data->lib_path, entry->lib_path, lib_path_len); + data->lib_path[lib_path_len] = '\0'; + + ia_params_len = strlen (entry->ia_params); + ia_params_size = (ia_params_len + 1) * sizeof (char); + + if ( NULL == (data->ia_params = dat_os_alloc (ia_params_size)) ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_strncpy (data->ia_params, entry->ia_params, ia_params_len); + data->ia_params[ia_params_len] = '\0'; + + data->info = entry->info; + data->lib_handle = entry->lib_handle; + data->ref_count = entry->ref_count; + data->next = NULL; + + dict_entry = NULL; + status = dat_dictionary_entry_create (&dict_entry); + if ( DAT_SUCCESS != status ) + { + goto bail; + } + + dat_os_lock (&g_sr_lock); + + status = dat_dictionary_search (g_sr_dictionary, + info, + (DAT_DICTIONARY_DATA *) &prev_data); + if ( DAT_SUCCESS == status ) + { + /* We already have a dictionary entry, so we don't need a new one. + * This means there are multiple duplicate names in dat.conf, + * but presumably they have different paths. Simply link the + * new entry at the end of the chain of like-named entries. + */ + (void) dat_dictionary_entry_destroy (dict_entry); + dict_entry = NULL; + + /* Find the next available slot in this chain */ + while (NULL != prev_data->next) + { + prev_data = prev_data->next; + } + dat_os_assert (NULL != prev_data); + prev_data->next = data; + } + else + { + status = dat_dictionary_insert (g_sr_dictionary, + dict_entry, + info, + (DAT_DICTIONARY_DATA *) data); + } + + dat_os_unlock (&g_sr_lock); + +bail: + if ( DAT_SUCCESS != status ) + { + if ( NULL != data ) + { + if ( NULL != data->lib_path ) + { + dat_os_free (data->lib_path, lib_path_size); + } + + if ( NULL != data->ia_params ) + { + dat_os_free (data->ia_params, ia_params_size); + } + + dat_os_free (data, sizeof (DAT_SR_ENTRY)); + } + + if ( NULL != dict_entry ) + { + (void) dat_dictionary_entry_destroy (dict_entry); + } + } + + return status; +} + + +//*********************************************************************** +// Function: dat_sr_size +//*********************************************************************** + +extern DAT_RETURN +dat_sr_size ( + OUT DAT_COUNT *size ) +{ + return dat_dictionary_size (g_sr_dictionary, size); +} + + +//*********************************************************************** +// Function: dat_sr_list +//*********************************************************************** + +extern DAT_RETURN +dat_sr_list ( + IN DAT_COUNT max_to_return, + OUT DAT_COUNT *entries_returned, + OUT DAT_PROVIDER_INFO * (dat_provider_list[]) ) +{ + DAT_SR_ENTRY **array; + DAT_COUNT array_size; + DAT_COUNT i; + DAT_RETURN status; + + array = NULL; + status = DAT_SUCCESS; + + /* The dictionary size may increase between the call to */ + /* dat_dictionary_size() and dat_dictionary_enumerate(). */ + /* Therefore we loop until a successful enumeration is made. */ + *entries_returned = 0; + for (;;) + { + status = dat_dictionary_size (g_sr_dictionary, &array_size); + if ( DAT_SUCCESS != status ) + { + goto bail; + } + + if (array_size == 0) + { + status = DAT_SUCCESS; + goto bail; + } + + array = dat_os_alloc (array_size * sizeof (DAT_SR_ENTRY *)); + if ( array == NULL ) + { + status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY); + goto bail; + } + + dat_os_lock (&g_sr_lock); + + status = dat_dictionary_enumerate (g_sr_dictionary, + (DAT_DICTIONARY_DATA *) array, + array_size); + + dat_os_unlock (&g_sr_lock); + + if ( DAT_SUCCESS == status ) + { + break; + } + else + { + dat_os_free (array, array_size * sizeof (DAT_SR_ENTRY *)); + array = NULL; + continue; + } + } + + for ( i = 0; (i < max_to_return) && (i < array_size); i++) + { + if ( NULL == dat_provider_list[i] ) + { + status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + goto bail; + } + + *dat_provider_list[i] = array[i]->info; + } + + *entries_returned = i; + +bail: + if ( NULL != array ) + { + dat_os_free (array, array_size * sizeof (DAT_SR_ENTRY *)); + } + + return status; +} + + + +//*********************************************************************** +// Function: dat_sr_provider_open +//*********************************************************************** + +extern DAT_RETURN +dat_sr_provider_open ( + IN const DAT_PROVIDER_INFO *info ) +{ + DAT_RETURN status; + DAT_SR_ENTRY *data; + + dat_os_lock (&g_sr_lock); + + status = dat_dictionary_search (g_sr_dictionary, + info, + (DAT_DICTIONARY_DATA *) &data); + + if ( DAT_SUCCESS == status ) + { + while (data != NULL) + { + if ( 0 == data->ref_count ) + { + /* + * Try to open the path. If it fails, try the next + * path in the chain. Only the first successful library + * open matters, the others will be unused. + */ + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT2 Registry: IA %s, trying to load library %s\n", + data->info.ia_name, data->lib_path); + + status = dat_os_library_load (data->lib_path, + &data->lib_handle); + + if ( status == DAT_SUCCESS ) + { +#ifdef DBG + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT2 Registry: IA %s, loaded library %s\n", + data->info.ia_name, data->lib_path); +#endif + data->ref_count++; + data->init_func = dat_os_library_sym (data->lib_handle, + DAT_PROVIDER_INIT_FUNC_STR); + data->fini_func = dat_os_library_sym (data->lib_handle, + DAT_PROVIDER_FINI_FUNC_STR); + /* Warning: DAT and DAPL libraries not ext compatible */ +#ifdef DAT_EXTENSIONS +{ + void *fncptr; + + fncptr = dat_os_library_sym(data->lib_handle, + "dapl_extensions"); + + if ((dat_os_library_error() != NULL) || (fncptr == NULL)) + { + dat_os_dbg_print(DAT_OS_DBG_TYPE_SR, + "DAT Registry: WARNING: library %s, " + "extended DAT expected extended uDAPL: %s\n", + data->lib_path, strerror(errno)); + } +} +#endif + if ( NULL != data->init_func ) + { + (*data->init_func) (&data->info, data->ia_params); + } + else + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT Registry: Cannot find library init func (%s)\n", + DAT_PROVIDER_INIT_FUNC_STR); + } + + /* exit after we find the first valid entry */ + break; + } + else + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT Registry: static registry unable to " + "load library %s\n", data->lib_path); + } + } + else + { + data->ref_count++; + break; + } + data = data->next; + } + } + + dat_os_unlock (&g_sr_lock); + + return status; +} + + +//*********************************************************************** +// Function: dat_sr_provider_close +//*********************************************************************** + +extern DAT_RETURN +dat_sr_provider_close ( + IN const DAT_PROVIDER_INFO *info ) +{ + DAT_RETURN status; + DAT_SR_ENTRY *data; + + dat_os_lock (&g_sr_lock); + + status = dat_dictionary_search (g_sr_dictionary, + info, + (DAT_DICTIONARY_DATA *) &data); + + if ( DAT_SUCCESS == status ) + { + while (data != NULL) + { + if ( 1 == data->ref_count ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT Registry: IA %s, unloading library %s\n", + data->info.ia_name, data->lib_path); + + if ( NULL != data->fini_func ) + { + (*data->fini_func) (&data->info); + } + + status = dat_os_library_unload (data->lib_handle); + if ( status == DAT_SUCCESS ) + { + data->ref_count--; + } + break; + } + else if ( data->ref_count > 0) + { + data->ref_count--; + break; + } + data = data->next; + } + } + + dat_os_unlock (&g_sr_lock); + + return status; +} diff --git a/trunk/ulp/dapl2/dat/common/dat_sr.h b/trunk/ulp/dapl2/dat/common/dat_sr.h new file mode 100644 index 00000000..6f94ff82 --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_sr.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_sr.h + * + * PURPOSE: static registry (SR) inteface declarations + * + * $Id: dat_sr.h,v 1.12 2005/03/24 05:58:28 jlentini Exp $ + **********************************************************************/ + +#ifndef _DAT_SR_H_ +#define _DAT_SR_H_ + + +#include +#include + +#include "dat_osd.h" + +/********************************************************************* + * * + * Strucutres * + * * + *********************************************************************/ + +typedef struct DAT_SR_ENTRY +{ + DAT_PROVIDER_INFO info; + char * lib_path; + char * ia_params; + DAT_OS_LIBRARY_HANDLE lib_handle; + DAT_PROVIDER_INIT_FUNC init_func; + DAT_PROVIDER_FINI_FUNC fini_func; + DAT_COUNT ref_count; + struct DAT_SR_ENTRY *next; +} DAT_SR_ENTRY; + + +/********************************************************************* + * * + * Function Declarations * + * * + *********************************************************************/ + +extern DAT_RETURN +dat_sr_init ( void ); + +extern DAT_RETURN +dat_sr_fini ( void ); + +extern DAT_RETURN +dat_sr_insert ( + IN const DAT_PROVIDER_INFO *info, + IN DAT_SR_ENTRY *entry ); + +extern DAT_RETURN +dat_sr_size ( + OUT DAT_COUNT *size); + +extern DAT_RETURN +dat_sr_list ( + IN DAT_COUNT max_to_return, + OUT DAT_COUNT *entries_returned, + OUT DAT_PROVIDER_INFO * (dat_provider_list[]) ); + +extern DAT_RETURN +dat_sr_provider_open ( + IN const DAT_PROVIDER_INFO *info ); + +extern DAT_RETURN +dat_sr_provider_close ( + IN const DAT_PROVIDER_INFO *info ); + + +#endif diff --git a/trunk/ulp/dapl2/dat/common/dat_strerror.c b/trunk/ulp/dapl2/dat/common/dat_strerror.c new file mode 100644 index 00000000..3c99986b --- /dev/null +++ b/trunk/ulp/dapl2/dat/common/dat_strerror.c @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_strerror.c + * + * PURPOSE: Convert DAT_RETURN values to humman readable string + * + * $Id: dat_strerror.c,v 1.10 2005/03/24 05:58:28 jlentini Exp $ + **********************************************************************/ + +#ifdef __KDAPL__ +#include +#else /*__UDAPL__*/ +#include +#endif /* __UDAPL__ */ + +/********************************************************************* + * * + * Internal Function Declarations * + * * + *********************************************************************/ + +static DAT_RETURN +dat_strerror_major ( + IN DAT_RETURN value, + OUT const char **message ); + +static DAT_RETURN +dat_strerror_minor ( + IN DAT_RETURN value, + OUT const char **message ); + + +/********************************************************************* + * * + * Internal Function Definitions * + * * + *********************************************************************/ + +static DAT_RETURN +dat_strerror_major ( + IN DAT_RETURN value, + OUT const char **message ) +{ + switch ( DAT_GET_TYPE (value) ) + { + case DAT_SUCCESS: + { + *message = "DAT_SUCCESS"; + return DAT_SUCCESS; + } + case DAT_ABORT: + { + *message = "DAT_ABORT"; + return DAT_SUCCESS; + } + case DAT_CONN_QUAL_IN_USE: + { + *message = "DAT_CONN_QUAL_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INSUFFICIENT_RESOURCES: + { + *message = "DAT_INSUFFICIENT_RESOURCES"; + return DAT_SUCCESS; + } + case DAT_INTERNAL_ERROR: + { + *message = "DAT_INTERNAL_ERROR"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE: + { + *message = "DAT_INVALID_HANDLE"; + return DAT_SUCCESS; + } + case DAT_INVALID_PARAMETER: + { + *message = "DAT_INVALID_PARAMETER"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE: + { + *message = "DAT_INVALID_STATE"; + return DAT_SUCCESS; + } + case DAT_LENGTH_ERROR: + { + *message = "DAT_LENGTH_ERROR"; + return DAT_SUCCESS; + } + case DAT_MODEL_NOT_SUPPORTED: + { + *message = "DAT_MODEL_NOT_SUPPORTED"; + return DAT_SUCCESS; + } + case DAT_PROVIDER_NOT_FOUND: + { + *message = "DAT_PROVIDER_NOT_FOUND"; + return DAT_SUCCESS; + } + case DAT_PRIVILEGES_VIOLATION: + { + *message = "DAT_PRIVILEGES_VIOLATION"; + return DAT_SUCCESS; + } + case DAT_PROTECTION_VIOLATION: + { + *message = "DAT_PROTECTION_VIOLATION"; + return DAT_SUCCESS; + } + case DAT_QUEUE_EMPTY: + { + *message = "DAT_QUEUE_EMPTY"; + return DAT_SUCCESS; + } + case DAT_QUEUE_FULL: + { + *message = "DAT_QUEUE_FULL"; + return DAT_SUCCESS; + } + case DAT_TIMEOUT_EXPIRED: + { + *message = "DAT_TIMEOUT_EXPIRED"; + return DAT_SUCCESS; + } + case DAT_PROVIDER_ALREADY_REGISTERED: + { + *message = "DAT_PROVIDER_ALREADY_REGISTERED"; + return DAT_SUCCESS; + } + case DAT_PROVIDER_IN_USE: + { + *message = "DAT_PROVIDER_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INVALID_ADDRESS: + { + *message = "DAT_INVALID_ADDRESS"; + return DAT_SUCCESS; + } + case DAT_INTERRUPTED_CALL: + { + *message = "DAT_INTERRUPTED_CALL"; + return DAT_SUCCESS; + } + case DAT_NOT_IMPLEMENTED: + { + *message = "DAT_NOT_IMPLEMENTED"; + return DAT_SUCCESS; + } + default: + { + *message = "unknown error"; + return DAT_INVALID_PARAMETER; + } + } +} + + +static DAT_RETURN +dat_strerror_minor ( + IN DAT_RETURN value, + OUT const char **message ) +{ + switch ( DAT_GET_SUBTYPE (value) ) + { + + case DAT_NO_SUBTYPE: /* NO subtype */ + { + *message = ""; + return DAT_SUCCESS; + } + case DAT_SUB_INTERRUPTED: + { + *message = "DAT_SUB_INTERRUPTED"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_MEMORY: + { + *message = "DAT_RESOURCE_MEMORY"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_DEVICE: + { + *message = "DAT_RESOURCE_DEVICE"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_TEP: + { + *message = "DAT_RESOURCE_TEP"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_TEVD: + { + *message = "DAT_RESOURCE_TEVD"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_PROTECTION_DOMAIN: + { + *message = "DAT_RESOURCE_PROTECTION_DOMAIN"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_MEMORY_REGION: + { + *message = "DAT_RESOURCE_MEMORY_REGION"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_ERROR_HANDLER: + { + *message = "DAT_RESOURCE_ERROR_HANDLER"; + return DAT_SUCCESS; + } + case DAT_RESOURCE_CREDITS: + { + *message = "DAT_RESOURCE_CREDITS"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_IA: + { + *message = "DAT_INVALID_HANDLE_IA"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_EP: + { + *message = "DAT_INVALID_HANDLE_EP"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_LMR: + { + *message = "DAT_INVALID_HANDLE_LMR"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_RMR: + { + *message = "DAT_INVALID_HANDLE_RMR"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_PZ: + { + *message = "DAT_INVALID_HANDLE_PZ"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_PSP: + { + *message = "DAT_INVALID_HANDLE_PSP"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_RSP: + { + *message = "DAT_INVALID_HANDLE_RSP"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_CR: + { + *message = "DAT_INVALID_HANDLE_CR"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_CNO: + { + *message = "DAT_INVALID_HANDLE_CNO"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_EVD_CR: + { + *message = "DAT_INVALID_HANDLE_EVD_CR"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_EVD_REQUEST: + { + *message = "DAT_INVALID_HANDLE_EVD_REQUEST"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_EVD_RECV: + { + *message = "DAT_INVALID_HANDLE_EVD_RECV"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_EVD_CONN: + { + *message = "DAT_INVALID_HANDLE_EVD_CONN"; + return DAT_SUCCESS; + } + case DAT_INVALID_HANDLE_EVD_ASYNC: + { + *message = "DAT_INVALID_HANDLE_EVD_ASYNC"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG1: + { + *message = "DAT_INVALID_ARG1"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG2: + { + *message = "DAT_INVALID_ARG2"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG3: + { + *message = "DAT_INVALID_ARG3"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG4: + { + *message = "DAT_INVALID_ARG4"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG5: + { + *message = "DAT_INVALID_ARG5"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG6: + { + *message = "DAT_INVALID_ARG6"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG7: + { + *message = "DAT_INVALID_ARG7"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG8: + { + *message = "DAT_INVALID_ARG8"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG9: + { + *message = "DAT_INVALID_ARG9"; + return DAT_SUCCESS; + } + case DAT_INVALID_ARG10: + { + *message = "DAT_INVALID_ARG10"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_UNCONNECTED: + { + *message = "DAT_INVALID_STATE_EP_UNCONNECTED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_ACTCONNPENDING: + { + *message = "DAT_INVALID_STATE_EP_ACTCONNPENDING"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_PASSCONNPENDING: + { + *message = "DAT_INVALID_STATE_EP_PASSCONNPENDING"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_TENTCONNPENDING: + { + *message = "DAT_INVALID_STATE_EP_TENTCONNPENDING"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_CONNECTED: + { + *message = "DAT_INVALID_STATE_EP_CONNECTED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_DISCONNECTED: + { + *message = "DAT_INVALID_STATE_EP_DISCONNECTED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_RESERVED: + { + *message = "DAT_INVALID_STATE_EP_RESERVED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_COMPLPENDING: + { + *message = "DAT_INVALID_STATE_EP_COMPLPENDING"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_DISCPENDING: + { + *message = "DAT_INVALID_STATE_EP_DISCPENDING"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_PROVIDERCONTROL: + { + *message = "DAT_INVALID_STATE_EP_PROVIDERCONTROL"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EP_NOTREADY: + { + *message = "DAT_INVALID_STATE_EP_NOTREADY"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_CNO_IN_USE: + { + *message = "DAT_INVALID_STATE_CNO_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_CNO_DEAD: + { + *message = "DAT_INVALID_STATE_CNO_DEAD"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_OPEN: + { + *message = "DAT_INVALID_STATE_EVD_OPEN"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_ENABLED: + { + *message = "DAT_INVALID_STATE_EVD_ENABLED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_DISABLED: + { + *message = "DAT_INVALID_STATE_EVD_DISABLED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_WAITABLE: + { + *message = "DAT_INVALID_STATE_EVD_WAITABLE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_UNWAITABLE: + { + *message = "DAT_INVALID_STATE_EVD_UNWAITABLE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_IN_USE: + { + *message = "DAT_INVALID_STATE_EVD_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_CONFIG_NOTIFY: + { + *message = "DAT_INVALID_STATE_EVD_CONFIG_NOTIFY"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_CONFIG_SOLICITED: + { + *message = "DAT_INVALID_STATE_EVD_CONFIG_SOLICITED"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_CONFIG_THRESHOLD: + { + *message = "DAT_INVALID_STATE_EVD_CONFIG_THRESHOLD"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_WAITER: + { + *message = "DAT_INVALID_STATE_EVD_WAITER"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_EVD_ASYNC: + { + *message = "DAT_INVALID_STATE_EVD_ASYNC"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_IA_IN_USE: + { + *message = "DAT_INVALID_STATE_IA_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_LMR_IN_USE: + { + *message = "DAT_INVALID_STATE_LMR_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_LMR_FREE: + { + *message = "DAT_INVALID_STATE_LMR_FREE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_PZ_IN_USE: + { + *message = "DAT_INVALID_STATE_PZ_IN_USE"; + return DAT_SUCCESS; + } + case DAT_INVALID_STATE_PZ_FREE: + { + *message = "DAT_INVALID_STATE_PZ_FREE"; + return DAT_SUCCESS; + } + case DAT_PRIVILEGES_READ: + { + *message = "DAT_PRIVILEGES_READ"; + return DAT_SUCCESS; + } + case DAT_PRIVILEGES_WRITE: + { + *message = "DAT_PRIVILEGES_WRITE"; + return DAT_SUCCESS; + } + case DAT_PRIVILEGES_RDMA_READ: + { + *message = "DAT_PRIVILEGES_RDMA_READ"; + return DAT_SUCCESS; + } + case DAT_PRIVILEGES_RDMA_WRITE: + { + *message = "DAT_PRIVILEGES_RDMA_WRITE"; + return DAT_SUCCESS; + } + case DAT_PROTECTION_READ: + { + *message = "DAT_PROTECTION_READ"; + return DAT_SUCCESS; + } + case DAT_PROTECTION_WRITE: + { + *message = "DAT_PROTECTION_WRITE"; + return DAT_SUCCESS; + } + case DAT_PROTECTION_RDMA_READ: + { + *message = "DAT_PROTECTION_RDMA_READ"; + return DAT_SUCCESS; + } + case DAT_PROTECTION_RDMA_WRITE: + { + *message = "DAT_PROTECTION_RDMA_WRITE"; + return DAT_SUCCESS; + } + case DAT_INVALID_ADDRESS_UNSUPPORTED: + { + *message = "DAT_INVALID_ADDRESS_UNSUPPORTED"; + return DAT_SUCCESS; + } + case DAT_INVALID_ADDRESS_UNREACHABLE: + { + *message = "DAT_INVALID_ADDRESS_UNREACHABLE"; + return DAT_SUCCESS; + } + case DAT_INVALID_ADDRESS_MALFORMED: + { + *message = "DAT_INVALID_ADDRESS_MALFORMED"; + return DAT_SUCCESS; + } + case DAT_NAME_NOT_REGISTERED: + { + *message = "DAT_NAME_NOT_REGISTERED"; + return DAT_SUCCESS; + } + case DAT_MAJOR_NOT_FOUND: + { + *message = "DAT_MAJOR_NOT_FOUND"; + return DAT_SUCCESS; + } + case DAT_MINOR_NOT_FOUND: + { + *message = "DAT_MINOR_NOT_FOUND"; + return DAT_SUCCESS; + } + case DAT_THREAD_SAFETY_NOT_FOUND: + { + *message = "DAT_THREAD_SAFETY_NOT_FOUND"; + return DAT_SUCCESS; + } + default: + { + *message = "unknown minor error"; + return DAT_INVALID_PARAMETER; + } + } +} + + +/********************************************************************* + * * + * External Function Definitions * + * * + *********************************************************************/ + +DAT_RETURN DAT_API +dat_strerror ( + IN DAT_RETURN value, + OUT const char **major_message, + OUT const char **minor_message ) +{ + /* + * The DAT specification contains a note to implementers + * suggesting that the consumer's DAT_RETURN value be used + * as an index into a table of text strings. However, + * the DAT_RETURN values are not consecutive. Therefore this + * implementation does not follow the suggested implementation. + */ + + if ( DAT_SUCCESS != dat_strerror_major (value, major_message) ) + { + return DAT_INVALID_PARAMETER; + } + else if (minor_message != NULL) + { + if ( DAT_SUCCESS != dat_strerror_minor (value, minor_message) ) + { + return DAT_INVALID_PARAMETER; + } + } + + return DAT_SUCCESS; +} diff --git a/trunk/ulp/dapl2/dat/dirs b/trunk/ulp/dapl2/dat/dirs new file mode 100644 index 00000000..97d2b67c --- /dev/null +++ b/trunk/ulp/dapl2/dat/dirs @@ -0,0 +1 @@ +DIRS=udat diff --git a/trunk/ulp/dapl2/dat/include/dat/dat.h b/trunk/ulp/dapl2/dat/include/dat/dat.h new file mode 100644 index 00000000..cdd5df4e --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat.h @@ -0,0 +1,1377 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*************************************************************** + * + * HEADER: dat.h + * + * PURPOSE: defines the common DAT API for uDAPL and kDAPL. + * + * Description: Header file for "DAPL: Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'EP' or 'LMR' + * The method table is in the provider definition structure. + * + * + ***************************************************************/ +#ifndef _DAT_H_ +#define _DAT_H_ + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Generic DAT types */ + +typedef char * DAT_NAME_PTR; /* Format for ia_name and attributes */ +#define DAT_NAME_MAX_LENGTH 256 + +/* + * Used for provider, vendor, transport, hardware-specific attributes + * definitions. + */ + +typedef struct dat_named_attr +{ + const char * name; /* Name of attribute */ + const char * value; /* Value of attribute */ +} DAT_NAMED_ATTR; + +typedef enum dat_boolean +{ + DAT_FALSE = 0, + DAT_TRUE = 1 +} DAT_BOOLEAN; + +#ifdef DAT_EXTENSIONS +#define DAT_IB_EXTENSION 1 +#define DAT_IW_EXTENSION 2 +#endif /* DAT_EXTENSIONS */ + +typedef DAT_UINT32 DAT_HA_LB; +#define DAT_HA_LB_NONE (DAT_HA_LB)0 +#define DAT_HA_LB_INTERCOMM (DAT_HA_LB)1 +#define DAT_HA_LB_INTRACOMM (DAT_HA_LB)2 + +typedef union dat_context +{ + DAT_PVOID as_ptr; + DAT_UINT64 as_64; + DAT_UVERYLONG as_index; +} DAT_CONTEXT; + +typedef DAT_CONTEXT DAT_DTO_COOKIE; +typedef DAT_CONTEXT DAT_RMR_COOKIE; + +typedef enum dat_completion_flags +{ + /* Completes with notification */ + DAT_COMPLETION_DEFAULT_FLAG = 0x00, + /* Completions suppressed if successful */ + DAT_COMPLETION_SUPPRESS_FLAG = 0x01, + /* Sender controlled notification for recv completion */ + DAT_COMPLETION_SOLICITED_WAIT_FLAG = 0x02, + /* Completions with unsignaled notifications */ + DAT_COMPLETION_UNSIGNALLED_FLAG = 0x04, + /* Do not start processing until all previous RDMA reads complete. */ + DAT_COMPLETION_BARRIER_FENCE_FLAG = 0x08, + /* Only valid for uDAPL as EP attribute for Recv Completion flags. + * Waiter unblocking is controlled by the Threshold value of + * dat_evd_wait. UNSIGNALLED for RECV is not allowed when EP has + * this attribute. */ + DAT_COMPLETION_EVD_THRESHOLD_FLAG = 0x10, + /* Only valid for kDAPL + * Do not start processing LMR invalidate until all + * previously posted DTOs to the EP Request Queue + * have been completed. + * The value for LMR Invalidate Fence does not + * conflict with uDAPL so it can be extended + * to uDAPL usage later. + */ + DAT_COMPLETION_LMR_INVALIDATE_FENCE_FLAG = 0x20 +} DAT_COMPLETION_FLAGS; + + +typedef DAT_UINT32 DAT_TIMEOUT; /* microseconds */ + +/* timeout = infinity */ +#define DAT_TIMEOUT_INFINITE ((DAT_TIMEOUT) ~0) + +/* dat handles */ +typedef DAT_PVOID DAT_HANDLE; +typedef DAT_HANDLE DAT_CR_HANDLE; +typedef DAT_HANDLE DAT_EP_HANDLE; +typedef DAT_HANDLE DAT_EVD_HANDLE; +typedef DAT_HANDLE DAT_IA_HANDLE; +typedef DAT_HANDLE DAT_LMR_HANDLE; +typedef DAT_HANDLE DAT_PSP_HANDLE; +typedef DAT_HANDLE DAT_PZ_HANDLE; +typedef DAT_HANDLE DAT_RMR_HANDLE; +typedef DAT_HANDLE DAT_RSP_HANDLE; +typedef DAT_HANDLE DAT_SRQ_HANDLE; +typedef DAT_HANDLE DAT_CSP_HANDLE; + +typedef enum dat_dtos +{ + DAT_DTO_SEND, + DAT_DTO_RDMA_WRITE, + DAT_DTO_RDMA_READ, + DAT_DTO_RECEIVE, + DAT_DTO_RECEIVE_WITH_INVALIDATE, + DAT_DTO_BIND_MW, /* DAT 2.0 review, binds are reported via DTO events */ + DAT_DTO_LMR_FMR, /* kdat specific */ + DAT_DTO_LMR_INVALIDATE /* kdat specific */ +#ifdef DAT_EXTENSIONS + ,DAT_DTO_EXTENSION_BASE /* To be used by DAT extensions + as a starting point of extension DTOs */ +#endif /* DAT_EXTENSIONS */ +} DAT_DTOS; + + +/* dat NULL handles */ +#define DAT_HANDLE_NULL ((DAT_HANDLE)NULL) + +typedef DAT_SOCK_ADDR* DAT_IA_ADDRESS_PTR; + +typedef DAT_UINT64 DAT_CONN_QUAL; +typedef DAT_UINT64 DAT_PORT_QUAL; + +/* QOS definitions */ +typedef enum dat_qos +{ + DAT_QOS_BEST_EFFORT = 0x00, + DAT_QOS_HIGH_THROUGHPUT = 0x01, + DAT_QOS_LOW_LATENCY = 0x02, + /* not low latency, nor high throughput */ + DAT_QOS_ECONOMY = 0x04, + /* both low latency and high throughput */ + DAT_QOS_PREMIUM = 0x08 +} DAT_QOS; + +/* + * FLAGS + */ + +/* for backward compatibility */ +#define DAT_CONNECT_MULTIPATH_REQUESTED_FLAG DAT_CONNECT_MULTIPATH_FLAG + +typedef enum dat_connect_flags +{ + DAT_CONNECT_DEFAULT_FLAG = 0x00, + DAT_CONNECT_MULTIPATH_REQUESTED_FLAG = 0x01, + DAT_CONNECT_MULTIPATH_REQUIRED_FLAG = 0x02 +} DAT_CONNECT_FLAGS; + +typedef enum dat_close_flags +{ + DAT_CLOSE_ABRUPT_FLAG = 0x00, + DAT_CLOSE_GRACEFUL_FLAG = 0x01 +} DAT_CLOSE_FLAGS; + +#define DAT_CLOSE_DEFAULT DAT_CLOSE_ABRUPT_FLAG + +typedef enum dat_evd_flags +{ + DAT_EVD_SOFTWARE_FLAG = 0x001, + DAT_EVD_CR_FLAG = 0x010, + DAT_EVD_DTO_FLAG = 0x020, + DAT_EVD_CONNECTION_FLAG = 0x040, + DAT_EVD_RMR_BIND_FLAG = 0x080, + DAT_EVD_ASYNC_FLAG = 0x100, + + /* DAT events only, no software events */ + DAT_EVD_DEFAULT_FLAG = 0x1F0 +#ifdef DAT_EXTENSIONS +/* To be used by DAT extensions as a starting point for extended evd flags */ + ,DAT_EVD_EXTENSION_BASE = 0x200 +#endif /* DAT_EXTENSIONS */ +} DAT_EVD_FLAGS; + +typedef enum dat_psp_flags +{ + DAT_PSP_CONSUMER_FLAG = 0x00, /* Consumer creates an Endpoint */ + DAT_PSP_PROVIDER_FLAG = 0x01 /* Provider creates an Endpoint */ +} DAT_PSP_FLAGS; + +/* + * Memory Buffers + * + * Both LMR and RMR triplets specify 64-bit addresses in the local host's byte + * order, even when that exceeds the size of a DAT_PVOID for the host + * architecture. + */ + +/* + * Both LMR and RMR Triplets specify 64-bit addresses in the local host + * order, even when that exceeds the size of a void pointer for the host + * architecture. The DAT_VADDR type that represents addresses is in the + * native byte-order of the local host. Helper macros that allow Consumers + * to convert DAT_VADDR into various orders that might be useful for + * inclusion of RMR Triplets into a payload of a message follow. + * + * DAT defines the following macros to convert the fields on an RMR Triplet + * to defined byte orders to allow their export by the Consumer over wire + * protocols. DAT does not define how the two peers decide which byte should be + * used. + * + * DAT_LMRC_TO_LSB(lmrc) returns the supplied LMR Context in ls-byte + * order. + * DAT_LMRC_TO_MSB(lmrc) returns the supplied LMR Context in ms-byte + * order. + * DAT_RMRC_TO_LSB(rmrc) returns the supplied RMR Context in ls-byte + * order. + * DAT_RMRC_TO_MSB(rmrc) returns the supplied RMR Context in ms-byte + * order. + * DAT_VADDR_TO_LSB(vaddr) returns the supplied Virtual Address in ls-byte + * order. + * DAT_VADDR_TO_MSB(vaddr) returns the supplied Virtual Address in + * ms-byte order. + * DAT_VLEN_TO_LSB(vlen) returns the supplied length in ls-byte order. + * DAT_VLEN_TO_MSB(vlen) returns the supplied length in ms-byte order. + * + * Consumers are free to use 64-bit or 32-bit arithmetic for local or remote + * memory address and length manipulation in their preferred byte-order. Only the + * LMR and RMR Triplets passed to a Provider as part of a Posted DTO are + * required to be in 64-bit address and local host order formats. Providers shall + * convert RMR_Triplets to a Transport-required wire format. + * + * For the best performance, Consumers should align each buffer segment to + * the boundary specified by the dat_optimal_alignment. + */ +typedef DAT_UINT32 DAT_LMR_CONTEXT; +typedef DAT_UINT32 DAT_RMR_CONTEXT; + +typedef DAT_UINT64 DAT_VLEN; +typedef DAT_UINT64 DAT_VADDR; +typedef DAT_UINT32 DAT_SEG_LENGTH; /* The maximum data segment length */ + +typedef struct dat_provider_attr DAT_PROVIDER_ATTR; +typedef struct dat_evd_param DAT_EVD_PARAM; +typedef struct dat_lmr_param DAT_LMR_PARAM; +typedef enum dat_lmr_param_mask DAT_LMR_PARAM_MASK; + +/* It is legal for the Consumer to specify zero for segment_length + * of the dat_lmr_triplet. When 0 is specified for the + * segment_length then the other two elements of the + * dat_lmr_triplet are irrelevant and can be invalid. + */ + +typedef struct dat_lmr_triplet +{ + DAT_VADDR virtual_address;/* 64-bit address */ + DAT_SEG_LENGTH segment_length; /* 32-bit length */ + DAT_LMR_CONTEXT lmr_context; /* 32-bit lmr_context */ +} DAT_LMR_TRIPLET; + +typedef struct dat_rmr_triplet +{ + DAT_VADDR virtual_address;/* 64-bit address */ + DAT_SEG_LENGTH segment_length; /* 32-bit length */ + DAT_RMR_CONTEXT rmr_context; /* 32-bit rmr_context */ +} DAT_RMR_TRIPLET; + +/* Memory privileges */ + +typedef enum dat_mem_priv_flags +{ + DAT_MEM_PRIV_NONE_FLAG = 0x00, + DAT_MEM_PRIV_LOCAL_READ_FLAG = 0x01, + DAT_MEM_PRIV_REMOTE_READ_FLAG = 0x02, + DAT_MEM_PRIV_LOCAL_WRITE_FLAG = 0x10, + DAT_MEM_PRIV_REMOTE_WRITE_FLAG = 0x20, + DAT_MEM_PRIV_ALL_FLAG = 0x33 +#ifdef DAT_EXTENSIONS +/* To be used by DAT extensions as a starting + point of extension memory privileges */ + ,DAT_MEM_PRIV_EXTENSION_BASE = 0x40 +#endif /* DAT_EXTENSIONS */ + +} DAT_MEM_PRIV_FLAGS; + +/* For backward compatibility with DAT-1.0, memory privileges values are + * supported + */ +#define DAT_MEM_PRIV_READ_FLAG (DAT_MEM_PRIV_LOCAL_READ_FLAG | DAT_MEM_PRIV_REMOTE_READ_FLAG) +#define DAT_MEM_PRIV_WRITE_FLAG (DAT_MEM_PRIV_LOCAL_WRITE_FLAG | DAT_MEM_PRIV_REMOTE_WRITE_FLAG) + +/* LMR VA types */ +typedef enum dat_va_type +{ + DAT_VA_TYPE_VA = 0x0, + DAT_VA_TYPE_ZB = 0x1 +} DAT_VA_TYPE; + +/* RMR Arguments & RMR Arguments Mask */ + +/* DAPL 2.0 addition */ +/* Defines RMR protection scope */ +typedef enum dat_rmr_scope +{ + DAT_RMR_SCOPE_EP, /* bound to at most one EP at a time. */ + DAT_RMR_SCOPE_PZ, /* bound to a Protection Zone */ + DAT_RMR_SCOPE_ANY /* Supports all types */ +} DAT_RMR_SCOPE; + +typedef struct dat_rmr_param +{ + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_LMR_TRIPLET lmr_triplet; + DAT_MEM_PRIV_FLAGS mem_priv; + DAT_RMR_CONTEXT rmr_context; + DAT_RMR_SCOPE rmr_scope; + DAT_VA_TYPE va_type; +} DAT_RMR_PARAM; + +typedef enum dat_rmr_param_mask +{ + DAT_RMR_FIELD_IA_HANDLE = 0x01, + DAT_RMR_FIELD_PZ_HANDLE = 0x02, + DAT_RMR_FIELD_LMR_TRIPLET = 0x04, + DAT_RMR_FIELD_MEM_PRIV = 0x08, + DAT_RMR_FIELD_RMR_CONTEXT = 0x10, + DAT_RMR_FIELD_RMR_SCOPE = 0x20, + DAT_RMR_FIELD_VA_TYPE = 0x40, + + DAT_RMR_FIELD_ALL = 0x7F +} DAT_RMR_PARAM_MASK; + +/* Provider attributes */ + +typedef enum dat_iov_ownership +{ + /* Not a modification by the Provider; the Consumer can use anytime. */ + DAT_IOV_CONSUMER = 0x0, + /* Provider does not modify returned IOV DTO on completion. */ + DAT_IOV_PROVIDER_NOMOD = 0x1, + /* Provider can modify IOV DTO on completion; can't trust it. */ + DAT_IOV_PROVIDER_MOD = 0x2 +} DAT_IOV_OWNERSHIP; + +typedef enum dat_ep_creator_for_psp +{ + DAT_PSP_CREATES_EP_NEVER, /* Provider never creates Endpoint. */ + DAT_PSP_CREATES_EP_IFASKED, /* Provider creates Endpoint if asked. */ + DAT_PSP_CREATES_EP_ALWAYS /* Provider always creates Endpoint. */ +} DAT_EP_CREATOR_FOR_PSP; + +/* General Interface Adapter attributes. These apply to both udat and kdat. */ + +/* To support backwards compatibility for DAPL-1.0 */ +#define max_rdma_read_per_ep max_rdma_read_per_ep_in +#define DAT_IA_FIELD_IA_MAX_DTO_PER_OP DAT_IA_FIELD_IA_MAX_DTO_PER_EP_IN + +/* To support backwards compatibility for DAPL-1.0 & DAPL-1.1 */ +#define max_mtu_size max_message_size + +/* Query for provider IA extension support */ +typedef enum dat_extension +{ + DAT_EXTENSION_NONE,/* no extension supported. */ + DAT_EXTENSION_IB, /* IB extension. */ + DAT_EXTENSION_IW /* iWARP extension. */ +} DAT_EXTENSION; + +typedef struct dat_ia_attr DAT_IA_ATTR; + +/* To support backwards compatibility for DAPL-1.0 & DAPL-1.1 */ +#define DAT_IA_FIELD_IA_MAX_MTU_SIZE DAT_IA_FIELD_IA_MAX_MESSAGE_SIZE + +typedef DAT_UINT64 DAT_IA_ATTR_MASK; + +#define DAT_IA_FIELD_IA_ADAPTER_NAME UINT64_C(0x000000001) +#define DAT_IA_FIELD_IA_VENDOR_NAME UINT64_C(0x000000002) +#define DAT_IA_FIELD_IA_HARDWARE_MAJOR_VERSION UINT64_C(0x000000004) +#define DAT_IA_FIELD_IA_HARDWARE_MINOR_VERSION UINT64_C(0x000000008) +#define DAT_IA_FIELD_IA_FIRMWARE_MAJOR_VERSION UINT64_C(0x000000010) +#define DAT_IA_FIELD_IA_FIRMWARE_MINOR_VERSION UINT64_C(0x000000020) +#define DAT_IA_FIELD_IA_ADDRESS_PTR UINT64_C(0x000000040) +#define DAT_IA_FIELD_IA_MAX_EPS UINT64_C(0x000000080) +#define DAT_IA_FIELD_IA_MAX_DTO_PER_EP UINT64_C(0x000000100) +#define DAT_IA_FIELD_IA_MAX_RDMA_READ_PER_EP_IN UINT64_C(0x000000200) +#define DAT_IA_FIELD_IA_MAX_RDMA_READ_PER_EP_OUT UINT64_C(0x000000400) +#define DAT_IA_FIELD_IA_MAX_EVDS UINT64_C(0x000000800) +#define DAT_IA_FIELD_IA_MAX_EVD_QLEN UINT64_C(0x000001000) +#define DAT_IA_FIELD_IA_MAX_IOV_SEGMENTS_PER_DTO UINT64_C(0x000002000) +#define DAT_IA_FIELD_IA_MAX_LMRS UINT64_C(0x000004000) +#define DAT_IA_FIELD_IA_MAX_LMR_BLOCK_SIZE UINT64_C(0x000008000) +#define DAT_IA_FIELD_IA_MAX_LMR_VIRTUAL_ADDRESS UINT64_C(0x000010000) +#define DAT_IA_FIELD_IA_MAX_PZS UINT64_C(0x000020000) +#define DAT_IA_FIELD_IA_MAX_MESSAGE_SIZE UINT64_C(0x000040000) +#define DAT_IA_FIELD_IA_MAX_RDMA_SIZE UINT64_C(0x000080000) +#define DAT_IA_FIELD_IA_MAX_RMRS UINT64_C(0x000100000) +#define DAT_IA_FIELD_IA_MAX_RMR_TARGET_ADDRESS UINT64_C(0x000200000) +#define DAT_IA_FIELD_IA_MAX_SRQS UINT64_C(0x000400000) +#define DAT_IA_FIELD_IA_MAX_EP_PER_SRQ UINT64_C(0x000800000) +#define DAT_IA_FIELD_IA_MAX_RECV_PER_SRQ UINT64_C(0x001000000) +#define DAT_IA_FIELD_IA_MAX_IOV_SEGMENTS_PER_RDMA_READ UINT64_C(0x002000000) +#define DAT_IA_FIELD_IA_MAX_IOV_SEGMENTS_PER_RDMA_WRITE UINT64_C(0x004000000) +#define DAT_IA_FIELD_IA_MAX_RDMA_READ_IN UINT64_C(0x008000000) +#define DAT_IA_FIELD_IA_MAX_RDMA_READ_OUT UINT64_C(0x010000000) +#define DAT_IA_FIELD_IA_MAX_RDMA_READ_PER_EP_IN_GUARANTEED UINT64_C(0x020000000) +#define DAT_IA_FIELD_IA_MAX_RDMA_READ_PER_EP_OUT_GUARANTEED UINT64_C(0x040000000) +#define DAT_IA_FIELD_IA_ZB_SUPPORTED UINT64_C(0x080000000) +#define DAT_IA_FIELD_IA_EXTENSIONS_SUPPORTED UINT64_C(0x100000000) + +/* To support backwards compatibility for DAPL-1.0 & DAPL-1.1 */ +#define DAT_IA_ALL DAT_IA_FIELD_ALL +#define DAT_IA_FIELD_NONE UINT64_C(0x0) + +/* Endpoint attributes */ + +typedef enum dat_service_type +{ + DAT_SERVICE_TYPE_RC /* reliable connections */ +} DAT_SERVICE_TYPE; + +typedef struct dat_ep_attr +{ + DAT_SERVICE_TYPE service_type; + DAT_SEG_LENGTH max_message_size; + DAT_SEG_LENGTH max_rdma_size; + DAT_QOS qos; + DAT_COMPLETION_FLAGS recv_completion_flags; + DAT_COMPLETION_FLAGS request_completion_flags; + DAT_COUNT max_recv_dtos; + DAT_COUNT max_request_dtos; + DAT_COUNT max_recv_iov; + DAT_COUNT max_request_iov; + DAT_COUNT max_rdma_read_in; + DAT_COUNT max_rdma_read_out; + DAT_COUNT srq_soft_hw; + DAT_COUNT max_rdma_read_iov; + DAT_COUNT max_rdma_write_iov; + DAT_COUNT ep_transport_specific_count; + DAT_NAMED_ATTR * ep_transport_specific; + DAT_COUNT ep_provider_specific_count; + DAT_NAMED_ATTR * ep_provider_specific; +} DAT_EP_ATTR; + +/* Endpoint Parameters */ + +/* For backwards compatibility */ +#define DAT_EP_STATE_ERROR DAT_EP_STATE_DISCONNECTED + +typedef enum dat_ep_state +{ + DAT_EP_STATE_UNCONNECTED, /* quiescent state */ + DAT_EP_STATE_UNCONFIGURED_UNCONNECTED, + DAT_EP_STATE_RESERVED, + DAT_EP_STATE_UNCONFIGURED_RESERVED, + DAT_EP_STATE_PASSIVE_CONNECTION_PENDING, + DAT_EP_STATE_UNCONFIGURED_PASSIVE, + DAT_EP_STATE_ACTIVE_CONNECTION_PENDING, + DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING, + DAT_EP_STATE_UNCONFIGURED_TENTATIVE, + DAT_EP_STATE_CONNECTED, + DAT_EP_STATE_DISCONNECT_PENDING, + DAT_EP_STATE_DISCONNECTED, + DAT_EP_STATE_COMPLETION_PENDING, + DAT_EP_STATE_CONNECTED_SINGLE_PATH, + DAT_EP_STATE_CONNECTED_MULTI_PATH +} DAT_EP_STATE; + +typedef struct dat_ep_param +{ + DAT_IA_HANDLE ia_handle; + DAT_EP_STATE ep_state; + DAT_COMM comm; + DAT_IA_ADDRESS_PTR local_ia_address_ptr; + DAT_PORT_QUAL local_port_qual; + DAT_IA_ADDRESS_PTR remote_ia_address_ptr; + DAT_PORT_QUAL remote_port_qual; + DAT_PZ_HANDLE pz_handle; + DAT_EVD_HANDLE recv_evd_handle; + DAT_EVD_HANDLE request_evd_handle; + DAT_EVD_HANDLE connect_evd_handle; + DAT_SRQ_HANDLE srq_handle; + DAT_EP_ATTR ep_attr; +} DAT_EP_PARAM; + +typedef DAT_UINT64 DAT_EP_PARAM_MASK; +#define DAT_EP_FIELD_IA_HANDLE UINT64_C(0x00000001) +#define DAT_EP_FIELD_EP_STATE UINT64_C(0x00000002) +#define DAT_EP_FIELD_COMM UINT64_C(0x00000004) +#define DAT_EP_FIELD_LOCAL_IA_ADDRESS_PTR UINT64_C(0x00000008) +#define DAT_EP_FIELD_LOCAL_PORT_QUAL UINT64_C(0x00000010) +#define DAT_EP_FIELD_REMOTE_IA_ADDRESS_PTR UINT64_C(0x00000020) +#define DAT_EP_FIELD_REMOTE_PORT_QUAL UINT64_C(0x00000040) +#define DAT_EP_FIELD_PZ_HANDLE UINT64_C(0x00000080) +#define DAT_EP_FIELD_RECV_EVD_HANDLE UINT64_C(0x00000100) +#define DAT_EP_FIELD_REQUEST_EVD_HANDLE UINT64_C(0x00000200) +#define DAT_EP_FIELD_CONNECT_EVD_HANDLE UINT64_C(0x00000400) +#define DAT_EP_FIELD_SRQ_HANDLE UINT64_C(0x00000800) + /* Remainder of values from EP_ATTR, 0x00001000 and up */ +#define DAT_EP_FIELD_EP_ATTR_SERVICE_TYPE UINT64_C(0x00001000) +#define DAT_EP_FIELD_EP_ATTR_MAX_MESSAGE_SIZE UINT64_C(0x00002000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RDMA_SIZE UINT64_C(0x00004000) +#define DAT_EP_FIELD_EP_ATTR_QOS UINT64_C(0x00008000) +#define DAT_EP_FIELD_EP_ATTR_RECV_COMPLETION_FLAGS UINT64_C(0x00010000) +#define DAT_EP_FIELD_EP_ATTR_REQUEST_COMPLETION_FLAGS UINT64_C(0x00020000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RECV_DTOS UINT64_C(0x00040000) +#define DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_DTOS UINT64_C(0x00080000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RECV_IOV UINT64_C(0x00100000) +#define DAT_EP_FIELD_EP_ATTR_MAX_REQUEST_IOV UINT64_C(0x00200000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RDMA_READ_IN UINT64_C(0x00400000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RDMA_READ_OUT UINT64_C(0x00800000) +#define DAT_EP_FIELD_EP_ATTR_SRQ_SOFT_HW UINT64_C(0x01000000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RDMA_READ_IOV UINT64_C(0x02000000) +#define DAT_EP_FIELD_EP_ATTR_MAX_RDMA_WRITE_IOV UINT64_C(0x04000000) +#define DAT_EP_FIELD_EP_ATTR_NUM_TRANSPORT_ATTR UINT64_C(0x08000000) +#define DAT_EP_FIELD_EP_ATTR_TRANSPORT_SPECIFIC_ATTR UINT64_C(0x10000000) +#define DAT_EP_FIELD_EP_ATTR_NUM_PROVIDER_ATTR UINT64_C(0x20000000) +#define DAT_EP_FIELD_EP_ATTR_PROVIDER_SPECIFIC_ATTR UINT64_C(0x40000000) +#define DAT_EP_FIELD_EP_ATTR_ALL UINT64_C(0x7FFFF000) + +#define DAT_EP_FIELD_ALL UINT64_C(0x7FFFFFFF) + +#define DAT_WATERMARK_INFINITE ((DAT_COUNT)~0) +#define DAT_HW_DEFAULT DAT_WATERMARK_INFINITE +#define DAT_SRQ_LW_DEFAULT 0x0 + +typedef enum dat_srq_state +{ + DAT_SRQ_STATE_OPERATIONAL, + DAT_SRQ_STATE_ERROR +} DAT_SRQ_STATE; + +#define DAT_VALUE_UNKNOWN (((DAT_COUNT) ~0)-1) + +typedef struct dat_srq_attr +{ + DAT_COUNT max_recv_dtos; + DAT_COUNT max_recv_iov; + DAT_COUNT low_watermark; +} DAT_SRQ_ATTR; + +typedef struct dat_srq_param +{ + DAT_IA_HANDLE ia_handle; + DAT_SRQ_STATE srq_state; + DAT_PZ_HANDLE pz_handle; + DAT_COUNT max_recv_dtos; + DAT_COUNT max_recv_iov; + DAT_COUNT low_watermark; + DAT_COUNT available_dto_count; + DAT_COUNT outstanding_dto_count; +} DAT_SRQ_PARAM; + +typedef enum dat_srq_param_mask +{ + DAT_SRQ_FIELD_IA_HANDLE = 0x001, + DAT_SRQ_FIELD_SRQ_STATE = 0x002, + DAT_SRQ_FIELD_PZ_HANDLE = 0x004, + DAT_SRQ_FIELD_MAX_RECV_DTO = 0x008, + DAT_SRQ_FIELD_MAX_RECV_IOV = 0x010, + DAT_SRQ_FIELD_LOW_WATERMARK = 0x020, + DAT_SRQ_FIELD_AVAILABLE_DTO_COUNT = 0x040, + DAT_SRQ_FIELD_OUTSTANDING_DTO_COUNT = 0x080, + + DAT_SRQ_FIELD_ALL = 0x0FF +} DAT_SRQ_PARAM_MASK; + +/* PZ Parameters */ + +typedef struct dat_pz_param +{ + DAT_IA_HANDLE ia_handle; +} DAT_PZ_PARAM; + +typedef enum dat_pz_param_mask +{ + DAT_PZ_FIELD_IA_HANDLE = 0x01, + + DAT_PZ_FIELD_ALL = 0x01 +} DAT_PZ_PARAM_MASK; + +/* PSP Parameters */ + +typedef struct dat_psp_param +{ + DAT_IA_HANDLE ia_handle; + DAT_CONN_QUAL conn_qual; + DAT_EVD_HANDLE evd_handle; + DAT_PSP_FLAGS psp_flags; +} DAT_PSP_PARAM; + +typedef enum dat_psp_param_mask +{ + DAT_PSP_FIELD_IA_HANDLE = 0x01, + DAT_PSP_FIELD_CONN_QUAL = 0x02, + DAT_PSP_FIELD_EVD_HANDLE = 0x04, + DAT_PSP_FIELD_PSP_FLAGS = 0x08, + + DAT_PSP_FIELD_ALL = 0x0F +} DAT_PSP_PARAM_MASK; + +/* RSP Parameters */ + +typedef struct dat_rsp_param +{ + DAT_IA_HANDLE ia_handle; + DAT_CONN_QUAL conn_qual; + DAT_EVD_HANDLE evd_handle; + DAT_EP_HANDLE ep_handle; +} DAT_RSP_PARAM; + +typedef enum dat_rsp_param_mask +{ + DAT_RSP_FIELD_IA_HANDLE = 0x01, + DAT_RSP_FIELD_CONN_QUAL = 0x02, + DAT_RSP_FIELD_EVD_HANDLE = 0x04, + DAT_RSP_FIELD_EP_HANDLE = 0x08, + + DAT_RSP_FIELD_ALL = 0x0F +} DAT_RSP_PARAM_MASK; + +/* CSP Parameters */ +typedef struct dat_csp_param +{ + DAT_IA_HANDLE ia_handle; + DAT_COMM *comm; + DAT_IA_ADDRESS_PTR address_ptr; + DAT_EVD_HANDLE evd_handle; +} DAT_CSP_PARAM; + +typedef enum dat_csp_param_mask +{ + DAT_CSP_FIELD_IA_HANDLE = 0x01, + DAT_CSP_FIELD_COMM = 0x02, + DAT_CSP_FIELD_IA_ADDRESS = 0x04, + DAT_CSP_FIELD_EVD_HANDLE = 0x08, + + DAT_CSP_FIELD_ALL = 0x0F +} DAT_CSP_PARAM_MASK; + +/* Connection Request Parameters. + * + * The Connection Request does not provide Remote Endpoint attributes. + * If a local Consumer needs this information, the remote Consumer should + * encode it into Private Data. + */ + +typedef struct dat_cr_param +{ + /* Remote IA whose Endpoint requested the connection. */ + DAT_IA_ADDRESS_PTR remote_ia_address_ptr; + + /* Port qualifier of the remote Endpoint of the requested connection.*/ + DAT_PORT_QUAL remote_port_qual; + + /* Size of the Private Data. */ + DAT_COUNT private_data_size; + + /* Pointer to the Private Data passed by remote side in the Connection + * Request. + */ + DAT_PVOID private_data; + /* The local Endpoint provided by the Service Point for the requested + * connection. It is the only Endpoint that can accept a Connection + * Request on this Service Point. The value DAT_HANDLE_NULL + * represents that there is no associated local Endpoint for the + * requested connection. + */ + DAT_EP_HANDLE local_ep_handle; +} DAT_CR_PARAM; + +typedef enum dat_cr_param_mask +{ + DAT_CR_FIELD_REMOTE_IA_ADDRESS_PTR = 0x01, + DAT_CR_FIELD_REMOTE_PORT_QUAL = 0x02, + DAT_CR_FIELD_PRIVATE_DATA_SIZE = 0x04, + DAT_CR_FIELD_PRIVATE_DATA = 0x08, + DAT_CR_FIELD_LOCAL_EP_HANDLE = 0x10, + + DAT_CR_FIELD_ALL = 0x1F +} DAT_CR_PARAM_MASK; + +/************************** Events ******************************/ + +/* Completion status flags */ + + /* DTO completion status */ + + /* For backwards compatibility */ +#define DAT_DTO_LENGTH_ERROR DAT_DTO_ERR_LOCAL_LENGTH +#define DAT_DTO_FAILURE DAT_DTO_ERR_FLUSHED + +typedef enum dat_dto_completion_status +{ + DAT_DTO_SUCCESS = 0, + DAT_DTO_ERR_FLUSHED = 1, + DAT_DTO_ERR_LOCAL_LENGTH = 2, + DAT_DTO_ERR_LOCAL_EP = 3, + DAT_DTO_ERR_LOCAL_PROTECTION = 4, + DAT_DTO_ERR_BAD_RESPONSE = 5, + DAT_DTO_ERR_REMOTE_ACCESS = 6, + DAT_DTO_ERR_REMOTE_RESPONDER = 7, + DAT_DTO_ERR_TRANSPORT = 8, + DAT_DTO_ERR_RECEIVER_NOT_READY = 9, + DAT_DTO_ERR_PARTIAL_PACKET = 10, + DAT_RMR_OPERATION_FAILED = 11, + DAT_DTO_ERR_LOCAL_MM_ERROR = 12 /* kdat specific */ +} DAT_DTO_COMPLETION_STATUS; + + /* RMR completion status */ + + /* For backwards compatibility */ +#define DAT_RMR_BIND_SUCCESS DAT_DTO_SUCCESS +#define DAT_RMR_BIND_FAILURE DAT_DTO_ERR_FLUSHED + +/* RMR completion status */ +#define DAT_RMR_BIND_COMPLETION_STATUS DAT_DTO_COMPLETION_STATUS + +/* Completion group structs (six total) */ + +/* DTO completion event data */ +/* transfered_length is not defined if status is not DAT_SUCCESS */ +/* invalidate_flag and rmr_context are not defined if status is not DAT_SUCCESS */ + +typedef struct dat_dto_completion_event_data +{ + DAT_EP_HANDLE ep_handle; + DAT_DTO_COOKIE user_cookie; + DAT_DTO_COMPLETION_STATUS status; + DAT_SEG_LENGTH transfered_length; + DAT_DTOS operation; + DAT_RMR_CONTEXT rmr_context; +} DAT_DTO_COMPLETION_EVENT_DATA; + + /* RMR bind completion event data */ +typedef struct dat_rmr_bind_completion_event_data +{ + DAT_RMR_HANDLE rmr_handle; + DAT_RMR_COOKIE user_cookie; + DAT_RMR_BIND_COMPLETION_STATUS status; +} DAT_RMR_BIND_COMPLETION_EVENT_DATA; + +typedef union dat_sp_handle +{ + DAT_RSP_HANDLE rsp_handle; + DAT_PSP_HANDLE psp_handle; + DAT_CSP_HANDLE csp_handle; +} DAT_SP_HANDLE; + + /* Connection Request Arrival event data */ +typedef struct dat_cr_arrival_event_data +{ + /* Handle to the Service Point that received the Connection Request + * from the remote side. If the Service Point was Reserved, sp_handle is + * DAT_HANDLE_NULL because the reserved Service Point is + * automatically destroyed upon generating this event. Can be PSP, + * CSP, or RSP. + */ + DAT_SP_HANDLE sp_handle; + + /* Address of the IA on which the Connection Request arrived. */ + DAT_IA_ADDRESS_PTR local_ia_address_ptr; + + /* Connection Qualifier of the IA on which the Service Point received a + * Connection Request. + */ + DAT_CONN_QUAL conn_qual; + + /* The Connection Request instance created by a Provider for the + * arrived Connection Request. Consumers can find out private_data + * passed by a remote Consumer from cr_handle. It is up to a Consumer + * to dat_cr_accept or dat_cr_reject of the Connection Request. + */ + DAT_CR_HANDLE cr_handle; + + /* The binary indicator whether the arrived privata data was trancated + * or not. + * The default value of 0 means not truncation of received private data. */ + DAT_BOOLEAN truncate_flag; + +} DAT_CR_ARRIVAL_EVENT_DATA; + + + /* Connection event data */ +typedef struct dat_connection_event_data +{ + DAT_EP_HANDLE ep_handle; + DAT_COUNT private_data_size; + DAT_PVOID private_data; +} DAT_CONNECTION_EVENT_DATA; + +/* Async Error event data */ + +/* For unaffiliated asynchronous event dat_handle is ia_handle. For + * Endpoint affiliated asynchronous event dat_handle is ep_handle. For + * EVD affiliated asynchronous event dat_handle is evd_handle. For SRQ + * affiliated asynchronous event dat_handle is srq_handle. For Memory + * affiliated asynchronous event dat_handle is either lmr_handle, + * rmr_handle or pz_handle. + */ +typedef struct dat_asynch_error_event_data +{ + DAT_HANDLE dat_handle; /* either IA, EP, EVD, SRQ, */ + /* LMR, RMR, or PZ handle */ + DAT_COUNT reason; /* object specific */ +} DAT_ASYNCH_ERROR_EVENT_DATA; + +/* The reason is object type specific and its values are defined below. */ +typedef enum ia_async_error_reason +{ + DAT_IA_CATASTROPHIC_ERROR, + DAT_IA_OTHER_ERROR +} DAT_IA_ASYNC_ERROR_REASON; + +typedef enum ep_async_error_reason +{ + DAT_EP_TRANSFER_TO_ERROR, + DAT_EP_OTHER_ERROR, + DAT_SRQ_SOFT_HIGH_WATERMARK_EVENT +} DAT_EP_ASYNC_ERROR_REASON; + +typedef enum evd_async_error_reason +{ + DAT_EVD_OVERFLOW_ERROR, + DAT_EVD_OTHER_ERROR +} DAT_EVD_ASYNC_ERROR_REASON; + +typedef enum srq_async_error_reason +{ + DAT_SRQ_TRANSFER_TO_ERROR, + DAT_SRQ_OTHER_ERROR, + DAT_SRQ_LOW_WATERMARK_EVENT +} DAT_SRQ_ASYNC_ERROR_REASON; + +typedef enum lmr_async_error_reason +{ + DAT_LMR_OTHER_ERROR +} DAT_LMR_ASYNC_ERROR_REASON; + +typedef enum rmr_async_error_reason +{ + DAT_RMR_OTHER_ERROR +} DAT_RMR_ASYNC_ERROR_REASON; + +typedef enum pz_async_error_reason +{ + DAT_PZ_OTHER_ERROR +} DAT_PZ_ASYNC_ERROR_REASON; + +/* Software event data */ +typedef struct dat_software_event_data +{ + DAT_PVOID pointer; +} DAT_SOFTWARE_EVENT_DATA; + +typedef enum dat_event_number +{ + DAT_DTO_COMPLETION_EVENT = 0x00001, + DAT_RMR_BIND_COMPLETION_EVENT = 0x01001, + DAT_CONNECTION_REQUEST_EVENT = 0x02001, + DAT_CONNECTION_EVENT_ESTABLISHED = 0x04001, + DAT_CONNECTION_EVENT_PEER_REJECTED = 0x04002, + DAT_CONNECTION_EVENT_NON_PEER_REJECTED = 0x04003, + DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR = 0x04004, + DAT_CONNECTION_EVENT_DISCONNECTED = 0x04005, + DAT_CONNECTION_EVENT_BROKEN = 0x04006, + DAT_CONNECTION_EVENT_TIMED_OUT = 0x04007, + DAT_CONNECTION_EVENT_UNREACHABLE = 0x04008, + DAT_ASYNC_ERROR_EVD_OVERFLOW = 0x08001, + DAT_ASYNC_ERROR_IA_CATASTROPHIC = 0x08002, + DAT_ASYNC_ERROR_EP_BROKEN = 0x08003, + DAT_ASYNC_ERROR_TIMED_OUT = 0x08004, + DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR = 0x08005, + DAT_HA_DOWN_TO_1 = 0x08101, + DAT_HA_UP_TO_MULTI_PATH = 0x08102, + + DAT_SOFTWARE_EVENT = 0x10001 +#ifdef DAT_EXTENSIONS + ,DAT_EXTENSION_EVENT = 0x20000, + DAT_IB_EXTENSION_RANGE_BASE = 0x40000, + DAT_IW_EXTENSION_RANGE_BASE = 0x80000 +#endif /* DAT_EXTENSIONS */ + +} DAT_EVENT_NUMBER; + +/* Union for event Data */ + +typedef union dat_event_data +{ + DAT_DTO_COMPLETION_EVENT_DATA dto_completion_event_data; + DAT_RMR_BIND_COMPLETION_EVENT_DATA rmr_completion_event_data; + DAT_CR_ARRIVAL_EVENT_DATA cr_arrival_event_data; + DAT_CONNECTION_EVENT_DATA connect_event_data; + DAT_ASYNCH_ERROR_EVENT_DATA asynch_error_event_data; + DAT_SOFTWARE_EVENT_DATA software_event_data; +} DAT_EVENT_DATA; + +/* Event struct that holds all event information */ + +typedef struct dat_event +{ + DAT_EVENT_NUMBER event_number; + DAT_EVD_HANDLE evd_handle; + DAT_EVENT_DATA event_data; +#ifdef DAT_EXTENSIONS + DAT_UINT64 event_extension_data[8]; +#endif /* DAT_EXTENSIONS */ +} DAT_EVENT; + +/* Provider/registration info */ + +typedef struct dat_provider_info +{ + char ia_name[DAT_NAME_MAX_LENGTH]; + DAT_UINT32 dapl_version_major; + DAT_UINT32 dapl_version_minor; + DAT_BOOLEAN is_thread_safe; +} DAT_PROVIDER_INFO; + +/*************************************************************** + * + * FUNCTION PROTOTYPES + ***************************************************************/ + +/* + * IA functions + * + * Note that there are actual 'dat_ia_open' and 'dat_ia_close' + * functions, it is not just a re-directing #define. That is + * because the functions may have to ensure that the provider + * library is loaded before it can call it, and may choose to + * unload the library after the last close. + */ + +extern DAT_RETURN DAT_API dat_ia_openv ( + IN const DAT_NAME_PTR, /* provider */ + IN DAT_COUNT, /* asynch_evd_min_qlen */ + INOUT DAT_EVD_HANDLE *, /* asynch_evd_handle */ + OUT DAT_IA_HANDLE *, /* ia_handle */ + IN DAT_UINT32, /* dat major version number */ + IN DAT_UINT32, /* dat minor version number */ + IN DAT_BOOLEAN); /* dat thread safety */ + +#define dat_ia_open(name, qlen, async_evd, ia) \ + dat_ia_openv((name), (qlen), (async_evd), (ia), \ + DAT_VERSION_MAJOR, DAT_VERSION_MINOR, \ + DAT_THREADSAFE) + +extern DAT_RETURN DAT_API dat_ia_query ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_EVD_HANDLE *, /* async_evd_handle */ + IN DAT_IA_ATTR_MASK, /* ia_attr_mask */ + OUT DAT_IA_ATTR *, /* ia_attr */ + IN DAT_PROVIDER_ATTR_MASK, /* provider_attr_mask */ + OUT DAT_PROVIDER_ATTR * ); /* provider_attr */ + +extern DAT_RETURN DAT_API dat_ia_close ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CLOSE_FLAGS ); /* close_flags */ + +/* helper functions */ + +extern DAT_RETURN DAT_API dat_set_consumer_context ( + IN DAT_HANDLE, /* dat_handle */ + IN DAT_CONTEXT); /* context */ + +extern DAT_RETURN DAT_API dat_get_consumer_context ( + IN DAT_HANDLE, /* dat_handle */ + OUT DAT_CONTEXT * ); /* context */ + +extern DAT_RETURN DAT_API dat_get_handle_type ( + IN DAT_HANDLE, /* dat_handle */ + OUT DAT_HANDLE_TYPE * ); /* handle_type */ + +/* CR functions */ + +extern DAT_RETURN DAT_API dat_cr_query ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_CR_PARAM_MASK, /* cr_param_mask */ + OUT DAT_CR_PARAM * ); /* cr_param */ + +extern DAT_RETURN DAT_API dat_cr_accept ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +extern DAT_RETURN DAT_API dat_cr_reject ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +/* For DAT-1.1 and above, this function is defined for both uDAPL and + * kDAPL. For DAT-1.0, it is only defined for uDAPL. + */ +extern DAT_RETURN DAT_API dat_cr_handoff ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_CONN_QUAL); /* handoff */ + +/* EVD functions */ + +extern DAT_RETURN DAT_API dat_evd_resize ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_COUNT ); /* evd_min_qlen */ + +extern DAT_RETURN DAT_API dat_evd_post_se ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN const DAT_EVENT * ); /* event */ + +extern DAT_RETURN DAT_API dat_evd_dequeue ( + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_EVENT * ); /* event */ + +extern DAT_RETURN DAT_API dat_evd_query ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_EVD_PARAM_MASK, /* evd_param_mask */ + OUT DAT_EVD_PARAM * ); /* evd_param */ + +extern DAT_RETURN DAT_API dat_evd_free ( + IN DAT_EVD_HANDLE ); /* evd_handle */ + +/* EP functions */ + +extern DAT_RETURN DAT_API dat_ep_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_EVD_HANDLE, /* recv_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* request_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* connect_evd_handle */ + IN const DAT_EP_ATTR *, /* ep_attributes */ + OUT DAT_EP_HANDLE * ); /* ep_handle */ + +extern DAT_RETURN DAT_API dat_ep_query ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_PARAM_MASK, /* ep_param_mask */ + OUT DAT_EP_PARAM * ); /* ep_param */ + +extern DAT_RETURN DAT_API dat_ep_modify ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_PARAM_MASK, /* ep_param_mask */ + IN const DAT_EP_PARAM * ); /* ep_param */ + +extern DAT_RETURN DAT_API dat_ep_connect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR, /* remote_ia_address */ + IN DAT_CONN_QUAL, /* remote_conn_qual */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID, /* private_data */ + IN DAT_QOS, /* quality_of_service */ + IN DAT_CONNECT_FLAGS ); /* connect_flags */ + +extern DAT_RETURN DAT_API dat_ep_dup_connect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_HANDLE, /* ep_dup_handle */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID, /* private_data */ + IN DAT_QOS); /* quality_of_service */ + +extern DAT_RETURN DAT_API dat_ep_common_connect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR, /* remote_ia_address */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +extern DAT_RETURN DAT_API dat_ep_disconnect ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_CLOSE_FLAGS ); /* close_flags */ + +extern DAT_RETURN DAT_API dat_ep_post_send ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dat_ep_post_send_with_invalidate ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + IN DAT_BOOLEAN, /* invalidate_flag */ + IN DAT_RMR_CONTEXT ); /* RMR to invalidate */ + +extern DAT_RETURN DAT_API dat_ep_post_recv ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dat_ep_post_rdma_read ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dat_ep_post_rdma_read_to_rmr ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN const DAT_RMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dat_ep_post_rdma_write ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN DAT_API dat_ep_get_status ( + IN DAT_EP_HANDLE, /* ep_handle */ + OUT DAT_EP_STATE *, /* ep_state */ + OUT DAT_BOOLEAN *, /* recv_idle */ + OUT DAT_BOOLEAN * ); /* request_idle */ + +extern DAT_RETURN DAT_API dat_ep_free ( + IN DAT_EP_HANDLE); /* ep_handle */ + +extern DAT_RETURN DAT_API dat_ep_reset ( + IN DAT_EP_HANDLE); /* ep_handle */ + +extern DAT_RETURN DAT_API dat_ep_create_with_srq ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_EVD_HANDLE, /* recv_evd_handle */ + IN DAT_EVD_HANDLE, /* request_evd_handle */ + IN DAT_EVD_HANDLE, /* connect_evd_handle */ + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN const DAT_EP_ATTR *, /* ep_attributes */ + OUT DAT_EP_HANDLE *); /* ep_handle */ + +extern DAT_RETURN DAT_API dat_ep_recv_query ( + IN DAT_EP_HANDLE, /* ep_handle */ + OUT DAT_COUNT *, /* nbufs_allocated */ + OUT DAT_COUNT *); /* bufs_alloc_span */ + +extern DAT_RETURN DAT_API dat_ep_set_watermark ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* soft_high_watermark */ + IN DAT_COUNT); /* hard_high_watermark */ + +/* LMR functions */ + +extern DAT_RETURN DAT_API dat_lmr_free ( + IN DAT_LMR_HANDLE); /* lmr_handle */ + +/* Non-coherent memory functions */ + +extern DAT_RETURN DAT_API dat_lmr_sync_rdma_read ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_LMR_TRIPLET *, /* local_segments */ + IN DAT_VLEN); /* num_segments */ + +extern DAT_RETURN DAT_API dat_lmr_sync_rdma_write ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_LMR_TRIPLET *, /* local_segments */ + IN DAT_VLEN); /* num_segments */ + +/* RMR functions */ + +extern DAT_RETURN DAT_API dat_rmr_create ( + IN DAT_PZ_HANDLE, /* pz_handle */ + OUT DAT_RMR_HANDLE *); /* rmr_handle */ + +extern DAT_RETURN DAT_API dat_rmr_create_for_ep ( + IN DAT_PZ_HANDLE, /* pz_handle */ + OUT DAT_RMR_HANDLE *); /* rmr_handle */ + +extern DAT_RETURN DAT_API dat_rmr_query ( + IN DAT_RMR_HANDLE, /* rmr_handle */ + IN DAT_RMR_PARAM_MASK, /* rmr_param_mask */ + OUT DAT_RMR_PARAM *); /* rmr_param */ + +extern DAT_RETURN DAT_API dat_rmr_bind ( + IN DAT_RMR_HANDLE, /* rmr_handle */ + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN const DAT_LMR_TRIPLET *,/* lmr_triplet */ + IN DAT_MEM_PRIV_FLAGS, /* mem_priv */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_RMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + OUT DAT_RMR_CONTEXT * ); /* context */ + +extern DAT_RETURN DAT_API dat_rmr_free ( + IN DAT_RMR_HANDLE); /* rmr_handle */ + +/* PSP functions */ + +extern DAT_RETURN DAT_API dat_psp_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CONN_QUAL, /* conn_qual */ + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_PSP_FLAGS, /* psp_flags */ + OUT DAT_PSP_HANDLE * ); /* psp_handle */ + +extern DAT_RETURN DAT_API dat_psp_create_any ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_CONN_QUAL *, /* conn_qual */ + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_PSP_FLAGS, /* psp_flags */ + OUT DAT_PSP_HANDLE * ); /* psp_handle */ + +extern DAT_RETURN DAT_API dat_psp_query ( + IN DAT_PSP_HANDLE, /* psp_handle */ + IN DAT_PSP_PARAM_MASK, /* psp_param_mask */ + OUT DAT_PSP_PARAM * ); /* psp_param */ + +extern DAT_RETURN DAT_API dat_psp_free ( + IN DAT_PSP_HANDLE ); /* psp_handle */ + +/* RSP functions */ + +extern DAT_RETURN DAT_API dat_rsp_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CONN_QUAL, /* conn_qual */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_RSP_HANDLE * ); /* rsp_handle */ + +extern DAT_RETURN DAT_API dat_rsp_query ( + IN DAT_RSP_HANDLE, /* rsp_handle */ + IN DAT_RSP_PARAM_MASK, /* rsp_param_mask */ + OUT DAT_RSP_PARAM * ); /* rsp_param */ + +extern DAT_RETURN DAT_API dat_rsp_free ( + IN DAT_RSP_HANDLE ); /* rsp_handle */ + +/* CSP functions */ + +extern DAT_RETURN DAT_API dat_csp_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COMM *, /* communicator */ + IN DAT_IA_ADDRESS_PTR, /* address */ + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_CSP_HANDLE * ); /* csp_handle */ + +extern DAT_RETURN DAT_API dat_csp_query ( + IN DAT_CSP_HANDLE, /* csp_handle */ + IN DAT_CSP_PARAM_MASK, /* csp_param_mask */ + OUT DAT_CSP_PARAM * ); /* csp_param */ + +extern DAT_RETURN DAT_API dat_csp_free ( + IN DAT_CSP_HANDLE ); /* csp_handle */ + +/* PZ functions */ + +extern DAT_RETURN DAT_API dat_pz_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_PZ_HANDLE * ); /* pz_handle */ + +extern DAT_RETURN DAT_API dat_pz_query ( + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_PZ_PARAM_MASK, /* pz_param_mask */ + OUT DAT_PZ_PARAM *); /* pz_param */ + +extern DAT_RETURN DAT_API dat_pz_free ( + IN DAT_PZ_HANDLE ); /* pz_handle */ + +/* SRQ functions */ + +extern DAT_RETURN DAT_API dat_srq_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_SRQ_ATTR *, /* srq_attr */ + OUT DAT_SRQ_HANDLE *); /* srq_handle */ + +extern DAT_RETURN DAT_API dat_srq_free ( + IN DAT_SRQ_HANDLE); /* srq_handle */ + +extern DAT_RETURN DAT_API dat_srq_post_recv ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE); /* user_cookie */ + +extern DAT_RETURN DAT_API dat_srq_query ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_SRQ_PARAM_MASK, /* srq_param_mask */ + OUT DAT_SRQ_PARAM *); /* srq_param */ + +extern DAT_RETURN DAT_API dat_srq_resize ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT); /* srq_max_recv_dto */ + +extern DAT_RETURN DAT_API dat_srq_set_lw ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT); /* low_watermark */ + +#ifdef DAT_EXTENSIONS +typedef int DAT_EXTENDED_OP; +extern DAT_RETURN DAT_API dat_extension_op( + IN DAT_HANDLE, /* handle */ + IN DAT_EXTENDED_OP, /* operation */ + IN ... ); /* args */ +#endif + +/* + * DAT registry functions. + * + * Note the dat_ia_open and dat_ia_close functions are linked to + * registration code which "redirects" to the appropriate provider. + */ +extern DAT_RETURN DAT_API dat_registry_list_providers ( + IN DAT_COUNT, /* max_to_return */ + OUT DAT_COUNT *, /* entries_returned */ + OUT DAT_PROVIDER_INFO *(dat_provider_list[]) ); /* dat_provider_list */ + +/* + * DAT error functions. + */ +extern DAT_RETURN DAT_API dat_strerror ( + IN DAT_RETURN, /* dat function return */ + OUT const char ** , /* major message string */ + OUT const char ** ); /* minor message string */ + +#ifdef __cplusplus +} +#endif + +#endif /* _DAT_H_ */ + diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_error.h b/trunk/ulp/dapl2/dat/include/dat/dat_error.h new file mode 100644 index 00000000..c87c5e77 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_error.h @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*********************************************************** + * + * HEADER: dat_error.h + * + * PURPOSE: DAT return codes + * + * Description: Header file for "DAPL: Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * Error types are compound types, as mapped out below. + * + *********************************************************/ + +#ifndef _DAT_ERROR_H_ +#define _DAT_ERROR_H_ + +/* + * + * All return codes are actually a 3-way tuple: + * + * type: DAT_RETURN_CLASS DAT_RETURN_TYPE DAT_RETURN_SUBTYPE + * bits: 31-30 29-16 15-0 + * + * 3 2 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | C | DAT_RETURN_TYPE | DAT_RETURN_SUBTYPE | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + */ + +/* + * Class Bits + */ +#define DAT_CLASS_ERROR 0x80000000 +#define DAT_CLASS_WARNING 0x40000000 +#define DAT_CLASS_SUCCESS 0x00000000 +/* + * DAT Error bits + */ +#define DAT_TYPE_MASK 0x3fff0000 /* mask for DAT_TYPE_STATUS bits */ +#define DAT_SUBTYPE_MASK 0x0000FFFF /* mask for DAT_SUBTYPE_STATUS bits */ + +/* + * Determining the success of an operation is best done with a macro; + * each of these returns a boolean value. + */ +#define DAT_IS_WARNING(status) ((DAT_UINT32)(status) & DAT_CLASS_WARNING) + +#define DAT_GET_TYPE(status) ((DAT_UINT32)(status) & DAT_TYPE_MASK) +#define DAT_GET_SUBTYPE(status) ((DAT_UINT32)(status) & DAT_SUBTYPE_MASK) + +/* + * DAT return types. The ERROR bit is enabled for these definitions + */ +typedef enum dat_return_type +{ + /* The operation was successful. */ + DAT_SUCCESS = 0x00000000, + + /* The operation was aborted because IA was closed or EVD was * + * destroyed. */ + DAT_ABORT = 0x00010000, + + /* The specified Connection Qualifier was in use. */ + DAT_CONN_QUAL_IN_USE = 0x00020000, + + /* The operation failed due to resource limitations. */ + DAT_INSUFFICIENT_RESOURCES = 0x00030000, + + /* Provider internal error. This error can be returned by any * + * operation when the Provider has detected an internal error.* + * This error does not mask any error caused by the Consumer. */ + DAT_INTERNAL_ERROR = 0x00040000, + + /* One of the DAT handles was invalid. */ + DAT_INVALID_HANDLE = 0x00050000, + + /* One of the parameters was invalid. */ + DAT_INVALID_PARAMETER = 0x00060000, + + /* One of the parameters was invalid for this operation. There* + * are Event Streams associated with the Event Dispatcher * + * feeding it. */ + DAT_INVALID_STATE = 0x00070000, + + /* The size of the receiving buffer is too small for sending * + * buffer data. The size of the local buffer is too small for* + * the data of the remote buffer. */ + DAT_LENGTH_ERROR = 0x00080000, + + /* The requested Model was not supported by the Provider. */ + DAT_MODEL_NOT_SUPPORTED = 0x00090000, + + /* The specified IA name was not found in the list of * + * registered Providers. */ + DAT_PROVIDER_NOT_FOUND = 0x000A0000, + + /* Protection violation for local or remote memory access * + * Protection Zone mismatch between an LMR of one of the * + * local_iov segments and the local Endpoint. */ + DAT_PRIVILEGES_VIOLATION = 0x000B0000, + + /* Privileges violation for local or remote memory access. One* + * of the LMRs used in local_iov was either invalid or did not* + * have local read privileges. */ + DAT_PROTECTION_VIOLATION = 0x000C0000, + + /* The operation timed out without a notification. */ + DAT_QUEUE_EMPTY = 0x000D0000, + + /* The Event Dispatcher queue is full. */ + DAT_QUEUE_FULL = 0x000E0000, + + /* The operation timed out. uDAPL ONLY */ + DAT_TIMEOUT_EXPIRED = 0x000F0000, + + /* The provider name was already registered */ + DAT_PROVIDER_ALREADY_REGISTERED = 0x00100000, + + /* The provider is "in-use" and cannot be closed at this time */ + DAT_PROVIDER_IN_USE = 0x00110000, + + /* The requested remote address is not valid or not reachable */ + DAT_INVALID_ADDRESS = 0x00120000, + + /* [Unix only] dat_evd_wait or dat_cno_wait has been * + * interrupted. */ + DAT_INTERRUPTED_CALL = 0x00130000, + + /* No Connection Qualifiers are available */ + DAT_CONN_QUAL_UNAVAILABLE = 0x00140000, + + /* kDAPL reserved LMR */ + DAT_RESERVED_LMR = 0x00150000, /* kdapl only */ + + /* The specified IP Port was in use. */ + DAT_PORT_IN_USE = 0x00160000, + /* The specified COMM not supported. */ + DAT_COMM_NOT_SUPPORTED = 0x00170000, + +#ifdef DAT_EXTENSIONS + /* The DAT extensions support. */ + DAT_EXTENSION_BASE = 0x10000000, + /* range 0x10000000 - 0x3FFF0000 is reserved for extensions */ +#endif /* DAT_EXTENSIONS */ + + /* Provider does not support the operation yet. */ + DAT_NOT_IMPLEMENTED = 0x3FFF0000 +} DAT_RETURN_TYPE; + +typedef DAT_UINT32 DAT_RETURN; + +/* Backward compatibility with DAT 1.0 */ +#define DAT_NAME_NOT_FOUND DAT_PROVIDER_NOT_FOUND + +/* + * DAT_RETURN_SUBTYPE listing + */ + +typedef enum dat_return_subtype +{ + /* First element is no subtype */ + DAT_NO_SUBTYPE, + /* ABORT sub types */ + /* call was interrupted by a signal, or otherwise */ + DAT_SUB_INTERRUPTED, + + /* DAT_CONN_QUAL_IN_USE has no subtypes */ + + /* INSUFFICIENT_RESOURCES subtypes */ + DAT_RESOURCE_MEMORY, + DAT_RESOURCE_DEVICE, + DAT_RESOURCE_TEP, /* transport endpoint, e.g. QP */ + DAT_RESOURCE_TEVD, /* transport EVD, e.g. CQ */ + DAT_RESOURCE_PROTECTION_DOMAIN, + DAT_RESOURCE_MEMORY_REGION, /* HCA memory for LMR or RMR */ + DAT_RESOURCE_ERROR_HANDLER, + DAT_RESOURCE_CREDITS, /* e.g outstanding RDMA Read credit as target */ + DAT_RESOURCE_SRQ, + + /* DAT_INTERNAL_ERROR has no subtypes */ + + /* INVALID_HANDLE subtypes */ + + DAT_INVALID_HANDLE_IA, + DAT_INVALID_HANDLE_EP, + DAT_INVALID_HANDLE_LMR, + DAT_INVALID_HANDLE_RMR, + DAT_INVALID_HANDLE_PZ, + DAT_INVALID_HANDLE_PSP, + DAT_INVALID_HANDLE_RSP, + DAT_INVALID_HANDLE_CR, + DAT_INVALID_HANDLE_CNO, + DAT_INVALID_HANDLE_EVD_CR, + DAT_INVALID_HANDLE_EVD_REQUEST, + DAT_INVALID_HANDLE_EVD_RECV, + DAT_INVALID_HANDLE_EVD_CONN, + DAT_INVALID_HANDLE_EVD_ASYNC, + DAT_INVALID_HANDLE_SRQ, + DAT_INVALID_HANDLE_CSP, + DAT_INVALID_HANDLE1, + DAT_INVALID_HANDLE2, + DAT_INVALID_HANDLE3, + DAT_INVALID_HANDLE4, + DAT_INVALID_HANDLE5, + DAT_INVALID_HANDLE6, + DAT_INVALID_HANDLE7, + DAT_INVALID_HANDLE8, + DAT_INVALID_HANDLE9, + DAT_INVALID_HANDLE10, + + /* DAT_INVALID_PARAMETER subtypes */ + DAT_INVALID_ARG1, + DAT_INVALID_ARG2, + DAT_INVALID_ARG3, + DAT_INVALID_ARG4, + DAT_INVALID_ARG5, + DAT_INVALID_ARG6, + DAT_INVALID_ARG7, + DAT_INVALID_ARG8, + DAT_INVALID_ARG9, + DAT_INVALID_ARG10, + + /* DAT_INVALID_EP_STATE subtypes */ + DAT_INVALID_STATE_EP_UNCONNECTED, + DAT_INVALID_STATE_EP_ACTCONNPENDING, + DAT_INVALID_STATE_EP_PASSCONNPENDING, + DAT_INVALID_STATE_EP_TENTCONNPENDING, + DAT_INVALID_STATE_EP_CONNECTED, + DAT_INVALID_STATE_EP_DISCONNECTED, + DAT_INVALID_STATE_EP_RESERVED, + DAT_INVALID_STATE_EP_COMPLPENDING, + DAT_INVALID_STATE_EP_DISCPENDING, + DAT_INVALID_STATE_EP_PROVIDERCONTROL, + DAT_INVALID_STATE_EP_NOTREADY, + DAT_INVALID_STATE_EP_RECV_WATERMARK, + DAT_INVALID_STATE_EP_PZ, + DAT_INVALID_STATE_EP_EVD_REQUEST, + DAT_INVALID_STATE_EP_EVD_RECV, + DAT_INVALID_STATE_EP_EVD_CONNECT, + DAT_INVALID_STATE_EP_UNCONFIGURED, + DAT_INVALID_STATE_EP_UNCONFRESERVED, + DAT_INVALID_STATE_EP_UNCONFPASSIVE, + DAT_INVALID_STATE_EP_UNCONFTENTATIVE, + DAT_INVALID_STATE_CNO_IN_USE, + DAT_INVALID_STATE_CNO_DEAD, + + /* EVD states. Enabled/Disabled, Waitable/Unwaitable, * + * and Notify/Solicited/Threshold are 3 orthogonal * + * bands of EVD state.The Threshold one is uDAPL specific. */ + DAT_INVALID_STATE_EVD_OPEN, + /* EVD can be either in enabled or disabled but not both * + * or neither at the same time */ + DAT_INVALID_STATE_EVD_ENABLED, + DAT_INVALID_STATE_EVD_DISABLED, + /* EVD can be either in waitable or unwaitable but not * + * both or neither at the same time */ + DAT_INVALID_STATE_EVD_WAITABLE, + DAT_INVALID_STATE_EVD_UNWAITABLE, + /* Do not release an EVD if it is in use */ + DAT_INVALID_STATE_EVD_IN_USE, + + /* EVD can be either in notify or solicited or threshold * + * but not any pair, or all, or none at the same time. * + * The threshold one is for uDAPL only */ + DAT_INVALID_STATE_EVD_CONFIG_NOTIFY, + DAT_INVALID_STATE_EVD_CONFIG_SOLICITED, + DAT_INVALID_STATE_EVD_CONFIG_THRESHOLD, + DAT_INVALID_STATE_EVD_WAITER, + DAT_INVALID_STATE_EVD_ASYNC, /* Async EVD required */ + DAT_INVALID_STATE_IA_IN_USE, + DAT_INVALID_STATE_LMR_IN_USE, + DAT_INVALID_STATE_LMR_FREE, + DAT_INVALID_STATE_PZ_IN_USE, + DAT_INVALID_STATE_PZ_FREE, + + /* DAT_INVALID_STATE_SRQ subtypes */ + DAT_INVALID_STATE_SRQ_OPERATIONAL, + DAT_INVALID_STATE_SRQ_ERROR, + DAT_INVALID_STATE_SRQ_IN_USE, + + /* DAT_LENGTH_ERROR has no subtypes */ + + /* DAT_MODEL_NOT_SUPPORTED has no subtypes */ + + /* DAT_PRIVILEGES_VIOLATION subtypes */ + DAT_PRIVILEGES_READ, + DAT_PRIVILEGES_WRITE, + DAT_PRIVILEGES_RDMA_READ, + DAT_PRIVILEGES_RDMA_WRITE, + + /* DAT_PROTECTION_VIOLATION subtypes */ + DAT_PROTECTION_READ, + DAT_PROTECTION_WRITE, + DAT_PROTECTION_RDMA_READ, + DAT_PROTECTION_RDMA_WRITE, + + /* DAT_QUEUE_EMPTY has no subtypes */ + + /* DAT_QUEUE_FULL has no subtypes */ + + /* DAT_TIMEOUT_EXPIRED has no subtypes */ + + /* DAT_PROVIDER_ALREADY_REGISTERED has no subtypes */ + + /* DAT_PROVIDER_IN_USE has no subtypes */ + + /* DAT_INVALID_ADDRESS subtypes */ + + /* Unsupported addresses - those that are not Malformed, but * + * are incorrect for use in DAT (regardless of local routing * + * capabilities): IPv6 Multicast Addresses (ff/8) IPv4 * + * Broadcast/Multicast Addresses */ + + DAT_INVALID_ADDRESS_UNSUPPORTED, + /* Unreachable addresses - A Provider might know that certain * + * addresses are unreachable immediately. One example would be * + * an IPv6 addresses on an IPv4-only system. This can also be * + * returned if it is known that there is no route to the host. * + * A Provider is not obligated to check for this condition. */ + DAT_INVALID_ADDRESS_UNREACHABLE, + /* Malformed addresses -- these cannot be valid in any context.* + * Those listed in RFC1884 section 2.3 as "Reserved" or * + * "Unassigned". */ + DAT_INVALID_ADDRESS_MALFORMED, + + /* DAT_INTERRUPTED_CALL has no subtypes */ + + /* DAT_CONN_QUAL_UNAVAILABLE has no subtypes */ + + /* DAT_PROVIDER_NOT_FOUND subtypes. Erratta to the 1.1 spec */ + + DAT_NAME_NOT_REGISTERED, + DAT_MAJOR_NOT_FOUND, + DAT_MINOR_NOT_FOUND, + DAT_THREAD_SAFETY_NOT_FOUND +} DAT_RETURN_SUBTYPE; + +#endif /* _DAT_ERROR_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_ib_extensions.h b/trunk/ulp/dapl2/dat/include/dat/dat_ib_extensions.h new file mode 100644 index 00000000..6799384b --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_ib_extensions.h @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_ib_extensions.h + * + * PURPOSE: extensions to the DAT API for IB transport specific services + * NOTE: Prototyped IB extension support in openib-cma 1.2 provider. + * Applications MUST recompile with new dat.h definitions + * and include this file. + * + * Description: Header file for "uDAPL: User Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * All global symbols are prepended with "DAT_" or "dat_" + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + * + **********************************************************************/ +#ifndef _DAT_IB_EXTENSIONS_H_ +#define _DAT_IB_EXTENSIONS_H_ + +/* + * Provider specific attribute strings for extension support + * returned with dat_ia_query() and + * DAT_PROVIDER_ATTR_MASK == DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR + * + * DAT_NAMED_ATTR name == extended operations and version, + * version_value = version number of extension API + */ +#define DAT_IB_EXTENSION_VERSION 201 /* 2.0.1 */ +#define DAT_IB_ATTR_FETCH_AND_ADD "DAT_IB_FETCH_AND_ADD" +#define DAT_IB_ATTR_CMP_AND_SWAP "DAT_IB_CMP_AND_SWAP" +#define DAT_IB_ATTR_IMMED_DATA "DAT_IB_IMMED_DATA" + +/* + * Definition for extended EVENT numbers, DAT_IB_EXTENSION_BASE_RANGE + * is used by these extensions as a starting point for extended event numbers + */ +typedef enum dat_ib_event_number +{ + DAT_IB_DTO_EVENT = DAT_IB_EXTENSION_RANGE_BASE, + +} DAT_IB_EVENT_NUMBER; + +/* + * Extension operations + */ +typedef enum dat_ib_op +{ + DAT_IB_FETCH_AND_ADD_OP, + DAT_IB_CMP_AND_SWAP_OP, + DAT_IB_RDMA_WRITE_IMMED_OP, + +} DAT_IB_OP; + +/* + * The DAT_IB_EXT_TYPE enum specifies the type of extension operation that just + * completed. All IB extended completion types both, DTO and NON-DTO, are + * reported in the extended operation type with the single DAT_IB_DTO_EVENT type. + * The specific extended DTO operation is reported with a DAT_IB_DTOS type in the + * operation field of the base DAT_EVENT structure. All other extended events are + * identified by unique DAT_IB_EVENT_NUMBER types. + */ +typedef enum dat_ib_ext_type +{ + DAT_IB_FETCH_AND_ADD, // 0 + DAT_IB_CMP_AND_SWAP, // 1 + DAT_IB_RDMA_WRITE_IMMED, // 2 + DAT_IB_RDMA_WRITE_IMMED_DATA, // 3 + +} DAT_IB_EXT_TYPE; + +/* + * Extension event status + */ +typedef enum dat_ib_status +{ + DAT_OP_SUCCESS = DAT_SUCCESS, + DAT_IB_OP_ERR, + +} DAT_IB_STATUS; + + +/* + * Definitions for additional extension type RETURN codes above + * standard DAT types. Included with standard DAT_TYPE_STATUS + * bits using a DAT_EXTENSION BASE as a starting point. + */ +typedef enum dat_ib_return +{ + DAT_IB_ERR = DAT_EXTENSION_BASE, + +} DAT_IB_RETURN; + +/* + * Definition for extended IB DTO operations, DAT_DTO_EXTENSION_BASE + * is used by DAT extensions as a starting point of extension DTOs + */ +typedef enum dat_ib_dtos +{ + DAT_IB_DTO_RDMA_WRITE_IMMED = DAT_DTO_EXTENSION_BASE, + DAT_IB_DTO_RECV_IMMED, + DAT_IB_DTO_FETCH_ADD, + DAT_IB_DTO_CMP_SWAP, + +} DAT_IB_DTOS; + +/* + * Definitions for additional extension handle types beyond + * standard DAT handle. New Bit definitions MUST start at + * DAT_HANDLE_TYPE_EXTENSION_BASE + */ +typedef enum dat_ib_handle_type +{ + DAT_IB_HANDLE_TYPE_EXT = DAT_HANDLE_TYPE_EXTENSION_BASE, + +} DAT_IB_HANDLE_TYPE; + +/* + * The DAT_IB_EVD_EXTENSION_FLAGS enum specifies the EVD extension flags that + * do not map directly to existing DAT_EVD_FLAGS. This new EVD flag has been + * added to identify an extended EVD that does not fit the existing stream + * types. + */ +typedef enum dat_ib_evd_extension_flags +{ + DAT_IB_EVD_EXTENSION_FLAG = DAT_EVD_EXTENSION_BASE + +} DAT_IB_EVD_EXTENSION_FLAGS; + +/* + * Definition for memory privilege extension flags. + * New privileges required for new atomic DTO type extensions. + * New Bit definitions MUST start at DAT_MEM_PRIV_EXTENSION + */ +typedef enum dat_ib_mem_priv_flags +{ + DAT_IB_MEM_PRIV_REMOTE_ATOMIC = DAT_MEM_PRIV_EXTENSION_BASE, + +} DAT_IB_MEM_PRIV_FLAGS; + + +/* + * Definitions for extended event data: + * When dat_event->event_number >= DAT_IB_EXTENSION_BASE_RANGE + * then dat_event->extension_data == DAT_IB_EXT_EVENT_DATA type + * and ((DAT_IB_EXT_EVENT_DATA*)dat_event->extension_data)->type + * specifies extension data values. + * NOTE: DAT_IB_EXT_EVENT_DATA cannot exceed 64 bytes as defined by + * "DAT_UINT64 extension_data[8]" in DAT_EVENT (dat.h) + */ +typedef struct dat_ib_immed_data +{ + DAT_UINT32 data; + +} DAT_IB_IMMED_DATA; + +/* + * Definitions for extended event data: + * When dat_event->event_number >= DAT_IB_EXTENSION_BASE_RANGE + * then dat_event->extension_data == DAT_EXTENSION_EVENT_DATA type + * and ((DAT_EXTENSION_EVENT_DATA*)dat_event->extension_data)->type + * specifies extension data values. + * NOTE: DAT_EXTENSION_EVENT_DATA cannot exceed 64 bytes as defined by + * "DAT_UINT64 extension_data[8]" in DAT_EVENT (dat.h) + */ +typedef struct dat_ib_extension_event_data +{ + DAT_IB_EXT_TYPE type; + DAT_IB_STATUS status; + union { + DAT_IB_IMMED_DATA immed; + } val; + +} DAT_IB_EXTENSION_EVENT_DATA; + +/* Extended RETURN and EVENT STATUS string helper functions */ + +/* DAT_EXT_RETURN error to string */ +static __inline__ DAT_RETURN +dat_strerror_extension ( + IN DAT_IB_RETURN value, + OUT const char **message ) +{ + switch( DAT_GET_TYPE(value) ) { + case DAT_IB_ERR: + *message = "DAT_IB_ERR"; + return DAT_SUCCESS; + default: + /* standard DAT return type */ + return(dat_strerror(value, message, NULL)); + } +} + +/* DAT_EXT_STATUS error to string */ +static __inline__ DAT_RETURN +dat_strerror_ext_status ( + IN DAT_IB_STATUS value, + OUT const char **message ) +{ + switch(value) { + case 0: + *message = " "; + return DAT_SUCCESS; + case DAT_IB_OP_ERR: + *message = "DAT_IB_OP_ERR"; + return DAT_SUCCESS; + default: + *message = "unknown extension status"; + return DAT_INVALID_PARAMETER; + } +} + +/* + * Extended IB transport specific APIs + * redirection via DAT extension function + */ + +/* + * This asynchronous call is modeled after the InfiniBand atomic + * Fetch and Add operation. The add_value is added to the 64 bit + * value stored at the remote memory location specified in remote_iov + * and the result is stored in the local_iov. + */ +#define dat_ib_post_fetch_and_add(ep, add_val, lbuf, cookie, rbuf, flgs) \ + dat_extension_op( ep, \ + DAT_IB_FETCH_AND_ADD_OP, \ + (add_val), \ + (lbuf), \ + (cookie), \ + (rbuf), \ + (flgs)) + +/* + * This asynchronous call is modeled after the InfiniBand atomic + * Compare and Swap operation. The cmp_value is compared to the 64 bit + * value stored at the remote memory location specified in remote_iov. + * If the two values are equal, the 64 bit swap_value is stored in + * the remote memory location. In all cases, the original 64 bit + * value stored in the remote memory location is copied to the local_iov. + */ +#define dat_ib_post_cmp_and_swap(ep, cmp_val, swap_val, lbuf, cookie, rbuf, flgs) \ + dat_extension_op( ep, \ + DAT_IB_CMP_AND_SWAP_OP, \ + (cmp_val), \ + (swap_val), \ + (lbuf), \ + (cookie), \ + (rbuf), \ + (flgs)) + +/* + * RDMA Write with IMMEDIATE: + * + * This asynchronous call is modeled after the InfiniBand rdma write with + * immediate data operation. Event completion for the request completes as an + * DAT_EXTENSION with extension type set to DAT_DTO_EXTENSION_IMMED_DATA. + * Event completion on the remote endpoint completes as receive DTO operation + * type of DAT_EXTENSION with operation set to DAT_DTO_EXTENSION_IMMED_DATA. + * The immediate data will be provided in the extented DTO event data structure. + * + * Note to Consumers: the immediate data will consume a receive + * buffer at the Data Sink. + * + * Other extension flags: + * n/a + */ +#define dat_ib_post_rdma_write_immed(ep, size, lbuf, cookie, rbuf, idata, flgs) \ + dat_extension_op( ep, \ + DAT_IB_RDMA_WRITE_IMMED_OP, \ + (size), \ + (lbuf), \ + (cookie), \ + (rbuf), \ + (idata), \ + (flgs)) + +#endif /* _DAT_IB_EXTENSIONS_H_ */ + diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_iw_extensions.h b/trunk/ulp/dapl2/dat/include/dat/dat_iw_extensions.h new file mode 100644 index 00000000..00676056 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_iw_extensions.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +/********************************************************************** + * + * HEADER: dat_iw_extensions.h + * + * PURPOSE: extensions to the DAT API for iWARP transport specific services + * NOTE: + * Applications MUST recompile with new dat.h definitions + * and include this file. + * + * Description: Header file for "uDAPL: User Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * All global symbols are prepended with "DAT_" or "dat_" + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + * + **********************************************************************/ +#ifndef _DAT_IW_EXTENSIONS_H_ +#define _DAT_IW_EXTENSIONS_H_ + +/* + * Provider specific attribute strings for extension support + * returned with dat_ia_query() and + * DAT_PROVIDER_ATTR_MASK == DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR + * + * DAT_NAMED_ATTR name == extended operations and version, + * version_value = version number of extension API + */ +#define DAT_EXTENSION_ATTR "DAT_EXTENSION_INTERFACE" +#define DAT_EXTENSION_ATTR_VERSION "DAT_EXTENSION_VERSION" +#define DAT_EXTENSION_ATTR_VERSION_VALUE "2.0.1" +#define DAT_IW_ATTR_SSP "DAT_IW_ATTR_SSP" + + +/* + * The iWARP extension supports Socket Service Points + * These enable establishing a TCP connection first, then converting + * it to RDMA usage. + */ +typedef enum dat_handle_iw_ext_type { + DAT_IW_HANDLE_TYPE_SSP = DAT_HANDLE_TYPE_EXTENSION_BASE, +} DAT_HANDLE_IW_EXT_TYPE; +typedef DAT_HANDLE DAT_IW_SSP_HANDLE; + +typedef enum dat_iw_ssp_state +{ + DAT_IW_SSP_STATE_OPERATIONAL, + DAT_IW_SSP_STATE_NON_OPERATIONAL +} DAT_IW_SSP_STATE; + +/* The Socket ID is a local identifier for the socket. It is used for Socket + * based connection model. The format of the DAT_IW_SOCKET follows the native + * sockets programming practice of each target Operating System. The Consumer must + * follow normal sockets programming procedures provided by the host platform. + * + * This include assumes that a socket handle is an int unless we are in + * Linux Kernel code, in which case it is a struct socket pointer. This + * distinction could be moved either to a distinct file (for iWARP specific, + * platform specific types) or to dat_platform_specific.h (where the fact + * that it was iWARP specific would be awkward). The coin flip was heads, + * so the awkwardness of platform specific typing is in the iWARP specific + * file instead. + */ +#if defined(__linux__) && defined(__KERNEL__) +typyedef struct socket *DAT_IW_SOCKET; +#else +typedef int DAT_IW_SOCKET; +#endif + +typedef struct dat_iw_ssp_param +{ + DAT_IA_HANDLE ia_handle; + DAT_IW_SOCKET socket_id; + DAT_EVD_HANDLE evd_handle; + DAT_EP_HANDLE ep_handle; + DAT_IW_SSP_STATE ssp_state; +} DAT_IW_SSP_PARAM; + +typedef enum dat_iw_ssp_param_mask +{ + DAT_IW_SSP_FIELD_IA_HANDLE = 0x01, + DAT_IW_SSP_FIELD_SOCKET_ID = 0x02, + DAT_IW_SSP_FIELD_EVD_HANDLE = 0x04, + DAT_IW_SSP_FIELD_EP_HANDLE = 0x08, + DAT_IW_SSP_FIELD_SSP_STATE = 0x10, + DAT_IW_SSP_FIELD_ALL = 0x1F +} DAT_IW_SSP_PARAM_MASK; + + +/* + * Extension operations + */ +typedef enum dat_iw_op +{ + DAT_IW_SSP_CREATE_OP, + DAT_IW_SSP_FREE_OP, + DAT_IW_SSP_QUERY_OP, + DAT_IW_SOCKET_CONNECT_OP +} DAT_IW_OP; + +/* + * Extended IW transport specific APIs + * redirection via DAT extension function + */ + +/* dat_iw_ssp_create creates an iWARP Socket Service Point given an + * already connected Socket + */ + +#define dat_iw_ssp_create (ia,socket_id,ep_handle,evd_handle,\ + final_sm_msg,final_sm_msg_len,ssp_handle) \ + dat_extension_op (ia,DAT_IW_SSP_CREATE_OP,ia,socket_id, \ + ep_handle,evd_handle,final_sm_msg, \ + final_sm_msg_len,ssp_handle) + +/* dat_iw_ssp_free destroys the specified instance of an SSP + */ +#define dat_iw_ssp_free (ssp_handle) \ + dat_extension_op (ssp_handle,DAT_IW_SSP_FREE_OP) + +/* dat_iw_ssq_query obtains the SSP_PARAMs for an ssp + */ +#define dat_iw_ssp_query (ssp_handle,ssp_param_mask,ssp_param) \ + dat_extension_op (ssp_handle,DAT_IW_SSP_QUERY_OP, \ + ssp_param_maks,ssp_param) + +/* dat_iw_socket_connect initates a connection using an SSP + */ +#define dat_iw_socket_connect (ep_handle,socket_id,timeout, \ + private_data_size,private_data) \ + dat_extension_op (ep_handle,DAT_IW_SOCKET_CONNECT_OP,\ + socket_id,timeout,private_data_size,\ + private_data) + +#endif /* _DAT_IW_EXTENSIONS_H_ */ + diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_platform_specific.h b/trunk/ulp/dapl2/dat/include/dat/dat_platform_specific.h new file mode 100644 index 00000000..d448674a --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_platform_specific.h @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*************************************************************** + * + * HEADER: dat_platform_specific.h + * + * PURPOSE: defines Platform-specific types. + * + * Description: Header file for "DAPL: Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * + ***************************************************************/ + +#ifndef _DAT_PLATFORM_SPECIFIC_H_ +#define _DAT_PLATFORM_SPECIFIC_H_ + +/* OS, processor, compiler type definitions. Add OSes as needed. */ + +/* + * This captures the alignment for the bus transfer from the HCA/IB chip + * to the main memory. + */ +#ifndef DAT_OPTIMAL_ALIGNMENT +#define DAT_OPTIMAL_ALIGNMENT 256 /* Performance optimal alignment */ +#endif /* DAT_OPTIMAL_ALIGNMENT */ + +/* Assume all OSes use sockaddr, for address family: IPv4 == AF_INET, + * IPv6 == AF_INET6. Use of "namelen" field indicated. + * + * The Interface Adapter Address names an Interface Adapter local or + * remote, that is used for connection management and Name + * Service. The format of the dat_ia_address_ptr follows the normal + * socket programming practice of struct sockaddr *. DAT supports both + * IPv4 and IPv6 address families. Allocation and initialization of + * DAT IA address structures must follow normal Sockets programming + * procedures. The underlying type of the DAT IA address is the native + * struct sockaddr for each target operating system. In all cases, + * storage appropriate for the address family in use by the target + * Provider must be allocated. For instance, when IPv6 addressing is + * in use, this should be allocated as struct sockaddr_net6. The + * sockaddr sa_family and, if present, sa_len fields must be + * initialized appropriately, as well as the address information. + * When passed across the DAPL API this storage is cast to the + * DAT_IA_ADDRESS_PTR type. It is the responsibility of the callee to + * verify that the sockaddr contains valid data for the requested + * operation. It is always the responsibility of the caller to manage + * the storage. + * + * uDAPL code example for Linux (kdapl would be similar): + * + * #include + * #include + * #include + * #include + * + * struct sockaddr_in6 addr; + * DAT_IA_ADDRESS_PTR ia_addr; + * + * // Note: linux pton requires explicit encoding of IPv4 in IPv6 + * + * addr.sin6_family = AF_INET6; + * if (inet_pton(AF_INET6, "0:0:0:0:0:FFFF:192.168.0.1", + * &addr.sin6_addr) <= 0) + * return(-1); // Bad address or no address family support + * + * // initialize other necessary fields such as port, flow, etc + * + * ia_addr = (DAT_IA_ADDRESS_PTR) &addr; + * dat_ep_connect(ep_handle, ia_addr, conn_qual, timeout, 0, NULL, + * qos, DAT_CONNECT_DEFAULT_FLAG); + * + */ + +/* Solaris begins */ +#if defined (sun) || defined(__sun) || defined(_sun_) || defined (__solaris__) + +#include +#include /* needed for UINT64_C() macro */ + +typedef uint32_t DAT_UINT32; /* Unsigned host order, 32 bits */ +typedef uint64_t DAT_UINT64; /* unsigned host order, 64 bits */ +typedef unsigned long long DAT_UVERYLONG; /* unsigned longest native to compiler */ + +typedef void * DAT_PVOID; +typedef int DAT_COUNT; + +#define DAT_IA_HANDLE_TO_UL(a) (unsigned long)(a) +#define UL_TO_DAT_IA_HANDLE(a) (DAT_IA_HANDLE)(a) + +#include +#include +typedef struct sockaddr DAT_SOCK_ADDR; /* Socket address header native to OS */ +typedef struct sockaddr_in6 DAT_SOCK_ADDR6; /* Socket address header native to OS */ + +#define DAT_AF_INET AF_INET +#define DAT_AF_INET6 AF_INET6 + +typedef DAT_UINT64 DAT_PADDR; + +#define DAT_API +#define DAT_EXPORT extern + +/* Solaris ends */ + + +/* Linux begins */ +#elif defined(__linux__) /* Linux */ +#if defined(__KERNEL__) +#include +#else +#include +#endif /* defined(__KERNEL__) */ + +typedef u_int32_t DAT_UINT32; /* unsigned host order, 32 bits */ +typedef u_int64_t DAT_UINT64; /* unsigned host order, 64 bits */ +typedef unsigned long long DAT_UVERYLONG; /* unsigned longest native to compiler */ + +typedef void * DAT_PVOID; +typedef int DAT_COUNT; +typedef DAT_UINT64 DAT_PADDR; + +#ifndef UINT64_C +#define UINT64_C(c) c ## ULL +#endif /* UINT64_C */ + +#define DAT_IA_HANDLE_TO_UL(a) (unsigned long)(a) +#define UL_TO_DAT_IA_HANDLE(a) (DAT_IA_HANDLE)(a) + +#if defined(__KERNEL__) +#include +#include +#include +#else +#include +#endif /* defined(__KERNEL__) */ + +typedef struct dat_comm { + int domain; + int type; + int protocol; +} DAT_COMM; + +typedef int DAT_FD; /* DAT File Descriptor */ + +typedef struct sockaddr DAT_SOCK_ADDR; /* Socket address header native to OS */ +typedef struct sockaddr_in6 DAT_SOCK_ADDR6; /* Socket address header native to OS */ +#define DAT_AF_INET AF_INET +#define DAT_AF_INET6 AF_INET6 + +#define DAT_API +#define DAT_EXPORT extern + +/* Linux ends */ + +/* Win32/64 begins */ +#elif defined(_MSC_VER) || defined(_WIN32) || defined(_WIN64) +/* NT. MSC compiler, Win32/64 platform */ + +typedef unsigned __int32 DAT_UINT32; /* Unsigned host order, 32 bits */ +typedef unsigned __int64 DAT_UINT64; /* unsigned host order, 64 bits */ +typedef unsigned long DAT_UVERYLONG; /* unsigned longest native to compiler */ + +#if defined(_WIN64) +#define DAT_IA_HANDLE_TO_UL(a) (unsigned long)((DAT_UINT64)(a)) +#define UL_TO_DAT_IA_HANDLE(a) (DAT_IA_HANDLE)((DAT_UINT64)(a)) +#else // _WIN32 +#define DAT_IA_HANDLE_TO_UL(a) (unsigned long)(a) +#define UL_TO_DAT_IA_HANDLE(a) (DAT_IA_HANDLE)(a) +#endif + +typedef void * DAT_PVOID; +typedef long DAT_COUNT; +typedef DAT_UINT64 DAT_PADDR; + +typedef struct dat_comm { + int domain; + int type; + int protocol; +} DAT_COMM; + +typedef int DAT_FD; /* DAT File Descriptor */ + +typedef struct sockaddr DAT_SOCK_ADDR; /* Sock addr header native to OS */ +typedef struct sockaddr_in6 DAT_SOCK_ADDR6;/* Sock addr header native to OS */ + +#ifndef UINT64_C +#define UINT64_C(c) c ## i64 +#endif /* UINT64_C */ + +#define DAT_AF_INET AF_INET +#define DAT_AF_INET6 AF_INET6 + +#if defined(EXPORT_DAT_SYMBOLS) +#define DAT_EXPORT __declspec(dllexport) +#else +#define DAT_EXPORT __declspec(dllimport) +#endif + +#define DAT_API __stdcall + +#ifndef __inline__ +#define __inline__ __inline +#endif + +#ifndef INLINE +#define INLINE __inline +#endif + +#if defined(__KDAPL__) +/* must have the DDK for this definition */ +typedef PHYSICAL_ADDRESS DAT_PADDR; +#endif /* __KDAPL__ */ + +/* Win32/64 ends */ + + +#else +#error dat_platform_specific.h : OS type not defined +#endif + +#ifndef IN +#define IN +#endif +#ifndef OUT +#define OUT +#endif +#ifndef INOUT +#define INOUT +#endif + +#endif /* _DAT_PLATFORM_SPECIFIC_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_redirection.h b/trunk/ulp/dapl2/dat/include/dat/dat_redirection.h new file mode 100644 index 00000000..fa019ece --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_redirection.h @@ -0,0 +1,858 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/**************************************************************** + * + * HEADER: dat_redirection.h + * + * PURPOSE: Defines the common redirection macros + * + * Description: Macros to invoke DAPL functions from the dat_registry + * + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + **********************************************************/ +#ifndef _DAT_REDIRECTION_H_ +#define _DAT_REDIRECTION_H_ + +typedef struct dat_provider DAT_PROVIDER; + +#ifndef DAT_HANDLE_TO_PROVIDER + +/* A utility macro to fetch the Provider Library for any object + * + * An alternate version could be defined for single library systems. + * it would look something like: + * extern const struct dat_ia my_single_ia_provider; + * #define DAT_HANDLE_TO_PROVIDER(ignore) &my_single_ia_provider + * + * This would allow a good compiler to avoid indirection overhead when + * making function calls. + */ + +#define DAT_HANDLE_TO_PROVIDER(handle) (*(DAT_PROVIDER **)(handle)) +#endif + +#define DAT_IA_QUERY(ia, evd, ia_msk, ia_ptr, p_msk, p_ptr) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->ia_query_func) (\ + (ia), \ + (evd), \ + (ia_msk), \ + (ia_ptr), \ + (p_msk), \ + (p_ptr)) + +#define DAT_SET_CONSUMER_CONTEXT(handle, context) \ + (*DAT_HANDLE_TO_PROVIDER (handle)->set_consumer_context_func) (\ + (handle), \ + (context)) + +#define DAT_GET_CONSUMER_CONTEXT(handle, context) \ + (*DAT_HANDLE_TO_PROVIDER (handle)->get_consumer_context_func) (\ + (handle), \ + (context)) + +#define DAT_GET_HANDLE_TYPE(handle, handle_type) \ + (*DAT_HANDLE_TO_PROVIDER (handle)->get_handle_type_func) (\ + (handle), \ + (handle_type)) + +#define DAT_CR_QUERY(cr, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (cr)->cr_query_func) (\ + (cr), \ + (mask), \ + (param)) + +#define DAT_CR_ACCEPT(cr, ep, size, pdata) \ + (*DAT_HANDLE_TO_PROVIDER (cr)->cr_accept_func) (\ + (cr), \ + (ep), \ + (size), \ + (pdata)) + +#define DAT_CR_REJECT(cr, size, pdata) \ + (*DAT_HANDLE_TO_PROVIDER (cr)->cr_reject_func) (\ + (cr), \ + (size), \ + (pdata)) + +#define DAT_CR_HANDOFF(cr,qual) \ + (*DAT_HANDLE_TO_PROVIDER(cr)->cr_handoff_func)(\ + (cr), \ + (qual)) + +#define DAT_EVD_QUERY(evd, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (evd)->evd_query_func) (\ + (evd), \ + (mask), \ + (param)) + +#define DAT_EVD_RESIZE(evd, qsize) \ + (*DAT_HANDLE_TO_PROVIDER (evd)->evd_resize_func) (\ + (evd), \ + (qsize)) + +#define DAT_EVD_POST_SE(evd, event) \ + (*DAT_HANDLE_TO_PROVIDER (evd)->evd_post_se_func) (\ + (evd), \ + (event)) + +#define DAT_EVD_DEQUEUE(evd, event) \ + (*DAT_HANDLE_TO_PROVIDER (evd)->evd_dequeue_func) (\ + (evd), \ + (event)) + +#define DAT_EVD_FREE(evd)\ + (*DAT_HANDLE_TO_PROVIDER (evd)->evd_free_func) (\ + (evd)) + +#define DAT_EP_CREATE(ia, pz, in_evd, out_evd, connect_evd, attr, ep) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->ep_create_func) (\ + (ia), \ + (pz), \ + (in_evd), \ + (out_evd), \ + (connect_evd), \ + (attr), \ + (ep)) + +#define DAT_EP_CREATE_WITH_SRQ(ia, pz, in_evd, out_evd, \ + connect_evd, srq, attr, ep) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->ep_create_with_srq_func) (\ + (ia), \ + (pz), \ + (in_evd), \ + (out_evd), \ + (connect_evd), \ + (srq), \ + (attr), \ + (ep)) + +#define DAT_EP_QUERY(ep, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_query_func) (\ + (ep), \ + (mask), \ + (param)) + +#define DAT_EP_MODIFY(ep, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_modify_func) (\ + (ep), \ + (mask), \ + (param)) + +#define DAT_EP_CONNECT(ep, ia_addr, conn_qual, \ + timeout, psize, pdata, qos, flags) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_connect_func) (\ + (ep), \ + (ia_addr), \ + (conn_qual), \ + (timeout), \ + (psize), \ + (pdata), \ + (qos), \ + (flags)) + +#define DAT_EP_COMMON_CONNECT(ep, addr, timeout, psize, pdata) \ + (*DAT_HANDLE_TO_PROVIDER(ep)->ep_common_connect_func)(\ + (ep), \ + (addr), \ + (timeout), \ + (psize), \ + (pdata)) + +#define DAT_EP_DUP_CONNECT(ep, dup, timeout, psize, pdata, qos) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_dup_connect_func) (\ + (ep), \ + (dup), \ + (timeout), \ + (psize), \ + (pdata), \ + (qos)) + +#define DAT_EP_DISCONNECT(ep, flags) \ + (*DAT_HANDLE_TO_PROVIDER(ep)->ep_disconnect_func) (\ + (ep), \ + (flags)) + +#define DAT_EP_POST_SEND(ep, size, lbuf, cookie, flags) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_post_send_func) (\ + (ep), \ + (size), \ + (lbuf), \ + (cookie), \ + (flags)) + +#define DAT_EP_POST_SEND_WITH_INVALIDATE( \ + ep,size,lbuf,cookie,flags,inv_flag,rmr_context) \ + (*DAT_HANDLE_TO_PROVIDER(ep)-> \ + ep_post_send_with_invalidate_func)(\ + (ep), \ + (size), \ + (lbuf), \ + (cookie), \ + (flags), \ + (inv_flag), \ + (rmr_context)) + +#define DAT_EP_POST_RECV(ep, size, lbuf, cookie, flags) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_post_recv_func) (\ + (ep), \ + (size), \ + (lbuf), \ + (cookie), \ + (flags)) + +#define DAT_EP_POST_RDMA_READ(ep, size, lbuf, cookie, rbuf, flags) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_post_rdma_read_func) (\ + (ep), \ + (size), \ + (lbuf), \ + (cookie), \ + (rbuf), \ + (flags)) + +#define DAT_EP_POST_RDMA_READ_TO_RMR(ep, lbuf, cookie, rbuf, flags) \ + (*DAT_HANDLE_TO_PROVIDER(ep)->ep_post_rdma_read_to_rmr_func)(\ + (ep), \ + (lbuf), \ + (cookie), \ + (rbuf), \ + (flags)) + +#define DAT_EP_POST_RDMA_WRITE(ep, size, lbuf, cookie, rbuf, flags) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_post_rdma_write_func) (\ + (ep), \ + (size), \ + (lbuf), \ + (cookie), \ + (rbuf), \ + (flags)) + +#define DAT_EP_GET_STATUS(ep, ep_state, recv_idle, request_idle) \ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_get_status_func) (\ + (ep), \ + (ep_state), \ + (recv_idle), \ + (request_idle)) + +#define DAT_EP_FREE(ep)\ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_free_func) (\ + (ep)) + +#define DAT_EP_RESET(ep)\ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_reset_func) (\ + (ep)) + +#define DAT_EP_RECV_QUERY(ep, nbuf_alloc, buf_span)\ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_recv_query_func) (\ + (ep), \ + (nbuf_alloc), \ + (buf_span)) + +#define DAT_EP_SET_WATERMARK(ep, soft_wm, hard_wm)\ + (*DAT_HANDLE_TO_PROVIDER (ep)->ep_set_watermark_func) (\ + (ep), \ + (soft_wm), \ + (hard_wm)) + +#define DAT_LMR_QUERY(lmr, mask, param)\ + (*DAT_HANDLE_TO_PROVIDER (lmr)->lmr_query_func) (\ + (lmr), \ + (mask), \ + (param)) + +#define DAT_LMR_FREE(lmr)\ + (*DAT_HANDLE_TO_PROVIDER (lmr)->lmr_free_func) (\ + (lmr)) + +#define DAT_LMR_SYNC_RDMA_READ(ia, lbuf, size)\ + (*DAT_HANDLE_TO_PROVIDER (ia)->lmr_sync_rdma_read_func) (\ + (ia), \ + (lbuf), \ + (size)) + +#define DAT_LMR_SYNC_RDMA_WRITE(ia, lbuf, size)\ + (*DAT_HANDLE_TO_PROVIDER (ia)->lmr_sync_rdma_write_func) (\ + (ia), \ + (lbuf), \ + (size)) + +#define DAT_RMR_CREATE(pz, rmr) \ + (*DAT_HANDLE_TO_PROVIDER (pz)->rmr_create_func) (\ + (pz), \ + (rmr)) + +#define DAT_RMR_CREATE_FOR_EP(pz, rmr) \ + (*DAT_HANDLE_TO_PROVIDER (pz)->rmr_create_for_ep_func) (\ + (pz), \ + (rmr)) + +#define DAT_RMR_QUERY(rmr, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (rmr)->rmr_query_func) (\ + (rmr), \ + (mask), \ + (param)) + +#define DAT_RMR_BIND(rmr, lmr, lmr_triplet, mem_priv, \ + va_type, ep, cookie, flags, context) \ + (*DAT_HANDLE_TO_PROVIDER (rmr)->rmr_bind_func) (\ + (rmr), \ + (lmr), \ + (lmr_triplet), \ + (mem_priv), \ + (va_type), \ + (ep), \ + (cookie), \ + (flags), \ + (context)) + +#define DAT_RMR_FREE(rmr)\ + (*DAT_HANDLE_TO_PROVIDER (rmr)->rmr_free_func) (\ + (rmr)) + +#define DAT_PSP_CREATE(ia, conn_qual, evd, flags, handle) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->psp_create_func) (\ + (ia), \ + (conn_qual), \ + (evd), \ + (flags), \ + (handle)) + +#define DAT_PSP_CREATE_ANY(ia, conn_qual, evd, flags, handle) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->psp_create_any_func) (\ + (ia), \ + (conn_qual), \ + (evd), \ + (flags), \ + (handle)) + +#define DAT_PSP_QUERY(psp, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (psp)->psp_query_func) (\ + (psp), \ + (mask), \ + (param)) + +#define DAT_PSP_FREE(psp)\ + (*DAT_HANDLE_TO_PROVIDER (psp)->psp_free_func) (\ + (psp)) + +#define DAT_RSP_CREATE(ia, conn_qual, ep, evd, handle) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->rsp_create_func) (\ + (ia), \ + (conn_qual), \ + (ep), \ + (evd), \ + (handle)) + +#define DAT_RSP_QUERY(rsp, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (rsp)->rsp_query_func) (\ + (rsp), \ + (mask), \ + (param)) + +#define DAT_RSP_FREE(rsp)\ + (*DAT_HANDLE_TO_PROVIDER (rsp)->rsp_free_func) (\ + (rsp)) + +#define DAT_CSP_CREATE(ia, comm, addr, evd, handle) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->csp_create_func)(\ + (ia),\ + (comm),\ + (addr),\ + (evd),\ + (handle)) + +#define DAT_CSP_QUERY(csp, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER(csp)->csp_query_func)(\ + (csp),\ + (mask),\ + (param)) + +#define DAT_CSP_FREE(csp)\ + (*DAT_HANDLE_TO_PROVIDER(csp)->csp_free_func)(\ + (csp)) + +#define DAT_PZ_CREATE(ia, pz) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->pz_create_func) (\ + (ia), \ + (pz)) + +#define DAT_PZ_QUERY(pz, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (pz)->pz_query_func) (\ + (pz), \ + (mask), \ + (param)) + +#define DAT_PZ_FREE(pz) \ + (*DAT_HANDLE_TO_PROVIDER (pz)->pz_free_func) (\ + (pz)) + +#define DAT_SRQ_CREATE(ia, pz, attr, srq) \ + (*DAT_HANDLE_TO_PROVIDER (ia)->srq_create_func) (\ + (ia), \ + (pz), \ + (attr), \ + (srq)) + +#define DAT_SRQ_SET_LW(srq, lw) \ + (*DAT_HANDLE_TO_PROVIDER (srq)->srq_set_lw_func) (\ + (srq), \ + (lw)) + +#define DAT_SRQ_FREE(srq) \ + (*DAT_HANDLE_TO_PROVIDER (srq)->srq_free_func) (\ + (srq)) + +#define DAT_SRQ_QUERY(srq, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER (srq)->srq_query_func) (\ + (srq), \ + (mask), \ + (param)) + +#define DAT_SRQ_RESIZE(srq, qsize) \ + (*DAT_HANDLE_TO_PROVIDER (srq)->srq_resize_func) (\ + (srq), \ + (qsize)) + +#define DAT_SRQ_POST_RECV(srq, size, lbuf, cookie) \ + (*DAT_HANDLE_TO_PROVIDER (srq)->srq_post_recv_func) (\ + (srq), \ + (size), \ + (lbuf), \ + (cookie)) + +#define DAT_IA_HA_RELATED (ia, name, answer) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->ia_ha_related) (\ + (ia), \ + (name), \ + (answer)) + +#ifdef DAT_EXTENSIONS +#define DAT_HANDLE_EXTENDEDOP(handle, op, args) \ + (*DAT_HANDLE_TO_PROVIDER (handle)->handle_extendedop_func) (\ + (handle), \ + (op), \ + (args)) +#endif + +/*************************************************************** + * + * FUNCTION PROTOTYPES + ****************************************************************/ + +typedef DAT_RETURN (*DAT_IA_OPEN_FUNC) ( + IN const DAT_NAME_PTR, /* provider */ + IN DAT_COUNT, /* asynch_evd_min_qlen */ + INOUT DAT_EVD_HANDLE *, /* asynch_evd_handle */ + OUT DAT_IA_HANDLE *); /* ia_handle */ + +typedef DAT_RETURN (*DAT_IA_OPENV_FUNC) ( + IN const DAT_NAME_PTR, /* provider */ + IN DAT_COUNT, /* asynch_evd_min_qlen */ + INOUT DAT_EVD_HANDLE *, /* asynch_evd_handle */ + OUT DAT_IA_HANDLE *, /* ia_handle */ + IN DAT_UINT32, /* dat_major_version number */ + IN DAT_UINT32, /* dat_minor_version number */ + IN DAT_BOOLEAN); /* dat_thread_safety */ + +typedef DAT_RETURN (*DAT_IA_CLOSE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CLOSE_FLAGS ); /* close_flags */ + +typedef DAT_RETURN (*DAT_IA_QUERY_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_EVD_HANDLE *, /* async_evd_handle */ + IN DAT_IA_ATTR_MASK, /* ia_attr_mask */ + OUT DAT_IA_ATTR *, /* ia_attr */ + IN DAT_PROVIDER_ATTR_MASK, /* provider_attr_mask */ + OUT DAT_PROVIDER_ATTR * ); /* provider_attr */ + +/* helper functions */ + +typedef DAT_RETURN (*DAT_SET_CONSUMER_CONTEXT_FUNC) ( + IN DAT_HANDLE, /* dat_handle */ + IN DAT_CONTEXT); /* context */ + +typedef DAT_RETURN (*DAT_GET_CONSUMER_CONTEXT_FUNC) ( + IN DAT_HANDLE, /* dat_handle */ + OUT DAT_CONTEXT * ); /* context */ + +typedef DAT_RETURN (*DAT_GET_HANDLE_TYPE_FUNC) ( + IN DAT_HANDLE, /* dat_handle */ + OUT DAT_HANDLE_TYPE * ); /* dat_handle_type */ + +/* CR functions */ + +typedef DAT_RETURN (*DAT_CR_QUERY_FUNC) ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_CR_PARAM_MASK, /* cr_param_mask */ + OUT DAT_CR_PARAM * ); /* cr_param */ + +typedef DAT_RETURN (*DAT_CR_ACCEPT_FUNC) ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +typedef DAT_RETURN (*DAT_CR_REJECT_FUNC) ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +/* For DAT-1.1 this function is defined for both uDAPL and kDAPL. + * For DAT-1.0 it was only defined for uDAPL. + */ + +typedef DAT_RETURN (*DAT_CR_HANDOFF_FUNC) ( + IN DAT_CR_HANDLE, /* cr_handle */ + IN DAT_CONN_QUAL); /* handoff */ + +/* EVD functions */ + +typedef DAT_RETURN (*DAT_EVD_RESIZE_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_COUNT ); /* evd_min_qlen */ + +typedef DAT_RETURN (*DAT_EVD_POST_SE_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN const DAT_EVENT * ); /* event */ + +typedef DAT_RETURN (*DAT_EVD_DEQUEUE_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_EVENT * ); /* event */ + +typedef DAT_RETURN (*DAT_EVD_FREE_FUNC) ( + IN DAT_EVD_HANDLE ); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_QUERY_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_EVD_PARAM_MASK, /* evd_param_mask */ + OUT DAT_EVD_PARAM * ); /* evd_param */ + +/* EP functions */ + +typedef DAT_RETURN (*DAT_EP_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_EVD_HANDLE, /* recv_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* request_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* connect_evd_handle */ + IN const DAT_EP_ATTR *, /* ep_attributes */ + OUT DAT_EP_HANDLE * ); /* ep_handle */ + +typedef DAT_RETURN (*DAT_EP_CREATE_WITH_SRQ_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_EVD_HANDLE, /* recv_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* request_completion_evd_handle */ + IN DAT_EVD_HANDLE, /* connect_evd_handle */ + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN const DAT_EP_ATTR *, /* ep_attributes */ + OUT DAT_EP_HANDLE * ); /* ep_handle */ + +typedef DAT_RETURN (*DAT_EP_QUERY_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_PARAM_MASK, /* ep_param_mask */ + OUT DAT_EP_PARAM * ); /* ep_param */ + +typedef DAT_RETURN (*DAT_EP_MODIFY_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_PARAM_MASK, /* ep_param_mask */ + IN const DAT_EP_PARAM * ); /* ep_param */ + +typedef DAT_RETURN (*DAT_EP_CONNECT_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR, /* remote_ia_address */ + IN DAT_CONN_QUAL, /* remote_conn_qual */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID, /* private_data */ + IN DAT_QOS, /* quality_of_service */ + IN DAT_CONNECT_FLAGS ); /* connect_flags */ + +typedef DAT_RETURN (*DAT_EP_COMMON_CONNECT_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_IA_ADDRESS_PTR, /* remote_ia_address */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID ); /* private_data */ + +typedef DAT_RETURN (*DAT_EP_DUP_CONNECT_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EP_HANDLE, /* ep_dup_handle */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* private_data_size */ + IN const DAT_PVOID, /* private_data */ + IN DAT_QOS); /* quality_of_service */ + +typedef DAT_RETURN (*DAT_EP_DISCONNECT_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_CLOSE_FLAGS ); /* close_flags */ + +typedef DAT_RETURN (*DAT_EP_POST_SEND_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +typedef DAT_RETURN (*DAT_EP_POST_SEND_WITH_INVALIDATE_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + IN DAT_BOOLEAN, /* invalidate_flag */ + IN DAT_RMR_CONTEXT ); /* RMR context */ + +typedef DAT_RETURN (*DAT_EP_POST_RECV_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +typedef DAT_RETURN (*DAT_EP_POST_RDMA_READ_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +typedef DAT_RETURN (*DAT_EP_POST_RDMA_READ_TO_RMR_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN const DAT_RMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +typedef DAT_RETURN (*DAT_EP_POST_RDMA_WRITE_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE, /* user_cookie */ + IN const DAT_RMR_TRIPLET *,/* remote_iov */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +typedef DAT_RETURN (*DAT_EP_GET_STATUS_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + OUT DAT_EP_STATE *, /* ep_state */ + OUT DAT_BOOLEAN *, /* recv_idle */ + OUT DAT_BOOLEAN * ); /* request_idle */ + +typedef DAT_RETURN (*DAT_EP_FREE_FUNC) ( + IN DAT_EP_HANDLE); /* ep_handle */ + +typedef DAT_RETURN (*DAT_EP_RESET_FUNC) ( + IN DAT_EP_HANDLE); /* ep_handle */ + +typedef DAT_RETURN (*DAT_EP_RECV_QUERY_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + OUT DAT_COUNT *, /* nbufs_allocated */ + OUT DAT_COUNT *); /* bufs_alloc_span */ + +typedef DAT_RETURN (*DAT_EP_SET_WATERMARK_FUNC) ( + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_COUNT, /* ep_soft_high_watermark*/ + IN DAT_COUNT ); /* ep_hard_high_watermark*/ + +/* LMR functions */ + +typedef DAT_RETURN (*DAT_LMR_FREE_FUNC) ( + IN DAT_LMR_HANDLE ); /* lmr_handle */ + +typedef DAT_RETURN (*DAT_LMR_SYNC_RDMA_READ_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_LMR_TRIPLET *,/* local segments */ + IN DAT_VLEN ); /* num_segments */ + +typedef DAT_RETURN (*DAT_LMR_SYNC_RDMA_WRITE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_LMR_TRIPLET *, /* local_segments */ + IN DAT_VLEN ); /* num_segments */ + +/* RMR functions */ + +typedef DAT_RETURN (*DAT_RMR_CREATE_FUNC) ( + IN DAT_PZ_HANDLE, /* pz_handle */ + OUT DAT_RMR_HANDLE *); /* rmr_handle */ + +typedef DAT_RETURN (*DAT_RMR_CREATE_FOR_EP_FUNC) ( + IN DAT_PZ_HANDLE, /* pz_handle */ + OUT DAT_RMR_HANDLE *); /* rmr_handle */ + +typedef DAT_RETURN (*DAT_RMR_QUERY_FUNC) ( + IN DAT_RMR_HANDLE, /* rmr_handle */ + IN DAT_RMR_PARAM_MASK, /* rmr_param_mask */ + OUT DAT_RMR_PARAM *); /* rmr_param */ + +typedef DAT_RETURN (*DAT_RMR_BIND_FUNC) ( + IN DAT_RMR_HANDLE, /* rmr_handle */ + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN const DAT_LMR_TRIPLET *,/* lmr_triplet */ + IN DAT_MEM_PRIV_FLAGS, /* mem_priv */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_RMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + OUT DAT_RMR_CONTEXT * ); /* context */ + +typedef DAT_RETURN (*DAT_RMR_FREE_FUNC) ( + IN DAT_RMR_HANDLE); /* rmr_handle */ + +/* PSP functions */ + +typedef DAT_RETURN (*DAT_PSP_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CONN_QUAL, /* conn_qual */ + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_PSP_FLAGS, /* psp_flags */ + OUT DAT_PSP_HANDLE * ); /* psp_handle */ + +typedef DAT_RETURN (*DAT_PSP_CREATE_ANY_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_CONN_QUAL *, /* conn_qual */ + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_PSP_FLAGS, /* psp_flags */ + OUT DAT_PSP_HANDLE * ); /* psp_handle */ + +typedef DAT_RETURN (*DAT_PSP_QUERY_FUNC) ( + IN DAT_PSP_HANDLE, /* psp_handle */ + IN DAT_PSP_PARAM_MASK, /* psp_param_mask */ + OUT DAT_PSP_PARAM * ); /* psp_param */ + +typedef DAT_RETURN (*DAT_PSP_FREE_FUNC) ( + IN DAT_PSP_HANDLE ); /* psp_handle */ + +/* RSP functions */ + +typedef DAT_RETURN (*DAT_RSP_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_CONN_QUAL, /* conn_qual */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_RSP_HANDLE * ); /* rsp_handle */ + +typedef DAT_RETURN (*DAT_RSP_QUERY_FUNC) ( + IN DAT_RSP_HANDLE, /* rsp_handle */ + IN DAT_RSP_PARAM_MASK, /* rsp_param_mask */ + OUT DAT_RSP_PARAM * ); /* rsp_param */ + +typedef DAT_RETURN (*DAT_RSP_FREE_FUNC) ( + IN DAT_RSP_HANDLE ); /* rsp_handle */ + + /* CSP functions functions - DAT 2.0 */ + +typedef DAT_RETURN (*DAT_CSP_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COMM *, /* communicator */ + IN DAT_IA_ADDRESS_PTR, /* address */ + IN DAT_EVD_HANDLE, /* evd_handle */ + OUT DAT_CSP_HANDLE * ); /* csp_handle */ + +typedef DAT_RETURN (*DAT_CSP_QUERY_FUNC) ( + IN DAT_CSP_HANDLE, /* csp_handle */ + IN DAT_CSP_PARAM_MASK, /* csp_param_mask */ + OUT DAT_CSP_PARAM * ); /* csp_param */ + +typedef DAT_RETURN (*DAT_CSP_FREE_FUNC) ( + IN DAT_CSP_HANDLE ); /* csp_handle */ + +/* PZ functions */ + +typedef DAT_RETURN (*DAT_PZ_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_PZ_HANDLE * ); /* pz_handle */ + +typedef DAT_RETURN (*DAT_PZ_QUERY_FUNC) ( + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_PZ_PARAM_MASK, /* pz_param_mask */ + OUT DAT_PZ_PARAM *); /* pz_param */ + +typedef DAT_RETURN (*DAT_PZ_FREE_FUNC) ( + IN DAT_PZ_HANDLE ); /* pz_handle */ + +/* SRQ functions */ + +typedef DAT_RETURN (*DAT_SRQ_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_SRQ_ATTR *, /* srq_attributes */ + OUT DAT_SRQ_HANDLE *); /* srq_handle */ + +typedef DAT_RETURN (*DAT_SRQ_SET_LW_FUNC) ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT ); /* srq_low_watermark */ + +typedef DAT_RETURN (*DAT_SRQ_FREE_FUNC) ( + IN DAT_SRQ_HANDLE); /* srq_handle */ + +typedef DAT_RETURN (*DAT_SRQ_QUERY_FUNC) ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_SRQ_PARAM_MASK, /* srq_param_mask */ + OUT DAT_SRQ_PARAM *); /* srq_param */ + +typedef DAT_RETURN (*DAT_SRQ_RESIZE_FUNC) ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT ); /* srq_queue_length */ + +typedef DAT_RETURN (*DAT_SRQ_POST_RECV_FUNC) ( + IN DAT_SRQ_HANDLE, /* srq_handle */ + IN DAT_COUNT, /* num_segments */ + IN DAT_LMR_TRIPLET *, /* local_iov */ + IN DAT_DTO_COOKIE ); /* user_cookie */ + +typedef DAT_RETURN (*DAT_IA_HA_RELATED_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN const DAT_NAME_PTR, /* provider */ + OUT DAT_BOOLEAN *); /* answer */ + +#ifdef DAT_EXTENSIONS +#include +typedef DAT_RETURN (*DAT_HANDLE_EXTENDEDOP_FUNC)( + IN DAT_HANDLE, /* handle */ + IN DAT_EXTENDED_OP, /* extended op */ + IN va_list); /* argument list */ +#endif /* DAT_EXTENSIONS */ + +#endif /* _DAT_REDIRECTION_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_registry.h b/trunk/ulp/dapl2/dat/include/dat/dat_registry.h new file mode 100644 index 00000000..021dca46 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_registry.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/**************************************************************** + * + * HEADER: dat_registry.h + * + * PURPOSE: DAT registration API signatures + * + * Description: Header file for "DAPL: Direct Access Programming + * Library, Version: 2.0" + * + * Contains registration external reference signatures + * for dat registry functions. This file is *only* + * included by providers, not consumers. + * + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + **********************************************************/ +#ifndef _DAT_REGISTRY_H_ +#define _DAT_REGISTRY_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#if defined(_UDAT_H_) +#include +#elif defined(_KDAT_H_) +#include +#else +#error Must include udat.h or kdat.h +#endif + +/* + * dat registration API. + * + * Technically the dat_ia_open is part of the registration API. This + * is so the registration module can map the device name to a provider + * structure and then call the provider dat_ia_open function. + * dat_is_close is also part of the registration API so that the + * registration code can be aware when an ia is no longer in use. + * + */ + +extern DAT_RETURN DAT_API dat_registry_add_provider ( + IN const DAT_PROVIDER *, /* provider */ + IN const DAT_PROVIDER_INFO* ); /* provider info */ + +extern DAT_RETURN DAT_API dat_registry_remove_provider ( + IN const DAT_PROVIDER *, /* provider */ + IN const DAT_PROVIDER_INFO* ); /* provider info */ + +/* + * Provider initialization APIs. + * + * Providers that support being automatically loaded by the Registry must + * implement these APIs and export them as public symbols. + */ + +#define DAT_PROVIDER_INIT_FUNC_NAME dat_provider_init +#define DAT_PROVIDER_FINI_FUNC_NAME dat_provider_fini + +#define DAT_PROVIDER_INIT_FUNC_STR "dat_provider_init" +#define DAT_PROVIDER_FINI_FUNC_STR "dat_provider_fini" + +typedef void ( DAT_API *DAT_PROVIDER_INIT_FUNC) ( + IN const DAT_PROVIDER_INFO *, /* provider info */ + IN const char *); /* instance data */ + +typedef void ( DAT_API *DAT_PROVIDER_FINI_FUNC) ( + IN const DAT_PROVIDER_INFO *); /* provider info */ + +typedef enum dat_ha_relationship +{ + DAT_HA_FALSE, /* two IAs are not related */ + DAT_HA_TRUE, /* two IAs are related */ + DAT_HA_UNKNOWN, /* relationship is not known */ + DAT_HA_CONFLICTING, /* 2 IAs do not agree on the relationship */ + DAT_HA_EXTENSION_BASE +} DAT_HA_RELATIONSHIP; + +extern DAT_RETURN DAT_API dat_registry_providers_related ( + IN const DAT_NAME_PTR, + IN const DAT_NAME_PTR, + OUT DAT_HA_RELATIONSHIP * ); + +#ifdef __cplusplus +} +#endif + +#endif /* _DAT_REGISTRY_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/dat_vendor_specific.h b/trunk/ulp/dapl2/dat/include/dat/dat_vendor_specific.h new file mode 100644 index 00000000..a204945c --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/dat_vendor_specific.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/*************************************************************** + * + * HEADER: dat_vendor_specific.h + * + * PURPOSE: + * + * Description: Header file for "DAPL: Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * + ***************************************************************/ + +#ifndef _DAT_VENDOR_SPECIFIC_H_ +#define _DAT_VENDOR_SPECIFIC_H_ + + + +/* Vendor-specific extensions */ + +#if defined(_AMMASSO) + +#elif defined(_BROADCOM) + +#elif defined(_CISCO) + +#elif defined(_IBM) + +#elif defined(_INTEL) + +#elif defined(_JNI) + +#elif defined(_MELLANOX) + +#elif defined(_MYRINET) + +#elif defined(_NETEFFECT) + +#elif defined(_QLOGIC) + +#elif defined(_SILVERSTORM) + +#elif defined(_VOLTAIRE) + +#endif + +#endif /* _DAT_VENDOR_SPECIFIC_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/kdat.h b/trunk/ulp/dapl2/dat/include/dat/kdat.h new file mode 100644 index 00000000..9eb2d854 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/kdat.h @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/**************************************************************** + * + * HEADER: kdat.h + * + * PURPOSE: defines the kernel DAT API + * + * Description: Header file for "kDAPL: Kernel Direct Access Programming + * Library, Version: 2.0" + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + * + * + ***************************************************************/ + +#ifndef _KDAT_H_ +#define _KDAT_H_ + +#include + +#include + +#if 1 +#define EXPORT_SYMBOL_NOVERS(sym) EXPORT_SYMBOL(sym) +#endif + +typedef enum dat_mem_type +{ + /* Shared between udat and kdat */ + DAT_MEM_TYPE_VIRTUAL = 0x00, + DAT_MEM_TYPE_LMR = 0x01, + /* kdat specific */ + DAT_MEM_TYPE_PHYSICAL = 0x10, + DAT_MEM_TYPE_PLATFORM = 0x20, + DAT_MEM_TYPE_IA = 0x40, + DAT_MEM_TYPE_BYPASS = 0x80, + DAT_MEM_TYPE_PHYSICAL_VAR_PAGES = 0x100 +} DAT_MEM_TYPE; + +/* dat handle types */ +typedef enum dat_handle_type +{ + DAT_HANDLE_TYPE_CR, + DAT_HANDLE_TYPE_EP, + DAT_HANDLE_TYPE_EVD, + DAT_HANDLE_TYPE_IA, + DAT_HANDLE_TYPE_LMR, + DAT_HANDLE_TYPE_PSP, + DAT_HANDLE_TYPE_PZ, + DAT_HANDLE_TYPE_RMR, + DAT_HANDLE_TYPE_RSP, + DAT_HANDLE_TYPE_SRQ, + DAT_HANDLE_TYPE_CSP +#ifdef DAT_EXTENSIONS + ,DAT_HANDLE_TYPE_EXTENSION_BASE +#endif +} DAT_HANDLE_TYPE; + +typedef enum dat_evd_param_mask +{ + DAT_EVD_FIELD_IA_HANDLE = 0x01, + DAT_EVD_FIELD_EVD_QLEN = 0x02, + DAT_EVD_FIELD_UPCALL_POLICY = 0x04, + DAT_EVD_FIELD_UPCALL = 0x08, + DAT_EVD_FIELD_EVD_FLAGS = 0x10, + DAT_EVD_FIELD_ALL = 0x1F +} DAT_EVD_PARAM_MASK; + +typedef DAT_UINT64 DAT_PROVIDER_ATTR_MASK; + +#include + +typedef DAT_CONTEXT DAT_LMR_COOKIE; + +/* Upcall support */ + +typedef enum dat_upcall_policy +{ + DAT_UPCALL_DISABLE = 0, /* support no_upcalls */ + DAT_UPCALL_SINGLE_INSTANCE = 1, /* support only one upcall */ + DAT_UPCALL_MANY = 100, /* support multiple upcalls */ + DAT_UPCALL_TEARDOWN = -1 /* support no upcalls & return * + * after all in progress * + * UpCalls return */ + ,DAT_UPCALL_NORIFY = 2 /* support single upcall with * + * notification only */ +} DAT_UPCALL_POLICY; + +typedef void (*DAT_UPCALL_FUNC)( + DAT_PVOID, /* instance_data */ + const DAT_EVENT *, /* event */ + DAT_BOOLEAN); /* more_events */ + +typedef struct dat_upcall_object +{ + DAT_PVOID instance_data; + DAT_UPCALL_FUNC upcall_func; +} DAT_UPCALL_OBJECT; + +/* Define NULL upcall */ + +#define DAT_UPCALL_NULL \ + (DAT_UPCALL_OBJECT) { (DAT_PVOID) NULL, \ + (DAT_UPCALL_FUNC) NULL} + +#define DAT_UPCALL_SAME ((DAT_UPCALL_OBJECT *) NULL) + + /* kDAT Upcall Invocation Policy */ +typedef enum dat_upcall_flag +{ + DAT_UPCALL_INVOC_NEW = 0, /* invoke on a new event arrival */ + DAT_UPCALL_INVOC_ANY = 1 /* invoke on any event on EVD */ +} DAT_UPCALL_FLAG; + +struct dat_evd_param +{ + DAT_IA_HANDLE ia_handle; + DAT_COUNT evd_qlen; + DAT_UPCALL_POLICY upcall_policy; + DAT_UPCALL_OBJECT upcall; + DAT_EVD_FLAGS evd_flags; +}; + +/* + * Memory types + * + * Specifying memory type for LMR create. A Consumer must use a single + * value when registering memory. The union of any of these + * flags is used in the Provider parameters to indicate what memory + * type Provider supports for LMR memory creation. + */ + +/* memory data types */ +#define DAT_MEM_OPT DAT_MEM_OPTIMIZE_FLAGS + +typedef enum dat_mem_optimize_flags +{ + DAT_MEM_OPTIMIZE_DONT_CARE = 0x00, + DAT_MEM_OPTIMIZE_IA = 0x01, + DAT_MEM_OPTIMIZE_MIN_EXPOSURE = 0x02, + DAT_MEM_OPTIMIZE_EXACT_EXPOSURE = 0x04 +} DAT_MEM_OPTIMIZE_FLAGS; + +typedef struct dat_physical_var_pages +{ + DAT_VLEN page_size; + void* pages; +} DAT_PHYSICAL_VAR_PAGES; + +typedef union dat_region_description +{ + DAT_PVOID for_va; + DAT_LMR_HANDLE for_lmr_handle; + void * for_platform; /* For kdapl only + for platform and bypass */ + DAT_PHYSICAL_VAR_PAGES for_array; /* for kdapl only */ + DAT_PADDR *for_physical; /* for kdapl only */ + /* array of physical pages of the same size */ + DAT_PADDR for_ia; /* for kdapl only */ +} DAT_REGION_DESCRIPTION; + +/* LMR Arguments */ +typedef struct dat_lmr_state +{ + DAT_BOOLEAN fmr_support; + DAT_BOOLEAN bound; +} DAT_LMR_STATE; + +struct dat_lmr_param +{ + DAT_IA_HANDLE ia_handle; + DAT_MEM_TYPE mem_type; + DAT_REGION_DESCRIPTION region_desc; + DAT_VLEN length; + DAT_PZ_HANDLE pz_handle; + DAT_MEM_PRIV_FLAGS mem_priv; + DAT_VA_TYPE va_type; + DAT_LMR_CONTEXT lmr_context; + DAT_RMR_CONTEXT rmr_context; + DAT_VLEN registered_size; + DAT_VADDR registered_address; + DAT_LMR_STATE lmr_state; /* kDAPL only */ +}; + +/* LMR Arguments Mask */ + +enum dat_lmr_param_mask +{ + DAT_LMR_FIELD_IA_HANDLE = 0x001, + DAT_LMR_FIELD_MEM_TYPE = 0x002, + DAT_LMR_FIELD_REGION_DESC = 0x004, + DAT_LMR_FIELD_LENGTH = 0x008, + DAT_LMR_FIELD_PZ_HANDLE = 0x010, + DAT_LMR_FIELD_MEM_PRIV = 0x020, + DAT_LMR_FIELD_VA_TYPE = 0x040, + DAT_LMR_FIELD_LMR_CONTEXT = 0x080, + DAT_LMR_FIELD_RMR_CONTEXT = 0x100, + DAT_LMR_FIELD_REGISTERED_SIZE = 0x200, + DAT_LMR_FIELD_REGISTERED_ADDRESS = 0x400, + DAT_LMR_FIELD_LMR_STATE = 0x800, + + DAT_LMR_FIELD_ALL = 0xFFF +}; + + /* kDAPL 1.3 addition */ + /* Defines LMR protection scope */ +typedef enum dat_lmr_scope +{ + DAT_LMR_SCOPE_EP, /* bound to at most one EP at a time. */ + DAT_LMR_SCOPE_PZ, /* bound to a Protection Zone */ + DAT_LMR_SCOPE_ANY /* Supports all types */ +} DAT_LMR_SCOPE; + +struct dat_ia_attr +{ + char adapter_name[DAT_NAME_MAX_LENGTH]; + char vendor_name[DAT_NAME_MAX_LENGTH]; + DAT_UINT32 hardware_version_major; + DAT_UINT32 hardware_version_minor; + DAT_UINT32 firmware_version_major; + DAT_UINT32 firmware_version_minor; + DAT_IA_ADDRESS_PTR ia_address_ptr; + DAT_COUNT max_eps; + DAT_COUNT max_dto_per_ep; + DAT_COUNT max_rdma_read_per_ep_in; + DAT_COUNT max_rdma_read_per_ep_out; + DAT_COUNT max_evds; + DAT_COUNT max_evd_qlen; + DAT_COUNT max_iov_segments_per_dto; + DAT_COUNT max_lmrs; + DAT_SEG_LENGTH max_lmr_block_size; + DAT_VADDR max_lmr_virtual_address; + DAT_COUNT max_pzs; + DAT_SEG_LENGTH max_message_size; + DAT_SEG_LENGTH max_rdma_size; + DAT_COUNT max_rmrs; + DAT_VADDR max_rmr_target_address; + DAT_COUNT max_srqs; + DAT_COUNT max_ep_per_srq; + DAT_COUNT max_recv_per_srq; + DAT_COUNT max_iov_segments_per_rdma_read; + DAT_COUNT max_iov_segments_per_rdma_write; + DAT_COUNT max_rdma_read_in; + DAT_COUNT max_rdma_read_out; + DAT_BOOLEAN max_rdma_read_per_ep_in_guaranteed; + DAT_BOOLEAN max_rdma_read_per_ep_out_guaranteed; + DAT_BOOLEAN zb_supported; + DAT_BOOLEAN reserved_lmr_supported; +#ifdef DAT_EXTENSIONS + DAT_EXTENSION extension_supported; + DAT_COUNT extension_version; +#endif /* DAT_EXTENSIONS */ + DAT_COUNT num_transport_attr; + DAT_NAMED_ATTR *transport_attr; + DAT_COUNT num_vendor_attr; + DAT_NAMED_ATTR *vendor_attr; +}; + +#define DAT_IA_FIELD_IA_RESERVED_LMR_SUPPORTED UINT64_C(0x100000000) +#ifdef DAT_EXTENSIONS +#define DAT_IA_FIELD_IA_EXTENSION UINT64_C(0x200000000) +#define DAT_IA_FIELD_IA_EXTENSION_VERSION UINT64_C(0x400000000) +#endif /* DAT_EXTENSIONS */ + +#define DAT_IA_FIELD_IA_NUM_TRANSPORT_ATTR UINT64_C(0x800000000) +#define DAT_IA_FIELD_IA_TRANSPORT_ATTR UINT64_C(0x1000000000) +#define DAT_IA_FIELD_IA_NUM_VENDOR_ATTR UINT64_C(0x2000000000) +#define DAT_IA_FIELD_IA_VENDOR_ATTR UINT64_C(0x4000000000) +#define DAT_IA_FIELD_ALL UINT64_C(0x7FFFFFFFFF) + +#define DAT_EVENT_NULL ((DAT_EVENT) NULL) + +/* General Provider attributes. kdat specific. */ + +#include + +/* Provider should support merging of all event stream types. Provider + * attribute specify support for merging different event stream types. + * It is a 2D binary matrix where each row and column represents an event + * stream type. Each binary entry is 1 if the event streams of its raw + * and column can fed the same EVD, and 0 otherwise. The order of event + * streams in row and column is the same as in the definition of + * DAT_EVD_FLAGS: index 0 - Software Event, 1- Connection Request, + * 2 - DTO Completion, 3 - Connection event, 4 - RMR Bind Completion, + * 5 - Asynchronous event. By definition each diagonal entry is 1. + * Consumer allocates an array for it and passes it IN as a pointer + * for the array that Provider fills. Provider must fill the array + * that Consumer passes. + */ + +struct dat_provider_attr +{ + char provider_name[DAT_NAME_MAX_LENGTH]; + DAT_UINT32 provider_version_major; + DAT_UINT32 provider_version_minor; + DAT_UINT32 dapl_version_major; + DAT_UINT32 dapl_version_minor; + DAT_MEM_TYPE lmr_mem_types_supported; + DAT_IOV_OWNERSHIP iov_ownership_on_return; + DAT_QOS dat_qos_supported; + DAT_COMPLETION_FLAGS completion_flags_supported; + DAT_BOOLEAN is_thread_safe; + DAT_COUNT max_private_data_size; + DAT_BOOLEAN supports_multipath; + DAT_EP_CREATOR_FOR_PSP ep_creator; + DAT_UPCALL_POLICY upcall_policy; + DAT_UINT32 optimal_buffer_alignment; + const DAT_BOOLEAN evd_stream_merging_supported[6][6]; + DAT_BOOLEAN srq_supported; + DAT_COUNT srq_watermarks_supported; + DAT_BOOLEAN srq_ep_pz_difference_supported; + DAT_COUNT srq_info_supported; + DAT_COUNT ep_recv_info_supported; + DAT_BOOLEAN lmr_sync_req; + DAT_BOOLEAN dto_async_return_guaranteed; + DAT_BOOLEAN rdma_write_for_rdma_read_req; + DAT_BOOLEAN rdma_read_lmr_rmr_context_exposure; + DAT_RMR_SCOPE rmr_scope_supported; + DAT_BOOLEAN is_interrupt_safe; + DAT_BOOLEAN fmr_supported; + DAT_LMR_SCOPE lmr_for_fmr_scope_supported; + DAT_BOOLEAN ha_supported; + DAT_HA_LB ha_loadbalancing; + + DAT_COUNT num_provider_specific_attr; + DAT_NAMED_ATTR * provider_specific_attr; +}; + + +#define DAT_PROVIDER_FIELD_PROVIDER_NAME UINT64_C(0x000000001) +#define DAT_PROVIDER_FIELD_PROVIDER_VERSION_MAJOR UINT64_C(0x000000002) +#define DAT_PROVIDER_FIELD_PROVIDER_VERSION_MINOR UINT64_C(0x000000004) +#define DAT_PROVIDER_FIELD_DAPL_VERSION_MAJOR UINT64_C(0x000000008) +#define DAT_PROVIDER_FIELD_DAPL_VERSION_MINOR UINT64_C(0x000000010) +#define DAT_PROVIDER_FIELD_LMR_MEM_TYPE_SUPPORTED UINT64_C(0x000000020) +#define DAT_PROVIDER_FIELD_IOV_OWNERSHIP UINT64_C(0x000000040) +#define DAT_PROVIDER_FIELD_DAT_QOS_SUPPORTED UINT64_C(0x000000080) +#define DAT_PROVIDER_FIELD_COMPLETION_FLAGS_SUPPORTED UINT64_C(0x000000100) +#define DAT_PROVIDER_FIELD_IS_THREAD_SAFE UINT64_C(0x000000200) +#define DAT_PROVIDER_FIELD_MAX_PRIVATE_DATA_SIZE UINT64_C(0x000000400) +#define DAT_PROVIDER_FIELD_SUPPORTS_MULTIPATH UINT64_C(0x000000800) +#define DAT_PROVIDER_FIELD_EP_CREATOR UINT64_C(0x000001000) +#define DAT_PROVIDER_FIELD_UPCALL_POLICY UINT64_C(0x000002000) +#define DAT_PROVIDER_FIELD_OPTIMAL_BUFFER_ALIGNMENT UINT64_C(0x000004000) +#define DAT_PROVIDER_FIELD_EVD_STREAM_MERGING_SUPPORTED UINT64_C(0x000008000) +#define DAT_PROVIDER_FIELD_SRQ_SUPPORTED UINT64_C(0x000010000) +#define DAT_PROVIDER_FIELD_SRQ_WATERMARKS_SUPPORTED UINT64_C(0x000020000) +#define DAT_PROVIDER_FIELD_SRQ_EP_PZ_DIFFERENCE_SUPPORTED UINT64_C(0x000040000) +#define DAT_PROVIDER_FIELD_SRQ_INFO_SUPPORTED UINT64_C(0x000080000) +#define DAT_PROVIDER_FIELD_EP_RECV_INFO_SUPPORTED UINT64_C(0x000100000) +#define DAT_PROVIDER_FIELD_LMR_SYNC_REQ UINT64_C(0x000200000) +#define DAT_PROVIDER_FIELD_DTO_ASYNC_RETURN_GUARANTEED UINT64_C(0x000400000) +#define DAT_PROVIDER_FIELD_RDMA_WRITE_FOR_RDMA_READ_REQ UINT64_C(0x000800000) +#define DAT_PROVIDER_FIELD_RDMA_READ_LMR_RMR_CONTEXT_EXPOSURE UINT64_C(0x001000000) +#define DAT_PROVIDER_FIELD_RMR_SCOPE_SUPPORTED UINT64_C(0x002000000) +#define DAT_PROVIDER_FIELD_IS_INTERRUPT_SAFE UINT64_C(0x004000000) +#define DAT_PROVIDER_FIELD_FMR_SUPPORTED UINT64_C(0x008000000) +#define DAT_PROVIDER_FIELD_LMR_FOR_FMR_SCOPE UINT64_C(0x010000000) +#define DAT_PROVIDER_FIELD_HA_SUPPORTED UINT64_C(0x020000000) +#define DAT_PROVIDER_FIELD_HA_LB UINT64_C(0x040000000) +#define DAT_PROVIDER_FIELD_NUM_PROVIDER_SPECIFIC_ATTR UINT64_C(0x080000000) +#define DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR UINT64_C(0x100000000) + +#define DAT_PROVIDER_FIELD_ALL UINT64_C(0x1FFFFFFFF) +#define DAT_PROVIDER_FIELD_NONE UINT64_C(0x0) + +/**********************************************************************/ + +/* + * Kernel DAT function call definitions, + */ + +extern DAT_RETURN dat_lmr_kcreate ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* length */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_MEM_PRIV_FLAGS, /* privileges */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_MEM_OPTIMIZE_FLAGS, /* mem_optimization */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT *, /* lmr_context */ + OUT DAT_RMR_CONTEXT *, /* rmr_context */ + OUT DAT_VLEN *, /* registered_length */ + OUT DAT_VADDR * ); /* registered_address */ + +extern DAT_RETURN dat_ia_memtype_hint ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_VLEN, /* length */ + IN DAT_MEM_OPTIMIZE_FLAGS, /* mem_optimization */ + OUT DAT_VLEN *, /* preferred_length */ + OUT DAT_VADDR * ); /* preferred_alignment */ + +extern DAT_RETURN dat_lmr_query ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_LMR_PARAM_MASK, /* lmr_param_mask */ + OUT DAT_LMR_PARAM *); /* lmr_param */ + +extern DAT_RETURN dat_lmr_allocate ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_COUNT, /* max # of physical pages */ + IN DAT_LMR_SCOPE, /* scope of lmr */ + OUT DAT_LMR_HANDLE * ); /*lmr handle */ + +extern DAT_RETURN dat_lmr_fmr ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* number of pages */ + IN DAT_MEM_PRIV_FLAGS, /* mem_privileges */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_LMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + OUT DAT_LMR_CONTEXT, /* lmr_context */ + OUT DAT_RMR_CONTEXT ); /* rmr_context */ + +extern DAT_RETURN dat_lmr_invalidate ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_LMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +extern DAT_RETURN dat_ia_reserved_lmr ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_LMR_HANDLE *, /* reserved_lmr_handle */ + OUT DAT_LMR_CONTEXT * ); /* reserved_lmr_context */ + +extern DAT_RETURN dat_evd_kcreate ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COUNT, /* evd_min_qlen */ + IN DAT_UPCALL_POLICY, /* upcall_policy */ + IN const DAT_UPCALL_OBJECT *, /* upcall */ + IN DAT_EVD_FLAGS, /* evd_flags */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +extern DAT_RETURN dat_evd_modify_upcall ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_UPCALL_POLICY, /* upcall_policy */ + IN const DAT_UPCALL_OBJECT* , /* upcall */ + IN DAT_UPCALL_FLAG ); /* upcall invocation policy */ + +#endif /* _KDAT_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/kdat_config.h b/trunk/ulp/dapl2/dat/include/dat/kdat_config.h new file mode 100644 index 00000000..bbc1d469 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/kdat_config.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*************************************************************** + * + * HEADER: kdat_config.h + * + * PURPOSE: provides kDAPL configuration information. + * + * Description: Header file for "kDAPL: Kernel Direct Access Programming + * Library, Version: 2.0" + * + ***************************************************************/ +#ifndef _KDAT_CONFIG_H_ +#define _KDAT_CONFIG_H_ + +#define DAT_VERSION_MAJOR 1 +#define DAT_VERSION_MINOR 2 + +/* + * The official header files will default DAT_THREADSAFE to DAT_TRUE. If + * your project does not wish to use this default, you must ensure that + * DAT_THREADSAFE will be set to DAT_FALSE. This may be done by an + * explicit #define in a common project header file that is included + * before any DAT header files, or through command line directives to the + * compiler (presumably controlled by the make environment). + */ + +/* + * A site, project or platform may consider setting an alternate default + * via their make rules, but are discouraged from doing so by editing + * the official header files. + */ + +/* + * The Reference Implementation is not Thread Safe. The Reference + * Implementation has chosen to go with the first method and define it + * explicitly in the header file. + */ + +#define DAT_THREADSAFE DAT_FALSE + +#ifndef DAT_THREADSAFE +#define DAT_THREADSAFE DAT_TRUE +#endif /* DAT_THREADSAFE */ + +#endif /* _KDAT_CONFIG_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/kdat_redirection.h b/trunk/ulp/dapl2/dat/include/dat/kdat_redirection.h new file mode 100644 index 00000000..5c88fec9 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/kdat_redirection.h @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/**************************************************************** + * + * HEADER: kdat_redirection.h + * + * PURPOSE: Kernel DAT macro definitions + * + * Description: Macros to invoke DAPL functions from the dat_registry + * + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + **********************************************************/ + +#ifndef _KDAT_REDIRECTION_H_ +#define _KDAT_REDIRECTION_H_ + +/* ia_memtype_hint macro */ + +#define DAT_IA_MEMTYPE_HINT(ia, mem_type, len, mem_opt, pref_len, pref_align) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->ia_memtype_hint_func)(\ + (ia), \ + (mem_type), \ + (len), \ + (mem_opt), \ + (pref_len), \ + (pref_align)) + +/* evd_modify_upcall macro */ + +#define DAT_EVD_MODIFY_UPCALL(evd, policy, upcall, flag) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_modify_upcall_func)(\ + (evd), \ + (policy), \ + (upcall), \ + (flag)) + +/* evd_create macro */ +#define DAT_EVD_KCREATE(ia,qlen,policy,upcall,flags,handle) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->evd_kcreate_func)(\ + (ia), \ + (qlen), \ + (policy), \ + (upcall), \ + (flags), \ + (handle)) + +/* lmr_kcreate macro */ +#define DAT_LMR_KCREATE(ia,mtype,reg_desc,len,pz,priv,mem_opt,\ + va_type,lmr,lmr_context,rmr_context,reg_len,reg_addr) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->lmr_kcreate_func)(\ + (ia), \ + (mtype), \ + (reg_desc), \ + (len), \ + (pz), \ + (priv), \ + (mem_opt), \ + (va_type), \ + (lmr), \ + (lmr_context), \ + (rmr_context), \ + (reg_len), \ + (reg_addr)) + +#define DAT_LMR_ALLOCATE(ia,pz,size,scope,lmr) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->lmr_allocate_func)(\ + (ia), \ + (pz), \ + (size), \ + (scope), \ + (lmr)) + +#define DAT_LMR_FMR(lmr,mtype,reg_desc,length,priv,\ + va_type,ep,cookie,flags,lmr_context,rmr_context) \ + (*DAT_HANDLE_TO_PROVIDER(lmr)->lmr_fmr_func)(\ + (lmr),\ + (mtype),\ + (reg_desc),\ + (length),\ + (priv),\ + (va_type),\ + (ep),\ + (cookie),\ + (flags),\ + (lmr_context),\ + (rmr_context)) + +#define DAT_LMR_INVALIDATE(lmr, ep,cookie,flags) \ + (*DAT_HANDLE_TO_PROVIDER(lmr)->lmr_invalidate_func)(\ + (lmr),\ + (ep),\ + (cookie),\ + (flags)) + +#define DAT_IA_RESERVED_LMR(ia, lmr, lmr_context) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->ia_reserved_lmr_func)(\ + (ia),\ + (lmr),\ + (lmr_context)) + +/*************************************************************** + * FUNCTION PROTOTYPES + * + * Kernel DAT function call definitions, + * + ****************************************************************/ + +typedef DAT_RETURN (*DAT_LMR_KCREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* length */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_MEM_PRIV_FLAGS, /* privileges */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_MEM_OPTIMIZE_FLAGS, /* mem_optimization */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT *, /* lmr_context */ + OUT DAT_RMR_CONTEXT *, /* rmr_context */ + OUT DAT_VLEN *, /* registered_length */ + OUT DAT_VADDR * ); /* registered_address */ + + +typedef DAT_RETURN (*DAT_IA_MEMTYPE_HINT_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_VLEN, /* length */ + IN DAT_MEM_OPTIMIZE_FLAGS, /* mem_optimization */ + OUT DAT_VLEN *, /* preferred_length */ + OUT DAT_VADDR * ); /* preferred_alignment */ + +typedef DAT_RETURN (*DAT_LMR_QUERY_FUNC) ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_LMR_PARAM_MASK, /* lmr_param_mask */ + OUT DAT_LMR_PARAM *); /* lmr_param */ + +typedef DAT_RETURN (*DAT_LMR_ALLOCATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_COUNT, /* max number of physical pages */ + IN DAT_LMR_SCOPE, /* scope of LMR */ + OUT DAT_LMR_HANDLE ); /* lmr_handle */ + +typedef DAT_RETURN (*DAT_LMR_FMR_FUNC) ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN const DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* number of pages */ + IN DAT_MEM_PRIV_FLAGS, /* mem_privileges */ + IN DAT_VA_TYPE, /* va_type */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_LMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS, /* completion_flags */ + OUT DAT_LMR_CONTEXT * , /* lmr_context */ + OUT DAT_RMR_CONTEXT * ); /* rmr_context */ + +typedef DAT_RETURN (*DAT_LMR_INVALIDATE_FUNC) ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_EP_HANDLE, /* ep_handle */ + IN DAT_LMR_COOKIE, /* user_cookie */ + IN DAT_COMPLETION_FLAGS ); /* completion_flags */ + +typedef DAT_RETURN (*DAT_EVD_KCREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COUNT, /* evd_min_qlen */ + IN DAT_UPCALL_POLICY, /* upcall_policy */ + IN const DAT_UPCALL_OBJECT *, /* upcall */ + IN DAT_EVD_FLAGS, /* evd_flags */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_MODIFY_UPCALL_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_UPCALL_POLICY, /* upcall_policy */ + IN const DAT_UPCALL_OBJECT *, /* upcall */ + IN DAT_UPCALL_FLAG ); /* upcall invocation flag */ + +typedef DAT_RETURN (*DAT_IA_RESERVED_LMR_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT * ); /* lmr_context */ + +#include + +struct dat_provider +{ + const char * device_name; + DAT_PVOID extension; + + DAT_IA_OPEN_FUNC ia_open_func; + DAT_IA_QUERY_FUNC ia_query_func; + DAT_IA_CLOSE_FUNC ia_close_func; + DAT_IA_MEMTYPE_HINT_FUNC ia_memtype_hint_func; /* kdat only */ + + DAT_SET_CONSUMER_CONTEXT_FUNC set_consumer_context_func; + DAT_GET_CONSUMER_CONTEXT_FUNC get_consumer_context_func; + DAT_GET_HANDLE_TYPE_FUNC get_handle_type_func; + + DAT_CR_QUERY_FUNC cr_query_func; + DAT_CR_ACCEPT_FUNC cr_accept_func; + DAT_CR_REJECT_FUNC cr_reject_func; + DAT_CR_HANDOFF_FUNC cr_handoff_func; + + DAT_EVD_KCREATE_FUNC evd_kcreate_func; + DAT_EVD_QUERY_FUNC evd_query_func; + + DAT_EVD_MODIFY_UPCALL_FUNC evd_modify_upcall_func; /* kdat only */ + + DAT_EVD_RESIZE_FUNC evd_resize_func; + DAT_EVD_POST_SE_FUNC evd_post_se_func; + DAT_EVD_DEQUEUE_FUNC evd_dequeue_func; + DAT_EVD_FREE_FUNC evd_free_func; + + DAT_EP_CREATE_FUNC ep_create_func; + DAT_EP_QUERY_FUNC ep_query_func; + DAT_EP_MODIFY_FUNC ep_modify_func; + DAT_EP_CONNECT_FUNC ep_connect_func; + DAT_EP_DUP_CONNECT_FUNC ep_dup_connect_func; + DAT_EP_DISCONNECT_FUNC ep_disconnect_func; + DAT_EP_POST_SEND_FUNC ep_post_send_func; + DAT_EP_POST_RECV_FUNC ep_post_recv_func; + DAT_EP_POST_RDMA_READ_FUNC ep_post_rdma_read_func; + DAT_EP_POST_RDMA_WRITE_FUNC ep_post_rdma_write_func; + DAT_EP_GET_STATUS_FUNC ep_get_status_func; + DAT_EP_FREE_FUNC ep_free_func; + + DAT_LMR_KCREATE_FUNC lmr_kcreate_func; + DAT_LMR_QUERY_FUNC lmr_query_func; + DAT_LMR_FREE_FUNC lmr_free_func; + + DAT_RMR_CREATE_FUNC rmr_create_func; + DAT_RMR_QUERY_FUNC rmr_query_func; + DAT_RMR_BIND_FUNC rmr_bind_func; + DAT_RMR_FREE_FUNC rmr_free_func; + + DAT_PSP_CREATE_FUNC psp_create_func; + DAT_PSP_QUERY_FUNC psp_query_func; + DAT_PSP_FREE_FUNC psp_free_func; + + DAT_RSP_CREATE_FUNC rsp_create_func; + DAT_RSP_QUERY_FUNC rsp_query_func; + DAT_RSP_FREE_FUNC rsp_free_func; + + DAT_PZ_CREATE_FUNC pz_create_func; + DAT_PZ_QUERY_FUNC pz_query_func; + DAT_PZ_FREE_FUNC pz_free_func; + + /* dat-1.1 */ + DAT_PSP_CREATE_ANY_FUNC psp_create_any_func; /* dat-1.1 */ + DAT_EP_RESET_FUNC ep_reset_func; /* dat-1.1 */ + + /* dat-1.2 */ + DAT_LMR_SYNC_RDMA_READ_FUNC lmr_sync_rdma_read_func; + DAT_LMR_SYNC_RDMA_WRITE_FUNC lmr_sync_rdma_write_func; + + DAT_EP_CREATE_WITH_SRQ_FUNC ep_create_with_srq_func; + DAT_EP_RECV_QUERY_FUNC ep_recv_query_func; + DAT_EP_SET_WATERMARK_FUNC ep_set_watermark_func; + DAT_SRQ_CREATE_FUNC srq_create_func; + DAT_SRQ_FREE_FUNC srq_free_func; + DAT_SRQ_POST_RECV_FUNC srq_post_recv_func; + DAT_SRQ_QUERY_FUNC srq_query_func; + DAT_SRQ_RESIZE_FUNC srq_resize_func; + DAT_SRQ_SET_LW_FUNC srq_set_lw_func; + +/* DAT 2.0 functions */ + DAT_CSP_CREATE_FUNC csp_create_func; + DAT_CSP_QUERY_FUNC csp_query_func; + DAT_CSP_FREE_FUNC csp_free_func; + + DAT_EP_COMMON_CONNECT_FUNC ep_common_connect_func; + DAT_RMR_CREATE_FOR_EP_FUNC rmr_create_for_ep_func; + + DAT_EP_POST_SEND_WITH_INVALIDATE_FUNC ep_post_send_with_invalidate_func; + DAT_EP_POST_RDMA_READ_TO_RMR_FUNC ep_post_rdma_read_to_rmr_func; + + /* kDAT 2.0 functions */ + DAT_LMR_ALLOCATE_FUNC lmr_allocate_func; + DAT_LMR_FMR_FUNC lmr_fmr_func; + DAT_LMR_INVALIDATE_FUNC lmr_invalidate_func; + DAT_IA_RESERVED_LMR_FUNC ia_reserved_lmr_func; + +#ifdef DAT_EXTENSIONS + DAT_HANDLE_EXTENDEDOP_FUNC handle_extendedop_func; +#endif + + DAT_IA_HA_RELATED_FUNC ia_ha_related_func; +}; + +#endif /* _KDAT_REDIRECTION_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/kdat_vendor_specific.h b/trunk/ulp/dapl2/dat/include/dat/kdat_vendor_specific.h new file mode 100644 index 00000000..2d7cd345 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/kdat_vendor_specific.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*************************************************************** + * + * HEADER: kdat_vendor_specific.h + * + * PURPOSE: Vendor defined macros & support. + * + * Description: Header file for "kDAPL: Kernel Direct Access Programming + * Library, Version: 2.0" + * + ***************************************************************/ +#ifndef _KDAT_VENDOR_SPECIFIC_H_ +#define _KDAT_VENDOR_SPECIFIC_H_ + +#include + +/* Vendor-specific extensions */ + +#if defined(_AMMASSO) + +#elif defined(_BROADCOM) + +#elif defined(_CISCO) + +#elif defined(_IBM) + +#elif defined(_INTEL) + +#elif defined(_JNI) + +#elif defined(_MELLANOX) + +#elif defined(_MYRINET) + +#elif defined(_NETEFFECT) + +#elif defined(_QLOGIC) + +#elif defined(_SILVERSTORM) + +#elif defined(_VOLTAIRE) + +#endif + +#endif /* _KDAT_VENDOR_SPECIFIC_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/udat.h b/trunk/ulp/dapl2/dat/include/dat/udat.h new file mode 100644 index 00000000..978aede7 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/udat.h @@ -0,0 +1,504 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/**************************************************************** + * + * HEADER: udat.h + * + * PURPOSE: defines the user DAT API + * + * Description: Header file for "uDAPL: User Direct Access Programming + * Library, Version: 2.0" + * + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + ***************************************************************/ + +#ifndef _UDAT_H_ +#define _UDAT_H_ + +#include + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +typedef enum dat_mem_type +{ + /* Shared between udat and kdat */ + DAT_MEM_TYPE_VIRTUAL = 0x00, + DAT_MEM_TYPE_LMR = 0x01, + /* udat specific */ + DAT_MEM_TYPE_SHARED_VIRTUAL = 0x02 +} DAT_MEM_TYPE; + +/* dat handle types */ +typedef enum dat_handle_type +{ + DAT_HANDLE_TYPE_CR, + DAT_HANDLE_TYPE_EP, + DAT_HANDLE_TYPE_EVD, + DAT_HANDLE_TYPE_IA, + DAT_HANDLE_TYPE_LMR, + DAT_HANDLE_TYPE_PSP, + DAT_HANDLE_TYPE_PZ, + DAT_HANDLE_TYPE_RMR, + DAT_HANDLE_TYPE_RSP, + DAT_HANDLE_TYPE_CNO, + DAT_HANDLE_TYPE_SRQ, + DAT_HANDLE_TYPE_CSP +#ifdef DAT_EXTENSIONS + ,DAT_HANDLE_TYPE_EXTENSION_BASE +#endif +} DAT_HANDLE_TYPE; + +/* EVD state consists of three orthogonal substates. One for + * enabled/disabled,one for waitable/unwaitable, and one for + * configuration. Within each substate the values are mutually + * exclusive. + */ +typedef enum dat_evd_state +{ + DAT_EVD_STATE_ENABLED = 0x01, + DAT_EVD_STATE_DISABLED = 0x02, + DAT_EVD_STATE_WAITABLE = 0x04, + DAT_EVD_STATE_UNWAITABLE = 0x08, + DAT_EVD_STATE_CONFIG_NOTIFY = 0x10, + DAT_EVD_STATE_CONFIG_SOLICITED = 0x20, + DAT_EVD_STATE_CONFIG_THRESHOLD = 0x30 +} DAT_EVD_STATE; + +typedef enum dat_evd_param_mask +{ + DAT_EVD_FIELD_IA_HANDLE = 0x01, + DAT_EVD_FIELD_EVD_QLEN = 0x02, + DAT_EVD_FIELD_EVD_STATE = 0x04, + DAT_EVD_FIELD_CNO = 0x08, + DAT_EVD_FIELD_EVD_FLAGS = 0x10, + DAT_EVD_FIELD_ALL = 0x1F +} DAT_EVD_PARAM_MASK; + +typedef DAT_UINT64 DAT_PROVIDER_ATTR_MASK; + +#include + +typedef DAT_HANDLE DAT_CNO_HANDLE; + +struct dat_evd_param +{ + DAT_IA_HANDLE ia_handle; + DAT_COUNT evd_qlen; + DAT_EVD_STATE evd_state; + DAT_CNO_HANDLE cno_handle; + DAT_EVD_FLAGS evd_flags; +}; + +#define DAT_LMR_COOKIE_SIZE 40 /* size of DAT_LMR_COOKIE in bytes */ +typedef char (* DAT_LMR_COOKIE)[DAT_LMR_COOKIE_SIZE]; + +/* Format for OS wait proxy agent function */ + +typedef void (DAT_API *DAT_AGENT_FUNC) +( + DAT_PVOID, /* instance data */ + DAT_EVD_HANDLE /* Event Dispatcher*/ +); + +/* Definition */ + +typedef struct dat_os_wait_proxy_agent +{ + DAT_PVOID instance_data; + DAT_AGENT_FUNC proxy_agent_func; +} DAT_OS_WAIT_PROXY_AGENT; + +/* Define NULL Proxy agent */ + +#define DAT_OS_WAIT_PROXY_AGENT_NULL \ + (DAT_OS_WAIT_PROXY_AGENT) { \ + (DAT_PVOID) NULL, \ + (DAT_AGENT_FUNC) NULL} + +/* Flags */ + +/* The value specified by the uDAPL Consumer for dat_ia_open to indicate + * that no async EVD should be created for the opening instance of an IA. + * The same IA has been open before that has the only async EVD to + * handle async errors for all open instances of the IA. + */ + +#define DAT_EVD_ASYNC_EXISTS (DAT_EVD_HANDLE) 0x1 + +/* + * The value returned by the dat_ia_query for the case when there is no + * async EVD for the IA instance. The Consumer specified the value of + * DAT_EVD_ASYNC_EXISTS for the async_evd_handle for dat_ia_open. + */ + +#define DAT_EVD_OUT_OF_SCOPE (DAT_EVD_HANDLE) 0x2 + +/* + * Memory types + * + * Specifying memory type for LMR create. A Consumer must use a single + * value when registering memory. The union of any of these + * flags is used in the Provider parameters to indicate what memory + * type Provider supports for LMR memory creation. + */ + +/* For udapl only */ + +typedef struct dat_shared_memory +{ + DAT_PVOID virtual_address; + DAT_LMR_COOKIE shared_memory_id; +} DAT_SHARED_MEMORY; + +typedef union dat_region_description +{ + DAT_PVOID for_va; + DAT_LMR_HANDLE for_lmr_handle; + DAT_SHARED_MEMORY for_shared_memory; /* For udapl only */ +} DAT_REGION_DESCRIPTION; + +/* LMR Arguments */ + +struct dat_lmr_param +{ + DAT_IA_HANDLE ia_handle; + DAT_MEM_TYPE mem_type; + DAT_REGION_DESCRIPTION region_desc; + DAT_VLEN length; + DAT_PZ_HANDLE pz_handle; + DAT_MEM_PRIV_FLAGS mem_priv; + DAT_VA_TYPE va_type; + DAT_LMR_CONTEXT lmr_context; + DAT_RMR_CONTEXT rmr_context; + DAT_VLEN registered_size; + DAT_VADDR registered_address; +}; + +enum dat_lmr_param_mask +{ + DAT_LMR_FIELD_IA_HANDLE = 0x001, + DAT_LMR_FIELD_MEM_TYPE = 0x002, + DAT_LMR_FIELD_REGION_DESC = 0x004, + DAT_LMR_FIELD_LENGTH = 0x008, + DAT_LMR_FIELD_PZ_HANDLE = 0x010, + DAT_LMR_FIELD_MEM_PRIV = 0x020, + DAT_LMR_FIELD_VA_TYPE = 0x040, + DAT_LMR_FIELD_LMR_CONTEXT = 0x080, + DAT_LMR_FIELD_RMR_CONTEXT = 0x100, + DAT_LMR_FIELD_REGISTERED_SIZE = 0x200, + DAT_LMR_FIELD_REGISTERED_ADDRESS = 0x400, + + DAT_LMR_FIELD_ALL = 0x7FF +}; + +typedef enum dat_proxy_type +{ + DAT_PROXY_TYPE_NONE = 0x0, + DAT_PROXY_TYPE_AGENT = 0x1, + DAT_PROXY_TYPE_FD = 0x2 +} DAT_PROXY_TYPE; + +typedef struct dat_cno_param +{ + DAT_IA_HANDLE ia_handle; + DAT_PROXY_TYPE proxy_type; + union { + DAT_OS_WAIT_PROXY_AGENT agent; + DAT_FD fd; + DAT_PVOID none; + } proxy; +} DAT_CNO_PARAM; + +typedef enum dat_cno_param_mask +{ + DAT_CNO_FIELD_IA_HANDLE = 0x1, + DAT_CNO_FIELD_PROXY_TYPE = 0x2, + DAT_CNO_FIELD_PROXY = 0x3, + DAT_CNO_FIELD_ALL = 0x4 +} DAT_CNO_PARAM_MASK; + +struct dat_ia_attr +{ + char adapter_name[DAT_NAME_MAX_LENGTH]; + char vendor_name[DAT_NAME_MAX_LENGTH]; + DAT_UINT32 hardware_version_major; + DAT_UINT32 hardware_version_minor; + DAT_UINT32 firmware_version_major; + DAT_UINT32 firmware_version_minor; + DAT_IA_ADDRESS_PTR ia_address_ptr; + DAT_COUNT max_eps; + DAT_COUNT max_dto_per_ep; + DAT_COUNT max_rdma_read_per_ep_in; + DAT_COUNT max_rdma_read_per_ep_out; + DAT_COUNT max_evds; + DAT_COUNT max_evd_qlen; + DAT_COUNT max_iov_segments_per_dto; + DAT_COUNT max_lmrs; + DAT_SEG_LENGTH max_lmr_block_size; + DAT_VADDR max_lmr_virtual_address; + DAT_COUNT max_pzs; + DAT_SEG_LENGTH max_message_size; + DAT_SEG_LENGTH max_rdma_size; + DAT_COUNT max_rmrs; + DAT_VADDR max_rmr_target_address; + DAT_COUNT max_srqs; + DAT_COUNT max_ep_per_srq; + DAT_COUNT max_recv_per_srq; + DAT_COUNT max_iov_segments_per_rdma_read; + DAT_COUNT max_iov_segments_per_rdma_write; + DAT_COUNT max_rdma_read_in; + DAT_COUNT max_rdma_read_out; + DAT_BOOLEAN max_rdma_read_per_ep_in_guaranteed; + DAT_BOOLEAN max_rdma_read_per_ep_out_guaranteed; + DAT_BOOLEAN zb_supported; + DAT_EXTENSION extension_supported; + DAT_COUNT extension_version; + DAT_COUNT num_transport_attr; + DAT_NAMED_ATTR *transport_attr; + DAT_COUNT num_vendor_attr; + DAT_NAMED_ATTR *vendor_attr; +}; + + +#define DAT_IA_FIELD_IA_EXTENSION UINT64_C(0x100000000) +#define DAT_IA_FIELD_IA_EXTENSION_VERSION UINT64_C(0x200000000) + +#define DAT_IA_FIELD_IA_NUM_TRANSPORT_ATTR UINT64_C(0x400000000) +#define DAT_IA_FIELD_IA_TRANSPORT_ATTR UINT64_C(0x800000000) +#define DAT_IA_FIELD_IA_NUM_VENDOR_ATTR UINT64_C(0x1000000000) +#define DAT_IA_FIELD_IA_VENDOR_ATTR UINT64_C(0x2000000000) +#define DAT_IA_FIELD_ALL UINT64_C(0x3FFFFFFFFF) + +/* General Provider attributes. udat specific. */ + +typedef enum dat_pz_support +{ + DAT_PZ_UNIQUE, + DAT_PZ_SHAREABLE +} DAT_PZ_SUPPORT; + +#include + +/* Provider should support merging of all event stream types. Provider + * attribute specify support for merging different event stream types. + * It is a 2D binary matrix where each row and column represents an event + * stream type. Each binary entry is 1 if the event streams of its raw + * and column can fed the same EVD, and 0 otherwise. The order of event + * streams in row and column is the same as in the definition of + * DAT_EVD_FLAGS: index 0 - Software Event, 1- Connection Request, + * 2 - DTO Completion, 3 - Connection event, 4 - RMR Bind Completion, + * 5 - Asynchronous event. By definition each diagonal entry is 1. + * Consumer allocates an array for it and passes it IN as a pointer + * for the array that Provider fills. Provider must fill the array + * that Consumer passes. + */ + +struct dat_provider_attr +{ + char provider_name[DAT_NAME_MAX_LENGTH]; + DAT_UINT32 provider_version_major; + DAT_UINT32 provider_version_minor; + DAT_UINT32 dapl_version_major; + DAT_UINT32 dapl_version_minor; + DAT_MEM_TYPE lmr_mem_types_supported; + DAT_IOV_OWNERSHIP iov_ownership_on_return; + DAT_QOS dat_qos_supported; + DAT_COMPLETION_FLAGS completion_flags_supported; + DAT_BOOLEAN is_thread_safe; + DAT_COUNT max_private_data_size; + DAT_BOOLEAN supports_multipath; + DAT_EP_CREATOR_FOR_PSP ep_creator; + DAT_PZ_SUPPORT pz_support; + DAT_UINT32 optimal_buffer_alignment; + const DAT_BOOLEAN evd_stream_merging_supported[6][6]; + DAT_BOOLEAN srq_supported; + DAT_COUNT srq_watermarks_supported; + DAT_BOOLEAN srq_ep_pz_difference_supported; + DAT_COUNT srq_info_supported; + DAT_COUNT ep_recv_info_supported; + DAT_BOOLEAN lmr_sync_req; + DAT_BOOLEAN dto_async_return_guaranteed; + DAT_BOOLEAN rdma_write_for_rdma_read_req; + DAT_BOOLEAN rdma_read_lmr_rmr_context_exposure; + DAT_RMR_SCOPE rmr_scope_supported; + DAT_BOOLEAN is_signal_safe; + DAT_BOOLEAN ha_supported; + DAT_HA_LB ha_loadbalancing; + DAT_COUNT num_provider_specific_attr; + DAT_NAMED_ATTR * provider_specific_attr; +}; + +#define DAT_PROVIDER_FIELD_PROVIDER_NAME UINT64_C(0x00000001) +#define DAT_PROVIDER_FIELD_PROVIDER_VERSION_MAJOR UINT64_C(0x00000002) +#define DAT_PROVIDER_FIELD_PROVIDER_VERSION_MINOR UINT64_C(0x00000004) +#define DAT_PROVIDER_FIELD_DAPL_VERSION_MAJOR UINT64_C(0x00000008) +#define DAT_PROVIDER_FIELD_DAPL_VERSION_MINOR UINT64_C(0x00000010) +#define DAT_PROVIDER_FIELD_LMR_MEM_TYPE_SUPPORTED UINT64_C(0x00000020) +#define DAT_PROVIDER_FIELD_IOV_OWNERSHIP UINT64_C(0x00000040) +#define DAT_PROVIDER_FIELD_DAT_QOS_SUPPORTED UINT64_C(0x00000080) +#define DAT_PROVIDER_FIELD_COMPLETION_FLAGS_SUPPORTED UINT64_C(0x00000100) +#define DAT_PROVIDER_FIELD_IS_THREAD_SAFE UINT64_C(0x00000200) +#define DAT_PROVIDER_FIELD_MAX_PRIVATE_DATA_SIZE UINT64_C(0x00000400) +#define DAT_PROVIDER_FIELD_SUPPORTS_MULTIPATH UINT64_C(0x00000800) +#define DAT_PROVIDER_FIELD_EP_CREATOR UINT64_C(0x00001000) +#define DAT_PROVIDER_FIELD_PZ_SUPPORT UINT64_C(0x00002000) +#define DAT_PROVIDER_FIELD_OPTIMAL_BUFFER_ALIGNMENT UINT64_C(0x00004000) +#define DAT_PROVIDER_FIELD_EVD_STREAM_MERGING_SUPPORTED UINT64_C(0x00008000) +#define DAT_PROVIDER_FIELD_SRQ_SUPPORTED UINT64_C(0x00010000) +#define DAT_PROVIDER_FIELD_SRQ_WATERMARKS_SUPPORTED UINT64_C(0x00020000) +#define DAT_PROVIDER_FIELD_SRQ_EP_PZ_DIFFERENCE_SUPPORTED UINT64_C(0x00040000) +#define DAT_PROVIDER_FIELD_SRQ_INFO_SUPPORTED UINT64_C(0x00080000) +#define DAT_PROVIDER_FIELD_EP_RECV_INFO_SUPPORTED UINT64_C(0x00100000) +#define DAT_PROVIDER_FIELD_LMR_SYNC_REQ UINT64_C(0x00200000) +#define DAT_PROVIDER_FIELD_DTO_ASYNC_RETURN_GUARANTEED UINT64_C(0x00400000) +#define DAT_PROVIDER_FIELD_RDMA_WRITE_FOR_RDMA_READ_REQ UINT64_C(0x00800000) +#define DAT_PROVIDER_FIELD_RDMA_READ_LMR_RMR_CONTEXT_EXPOSURE UINT64_C(0x01000000) +#define DAT_PROVIDER_FIELD_RMR_SCOPE_SUPPORTED UINT64_C(0x02000000) +#define DAT_PROVIDER_FIELD_IS_SIGNAL_SAFE UINT64_C(0x04000000) +#define DAT_PROVIDER_FIELD_HA_SUPPORTED UINT64_C(0x08000000) +#define DAT_PROVIDER_FIELD_HA_LB UINT64_C(0x10000000) +#define DAT_PROVIDER_FIELD_NUM_PROVIDER_SPECIFIC_ATTR UINT64_C(0x20000000) +#define DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR UINT64_C(0x40000000) + +#define DAT_PROVIDER_FIELD_ALL UINT64_C(0x7FFFFFFF) +#define DAT_PROVIDER_FIELD_NONE UINT64_C(0x0) + +/**************************************************************/ + +/* + * User DAT function call definitions, + */ + +extern DAT_RETURN DAT_API dat_lmr_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* length */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_MEM_PRIV_FLAGS, /* privileges */ + IN DAT_VA_TYPE, /* va_type */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT *, /* lmr_context */ + OUT DAT_RMR_CONTEXT *, /* rmr_context */ + OUT DAT_VLEN *, /* registered_length */ + OUT DAT_VADDR * ); /* registered_address */ + +extern DAT_RETURN DAT_API dat_lmr_query ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_LMR_PARAM_MASK, /* lmr_param_mask */ + OUT DAT_LMR_PARAM * ); /* lmr_param */ + +/* Event Functions */ + +extern DAT_RETURN DAT_API dat_evd_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COUNT, /* evd_min_qlen */ + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_EVD_FLAGS, /* evd_flags */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +extern DAT_RETURN DAT_API dat_evd_modify_cno ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_CNO_HANDLE); /* cno_handle */ + +extern DAT_RETURN DAT_API dat_cno_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_OS_WAIT_PROXY_AGENT,/* agent */ + OUT DAT_CNO_HANDLE *); /* cno_handle */ + +extern DAT_RETURN DAT_API dat_cno_modify_agent ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_OS_WAIT_PROXY_AGENT);/* agent */ + +extern DAT_RETURN DAT_API dat_cno_query ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_CNO_PARAM_MASK, /* cno_param_mask */ + OUT DAT_CNO_PARAM * ); /* cno_param */ + +extern DAT_RETURN DAT_API dat_cno_free ( + IN DAT_CNO_HANDLE); /* cno_handle */ + +extern DAT_RETURN DAT_API dat_cno_wait ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_TIMEOUT, /* timeout */ + OUT DAT_EVD_HANDLE *); /* evd_handle */ + +extern DAT_RETURN DAT_API dat_evd_enable ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +extern DAT_RETURN DAT_API dat_evd_wait ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_TIMEOUT, /* timeout */ + IN DAT_COUNT, /* threshold */ + OUT DAT_EVENT *, /* event */ + OUT DAT_COUNT * ); /* n_more_events */ + +extern DAT_RETURN DAT_API dat_evd_disable ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +extern DAT_RETURN DAT_API dat_evd_set_unwaitable ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +extern DAT_RETURN DAT_API dat_evd_clear_unwaitable ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +extern DAT_RETURN DAT_API dat_cno_fd_create ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_FD *, /* file descriptor */ + OUT DAT_CNO_HANDLE * ); /* cno_handle */ + +extern DAT_RETURN DAT_API dat_cno_trigger ( + IN DAT_CNO_HANDLE, /* cno_handle */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +#ifdef __cplusplus +} +#endif + +#endif /* _UDAT_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/udat_config.h b/trunk/ulp/dapl2/dat/include/dat/udat_config.h new file mode 100644 index 00000000..4f27e39f --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/udat_config.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*************************************************************** + * + * HEADER: udat_config.h + * + * PURPOSE: provides uDAPL configuration information. + * + * Description: Header file for "uDAPL: User Direct Access Programming + * Library, Version: 2.0" + * + ***************************************************************/ +#ifndef _UDAT_CONFIG_H_ +#define _UDAT_CONFIG_H_ + +#define DAT_VERSION_MAJOR 2 +#define DAT_VERSION_MINOR 0 + +/* + * The official header files will default DAT_THREADSAFE to DAT_TRUE. If + * your project does not wish to use this default, you must ensure that + * DAT_THREADSAFE will be set to DAT_FALSE. This may be done by an + * explicit #define in a common project header file that is included + * before any DAT header files, or through command line directives to the + * compiler (presumably controlled by the make environment). + */ + +/* + * A site, project or platform may consider setting an alternate default + * via their make rules, but are discouraged from doing so by editing + * the official header files. + */ + +/* + * The Reference Implementation is not Thread Safe. The Reference + * Implementation has chosen to go with the first method and define it + * explicitly in the header file. + */ +#define DAT_THREADSAFE DAT_FALSE + +#ifndef DAT_THREADSAFE +#define DAT_THREADSAFE DAT_TRUE +#endif /* DAT_THREADSAFE */ + +#endif /* _UDAT_CONFIG_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/udat_redirection.h b/trunk/ulp/dapl2/dat/include/dat/udat_redirection.h new file mode 100644 index 00000000..0a1ae106 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/udat_redirection.h @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/**************************************************************** + * + * HEADER: udat_redirection.h + * + * PURPOSE: User DAT macro definitions + * + * Description: Macros to invoke DAPL functions from the dat_registry + * + * Mapping rules: + * All global symbols are prepended with DAT_ or dat_ + * All DAT objects have an 'api' tag which, such as 'ep' or 'lmr' + * The method table is in the provider definition structure. + * + **********************************************************/ +#ifndef _UDAT_REDIRECTION_H_ +#define _UDAT_REDIRECTION_H_ + + +#define DAT_LMR_CREATE(ia, mem_type, reg_desc, len, pz, priv, va_type, \ + lmr, lmr_context, rmr_context, reg_len, reg_addr) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->lmr_create_func)(\ + (ia), \ + (mem_type), \ + (reg_desc), \ + (len), \ + (pz), \ + (priv), \ + (va_type), \ + (lmr), \ + (lmr_context), \ + (rmr_context), \ + (reg_len), \ + (reg_addr)) + +#define DAT_EVD_CREATE(ia, qlen, cno, flags, handle) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->evd_create_func)(\ + (ia), \ + (qlen), \ + (cno), \ + (flags), \ + (handle)) + +#define DAT_EVD_ENABLE(evd) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_enable_func)(\ + (evd)) + +#define DAT_EVD_WAIT(evd, timeout, threshold, event, nmore) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_wait_func)(\ + (evd), \ + (timeout), \ + (threshold), \ + (event), \ + (nmore)) + +#define DAT_EVD_DISABLE(evd) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_disable_func)(\ + (evd)) + +#define DAT_EVD_SET_UNWAITABLE(evd) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_set_unwaitable_func)(\ + (evd)) + +#define DAT_EVD_CLEAR_UNWAITABLE(evd) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_clear_unwaitable_func)(\ + (evd)) + +#define DAT_EVD_MODIFY_CNO(evd, cno) \ + (*DAT_HANDLE_TO_PROVIDER(evd)->evd_modify_cno_func)(\ + (evd), \ + (cno)) + +#define DAT_CNO_CREATE(ia, proxy, cno) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->cno_create_func)(\ + (ia), \ + (proxy), \ + (cno)) + +#define DAT_CNO_FD_CREATE(ia, fd, cno) \ + (*DAT_HANDLE_TO_PROVIDER(ia)->cno_fd_create_func)(\ + (ia), \ + (fd), \ + (cno)) + +#define DAT_CNO_TRIGGER(cno, evd) \ + (*DAT_HANDLE_TO_PROVIDER(cno)->cno_trigger_func)(\ + (cno), \ + (evd)) + + +#define DAT_CNO_MODIFY_AGENT(cno, proxy) \ + (*DAT_HANDLE_TO_PROVIDER(cno)->cno_modify_agent_func)(\ + (cno), \ + (proxy)) + +#define DAT_CNO_QUERY(cno, mask, param) \ + (*DAT_HANDLE_TO_PROVIDER(cno)->cno_query_func)(\ + (cno), \ + (mask), \ + (param)) + +#define DAT_CNO_FREE(cno) \ + (*DAT_HANDLE_TO_PROVIDER(cno)->cno_free_func)(\ + (cno)) + +#define DAT_CNO_WAIT(cno,timeout,evd) \ + (*DAT_HANDLE_TO_PROVIDER(cno)->cno_wait_func)(\ + (cno), \ + (timeout), \ + (evd)) + +/*************************************************************** + * FUNCTION PROTOTYPES + * + * User DAT function call definitions, + * + ****************************************************************/ + +typedef DAT_RETURN (*DAT_LMR_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_MEM_TYPE, /* mem_type */ + IN DAT_REGION_DESCRIPTION, /* region_description */ + IN DAT_VLEN, /* length */ + IN DAT_PZ_HANDLE, /* pz_handle */ + IN DAT_MEM_PRIV_FLAGS, /* privileges */ + IN DAT_VA_TYPE, /* va_type */ + OUT DAT_LMR_HANDLE *, /* lmr_handle */ + OUT DAT_LMR_CONTEXT *, /* lmr_context */ + OUT DAT_RMR_CONTEXT *, /* rmr_context */ + OUT DAT_VLEN *, /* registered_length */ + OUT DAT_VADDR * ); /* registered_address */ + +typedef DAT_RETURN (*DAT_LMR_QUERY_FUNC) ( + IN DAT_LMR_HANDLE, /* lmr_handle */ + IN DAT_LMR_PARAM_MASK, /* lmr_param_mask */ + OUT DAT_LMR_PARAM *); /* lmr_param */ + +/* Event functions */ + +typedef DAT_RETURN (*DAT_EVD_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_COUNT, /* evd_min_qlen */ + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_EVD_FLAGS, /* evd_flags */ + OUT DAT_EVD_HANDLE * ); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_MODIFY_CNO_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_CNO_HANDLE); /* cno_handle */ + +typedef DAT_RETURN (*DAT_CNO_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + IN DAT_OS_WAIT_PROXY_AGENT,/* agent */ + OUT DAT_CNO_HANDLE *); /* cno_handle */ + +typedef DAT_RETURN (*DAT_CNO_FD_CREATE_FUNC) ( + IN DAT_IA_HANDLE, /* ia_handle */ + OUT DAT_FD *, /* file_descriptor */ + OUT DAT_CNO_HANDLE *); /* cno_handle */ + +typedef DAT_RETURN (*DAT_CNO_TRIGGER_FUNC) ( + IN DAT_CNO_HANDLE, /* cno_handle */ + OUT DAT_EVD_HANDLE *); /* trigger */ + +typedef DAT_RETURN (*DAT_CNO_MODIFY_AGENT_FUNC) ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_OS_WAIT_PROXY_AGENT);/* agent */ + +typedef DAT_RETURN (*DAT_CNO_QUERY_FUNC) ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_CNO_PARAM_MASK, /* cno_param_mask */ + OUT DAT_CNO_PARAM * ); /* cno_param */ + +typedef DAT_RETURN (*DAT_CNO_FREE_FUNC) ( + IN DAT_CNO_HANDLE); /* cno_handle */ + +typedef DAT_RETURN (*DAT_CNO_WAIT_FUNC) ( + IN DAT_CNO_HANDLE, /* cno_handle */ + IN DAT_TIMEOUT, /* timeout */ + OUT DAT_EVD_HANDLE *); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_ENABLE_FUNC) ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_WAIT_FUNC) ( + IN DAT_EVD_HANDLE, /* evd_handle */ + IN DAT_TIMEOUT, /* Timeout */ + IN DAT_COUNT, /* Threshold */ + OUT DAT_EVENT *, /* event */ + OUT DAT_COUNT * ); /* N more events */ + +typedef DAT_RETURN (*DAT_EVD_DISABLE_FUNC) ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_SET_UNWAITABLE_FUNC) ( + IN DAT_EVD_HANDLE); /* evd_handle */ + +typedef DAT_RETURN (*DAT_EVD_CLEAR_UNWAITABLE_FUNC) ( + IN DAT_EVD_HANDLE); /* evd_handle */ + + +#include + +struct dat_provider +{ + const char * device_name; + DAT_PVOID extension; + + DAT_IA_OPEN_FUNC ia_open_func; + DAT_IA_QUERY_FUNC ia_query_func; + DAT_IA_CLOSE_FUNC ia_close_func; + + DAT_SET_CONSUMER_CONTEXT_FUNC set_consumer_context_func; + DAT_GET_CONSUMER_CONTEXT_FUNC get_consumer_context_func; + DAT_GET_HANDLE_TYPE_FUNC get_handle_type_func; + + DAT_CNO_CREATE_FUNC cno_create_func; /* udat only */ + DAT_CNO_MODIFY_AGENT_FUNC cno_modify_agent_func; /* udat only */ + DAT_CNO_QUERY_FUNC cno_query_func; /* udat only */ + DAT_CNO_FREE_FUNC cno_free_func; /* udat only */ + DAT_CNO_WAIT_FUNC cno_wait_func; /* udat only */ + + DAT_CR_QUERY_FUNC cr_query_func; + DAT_CR_ACCEPT_FUNC cr_accept_func; + DAT_CR_REJECT_FUNC cr_reject_func; + DAT_CR_HANDOFF_FUNC cr_handoff_func; + + DAT_EVD_CREATE_FUNC evd_create_func; + DAT_EVD_QUERY_FUNC evd_query_func; + + DAT_EVD_MODIFY_CNO_FUNC evd_modify_cno_func; /* udat only */ + DAT_EVD_ENABLE_FUNC evd_enable_func; /* udat only */ + DAT_EVD_DISABLE_FUNC evd_disable_func; /* udat only */ + DAT_EVD_WAIT_FUNC evd_wait_func; /* udat only */ + + DAT_EVD_RESIZE_FUNC evd_resize_func; + DAT_EVD_POST_SE_FUNC evd_post_se_func; + DAT_EVD_DEQUEUE_FUNC evd_dequeue_func; + DAT_EVD_FREE_FUNC evd_free_func; + + DAT_EP_CREATE_FUNC ep_create_func; + DAT_EP_QUERY_FUNC ep_query_func; + DAT_EP_MODIFY_FUNC ep_modify_func; + DAT_EP_CONNECT_FUNC ep_connect_func; + DAT_EP_DUP_CONNECT_FUNC ep_dup_connect_func; + DAT_EP_DISCONNECT_FUNC ep_disconnect_func; + DAT_EP_POST_SEND_FUNC ep_post_send_func; + DAT_EP_POST_RECV_FUNC ep_post_recv_func; + DAT_EP_POST_RDMA_READ_FUNC ep_post_rdma_read_func; + DAT_EP_POST_RDMA_WRITE_FUNC ep_post_rdma_write_func; + DAT_EP_GET_STATUS_FUNC ep_get_status_func; + DAT_EP_FREE_FUNC ep_free_func; + + DAT_LMR_CREATE_FUNC lmr_create_func; + DAT_LMR_QUERY_FUNC lmr_query_func; + + DAT_LMR_FREE_FUNC lmr_free_func; + + DAT_RMR_CREATE_FUNC rmr_create_func; + DAT_RMR_QUERY_FUNC rmr_query_func; + DAT_RMR_BIND_FUNC rmr_bind_func; + DAT_RMR_FREE_FUNC rmr_free_func; + + DAT_PSP_CREATE_FUNC psp_create_func; + DAT_PSP_QUERY_FUNC psp_query_func; + DAT_PSP_FREE_FUNC psp_free_func; + + DAT_RSP_CREATE_FUNC rsp_create_func; + DAT_RSP_QUERY_FUNC rsp_query_func; + DAT_RSP_FREE_FUNC rsp_free_func; + + DAT_PZ_CREATE_FUNC pz_create_func; + DAT_PZ_QUERY_FUNC pz_query_func; + DAT_PZ_FREE_FUNC pz_free_func; + + /* dat-1.1 */ + DAT_PSP_CREATE_ANY_FUNC psp_create_any_func; /* dat-1.1 */ + DAT_EP_RESET_FUNC ep_reset_func; /* dat-1.1 */ + + /* udat-1.1 */ + DAT_EVD_SET_UNWAITABLE_FUNC evd_set_unwaitable_func; /* udat-1.1 */ + DAT_EVD_CLEAR_UNWAITABLE_FUNC evd_clear_unwaitable_func; /* udat-1.1 */ + /* dat-1.2 */ + DAT_LMR_SYNC_RDMA_READ_FUNC lmr_sync_rdma_read_func; + DAT_LMR_SYNC_RDMA_WRITE_FUNC lmr_sync_rdma_write_func; + + DAT_EP_CREATE_WITH_SRQ_FUNC ep_create_with_srq_func; + DAT_EP_RECV_QUERY_FUNC ep_recv_query_func; + DAT_EP_SET_WATERMARK_FUNC ep_set_watermark_func; + DAT_SRQ_CREATE_FUNC srq_create_func; + DAT_SRQ_FREE_FUNC srq_free_func; + DAT_SRQ_POST_RECV_FUNC srq_post_recv_func; + DAT_SRQ_QUERY_FUNC srq_query_func; + DAT_SRQ_RESIZE_FUNC srq_resize_func; + DAT_SRQ_SET_LW_FUNC srq_set_lw_func; + + /* DAT 2.0 functions */ + DAT_CSP_CREATE_FUNC csp_create_func; + DAT_CSP_QUERY_FUNC csp_query_func; + DAT_CSP_FREE_FUNC csp_free_func; + + DAT_EP_COMMON_CONNECT_FUNC ep_common_connect_func; + + DAT_RMR_CREATE_FOR_EP_FUNC rmr_create_for_ep_func; + DAT_EP_POST_SEND_WITH_INVALIDATE_FUNC ep_post_send_with_invalidate_func; + DAT_EP_POST_RDMA_READ_TO_RMR_FUNC ep_post_rdma_read_to_rmr_func; + + DAT_CNO_FD_CREATE_FUNC cno_fd_create_func; + DAT_CNO_TRIGGER_FUNC cno_trigger_func; + + DAT_IA_HA_RELATED_FUNC ia_ha_related_func; + +#ifdef DAT_EXTENSIONS + DAT_HANDLE_EXTENDEDOP_FUNC handle_extendedop_func; +#endif /* DAT_EXTENSIONS */ +}; + +#endif /* _UDAT_REDIRECTION_H_ */ diff --git a/trunk/ulp/dapl2/dat/include/dat/udat_vendor_specific.h b/trunk/ulp/dapl2/dat/include/dat/udat_vendor_specific.h new file mode 100644 index 00000000..f3a4f4a2 --- /dev/null +++ b/trunk/ulp/dapl2/dat/include/dat/udat_vendor_specific.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain both the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/*************************************************************** + * + * HEADER: udat_vendor_specific.h + * + * PURPOSE: Vendor defined macros & support. + * + * Description: Header file for "uDAPL: User Direct Access Programming + * Library, Version: 2.0" + * + ***************************************************************/ +#ifndef _UDAT_VENDOR_SPECIFIC_H_ +#define _UDAT_VENDOR_SPECIFIC_H_ + +#include + +/* Vendor-specific extensions */ + +#if defined(_AMMASSO) + +#elif defined(_BROADCOM) + +#elif defined(_CISCO) + +#elif defined(_IBM) + +#elif defined(_INTEL) + +#elif defined(_JNI) + +#elif defined(_MELLANOX) + +#elif defined(_MYRINET) + +#elif defined(_NETEFFECT) + +#elif defined(_QLOGIC) + +#elif defined(_SILVERSTORM) + +#elif defined(_VOLTAIRE) + +#endif + +#endif /* _UDAT_VENDOR_SPECIFIC_H_ */ diff --git a/trunk/ulp/dapl2/dat/udat/Makefile.linux b/trunk/ulp/dapl2/dat/udat/Makefile.linux new file mode 100644 index 00000000..3947588e --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/Makefile.linux @@ -0,0 +1,256 @@ +# +# Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. +# +# This Software is licensed under one of the following licenses: +# +# 1) under the terms of the "Common Public License 1.0" a copy of which is +# in the file LICENSE.txt in the root directory. The license is also +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/cpl.php. +# +# 2) under the terms of the "The BSD License" a copy of which is in the file +# LICENSE2.txt in the root directory. The license is also available from +# the Open Source Initiative, see +# http://www.opensource.org/licenses/bsd-license.php. +# +# 3) under the terms of the "GNU General Public License (GPL) Version 2" a +# copy of which is in the file LICENSE3.txt in the root directory. The +# license is also available from the Open Source Initiative, see +# http://www.opensource.org/licenses/gpl-license.php. +# +# Licensee has the right to choose one of the above licenses. +# +# Redistributions of source code must retain the above copyright +# notice and one of the license notices. +# +# Redistributions in binary form must reproduce both the above copyright +# notice, one of the license notices in the documentation +# and/or other materials provided with the distribution. +# + +#********************************************************************* +# +# MODULE: Makefile +# +# PURPOSE: Makefile for DAT registration module +# +# $Id: Makefile 1301 2005-03-24 05:58:55Z jlentini $ +#*********************************************************************/ + + +#AT = @ +ARCH := $(shell uname -m) +UDAT_ROOT := $(shell /bin/pwd) +UDAT_LINUX := $(UDAT_ROOT)/linux +UDAT_COMMON := $(UDAT_ROOT)/../common + +DAT_ROOT := $(UDAT_ROOT)/.. +DAT_HEADERS := $(DAT_ROOT)/include +DAT_HEADERS_SYSTEM_PATH := $(PREFIX)/usr/include/dat + +VPATH = $(UDAT_ROOT) $(UDAT_LINUX) $(UDAT_COMMON) + +OBJ_PATH = $(UDAT_ROOT)/Obj/$(ARCH) +TARGET_PATH = $(UDAT_ROOT)/Target/$(ARCH) +OBJ_PATH32 = $(UDAT_ROOT)/Obj/$(ARCH).32bit +TARGET_PATH32 = $(UDAT_ROOT)/Target/$(ARCH).32bit + +STATIC = $(TARGET_PATH)/libdat.a +DYNAMIC = $(TARGET_PATH)/libdat.so +STATIC32 = $(TARGET_PATH32)/libdat.a +DYNAMIC32 = $(TARGET_PATH32)/libdat.so + +ifeq "$(ARCH)" "x86_64" +#DUAL_ARCH = true +endif + +OBJS = $(OBJ_PATH)/udat.o \ + $(OBJ_PATH)/udat_api.o \ + $(OBJ_PATH)/dat_api.o \ + $(OBJ_PATH)/dat_osd.o \ + $(OBJ_PATH)/dat_dictionary.o \ + $(OBJ_PATH)/dat_dr.o \ + $(OBJ_PATH)/dat_init.o \ + $(OBJ_PATH)/dat_sr.o \ + $(OBJ_PATH)/udat_sr_parser.o \ + $(OBJ_PATH)/dat_strerror.o + +OBJS32= $(OBJ_PATH32)/udat.o \ + $(OBJ_PATH32)/udat_api.o \ + $(OBJ_PATH32)/dat_api.o \ + $(OBJ_PATH32)/dat_osd.o \ + $(OBJ_PATH32)/dat_dictionary.o \ + $(OBJ_PATH32)/dat_dr.o \ + $(OBJ_PATH32)/dat_init.o \ + $(OBJ_PATH32)/dat_sr.o \ + $(OBJ_PATH32)/udat_sr_parser.o \ + $(OBJ_PATH32)/dat_strerror.o + +# +# CC definitions +# + +CC = gcc + +CFLAGS = -O $(CPPFLAGS) +CFLAGS += -g3 +CFLAGS += -fPIC +CFLAGS += -I. +CFLAGS += -I../common +CFLAGS += -I./linux +CFLAGS += -I$(DAT_HEADERS) +CFLAGS += -Wall +CFLAGS += -Wstrict-prototypes +CFLAGS += -Wmissing-prototypes +CFLAGS += -Wmissing-declarations +CFLAGS += -Werror +ifdef GPROF +CFLAGS += -pg +endif +ifeq "$(ARCH)" "x86_64" +CFLAGS32 = -m32 +endif + +CFLAGS += -DDAT_EXTENSIONS + +# +# LD definitions +# + +LD = gcc + +LDFLAGS = -shared +LDFLAGS += -ldl +ifeq "$(ARCH)" "x86_64" +LDFLAGS32 = -m32 +endif + + +# +# AR definitions +# + +AR = ar + +ARFLAGS = r + + +# +# install definitions +# + +INSTALL = install + +INSTALL_FLAGS += -p + + +# +# rpm definitions +# + +RPM_FILE = dat-registry-1.1 +RPM_PATH = /tmp/$(RPM_FILE)-build + + +# +# Rules +# + +ifdef DUAL_ARCH +all: mkdirs $(DYNAMIC) $(STATIC) $(DYNAMIC32) $(STATIC32) +else +all: mkdirs $(DYNAMIC) $(STATIC) +endif + +mkdirs: + $(AT)[ -d $(TARGET_PATH) ] || /bin/mkdir -p $(TARGET_PATH) + $(AT)[ -d $(OBJ_PATH) ] || /bin/mkdir -p $(OBJ_PATH) +ifdef DUAL_ARCH + $(AT)[ -d $(TARGET_PATH32) ] || /bin/mkdir -p $(TARGET_PATH32) + $(AT)[ -d $(OBJ_PATH32) ] || /bin/mkdir -p $(OBJ_PATH32) +endif + +$(OBJ_PATH)/%.o : %.c + @echo Compiling $< + $(AT)$(CC) $(CFLAGS) -c $< -o $@ + +$(OBJ_PATH32)/%.o : %.c + @echo Compiling $< + $(AT)$(CC) $(CFLAGS) $(CFLAGS32) -c $< -o $@ + +$(DYNAMIC): $(OBJS) + @echo Linking $(DYNAMIC) + $(AT)$(LD) $(LDFLAGS) $^ -o $(DYNAMIC) + +$(DYNAMIC32): $(OBJS32) + @echo Linking $(DYNAMIC32) + $(AT)$(LD) $(LDFLAGS) $(LDFLAGS32) $^ -o $(DYNAMIC32) + +$(STATIC): $(OBJS) + @echo Archiving $(STATIC) + $(AT)$(AR) $(ARFLAGS) $(STATIC) $^ + +$(STATIC32): $(OBJS32) + @echo Archiving $(STATIC32) + $(AT)$(AR) $(ARFLAGS) $(STATIC32) $^ + +rpm: mkdirs + @echo Creating RPM + $(AT)if [ -d $(RPM_PATH) ]; then rm -rf $(RPM_PATH); fi; \ + mkdir $(RPM_PATH); \ + mkdir $(RPM_PATH)/SOURCES; \ + mkdir $(RPM_PATH)/BUILD; \ + mkdir $(RPM_PATH)/RPMS; \ + mkdir $(RPM_PATH)/SPECS; \ + case `uname -m` in \ + i?86*) ARCH_DIR=i386;;\ + ia64*) ARCH_DIR=ia64;;\ + x86_64*) ARCH_DIR=x86_64;;\ + *) echo "Unknown architecture: `uname -m`"; exit 1;;\ + esac;\ + mkdir $(RPM_PATH)/RPMS/$$ARCH_DIR; \ + echo "%_topdir $(RPM_PATH)" > $(RPM_PATH)/rpmmacros; \ + echo "macrofiles: /usr/lib/rpm/macros:/etc/rpm/macros:$(RPM_PATH)/rpmmacros" > $(RPM_PATH)/rpmrc; \ + cd $(DAT_ROOT)/..; tar zcvf $(RPM_PATH)/SOURCES/$(RPM_FILE).tgz dat/* --exclude dat/udat/Target --exclude dat/udat/Obj; \ + install -m 644 $(UDAT_LINUX)/$(RPM_FILE).spec $(RPM_PATH)/SPECS/; \ + if [ -f /usr/bin/rpmbuild ]; then \ + RPMBUILD=/usr/bin/rpmbuild;\ + else\ + RPMBUILD=rpm;\ + fi;\ + $$RPMBUILD -bb --rcfile $(RPM_PATH)/rpmrc --target=$$ARCH_DIR $(RPM_PATH)/SPECS/$(RPM_FILE).spec; \ + if [ $$? != 0 ]; then \ + echo "### ERROR: rpm failed:"; \ + exit 1; \ + fi; \ + mv $(RPM_PATH)/RPMS/$$ARCH_DIR/*.rpm $(TARGET_PATH); \ + rm -rf $(RPM_PATH) + +install: all +ifdef DUAL_ARCH + $(INSTALL) $(INSTALL_FLAGS) -m 755 $(DYNAMIC32) $(PREFIX)/usr/lib/libdat.so + $(INSTALL) $(INSTALL_FLAGS) -m 644 $(STATIC32) $(PREFIX)/usr/lib/libdat.a + $(INSTALL) $(INSTALL_FLAGS) -m 755 $(DYNAMIC) $(PREFIX)/usr/lib64/libdat.so + $(INSTALL) $(INSTALL_FLAGS) -m 644 $(STATIC) $(PREFIX)/usr/lib64/libdat.a +else + $(INSTALL) $(INSTALL_FLAGS) -m 755 $(DYNAMIC) $(PREFIX)/usr/lib + $(INSTALL) $(INSTALL_FLAGS) -m 644 $(STATIC) $(PREFIX)/usr/lib +endif + $(INSTALL) $(INSTALL_FLAGS) -m 755 -d $(DAT_HEADERS_SYSTEM_PATH) + $(INSTALL) $(INSTALL_FLAGS) -m 644 $(DAT_HEADERS)/dat/*.h $(DAT_HEADERS_SYSTEM_PATH) + +clean: + rm -f $(OBJ_PATH)/*.o + rm -f $(TARGET_PATH)/*.rpm + rm -f $(DYNAMIC) + rm -f $(STATIC) +ifdef DUAL_ARCH + rm -f $(OBJ_PATH32)/*.o + rm -f $(DYNAMIC32) + rm -f $(STATIC32) +endif + +tidy: + rm -f $(UDAT_ROOT)/*~ + rm -f $(UDAT_LINUX)/*~ + rm -f $(UDAT_COMMON)/*~ diff --git a/trunk/ulp/dapl2/dat/udat/SOURCES b/trunk/ulp/dapl2/dat/udat/SOURCES new file mode 100644 index 00000000..d64d88fc --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/SOURCES @@ -0,0 +1,32 @@ +!if $(FREEBUILD) +TARGETNAME=dat2 +!else +TARGETNAME=dat2d +!endif +TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=DYNLINK +DLLENTRY=_DllMainCRTStartup +DLLDEF=$O\udat_exports.def +USE_CRTDLL=1 + +SOURCES=udat.rc \ + udat.c \ + udat_api.c \ + udat_sr_parser.c \ + udat_sources.c + +INCLUDES=windows;..\common;..\include; +RCOPTIONS=/I..\..\..\..\inc; + +# FIX ME!!! XXX specifically the 4113 warning prior to testing. +#MSC_WARNING_LEVEL= /W3 + +MSC_WARNING_LEVEL= /W1 /wd4113 + +DAT_OPTS= $(DAT_OPTS) -DEXPORT_DAT_SYMBOLS #-DDAT_EXTENSIONS + +USER_C_FLAGS=$(USER_C_FLAGS) $(DAT_OPTS) $(MSC_WARNING_LEVEL) + +TARGETLIBS= \ + $(SDK_LIB_PATH)\kernel32.lib + diff --git a/trunk/ulp/dapl2/dat/udat/linux/dat-registry-1.1.spec b/trunk/ulp/dapl2/dat/udat/linux/dat-registry-1.1.spec new file mode 100644 index 00000000..838c89f3 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/linux/dat-registry-1.1.spec @@ -0,0 +1,106 @@ +# Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. +# +# This Software is licensed under one of the following licenses: +# +# 1) under the terms of the "Common Public License 1.0" a copy of which is +# in the file LICENSE.txt in the root directory. The license is also +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/cpl.php. +# +# 2) under the terms of the "The BSD License" a copy of which is in the file +# LICENSE2.txt in the root directory. The license is also available from +# the Open Source Initiative, see +# http://www.opensource.org/licenses/bsd-license.php. +# +# 3) under the terms of the "GNU General Public License (GPL) Version 2" a +# copy of which is in the file LICENSE3.txt in the root directory. The +# license is also available from the Open Source Initiative, see +# http://www.opensource.org/licenses/gpl-license.php. +# +# Licensee has the right to choose one of the above licenses. +# +# Redistributions of source code must retain the above copyright +# notice and one of the license notices. +# +# Redistributions in binary form must reproduce both the above copyright +# notice, one of the license notices in the documentation +# and/or other materials provided with the distribution. +# +# +# DAT Registry RPM SPEC file +# + +%define make_dir udat + +# Defining these to nothing overrides the stupid automatic RH9 +# functionality of making "debuginfo" RPMs. + +%define debug_package %{nil} +%define __check_files %{nil} +%define dual_arch 0 + +%ifarch x86_64 +%define dual_arch 1 +%endif + +# +# Preamble +# + +Summary: DAT Registry +Name: dat-registry +Version: 1.1 +Release: 0 +Vendor: Dat Collaborative +Exclusiveos: Linux +License: BSD and CPL +Group: System/Libraries +Source: %{name}-%{version}.tgz +URL: http://www.datcollaborative.org +BuildRoot: /var/tmp/%{name}-%{version} + +%description +This package contains the DAT Registry. + +# +# Preparation +# + +%prep +%setup -n dat + +# +# Build +# + +%build +cd %{make_dir} +make + +# +# Install +# + +%install +if [ -d %{buildroot} ]; then rm -rf %{buildroot}; fi +mkdir -p %{buildroot}/usr/include/dat +mkdir -p %{buildroot}/usr/lib +%if %{dual_arch} +mkdir -p %{buildroot}/usr/lib64 +%endif +cd %{make_dir} +make install PREFIX=%{buildroot} + +# +# Files +# + +%files +%defattr(-,root,root) +/usr/include/dat +/usr/lib/libdat.so +/usr/lib/libdat.a +%if %{dual_arch} +/usr/lib64/libdat.so +/usr/lib64/libdat.a +%endif diff --git a/trunk/ulp/dapl2/dat/udat/linux/dat_osd.c b/trunk/ulp/dapl2/dat/udat/linux/dat_osd.c new file mode 100644 index 00000000..c3a18e60 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/linux/dat_osd.c @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_osd.c + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent functions with a canonical DAPL + * interface. Designed to be portable and hide OS specific quirks + * of common functions. + * + * $Id: dat_osd.c,v 1.14 2005/04/25 17:29:41 jlentini Exp $ + **********************************************************************/ + +#include "dat_osd.h" + + +/********************************************************************* + * * + * Constants * + * * + *********************************************************************/ + +#define DAT_DBG_TYPE_ENV "DAT_DBG_TYPE" +#define DAT_DBG_DEST_ENV "DAT_DBG_DEST" + + +/********************************************************************* + * * + * Enumerations * + * * + *********************************************************************/ + +typedef int DAT_OS_DBG_DEST; + +typedef enum +{ + DAT_OS_DBG_DEST_STDOUT = 0x1, + DAT_OS_DBG_DEST_SYSLOG = 0x2, + DAT_OS_DBG_DEST_ALL = 0x3 +} DAT_OS_DBG_DEST_TYPE; + + +/********************************************************************* + * * + * Global Variables * + * * + *********************************************************************/ + +static DAT_OS_DBG_TYPE_VAL g_dbg_type = 0; +static DAT_OS_DBG_DEST g_dbg_dest = DAT_OS_DBG_DEST_STDOUT; + + +/*********************************************************************** + * Function: dat_os_dbg_init + ***********************************************************************/ + +void +dat_os_dbg_init ( void ) +{ + char *dbg_type; + char *dbg_dest; + + if ( NULL != (dbg_type = dat_os_getenv (DAT_DBG_TYPE_ENV)) ) + { + g_dbg_type = dat_os_strtol (dbg_type, NULL, 0); + } + + if ( NULL != (dbg_dest = dat_os_getenv (DAT_DBG_DEST_ENV)) ) + { + g_dbg_dest = dat_os_strtol (dbg_dest, NULL, 0); + } +} + + +/*********************************************************************** + * Function: dat_os_dbg_print + ***********************************************************************/ + +void +dat_os_dbg_print ( + DAT_OS_DBG_TYPE_VAL type, + const char * fmt, + ...) +{ + if ( (DAT_OS_DBG_TYPE_ERROR == type) || (type & g_dbg_type) ) + { + va_list args; + + va_start (args, fmt); + + if ( DAT_OS_DBG_DEST_STDOUT & g_dbg_dest ) + { + vfprintf (stdout, fmt, args); + fflush (stdout); + } + + if ( DAT_OS_DBG_DEST_SYSLOG & g_dbg_dest ) + { + vsyslog (LOG_USER | LOG_DEBUG, fmt, args); + } + + va_end (args); + } +} + + +/*********************************************************************** + * Function: dat_os_library_load + ***********************************************************************/ + +DAT_RETURN +dat_os_library_load ( + const char *library_path, + DAT_OS_LIBRARY_HANDLE *library_handle_ptr) +{ + DAT_OS_LIBRARY_HANDLE library_handle; + + if ( NULL != (library_handle = dlopen (library_path, RTLD_NOW | RTLD_GLOBAL)) ) + { + if ( NULL != library_handle_ptr ) + { + *library_handle_ptr = library_handle; + } + + return DAT_SUCCESS; + } + else + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT: library load failure: %s\n", + dlerror ()); + return DAT_INTERNAL_ERROR; + } +} + + +/*********************************************************************** + * Function: dat_os_library_unload + ***********************************************************************/ + +DAT_RETURN +dat_os_library_unload ( + const DAT_OS_LIBRARY_HANDLE library_handle) +{ + if ( 0 != dlclose (library_handle) ) + { + return DAT_INTERNAL_ERROR; + } + else + { + return DAT_SUCCESS; + } +} diff --git a/trunk/ulp/dapl2/dat/udat/linux/dat_osd.h b/trunk/ulp/dapl2/dat/udat/linux/dat_osd.h new file mode 100644 index 00000000..919a505d --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/linux/dat_osd.h @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_osd.h + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent data structures & functions with + * a canonical DAT interface. Designed to be portable + * and hide OS specific quirks of common functions. + * + * $Id: dat_osd.h,v 1.18 2005/03/24 05:58:36 jlentini Exp $ + **********************************************************************/ + +#ifndef _DAT_OSD_H_ +#define _DAT_OSD_H_ + +/* + * This file is defined for Linux systems only, including it on any + * other build will cause an error + */ +#ifndef __linux__ +#error "UNDEFINED OS TYPE" +#endif /* __linux__ */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef STATIC +#define STATIC static +#endif /* STATIC */ + +#ifndef INLINE +#define INLINE __inline__ +#endif /* INLINE */ + + +/********************************************************************* + * * + * Debuging * + * * + *********************************************************************/ + +#define dat_os_assert(expr) assert(expr) + +typedef int DAT_OS_DBG_TYPE_VAL; + +typedef enum +{ + DAT_OS_DBG_TYPE_ERROR = 0x1, + DAT_OS_DBG_TYPE_GENERIC = 0x2, + DAT_OS_DBG_TYPE_SR = 0x4, + DAT_OS_DBG_TYPE_DR = 0x8, + DAT_OS_DBG_TYPE_PROVIDER_API = 0x10, + DAT_OS_DBG_TYPE_CONSUMER_API = 0x20, + DAT_OS_DBG_TYPE_ALL = 0xff +} DAT_OS_DBG_TYPE; + +extern void +dat_os_dbg_init ( void ); + +extern void +dat_os_dbg_print ( + DAT_OS_DBG_TYPE_VAL type, + const char * fmt, + ...); + + +/********************************************************************* + * * + * Utility Functions * + * * + *********************************************************************/ + +#define DAT_ERROR(Type, SubType) ((DAT_RETURN)(DAT_CLASS_ERROR | Type | SubType)) + +typedef size_t DAT_OS_SIZE; +typedef void * DAT_OS_LIBRARY_HANDLE; + +extern DAT_RETURN +dat_os_library_load ( + const char *library_path, + DAT_OS_LIBRARY_HANDLE *library_handle_ptr ); + +STATIC INLINE void * +dat_os_library_sym ( + DAT_OS_LIBRARY_HANDLE library_handle, + char *sym) +{ + return dlsym (library_handle, sym); +} + +extern DAT_RETURN +dat_os_library_unload ( + const DAT_OS_LIBRARY_HANDLE library_handle ); + +STATIC INLINE char * +dat_os_getenv ( + const char *name) +{ + return getenv (name); +} + +STATIC INLINE long int +dat_os_strtol ( + const char *nptr, + char **endptr, + int base) +{ + return strtol (nptr, endptr, base); +} + +STATIC INLINE DAT_OS_SIZE +dat_os_strlen ( + const char *s ) +{ + return strlen (s); +} + +STATIC INLINE int +dat_os_strncmp ( + const char *s1, + const char *s2, + DAT_OS_SIZE n) +{ + return strncmp (s1, s2, n); +} + +STATIC INLINE void * +dat_os_strncpy ( + char *dest, + const char *src, + DAT_OS_SIZE len) +{ + return strncpy (dest, src, len); +} + +STATIC INLINE DAT_BOOLEAN +dat_os_isblank ( + int c) +{ + if ( (' ' == c) || ('\t' == c) ) + { + return DAT_TRUE; + } + else + { + return DAT_FALSE; + } +} + +STATIC INLINE DAT_BOOLEAN +dat_os_isdigit ( + int c) +{ + if ( isdigit (c) ) + { + return DAT_TRUE; + } + else + { + return DAT_FALSE; + } +} + +STATIC INLINE void +dat_os_usleep ( + unsigned long usec) +{ + usleep (usec); +} + + +/********************************************************************* + * * + * Memory Functions * + * * + *********************************************************************/ + +STATIC INLINE void * +dat_os_alloc ( + int size) +{ + return malloc (size); +} + +STATIC INLINE void +dat_os_free ( + void *ptr, + int size) +{ + free (ptr); +} + +STATIC INLINE void * +dat_os_memset (void *loc, int c, DAT_OS_SIZE size) +{ + return memset (loc, c, size); +} + + +/********************************************************************* + * * + * File I/O * + * * + *********************************************************************/ + +typedef FILE DAT_OS_FILE; +typedef fpos_t DAT_OS_FILE_POS; + + +STATIC INLINE DAT_OS_FILE * +dat_os_fopen ( + const char * path) +{ + /* always open files in read only mode*/ + return fopen (path, "r"); +} + +STATIC INLINE DAT_RETURN +dat_os_fgetpos ( + DAT_OS_FILE *file, + DAT_OS_FILE_POS *pos) +{ + if ( 0 == fgetpos (file, pos) ) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + +STATIC INLINE DAT_RETURN +dat_os_fsetpos ( + DAT_OS_FILE *file, + DAT_OS_FILE_POS *pos) +{ + if ( 0 == fsetpos (file, pos) ) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + +/* dat_os_fgetc() returns EOF on error or end of file. */ +STATIC INLINE int +dat_os_fgetc ( + DAT_OS_FILE *file) +{ + return fgetc (file); +} + +/* dat_os_fgetc() returns EOF on error or end of file. */ +STATIC INLINE int +dat_os_fputc ( + DAT_OS_FILE *file, int c) +{ + return fputc (c, file); +} + +/* dat_os_fread returns the number of bytes read from the file. */ +STATIC INLINE DAT_OS_SIZE +dat_os_fread ( + DAT_OS_FILE *file, + char *buf, + DAT_OS_SIZE len) +{ + return fread (buf, sizeof (char), len, file); +} + +STATIC INLINE DAT_RETURN +dat_os_fclose ( + DAT_OS_FILE *file) +{ + if ( 0 == fclose (file) ) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + + +/********************************************************************* + * * + * Locks * + * * + *********************************************************************/ + +typedef pthread_mutex_t DAT_OS_LOCK; + + +/* lock functions */ +STATIC INLINE DAT_RETURN +dat_os_lock_init ( + IN DAT_OS_LOCK *m) +{ + /* pthread_mutex_init always returns 0 */ + pthread_mutex_init (m, NULL); + + return DAT_SUCCESS; +} + +STATIC INLINE DAT_RETURN +dat_os_lock ( + IN DAT_OS_LOCK *m) +{ + if (0 == pthread_mutex_lock (m)) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + +STATIC INLINE DAT_RETURN +dat_os_unlock ( + IN DAT_OS_LOCK *m) +{ + if (0 == pthread_mutex_unlock (m)) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + +STATIC INLINE DAT_RETURN +dat_os_lock_destroy ( + IN DAT_OS_LOCK *m) +{ + if (0 == pthread_mutex_destroy (m)) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + + +#endif /* _DAT_OSD_H_ */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dat/udat/makefile b/trunk/ulp/dapl2/dat/udat/makefile new file mode 100644 index 00000000..a0c06273 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\inc\openib.def diff --git a/trunk/ulp/dapl2/dat/udat/udat.c b/trunk/ulp/dapl2/dat/udat/udat.c new file mode 100644 index 00000000..a27618cd --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat.c @@ -0,0 +1,473 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: udat.c + * + * PURPOSE: DAT Provider and Consumer registry functions. + * + * $Id: udat.c,v 1.22 2005/03/24 05:58:35 jlentini Exp $ + **********************************************************************/ + +#include +#include /* Provider API function prototypes */ + +#include "dat_dr.h" +#include "dat_init.h" +#include "dat_osd.h" +#ifndef DAT_NO_STATIC_REGISTRY +#include "dat_sr.h" +#endif + + +#define UDAT_IS_BAD_POINTER(p) ( NULL == (p) ) + +/********************************************************************* + * * + * Internal Function Declarations * + * * + *********************************************************************/ + +DAT_BOOLEAN +udat_check_state ( void ); + + +/********************************************************************* + * * + * External Function Definitions * + * * + *********************************************************************/ + +/* + * Use a global to get an unresolved when run with pre-extension library + */ +int g_dat_extensions = 0; + +/* + * + * Provider API + * + */ + + +/*********************************************************************** + * Function: dat_registry_add_provider + ***********************************************************************/ + +DAT_RETURN DAT_API +dat_registry_add_provider ( + IN const DAT_PROVIDER *provider, + IN const DAT_PROVIDER_INFO *provider_info ) +{ + DAT_DR_ENTRY entry; + + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "DAT Registry: %s (%s,%x:%x,%x)\n", __FUNCTION__, + provider_info->ia_name, + provider_info->dapl_version_major, + provider_info->dapl_version_minor, + provider_info->is_thread_safe); + + if ( UDAT_IS_BAD_POINTER (provider) ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG1); + } + + if ( UDAT_IS_BAD_POINTER (provider_info) ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + } + + if ( DAT_FALSE == udat_check_state () ) + { + return DAT_ERROR (DAT_INVALID_STATE, 0); + } + + entry.ref_count = 0; + entry.ia_open_func = provider->ia_open_func; + entry.info = *provider_info; + + return dat_dr_insert (provider_info, &entry); +} + + +//*********************************************************************** +// Function: dat_registry_remove_provider +//*********************************************************************** + +DAT_RETURN DAT_API +dat_registry_remove_provider ( + IN const DAT_PROVIDER *provider, + IN const DAT_PROVIDER_INFO *provider_info ) +{ + dat_os_dbg_print (DAT_OS_DBG_TYPE_PROVIDER_API, + "DAT Registry: dat_registry_remove_provider () called\n"); + + if ( UDAT_IS_BAD_POINTER (provider) ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG1); + } + + if ( DAT_FALSE == udat_check_state () ) + { + return DAT_ERROR (DAT_INVALID_STATE, 0); + } + + return dat_dr_remove (provider_info); +} + + +/* + * + * Consumer API + * + */ + +/*********************************************************************** + * Function: dat_ia_open + ***********************************************************************/ + +DAT_RETURN DAT_API +dat_ia_openv ( + IN const DAT_NAME_PTR name, + IN DAT_COUNT async_event_qlen, + INOUT DAT_EVD_HANDLE *async_event_handle, + OUT DAT_IA_HANDLE *ia_handle, + IN DAT_UINT32 dapl_major, + IN DAT_UINT32 dapl_minor, + IN DAT_BOOLEAN thread_safety ) +{ + DAT_IA_OPEN_FUNC ia_open_func; + DAT_PROVIDER_INFO info; + DAT_OS_SIZE len; + DAT_RETURN dat_status; + + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: dat_ia_openv (%s,%x:%x,%x) called\n", + name, + dapl_major, + dapl_minor, + thread_safety); + + if ( UDAT_IS_BAD_POINTER (name) ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG1); + } + + len = dat_os_strlen (name); + + if ( DAT_NAME_MAX_LENGTH < len ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG1); + } + + if ( UDAT_IS_BAD_POINTER (ia_handle) ) + { + return DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + } + + if ( DAT_FALSE == udat_check_state () ) + { + return DAT_ERROR (DAT_INVALID_STATE, 0); + } + + dat_os_strncpy (info.ia_name, name, len); + info.ia_name[len] = '\0'; + + info.dapl_version_major = dapl_major; + info.dapl_version_minor = dapl_minor; + info.is_thread_safe = thread_safety; + + /* + * Since DAT allows providers to be loaded by either the static + * registry or explicitly through OS dependent methods, do not + * return an error if no providers are loaded via the static registry. + * Don't even bother calling the static registry if DAT is compiled + * with no static registry support. + */ + +#ifndef DAT_NO_STATIC_REGISTRY + (void) dat_sr_provider_open ( &info ); +#endif + + dat_status = dat_dr_provider_open ( &info, &ia_open_func); + if ( dat_status != DAT_SUCCESS ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: dat_ia_open () provider information " + "for IA name %s not found in dynamic registry\n", + name); + + return dat_status; + } + + dat_status = (*ia_open_func) (name, + async_event_qlen, + async_event_handle, + ia_handle); + if (dat_status == DAT_SUCCESS) + { + *ia_handle = (DAT_IA_HANDLE) dats_set_ia_handle (*ia_handle); + } + + /* + * See if provider supports extensions + */ + if (dat_status == DAT_SUCCESS) + { + DAT_IA_ATTR ia_attr; + + if ( dat_ia_query( *ia_handle, + NULL, + DAT_IA_ALL, + &ia_attr, + 0, + NULL ) == DAT_SUCCESS ) + { + if (ia_attr.extension_supported) + { + dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: dat_ia_open () " + "Extensions (Type=%d, ver=%d) supported!\n", + ia_attr.extension_supported, + ia_attr.extension_version); + g_dat_extensions = 1; + } + } + } + return dat_status; +} + + +/************************************************************************ + * Function: dat_ia_close + ***********************************************************************/ + +DAT_RETURN DAT_API +dat_ia_close ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_CLOSE_FLAGS ia_flags) +{ + DAT_PROVIDER *provider; + DAT_PROVIDER_ATTR provider_attr; + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + const char *ia_name; + + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: dat_ia_close () called\n"); + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status != DAT_SUCCESS) + { + return DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); + } + + if ( DAT_FALSE == udat_check_state () ) + { + return DAT_ERROR (DAT_INVALID_STATE, 0); + } + + provider = DAT_HANDLE_TO_PROVIDER (dapl_ia_handle); + ia_name = provider->device_name; + + if ( DAT_SUCCESS != (dat_status = dat_ia_query (ia_handle, + NULL, + 0, + NULL, + DAT_PROVIDER_FIELD_ALL, + &provider_attr)) ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: query function for %s provider failed\n", + ia_name); + } + else if ( DAT_SUCCESS != (dat_status = + (*provider->ia_close_func) (dapl_ia_handle, ia_flags)) ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: close function for %s provider failed\n", + ia_name); + } + else + { + DAT_PROVIDER_INFO info; + DAT_OS_SIZE len; + + len = dat_os_strlen (ia_name); + + dat_os_assert ( len <= DAT_NAME_MAX_LENGTH ); + + dat_os_strncpy (info.ia_name, ia_name, len); + info.ia_name[len] = '\0'; + + info.dapl_version_major = provider_attr.dapl_version_major; + info.dapl_version_minor = provider_attr.dapl_version_minor; + info.is_thread_safe = provider_attr.is_thread_safe; + + dat_status = dat_dr_provider_close ( &info ); + if ( DAT_SUCCESS != dat_status ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: dynamic registry unable to close " + "provider for IA name %s\n", + ia_name); + } + + /* Remove the handle from the handle table */ + (void) dats_free_ia_handle(ia_handle); + +#ifndef DAT_NO_STATIC_REGISTRY + dat_status = dat_sr_provider_close ( &info ); + if ( DAT_SUCCESS != dat_status ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: static registry unable to close " + "provider for IA name %s\n", + ia_name); + } +#endif + } + + return dat_status; +} + + +//*********************************************************************** +// Function: dat_registry_list_providers +//*********************************************************************** + +DAT_RETURN DAT_API +dat_registry_list_providers ( + IN DAT_COUNT max_to_return, + OUT DAT_COUNT *entries_returned, + OUT DAT_PROVIDER_INFO * (dat_provider_list[])) +{ + DAT_RETURN dat_status; + + dat_status = DAT_SUCCESS; + dat_os_dbg_print (DAT_OS_DBG_TYPE_CONSUMER_API, + "DAT Registry: dat_registry_list_providers () called\n"); + + if ( DAT_FALSE == udat_check_state () ) + { + return DAT_ERROR (DAT_INVALID_STATE, 0); + } + + if ( ( UDAT_IS_BAD_POINTER (entries_returned) ) ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG2); + } + + if (0 != max_to_return && ( UDAT_IS_BAD_POINTER (dat_provider_list) ) ) + { + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + } + + if ( 0 == max_to_return ) + { + /* the user is allowed to call with max_to_return set to zero. + * in which case we simply return (in *entries_returned) the + * number of providers currently installed. We must also + * (per spec) return an error + */ +#ifndef DAT_NO_STATIC_REGISTRY + (void) dat_sr_size ( entries_returned ); +#else + (void) dat_dr_size ( entries_returned ); +#endif + return DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG1); + } + else + { +#ifndef DAT_NO_STATIC_REGISTRY + dat_status = dat_sr_list (max_to_return, + entries_returned, + dat_provider_list); +#else + dat_status = dat_dr_list (max_to_return, + entries_returned, + dat_provider_list); +#endif + } + return dat_status; +} + + +/********************************************************************* + * * + * Internal Function Definitions * + * * + *********************************************************************/ + + +//*********************************************************************** +// Function: udat_check_state +//*********************************************************************** + +/* + * This function returns TRUE if the DAT registry is in a state capable + * of handling DAT API calls and false otherwise. + */ + +DAT_BOOLEAN +udat_check_state ( void ) +{ + DAT_MODULE_STATE state; + DAT_BOOLEAN status; + + state = dat_module_get_state (); + + if ( DAT_MODULE_STATE_UNINITIALIZED == state ) + { + dat_init (); + status = DAT_TRUE; + } + else if ( DAT_MODULE_STATE_DEINITIALIZED == state ) + { + status = DAT_FALSE; + } + else + { + status = DAT_TRUE; + } + + return status; +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dat/udat/udat.rc b/trunk/ulp/dapl2/dat/udat/udat.rc new file mode 100644 index 00000000..01195692 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat.rc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id$ + */ + + +#include + +#define VER_FILETYPE VFT_DLL +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#if DBG +#define VER_FILEDESCRIPTION_STR "Direct Access Transport Library v2.0 (Debug)" +#define VER_INTERNALNAME_STR "dat2d.dll" +#define VER_ORIGINALFILENAME_STR "dat2d.dll" +#else +#define VER_FILEDESCRIPTION_STR "Direct Access Transport Library v2.0" +#define VER_INTERNALNAME_STR "dat2.dll" +#define VER_ORIGINALFILENAME_STR "dat2.dll" +#endif + +#include diff --git a/trunk/ulp/dapl2/dat/udat/udat_api.c b/trunk/ulp/dapl2/dat/udat/udat_api.c new file mode 100644 index 00000000..c11bbd9a --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat_api.c @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2002-2006, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under all of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Network Appliance, Inc. nor the names of other DAT + * Collaborative contributors may be used to endorse or promote + * products derived from this software without specific prior written + * permission. + */ + +/********************************************************************** + * + * MODULE: udat.c + * + * PURPOSE: DAT Provider and Consumer registry functions. + * + * $Id: udat_api.c 1301 2005-03-24 05:58:55Z jlentini $ + **********************************************************************/ + +#include "dat_osd.h" +#include +#include "dat_init.h" + +#define UDAT_IS_BAD_HANDLE(h) ( NULL == (p) ) + +DAT_RETURN DAT_API dat_lmr_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_MEM_TYPE mem_type, + IN DAT_REGION_DESCRIPTION region_description, + IN DAT_VLEN length, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_LMR_CREATE (dapl_ia_handle, + mem_type, + region_description, + length, + pz_handle, + privileges, + va_type, + lmr_handle, + lmr_context, + rmr_context, + registered_length, + registered_address); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_evd_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_COUNT evd_min_qlen, + IN DAT_CNO_HANDLE cno_handle, + IN DAT_EVD_FLAGS evd_flags, + OUT DAT_EVD_HANDLE *evd_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_EVD_CREATE (dapl_ia_handle, + evd_min_qlen, + cno_handle, + evd_flags, + evd_handle); + } + + return dat_status; +} + + +DAT_RETURN DAT_API dat_evd_modify_cno ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_CNO_HANDLE cno_handle) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_MODIFY_CNO (evd_handle, + cno_handle); +} + + +DAT_RETURN DAT_API dat_cno_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_OS_WAIT_PROXY_AGENT agent, + OUT DAT_CNO_HANDLE *cno_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_CNO_CREATE (dapl_ia_handle, + agent, + cno_handle); + } + + return dat_status; +} + +DAT_RETURN DAT_API dat_cno_fd_create ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_FD *fd, + OUT DAT_CNO_HANDLE *cno_handle) +{ + DAT_IA_HANDLE dapl_ia_handle; + DAT_RETURN dat_status; + + dat_status = dats_get_ia_handle(ia_handle, &dapl_ia_handle); + if (dat_status == DAT_SUCCESS) + { + dat_status = DAT_CNO_FD_CREATE (dapl_ia_handle, + (DAT_FD *)fd, + cno_handle); + } + + return dat_status; +} + +DAT_RETURN DAT_API dat_cno_modify_agent ( + IN DAT_CNO_HANDLE cno_handle, + IN DAT_OS_WAIT_PROXY_AGENT agent) +{ + if (cno_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CNO); + } + return DAT_CNO_MODIFY_AGENT (cno_handle, + agent); +} + + +DAT_RETURN DAT_API dat_cno_query ( + IN DAT_CNO_HANDLE cno_handle, + IN DAT_CNO_PARAM_MASK cno_param_mask, + OUT DAT_CNO_PARAM *cno_param) +{ + return DAT_CNO_QUERY (cno_handle, + cno_param_mask, + cno_param); +} + + +DAT_RETURN DAT_API dat_cno_free ( + IN DAT_CNO_HANDLE cno_handle) +{ + if (cno_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CNO); + } + return DAT_CNO_FREE (cno_handle); +} + + +DAT_RETURN DAT_API dat_cno_wait ( + IN DAT_CNO_HANDLE cno_handle, + IN DAT_TIMEOUT timeout, + OUT DAT_EVD_HANDLE *evd_handle) +{ + if (cno_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CNO); + } + return DAT_CNO_WAIT (cno_handle, + timeout, + evd_handle); +} + + +DAT_RETURN DAT_API dat_evd_enable ( + IN DAT_EVD_HANDLE evd_handle) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_ENABLE (evd_handle); +} + + +DAT_RETURN DAT_API dat_evd_wait ( + IN DAT_EVD_HANDLE evd_handle, + IN DAT_TIMEOUT Timeout, + IN DAT_COUNT Threshold, + OUT DAT_EVENT *event, + OUT DAT_COUNT *n_more_events) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_WAIT (evd_handle, + Timeout, + Threshold, + event, + n_more_events); +} + + +DAT_RETURN DAT_API dat_evd_disable ( + IN DAT_EVD_HANDLE evd_handle) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_DISABLE (evd_handle); +} + + +DAT_RETURN DAT_API dat_evd_set_unwaitable ( + IN DAT_EVD_HANDLE evd_handle) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_SET_UNWAITABLE (evd_handle); +} + + +DAT_RETURN DAT_API dat_evd_clear_unwaitable ( + IN DAT_EVD_HANDLE evd_handle) +{ + if (evd_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EVD_REQUEST); + } + return DAT_EVD_CLEAR_UNWAITABLE (evd_handle); +} + + + +DAT_RETURN DAT_API dat_cr_handoff ( + IN DAT_CR_HANDLE cr_handle, + IN DAT_CONN_QUAL handoff) +{ + if (cr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CR); + } + return DAT_CR_HANDOFF (cr_handle, + handoff); +} + + +DAT_RETURN DAT_API dat_lmr_query ( + IN DAT_LMR_HANDLE lmr_handle, + IN DAT_LMR_PARAM_MASK lmr_param_mask, + OUT DAT_LMR_PARAM *lmr_param) +{ + if (lmr_handle == NULL) + { + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_LMR); + } + return DAT_LMR_QUERY (lmr_handle, + lmr_param_mask, + lmr_param); +} + + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/trunk/ulp/dapl2/dat/udat/udat_exports.src b/trunk/ulp/dapl2/dat/udat/udat_exports.src new file mode 100644 index 00000000..cb542011 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat_exports.src @@ -0,0 +1,53 @@ +#if DBG +LIBRARY dat2d.dll +#else +LIBRARY dat2.dll +#endif + +EXPORTS +dat_cno_create +dat_cno_query +dat_cno_wait +dat_cno_free +dat_cr_accept +dat_cr_reject +dat_cr_query +dat_ep_create +dat_ep_free +dat_ep_post_recv +dat_ep_post_rdma_read +dat_ep_post_rdma_write +dat_ep_post_send +dat_ep_reset +dat_ep_connect +dat_ep_disconnect +dat_ep_get_status +dat_ep_query +dat_evd_create +dat_evd_query +dat_evd_free +dat_evd_wait +dat_evd_dequeue +dat_ia_close +dat_ia_openv +dat_ia_query +dat_lmr_create +dat_lmr_free +dat_lmr_query +dat_psp_create +dat_psp_free +dat_psp_query +dat_pz_create +dat_pz_free +dat_pz_query +dat_registry_add_provider +dat_registry_remove_provider +dat_rmr_bind +dat_rmr_create +dat_rmr_free +dat_rmr_query +dat_rsp_create +dat_rsp_free +dat_rsp_query +dat_strerror + diff --git a/trunk/ulp/dapl2/dat/udat/udat_sources.c b/trunk/ulp/dapl2/dat/udat/udat_sources.c new file mode 100644 index 00000000..aeb91da9 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat_sources.c @@ -0,0 +1,11 @@ +/* + * Include all files that are not in children directories. + */ + +#include "../common/dat_api.c" +#include "../common/dat_dictionary.c" +#include "../common/dat_dr.c" +#include "../common/dat_init.c" +#include "../common/dat_sr.c" +#include "../common/dat_strerror.c" +#include "windows/dat_osd.c" diff --git a/trunk/ulp/dapl2/dat/udat/udat_sr_parser.c b/trunk/ulp/dapl2/dat/udat/udat_sr_parser.c new file mode 100644 index 00000000..44c068b4 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat_sr_parser.c @@ -0,0 +1,1525 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_sr_parser.c + * + * PURPOSE: static registry parser + * + * $Id: udat_sr_parser.c,v 1.6 2005/03/24 05:58:36 jlentini Exp $ + **********************************************************************/ + + +#include "udat_sr_parser.h" +#include "dat_sr.h" + + +/********************************************************************* + * * + * Constants * + * * + *********************************************************************/ + +#define DAT_SR_CONF_ENV "DAT_OVERRIDE" +#define DAT_SR_CONF_DEFAULT "/etc/dat.conf" + +#define DAT_SR_TOKEN_THREADSAFE "threadsafe" +#define DAT_SR_TOKEN_NONTHREADSAFE "nonthreadsafe" +#define DAT_SR_TOKEN_DEFAULT "default" +#define DAT_SR_TOKEN_NONDEFAULT "nondefault" + +#define DAT_SR_CHAR_NEWLINE '\n' +#define DAT_SR_CHAR_COMMENT '#' +#define DAT_SR_CHAR_QUOTE '"' +#define DAT_SR_CHAR_BACKSLASH '\\' + + +/********************************************************************* + * * + * Enumerations * + * * + *********************************************************************/ + +typedef enum +{ + DAT_SR_TOKEN_STRING, /* text field (both quoted or unquoted) */ + DAT_SR_TOKEN_EOR, /* end of record (newline) */ + DAT_SR_TOKEN_EOF /* end of file */ +} DAT_SR_TOKEN_TYPE; + +typedef enum +{ + DAT_SR_API_UDAT, + DAT_SR_API_KDAT +} DAT_SR_API_TYPE; + + +/********************************************************************* + * * + * Structures * + * * + *********************************************************************/ + +typedef struct +{ + DAT_SR_TOKEN_TYPE type; + char * value; /* valid if type is DAT_SR_TOKEN_STRING */ + DAT_OS_SIZE value_len; +} DAT_SR_TOKEN; + +typedef struct DAT_SR_STACK_NODE +{ + DAT_SR_TOKEN token; + struct DAT_SR_STACK_NODE *next; +} DAT_SR_STACK_NODE; + +typedef struct +{ + DAT_UINT32 major; + DAT_UINT32 minor; +} DAT_SR_VERSION; + +typedef struct +{ + char * id; + DAT_SR_VERSION version; +} DAT_SR_PROVIDER_VERSION; + +typedef struct +{ + DAT_SR_API_TYPE type; + DAT_SR_VERSION version; +} DAT_SR_API_VERSION; + +typedef struct +{ + char * ia_name; + DAT_SR_API_VERSION api_version; + DAT_BOOLEAN is_thread_safe; + DAT_BOOLEAN is_default; + char * lib_path; + DAT_SR_PROVIDER_VERSION provider_version; + char * ia_params; + char * platform_params; +} DAT_SR_CONF_ENTRY; + + +/********************************************************************* + * * + * Internal Function Declarations * + * * + *********************************************************************/ + +static DAT_RETURN +dat_sr_load_entry ( + DAT_SR_CONF_ENTRY *entry); + +static DAT_BOOLEAN +dat_sr_is_valid_entry ( + DAT_SR_CONF_ENTRY *entry); + +static char * +dat_sr_type_to_str ( + DAT_SR_TOKEN_TYPE type); + +static DAT_RETURN +dat_sr_parse_eof ( + DAT_OS_FILE *file); + +static DAT_RETURN +dat_sr_parse_entry ( + DAT_OS_FILE *file); + +static DAT_RETURN +dat_sr_parse_ia_name ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_api ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_thread_safety ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_default ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_lib_path ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_provider_version ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_ia_params ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_platform_params ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_parse_eoe ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry); + +static DAT_RETURN +dat_sr_convert_api ( + char *str, + DAT_SR_API_VERSION *api_version); + +static DAT_RETURN +dat_sr_convert_thread_safety ( + char *str, + DAT_BOOLEAN *is_thread_safe); + +static DAT_RETURN +dat_sr_convert_default ( + char *str, + DAT_BOOLEAN *is_default); + +static DAT_RETURN +dat_sr_convert_provider_version ( + char *str, + DAT_SR_PROVIDER_VERSION *provider_version); + +static DAT_RETURN +dat_sr_get_token ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token); + +static DAT_RETURN +dat_sr_put_token ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token); + +static DAT_RETURN +dat_sr_read_token ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token); + +static DAT_RETURN +dat_sr_read_str ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token, + DAT_OS_SIZE token_len); + +static DAT_RETURN +dat_sr_read_quoted_str ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token, + DAT_OS_SIZE token_len, + DAT_COUNT num_escape_seq); + +static void +dat_sr_read_comment ( + DAT_OS_FILE *file); + + +/********************************************************************* + * * + * Global Variables * + * * + *********************************************************************/ + +static DAT_SR_STACK_NODE *g_token_stack = NULL; + + +/********************************************************************* + * * + * External Function Definitions * + * * + *********************************************************************/ + +/*********************************************************************** + * Function: dat_sr_load + ***********************************************************************/ + +DAT_RETURN +dat_sr_load (void) +{ + char *sr_path; + DAT_OS_FILE *sr_file; + + sr_path = dat_os_getenv (DAT_SR_CONF_ENV); + if ( sr_path == NULL ) + { + sr_path = DAT_SR_CONF_DEFAULT; + } + + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT Registry: static registry file <%s> \n", sr_path); + + sr_file = dat_os_fopen (sr_path); + if ( sr_file == NULL ) + { + return DAT_INTERNAL_ERROR; + } + + for (;;) + { + if ( DAT_SUCCESS == dat_sr_parse_eof (sr_file) ) + { + break; + } + else if ( DAT_SUCCESS == dat_sr_parse_entry (sr_file) ) + { + continue; + } + else + { + dat_os_assert (!"unable to parse static registry file"); + break; + } + } + + if ( 0 != dat_os_fclose (sr_file) ) + { + return DAT_INTERNAL_ERROR; + } + + return DAT_SUCCESS; +} + + +/********************************************************************* + * * + * Internal Function Definitions * + * * + *********************************************************************/ + +/*********************************************************************** + * Function: dat_sr_is_valid_entry + ***********************************************************************/ + +DAT_BOOLEAN +dat_sr_is_valid_entry ( + DAT_SR_CONF_ENTRY *entry) +{ + if ( ( DAT_SR_API_UDAT == entry->api_version.type ) && + (entry->is_default) ) + { + return DAT_TRUE; + } + else + { + return DAT_FALSE; + } +} + + +/*********************************************************************** + * Function: dat_sr_load_entry + ***********************************************************************/ + +DAT_RETURN +dat_sr_load_entry ( + DAT_SR_CONF_ENTRY *conf_entry) +{ + DAT_SR_ENTRY entry; + + if ( DAT_NAME_MAX_LENGTH < (strlen (conf_entry->ia_name) + 1) ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT Registry: ia name %s is longer than " + "DAT_NAME_MAX_LENGTH (%i)\n", + conf_entry->ia_name, DAT_NAME_MAX_LENGTH); + + return DAT_INSUFFICIENT_RESOURCES; + } + + dat_os_strncpy (entry.info.ia_name, conf_entry->ia_name, DAT_NAME_MAX_LENGTH); + entry.info.dapl_version_major = conf_entry->api_version.version.major; + entry.info.dapl_version_minor = conf_entry->api_version.version.minor; + entry.info.is_thread_safe = conf_entry->is_thread_safe; + entry.lib_path = conf_entry->lib_path; + entry.ia_params = conf_entry->ia_params; + entry.lib_handle = NULL; + entry.ref_count = 0; + + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "DAT Registry: loading provider for %s\n", + conf_entry->ia_name); + + return dat_sr_insert (&entry.info, &entry); +} + + +/*********************************************************************** + * Function: dat_sr_type_to_str + ***********************************************************************/ + +char * +dat_sr_type_to_str ( + DAT_SR_TOKEN_TYPE type) +{ + static char *str_array[] = { "string", "eor", "eof" }; + + if ( (type < 0) || (2 < type) ) + { + return "error: invalid token type"; + } + + return str_array[type]; +} + + +/*********************************************************************** + * Function: dat_sr_parse_eof + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_eof ( + DAT_OS_FILE *file) +{ + DAT_SR_TOKEN token; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_EOF == token.type ) + { + return DAT_SUCCESS; + } + else + { + dat_sr_put_token (file, &token); + return DAT_INTERNAL_ERROR; + } +} + + +/*********************************************************************** + * Function: dat_sr_parse_ia_name + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_entry ( + DAT_OS_FILE *file) +{ + DAT_SR_CONF_ENTRY entry; + DAT_RETURN status; + + dat_os_memset (&entry, 0, sizeof (DAT_SR_CONF_ENTRY)); + + if ( (DAT_SUCCESS == dat_sr_parse_ia_name (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_api (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_thread_safety (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_default (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_lib_path (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_provider_version (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_ia_params (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_platform_params (file, &entry)) && + (DAT_SUCCESS == dat_sr_parse_eoe (file, &entry)) ) + { + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "\n" + "DAT Registry: entry \n" + " ia_name %s\n" + " api_version\n" + " type 0x%X\n" + " major.minor %d.%d\n" + " is_thread_safe %d\n" + " is_default %d\n" + " lib_path %s\n" + " provider_version\n" + " id %s\n" + " major.minor %d.%d\n" + " ia_params %s\n" + "\n", + entry.ia_name, + entry.api_version.type, + entry.api_version.version.major, + entry.api_version.version.minor, + entry.is_thread_safe, + entry.is_default, + entry.lib_path, + entry.provider_version.id, + entry.provider_version.version.major, + entry.provider_version.version.minor, + entry.ia_params); + + if ( DAT_TRUE == dat_sr_is_valid_entry (&entry) ) + { + /* + * The static registry configuration file may have multiple + * entries with the same IA name. The first entry will be + * installed in the static registry causing subsequent attempts + * to register the same IA name to fail. Therefore the return code + * from dat_sr_load_entry() is ignored. + */ + (void) dat_sr_load_entry (&entry); + } + + status = DAT_SUCCESS; + } + else /* resync */ + { + DAT_SR_TOKEN token; + + /* + * The static registry format is specified in the DAT specification. + * While the registry file's contents may change between revisions of + * the specification, there is no way to determine the specification + * version to which the configuration file conforms. If an entry is + * found that does not match the expected format, the entry is discarded + * and the parsing of the file continues. There is no way to determine if + * the entry was an error or an entry confirming to an alternate version + * of specification. + */ + + for (;;) + { + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + status = DAT_INTERNAL_ERROR; + break; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_SUCCESS; + break; + } + else + { + dat_os_free (token.value, + (sizeof (char) * dat_os_strlen (token.value)) + 1); + continue; + } + } + } + + /* free resources */ + if ( NULL != entry.ia_name ) + { + dat_os_free (entry.ia_name, + sizeof (char) * (dat_os_strlen (entry.ia_name) + 1)); + } + if ( NULL != entry.lib_path ) + { + dat_os_free (entry.lib_path, + sizeof (char) * (dat_os_strlen (entry.lib_path) + 1)); + } + + if ( NULL != entry.provider_version.id ) + { + dat_os_free (entry.provider_version.id, + sizeof (char) * (dat_os_strlen (entry.provider_version.id) + 1)); + } + + if ( NULL != entry.ia_params ) + { + dat_os_free (entry.ia_params, + sizeof (char) * (dat_os_strlen (entry.ia_params) + 1)); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_ia_name + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_ia_name ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + entry->ia_name = token.value; + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_ia_name + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_api ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else if ( DAT_SUCCESS != dat_sr_convert_api ( + token.value, &entry->api_version) ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + dat_os_free (token.value, + (sizeof (char) * dat_os_strlen (token.value)) + 1); + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_thread_safety + ***********************************************************************/ + +static DAT_RETURN +dat_sr_parse_thread_safety ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else if ( DAT_SUCCESS != dat_sr_convert_thread_safety ( + token.value, &entry->is_thread_safe) ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + dat_os_free (token.value, + (sizeof (char) * dat_os_strlen (token.value)) + 1); + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_default + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_default ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else if ( DAT_SUCCESS != dat_sr_convert_default ( + token.value, &entry->is_default) ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + dat_os_free (token.value, + (sizeof (char) * dat_os_strlen (token.value)) + 1); + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_lib_path + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_lib_path ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + entry->lib_path = token.value; + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + +/*********************************************************************** + * Function: dat_sr_parse_provider_version + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_provider_version ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else if ( DAT_SUCCESS != dat_sr_convert_provider_version ( + token.value, &entry->provider_version) ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + dat_os_free (token.value, + (sizeof (char) * dat_os_strlen (token.value)) + 1); + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_ia_params + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_ia_params ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + entry->ia_params = token.value; + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_platform_params + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_platform_params ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( DAT_SR_TOKEN_STRING != token.type ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + entry->platform_params = token.value; + + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_parse_eoe + ***********************************************************************/ + +DAT_RETURN +dat_sr_parse_eoe ( + DAT_OS_FILE *file, + DAT_SR_CONF_ENTRY *entry) +{ + DAT_SR_TOKEN token; + DAT_RETURN status; + + if ( DAT_SUCCESS != dat_sr_get_token (file, &token) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( (DAT_SR_TOKEN_EOF != token.type) && + (DAT_SR_TOKEN_EOR != token.type) ) + { + status = DAT_INTERNAL_ERROR; + } + else + { + status = DAT_SUCCESS; + } + + if ( DAT_SUCCESS != status ) + { + DAT_RETURN status_success; + + status_success = dat_sr_put_token (file, &token); + dat_os_assert ( DAT_SUCCESS == status_success); + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_convert_api + ***********************************************************************/ + +DAT_RETURN +dat_sr_convert_api ( + char *str, + DAT_SR_API_VERSION *api_version) +{ + int i; + int minor_i; + + dat_os_assert ( 0 < dat_os_strlen (str) ); + + if ( 'u' == str[0] ) + { + api_version->type = DAT_SR_API_UDAT; + } + else if ( 'k' == str[0] ) + { + api_version->type = DAT_SR_API_KDAT; + } + else + { + return DAT_INTERNAL_ERROR; + } + + for ( i = 1 /* move past initial [u|k] */; '\0' != str[i]; i++ ) + { + if ( '.' == str[i] ) + { + break; + } + else if ( DAT_TRUE != dat_os_isdigit (str[i]) ) + { + return DAT_INTERNAL_ERROR; + } + } + + api_version->version.major = (DAT_UINT32) dat_os_strtol (str + 1, NULL, 10); + + /* move past '.' */ + minor_i = ++i; + + for ( ; '\0' != str[i]; i++ ) + { + if ( DAT_TRUE != dat_os_isdigit (str[i]) ) + { + return DAT_INTERNAL_ERROR; + } + } + + api_version->version.minor = (DAT_UINT32) dat_os_strtol (str + minor_i, NULL, 10); + + if ( '\0' != str[i] ) + { + return DAT_INTERNAL_ERROR; + } + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_sr_convert_thread_safety + ***********************************************************************/ + +static DAT_RETURN +dat_sr_convert_thread_safety ( + char *str, + DAT_BOOLEAN *is_thread_safe) +{ + if ( !dat_os_strncmp (str, + DAT_SR_TOKEN_THREADSAFE, + dat_os_strlen (DAT_SR_TOKEN_THREADSAFE)) ) + { + *is_thread_safe = DAT_TRUE; + return DAT_SUCCESS; + } + else if ( !dat_os_strncmp (str, + DAT_SR_TOKEN_NONTHREADSAFE, + dat_os_strlen (DAT_SR_TOKEN_NONTHREADSAFE)) ) + { + *is_thread_safe = DAT_FALSE; + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + + +/*********************************************************************** + * Function: dat_sr_convert_default + ***********************************************************************/ + +static DAT_RETURN +dat_sr_convert_default ( + char *str, + DAT_BOOLEAN *is_default) +{ + if ( !dat_os_strncmp (str, + DAT_SR_TOKEN_DEFAULT, + dat_os_strlen (DAT_SR_TOKEN_DEFAULT)) ) + { + *is_default = DAT_TRUE; + return DAT_SUCCESS; + } + else if ( !dat_os_strncmp (str, + DAT_SR_TOKEN_NONDEFAULT, + dat_os_strlen (DAT_SR_TOKEN_NONDEFAULT)) ) + { + *is_default = DAT_FALSE; + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + + +/*********************************************************************** + * Function: dat_sr_convert_provider_version + ***********************************************************************/ + +DAT_RETURN +dat_sr_convert_provider_version ( + char *str, + DAT_SR_PROVIDER_VERSION *provider_version) +{ + DAT_RETURN status; + int i; + int decimal_i; + + dat_os_assert ( 0 < dat_os_strlen (str) ); + dat_os_assert ( NULL == provider_version->id ); + + status = DAT_SUCCESS; + + for ( i = 0; '\0' != str[i]; i++ ) + { + if ( '.' == str[i] ) + { + break; + } + } + + /* if no id value was found */ + if ( 0 == i ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + + if ( NULL == (provider_version->id = dat_os_alloc (sizeof (char) * (i + 1))) ) + { + status = DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); + goto exit; + } + + dat_os_strncpy (provider_version->id, str, i); + provider_version->id[i] = '\0'; + + /* move past '.' */ + decimal_i = ++i; + + for ( ; '\0' != str[i]; i++ ) + { + if ( '.' == str[i] ) + { + break; + } + else if ( DAT_TRUE != dat_os_isdigit (str[i]) ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + } + + /* if no version value was found */ + if ( decimal_i == i ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + + provider_version->version.major = (DAT_UINT32) + dat_os_strtol (str + decimal_i, NULL, 10); + + /* move past '.' */ + decimal_i = ++i; + + for ( ; '\0' != str[i]; i++ ) + { + if ( DAT_TRUE != dat_os_isdigit (str[i]) ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + } + + /* if no version value was found */ + if ( decimal_i == i ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + + provider_version->version.minor = (DAT_UINT32) + dat_os_strtol (str + decimal_i, NULL, 10); + + if ( '\0' != str[i] ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + +exit: + if ( DAT_SUCCESS != status ) + { + if ( NULL != provider_version->id ) + { + dat_os_free (provider_version->id, + sizeof (char) * (dat_os_strlen (provider_version->id) + 1)); + provider_version->id = NULL; + } + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_get_token + ***********************************************************************/ + +DAT_RETURN +dat_sr_get_token ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token) +{ + if ( NULL == g_token_stack ) + { + return dat_sr_read_token (file, token); + } + else + { + DAT_SR_STACK_NODE *top; + + top = g_token_stack; + + *token = top->token; + g_token_stack = top->next; + + dat_os_free (top, sizeof (DAT_SR_STACK_NODE)); + + return DAT_SUCCESS; + } +} + + +/*********************************************************************** + * Function: dat_sr_put_token + ***********************************************************************/ + +DAT_RETURN +dat_sr_put_token ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token) +{ + DAT_SR_STACK_NODE *top; + + if ( NULL == (top = dat_os_alloc (sizeof (DAT_SR_STACK_NODE))) ) + { + return DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY; + } + + top->token = *token; + top->next = g_token_stack; + g_token_stack = top; + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_sr_read_token + ***********************************************************************/ + +DAT_RETURN +dat_sr_read_token ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token) +{ + DAT_OS_FILE_POS pos; + DAT_OS_SIZE token_len; + DAT_COUNT num_escape_seq; + DAT_BOOLEAN is_quoted_str; + DAT_BOOLEAN is_prev_char_backslash; + + /* + * The DAT standard does not specify a maximum size for quoted strings. + * Therefore the tokenizer must be able to read in a token of arbitrary + * size. Instead of allocating a fixed length buffer, the tokenizer first + * scans the input a single character at a time looking for the begining + * and end of the token. Once the these positions are found, the entire + * token is read into memory. By using this algorithm,the implementation + * does not place an arbitrary maximum on the token size. + */ + + token_len = 0; + num_escape_seq = 0; + is_quoted_str = DAT_FALSE; + is_prev_char_backslash = DAT_FALSE; + + for (;;) + { + DAT_OS_FILE_POS cur_pos; + int c; + + /* if looking for start of the token */ + if ( 0 == token_len ) + { + if ( DAT_SUCCESS != dat_os_fgetpos (file, &cur_pos) ) + { + return DAT_INTERNAL_ERROR; + } + } + + c = dat_os_fgetc (file); + + /* if looking for start of the token */ + if ( 0 == token_len ) + { + if ( EOF == c ) + { + token->type = DAT_SR_TOKEN_EOF; + token->value = NULL; + token->value_len = 0; + goto success; + } + else if ( DAT_SR_CHAR_NEWLINE == c ) + { + token->type = DAT_SR_TOKEN_EOR; + token->value = NULL; + token->value_len = 0; + goto success; + } + else if ( dat_os_isblank (c) ) + { + continue; + } + else if ( DAT_SR_CHAR_COMMENT == c ) + { + dat_sr_read_comment (file); + continue; + } + else + { + if ( DAT_SR_CHAR_QUOTE == c ) + { + is_quoted_str = DAT_TRUE; + } + + pos = cur_pos; + token_len++; + } + } + else /* looking for the end of the token */ + { + if ( EOF == c ) + { + break; + } + else if ( DAT_SR_CHAR_NEWLINE == c ) + { + /* put back the newline */ + dat_os_fputc (file, c); + break; + } + else if ( !is_quoted_str && dat_os_isblank (c) ) + { + break; + } + else + { + token_len++; + + if ( (DAT_SR_CHAR_QUOTE == c) && !is_prev_char_backslash ) + { + break; + } + else if ( (DAT_SR_CHAR_BACKSLASH == c) && !is_prev_char_backslash ) + { + is_prev_char_backslash = DAT_TRUE; + num_escape_seq++; + } + else + { + is_prev_char_backslash = DAT_FALSE; + } + } + } + } + + /* the token was a string */ + if ( DAT_SUCCESS != dat_os_fsetpos (file, &pos) ) + { + return DAT_INTERNAL_ERROR; + } + + if ( is_quoted_str ) + { + if ( DAT_SUCCESS != dat_sr_read_quoted_str (file, + token, + token_len, + num_escape_seq) ) + { + return DAT_INTERNAL_ERROR; + } + } + else + { + if ( DAT_SUCCESS != dat_sr_read_str (file, + token, + token_len) ) + { + return DAT_INTERNAL_ERROR; + } + } + +success: + dat_os_dbg_print (DAT_OS_DBG_TYPE_SR, + "\n" + "DAT Registry: token\n" + " type %s\n" + " value <%s>\n" + "\n", + dat_sr_type_to_str (token->type), + ((DAT_SR_TOKEN_STRING == token->type) ? token->value : "") ); + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_sr_read_str + ***********************************************************************/ + +DAT_RETURN +dat_sr_read_str ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token, + DAT_OS_SIZE token_len) +{ + token->type = DAT_SR_TOKEN_STRING; + token->value_len = sizeof (char) * (token_len + 1); /* +1 for null termination */ + if ( NULL == (token->value = dat_os_alloc (token->value_len)) ) + { + return DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY; + } + + if ( token_len != dat_os_fread (file, token->value, token_len) ) + { + dat_os_free (token->value, token->value_len); + token->value = NULL; + + return DAT_INTERNAL_ERROR; + } + + token->value[token->value_len - 1] = '\0'; + + return DAT_SUCCESS; +} + + +/*********************************************************************** + * Function: dat_sr_read_quoted_str + ***********************************************************************/ + +DAT_RETURN +dat_sr_read_quoted_str ( + DAT_OS_FILE *file, + DAT_SR_TOKEN *token, + DAT_OS_SIZE token_len, + DAT_COUNT num_escape_seq) +{ + DAT_OS_SIZE str_len; + DAT_OS_SIZE i; + DAT_OS_SIZE j; + int c; + DAT_RETURN status; + DAT_BOOLEAN is_prev_char_backslash; + + str_len = token_len - 2; /* minus 2 " characters */ + is_prev_char_backslash = DAT_FALSE; + status = DAT_SUCCESS; + + token->type = DAT_SR_TOKEN_STRING; + /* +1 for null termination */ + token->value_len = sizeof (char) * (str_len - num_escape_seq + 1); + + if ( NULL == (token->value = dat_os_alloc (token->value_len)) ) + { + status = DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY; + goto exit; + } + + /* throw away " */ + if ( DAT_SR_CHAR_QUOTE != dat_os_fgetc (file) ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + + for ( i = 0, j = 0; i < str_len; i++ ) + { + c = dat_os_fgetc (file); + + if ( EOF == c ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + else if ( (DAT_SR_CHAR_BACKSLASH == c) && !is_prev_char_backslash ) + { + is_prev_char_backslash = DAT_TRUE; + } + else + { + token->value[j] = (char)c; + j++; + + is_prev_char_backslash = DAT_FALSE; + } + } + + /* throw away " */ + if ( DAT_SR_CHAR_QUOTE != dat_os_fgetc (file) ) + { + status = DAT_INTERNAL_ERROR; + goto exit; + } + + token->value[token->value_len - 1] = '\0'; + +exit: + if ( DAT_SUCCESS != status ) + { + if ( NULL != token->value ) + { + dat_os_free (token->value, token->value_len); + token->value = NULL; + } + } + + return status; +} + + +/*********************************************************************** + * Function: dat_sr_read_comment + ***********************************************************************/ + +void +dat_sr_read_comment ( + DAT_OS_FILE *file) +{ + int c; + + /* read up to an EOR or EOF to move past the comment */ + do + { + c = dat_os_fgetc (file); + } while ( (DAT_SR_CHAR_NEWLINE != c) && (EOF != c) ); + + /* put back the newline */ + dat_os_fputc (file, c); +} diff --git a/trunk/ulp/dapl2/dat/udat/udat_sr_parser.h b/trunk/ulp/dapl2/dat/udat/udat_sr_parser.h new file mode 100644 index 00000000..3e83beb1 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/udat_sr_parser.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_sr_parser.h + * + * PURPOSE: static registry (SR) parser inteface declarations + * + * $Id: udat_sr_parser.h,v 1.4 2005/03/24 05:58:36 jlentini Exp $ + **********************************************************************/ + +#ifndef _DAT_SR_PARSER_H_ +#define _DAT_SR_PARSER_H_ + + +#include "dat_osd.h" + + +/********************************************************************* + * * + * Function Declarations * + * * + *********************************************************************/ + +/* + * The static registry exports the same interface regardless of + * platform. The particular implementation of dat_sr_load() is + * found with other platform dependent sources. + */ + +extern DAT_RETURN +dat_sr_load (void); + + +#endif /* _DAT_SR_PARSER_H_ */ diff --git a/trunk/ulp/dapl2/dat/udat/windows/dat_osd.c b/trunk/ulp/dapl2/dat/udat/windows/dat_osd.c new file mode 100644 index 00000000..0c2fe35e --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/windows/dat_osd.c @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under either one of the following two licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * OR + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * Licensee has the right to choose either one of the above two licenses. + * + * Redistributions of source code must retain both the above copyright + * notice and either one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, either one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dat_osd.c + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent functions with a canonical DAPL + * interface. Designed to be portable and hide OS specific quirks + * of common functions. + * + * $Id: dat_osd.c 33 2005-07-11 19:51:17Z ftillier $ + **********************************************************************/ + +#include "dat_osd.h" +#include "dat_init.h" + + +/********************************************************************* + * * + * Constants * + * * + *********************************************************************/ + +#define DAT_DBG_LEVEL_ENV "DAT_DBG_LEVEL" +#define DAT_DBG_DEST_ENV "DAT_DBG_DEST" + + +/********************************************************************* + * * + * Enumerations * + * * + *********************************************************************/ + +typedef int DAT_OS_DBG_DEST; + +typedef enum +{ + DAT_OS_DBG_DEST_STDOUT = 0x1, +} DAT_OS_DBG_DEST_TYPE; + + +/********************************************************************* + * * + * Global Variables * + * * + *********************************************************************/ + +static DAT_OS_DBG_TYPE_VAL g_dbg_type = DAT_OS_DBG_TYPE_GENERIC; +static DAT_OS_DBG_DEST g_dbg_dest = DAT_OS_DBG_DEST_STDOUT; + + +/*********************************************************************** + * Function: dat_os_dbg_set_level + ***********************************************************************/ + +void +dat_os_dbg_init ( void ) +{ + char *dbg_type; + char *dbg_dest; + + dbg_type = dat_os_getenv (DAT_DBG_LEVEL_ENV); + if ( dbg_type != NULL ) + { + g_dbg_type = dat_os_strtol(dbg_type, NULL, 0); + } + + dbg_dest = dat_os_getenv (DAT_DBG_DEST_ENV); + if ( dbg_dest != NULL ) + { + g_dbg_dest = dat_os_strtol(dbg_dest, NULL, 0); + } +} + + +/*********************************************************************** + * Function: dat_os_dbg_print + ***********************************************************************/ + +void +dat_os_dbg_print ( + DAT_OS_DBG_TYPE_VAL type, + const char * fmt, + ...) +{ + if ( (DAT_OS_DBG_TYPE_ERROR == type) || (type & g_dbg_type) ) + { + va_list args; + + va_start(args, fmt); + + if ( DAT_OS_DBG_DEST_STDOUT & g_dbg_dest ) + { + vfprintf(stdout, fmt, args); + } + + va_end(args); + + fflush(stdout); + } +} + + +BOOL APIENTRY +DllMain( + IN HINSTANCE h_module, + IN DWORD ul_reason_for_call, + IN LPVOID lp_reserved ) +{ + extern DAT_BOOLEAN udat_check_state ( void ); + + UNREFERENCED_PARAMETER( lp_reserved ); + + switch( ul_reason_for_call ) + { + case DLL_PROCESS_ATTACH: + DisableThreadLibraryCalls( h_module ); + udat_check_state(); + break; + + case DLL_PROCESS_DETACH: + dat_fini(); + } + + return TRUE; +} + +char * +dat_os_library_error(void) +{ + DWORD rc; + LPVOID lpMsgBuf; + + if (errno == 0) + return NULL; + + // consider formatmessage()? + rc = GetLastError(); + + FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + rc, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR) &lpMsgBuf, + 0, NULL ); + + //LocalFree(lpMsgBuf); error condition - will exit anyway. + + return (char*)lpMsgBuf; +} + +#ifndef INLINE_LIB_LOAD + +DAT_RETURN +dat_os_library_load ( const char *library_path, + DAT_OS_LIBRARY_HANDLE *library_handle_ptr ) +{ + DAT_OS_LIBRARY_HANDLE library_handle; + DAT_RETURN rc = DAT_SUCCESS; + + if ( NULL != (library_handle = LoadLibrary(library_path)) ) + { + if ( NULL != library_handle_ptr ) + { + *library_handle_ptr = library_handle; + } + } + else + { + dat_os_dbg_print(DAT_OS_DBG_TYPE_ERROR, + "DAT: library %s load failure\n",library_path); + rc = DAT_INTERNAL_ERROR; + } + return rc; +} +#endif diff --git a/trunk/ulp/dapl2/dat/udat/windows/dat_osd.h b/trunk/ulp/dapl2/dat/udat/windows/dat_osd.h new file mode 100644 index 00000000..4f2b2c24 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/windows/dat_osd.h @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under either one of the following two licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * OR + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * Licensee has the right to choose either one of the above two licenses. + * + * Redistributions of source code must retain both the above copyright + * notice and either one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, either one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * HEADER: dat_osd.h + * + * PURPOSE: Operating System Dependent layer + * Description: + * Provide OS dependent data structures & functions with + * a canonical DAPL interface. Designed to be portable + * and hide OS specific quirks of common functions. + * + * $Id: dat_osd.h 33 2005-07-11 19:51:17Z ftillier $ + **********************************************************************/ + +#ifndef _DAT_OSD_H_ +#define _DAT_OSD_H_ + +/* + * This file is defined for Windows systems only, including it on any + * other build will cause an error + */ +#if !defined(WIN32) && !defined(WIN64) +#error "UNDEFINED OS TYPE" +#endif /* WIN32/64 */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifndef STATIC +#define STATIC static +#endif /* STATIC */ + +#ifndef INLINE +#define INLINE __inline +#endif /* INLINE */ + +#ifndef __inline__ +#define __inline__ __inline +#endif /* __inline */ + +/********************************************************************* + * * + * Debuging * + * * + *********************************************************************/ + +#define dat_os_assert(expr) assert(expr) + +typedef int DAT_OS_DBG_TYPE_VAL; + +typedef enum +{ + DAT_OS_DBG_TYPE_ERROR = 0x1, + DAT_OS_DBG_TYPE_GENERIC = 0x2, + DAT_OS_DBG_TYPE_SR = 0x4, + DAT_OS_DBG_TYPE_DR = 0x8, + DAT_OS_DBG_TYPE_PROVIDER_API = 0x10, + DAT_OS_DBG_TYPE_CONSUMER_API = 0x20, + DAT_OS_DBG_TYPE_ALL = 0xff +} DAT_OS_DBG_TYPE_TYPE; + +extern void +dat_os_dbg_init ( void ); + +extern void +dat_os_dbg_print ( + DAT_OS_DBG_TYPE_VAL type, + const char * fmt, + ...); + + +/********************************************************************* + * * + * Utility Functions * + * * + *********************************************************************/ + +#define DAT_ERROR(Type,SubType) ((DAT_RETURN)(DAT_CLASS_ERROR | Type | SubType)) + +typedef size_t DAT_OS_SIZE; +typedef HMODULE DAT_OS_LIBRARY_HANDLE; + +#ifdef INLINE_LIB_LOAD + +STATIC INLINE DAT_RETURN +dat_os_library_load ( + const char *library_path, + DAT_OS_LIBRARY_HANDLE *library_handle_ptr) +{ + DAT_OS_LIBRARY_HANDLE library_handle; + + if ( NULL != (library_handle = LoadLibrary(library_path)) ) + { + if ( NULL != library_handle_ptr ) + { + *library_handle_ptr = library_handle; + } + return DAT_SUCCESS; + } + else + { + dat_os_dbg_print(DAT_OS_DBG_TYPE_ERROR, + "DAT: library load failure\n"); + return DAT_INTERNAL_ERROR; + } +} +#else + +DAT_RETURN +dat_os_library_load ( + const char *library_path, + DAT_OS_LIBRARY_HANDLE *library_handle_ptr); + +#endif + +extern char *dat_os_library_error(void); + +#ifdef WIN32_LEAN_AND_MEAN +#define dat_os_library_sym GetProcAddress +#else +STATIC INLINE +dat_os_library_sym ( + DAT_OS_LIBRARY_HANDLE library_handle, + const char *sym) +{ + return GetProcAddress(library_handle, (LPCSTR)sym); +} +#endif /* WIN32_LEAN_AND_MEAN */ + +STATIC INLINE DAT_RETURN +dat_os_library_unload ( + const DAT_OS_LIBRARY_HANDLE library_handle) +{ + if ( 0 == FreeLibrary(library_handle) ) + { + return DAT_INTERNAL_ERROR; + } + else + { + return DAT_SUCCESS; + } +} + +STATIC INLINE char * +dat_os_getenv ( + const char *name) +{ + return getenv(name); +} + +STATIC INLINE long int +dat_os_strtol ( + const char *nptr, + char **endptr, + int base) +{ + return strtol(nptr, endptr, base); +} + +STATIC INLINE DAT_OS_SIZE +dat_os_strlen ( + const char *s ) +{ + return strlen(s); +} + +STATIC INLINE int +dat_os_strncmp ( + const char *s1, + const char *s2, + DAT_OS_SIZE n) +{ + return strncmp(s1, s2, n); +} + +STATIC INLINE void * +dat_os_strncpy ( + char *dest, + const char *src, + DAT_OS_SIZE len) +{ + return strncpy (dest, src, len); +} + +STATIC INLINE DAT_BOOLEAN +dat_os_isblank( + int c) +{ + if ( (' ' == c) || ('\t' == c) ) { return DAT_TRUE; } + else { return DAT_FALSE; } +} + +STATIC INLINE DAT_BOOLEAN +dat_os_isdigit( + int c) +{ + if ( isdigit(c) ) { return DAT_TRUE; } + else { return DAT_FALSE; } +} + +STATIC INLINE void +dat_os_usleep( + unsigned long usec) +{ + Sleep(usec/1000); +} + + +/********************************************************************* + * * + * Memory Functions * + * * + *********************************************************************/ + +STATIC INLINE void * +dat_os_alloc ( + int size) +{ + return malloc (size); +} + +STATIC INLINE void +dat_os_free ( + void *ptr, + int size) +{ + free (ptr); +} + +STATIC INLINE void * +dat_os_memset (void *loc, int c, DAT_OS_SIZE size) +{ + return memset (loc, c, size); +} + + +/********************************************************************* + * * + * File I/O * + * * + *********************************************************************/ + +typedef FILE DAT_OS_FILE; +typedef fpos_t DAT_OS_FILE_POS; + + +STATIC INLINE DAT_OS_FILE * +dat_os_fopen ( + const char * path) +{ + /* always open files in read only mode*/ + return fopen(path, "r"); +} + +STATIC INLINE DAT_RETURN +dat_os_fgetpos ( + DAT_OS_FILE *file, + DAT_OS_FILE_POS *pos) +{ + if ( 0 == fgetpos(file, pos) ) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + +STATIC INLINE DAT_RETURN +dat_os_fsetpos ( + DAT_OS_FILE *file, + DAT_OS_FILE_POS *pos) +{ + if ( 0 == fsetpos(file, pos) ) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + +/* dat_os_fgetc() returns EOF on error or end of file. */ +STATIC INLINE int +dat_os_fgetc ( + DAT_OS_FILE *file) +{ + return fgetc(file); +} + +/* dat_os_fgetc() returns EOF on error or end of file. */ +STATIC INLINE int +dat_os_fputc ( + DAT_OS_FILE *file, int c) +{ + return fputc(c, file); +} + +/* dat_os_fread returns the number of bytes read from the file. */ +STATIC INLINE DAT_OS_SIZE +dat_os_fread ( + DAT_OS_FILE *file, + char *buf, + DAT_OS_SIZE len) +{ + return fread(buf, sizeof(char), len, file); +} + +STATIC INLINE DAT_RETURN +dat_os_fclose ( + DAT_OS_FILE *file) +{ + if ( 0 == fclose(file) ) + { + return DAT_SUCCESS; + } + else + { + return DAT_INTERNAL_ERROR; + } +} + + +/********************************************************************* + * * + * Locks * + * * + *********************************************************************/ + +typedef HANDLE DAT_OS_LOCK; + +/* lock functions */ +STATIC INLINE DAT_RETURN +dat_os_lock_init ( + IN DAT_OS_LOCK *m) +{ + *m = CreateMutex (0, FALSE, 0); + if (*(HANDLE *)m == NULL) + { + return DAT_INTERNAL_ERROR; + } + return DAT_SUCCESS; +} + +STATIC INLINE DAT_RETURN +dat_os_lock ( + IN DAT_OS_LOCK *m) +{ + WaitForSingleObject(*m, INFINITE); + + return DAT_SUCCESS; +} + +STATIC INLINE DAT_RETURN +dat_os_unlock ( + IN DAT_OS_LOCK *m) +{ + ReleaseMutex (*m); + + return DAT_SUCCESS; +} + +STATIC INLINE DAT_RETURN +dat_os_lock_destroy ( + IN DAT_OS_LOCK *m) +{ + CloseHandle (*m); + + return DAT_SUCCESS; +} + + +#endif /* _DAT_OSD_H_ */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ + diff --git a/trunk/ulp/dapl2/dat/udat/windows/dat_osd_sr.h b/trunk/ulp/dapl2/dat/udat/windows/dat_osd_sr.h new file mode 100644 index 00000000..dab9df02 --- /dev/null +++ b/trunk/ulp/dapl2/dat/udat/windows/dat_osd_sr.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2002, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under the terms of the "IBM Common Public + * License 1.0" a copy of which is in the file LICENSE.txt in the + * root directory. The license is also available from the Open Source + * Initiative, see http://www.opensource.org/licenses/ibmpl.html. + * + */ + +/********************************************************************** + * + * MODULE: dat_osd_sr.h + * + * PURPOSE: static registry (SR) platform specific inteface declarations + * + * $Id: dat_osd_sr.h 33 2005-07-11 19:51:17Z ftillier $ + **********************************************************************/ + +#ifndef _DAT_OSD_SR_H_ +#define _DAT_OSD_SR_H_ + + +#include "dat_osd.h" + + +/********************************************************************* + * * + * Function Declarations * + * * + *********************************************************************/ + +/* + * The static registry exports the same interface regardless of + * platform. The particular implementation of dat_sr_load() is + * found with other platform dependent sources. + */ + +extern DAT_RETURN +dat_sr_load (void); + + +#endif /* _DAT_OSD_SR_H_ */ diff --git a/trunk/ulp/dapl2/dirs b/trunk/ulp/dapl2/dirs new file mode 100644 index 00000000..93892b63 --- /dev/null +++ b/trunk/ulp/dapl2/dirs @@ -0,0 +1,4 @@ +DIRS=\ + dat \ + dapl \ + test diff --git a/trunk/ulp/dapl2/doc/dapl_coding_style.txt b/trunk/ulp/dapl2/doc/dapl_coding_style.txt new file mode 100644 index 00000000..cf41ae68 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_coding_style.txt @@ -0,0 +1,264 @@ +####################################################################### +# # +# DAPL Coding style reference # +# # +# Steve Sears # +# sjs2 at users.sourceforge.net # +# # +# 12/13/2002 # +# # +####################################################################### + +====================================================================== +Introduction +====================================================================== + +The purpose of this document is to establish the coding style adopted by +the team implementing the DAPL reference implementation. The rules +presented here were arrived at by consensus, they are intended to +provide consistency of implementation and make it intuitive to work with +the source code. + +====================================================================== +Source code conventions +====================================================================== + +1. Brackets + + Brackets should follow C99 conventions and declare a block. The + following convention is followed: + + if (x) + { + statement; + statement; + } + + The following bracket styles are to be avoided: + + K&R style: + + if (x) { /* DON'T DO THIS */ + statement; + } + + GNU style: + + if (x) /* DON'T DO THIS */ + { + statement; + } + + Statements are always indented from brackets. + + Brackets are always used for any statement in order to avoid dangling + clause bugs. E.g. + + RIGHT: + if ( x ) + { + j = 0; + } + + WRONG: + if ( x ) + j = 0; + +2. Indents + + Indents are always 4, tabs 8. A tab may serve as a double + indent. Many of the reference implementation file have an emacs + format statement at the bottom. + +3. Comments + + Comments are always full C style comments, and never C++ + style. Comments take the form: + + /* + * comment + */ + +4. Variable Declarations + + Variables are always declared on their own line, we do not declare + multiple variables on the same line. + + Variables are never initialized in their declaration, they are + initialized in the body of the code. + +5. Function Declarations + + The return type of a function is declared on a separate line from the + function name. + + Parameters each receive a line and should be clearly labeled as IN + or OUT or INOUT. Parameter declarations begin one tab stop from the + margin. + + For example: + + DAT_RETURN + dapl_function ( + IN DAT_IA_HANDLE ia_handle, + OUT DAT_EP_HANDLE *ep_handle ) + { + ... function body ... + } + +5. White space + + Don't be afraid of white space, the goal is to make the code readable + and maintainable. We use white space: + + - One space following function names or conditional expressions. It + might be better to say one space before any open parenthesis. + + - Suggestion: One space following open parens and one space before + closing parens. Not all of the code follows this convention, use + your best judgment. + + Example: + + foo ( x1, x2 ); + +6. Conditional code + + We generally try to avoid conditional compilation, but there are + certain places where it cannot be avoided. Whenever possible, move + the conditional code into a macro or otherwise work to put it into an + include file that can be used by the platform (e.g. Linux or Windows + osd files), or by the underlying provider (e.g. IBM Torrent or + Mellanox Tavor). + + Conditionals should be descriptive, and the associated #endif should + contain the declaration. E.g. + + #ifdef THIS_IS_AN_EXAMPLE + + /* code */ + + #endif /* THIS_IS_AN_EXAMPLE */ + + You may change the ending comment if a #else clause is present. E.g. + + #ifdef THIS_IS_AN_EXAMPLE + /* code */ + + #else + /* other code */ + + #endif /* !THIS_IS_AN_EXAMPLE */ + + +====================================================================== +Naming conventions +====================================================================== + +1. Variable Names + + Variable names for DAPL data structures generally follow their type + and should be the same in all source files. A few examples: + + Handles + DAT_IA_HANDLE ia_handle + DAT_EP_HANDLE ep_handle + + Pointers + + DAPL_IA *ia_ptr; + DAPL_EP *ep_ptr; + +2. Return Code Names + + There are at least two different subsystems supported in the DAPL + reference implementation. In order to bring sanity to the error + space, return codes are named and used for their appropriate + subsystem. E.g. + + ib_status: InfiniBand status return code + dat_status: DAT/DAPL return code + +3. Function Names + + Function names describe the scope to which they apply. There are + essentially three names in the reference implementation: + + dapl_* Name of an exported function visible externally. + These functions have a 1 to 1 correspondence to + their DAT counterparts. + + dapls_* Name of a function that is called from more than one + source file, but is limited to a subsystem. + + dapli_* Local function, internal to a file. Should always be + of type STATIC. + + +====================================================================== +Util files +====================================================================== + +The Reference implementation is organized such that a single, exported +function is located in its' own file. If you are trying to find the DAPL +function to create and End Point, it will be found in the dapl version +of the DAT function in the spec. E.g. + +dapl_ep_create() is found in dapl_ep_create.c +dapl_evd_free() is found in dapl_evd_free.c + +It is often the case that the implementation must interact with data +structures or call into other subsystems. All utility functions for a +subsystem are gathered into the appropriate "util" file. + +For example, dapl_ep_create must allocate a DAPL_EP structure. The +routine to allocate and initialize memory is found in the +dapl_ep_util.c file and is named dapl_ep_alloc(). Appropriate routines +for the util file are + + - Alloc + - Free + - Assign defaults + - linking routines + - Check restrictions + - Perform operations on a data structure. + +The idea of a util file is an object oriented idea for a non OO +language. It encourages a clean implementation. + +For each util.c file, there is also a util.h file. The purpose of the +util include file is to define the prototypes for the util file, and to +supply any local flags or values necessary to the subsystem. + +====================================================================== +Include files, prototypes +====================================================================== + +Include files are organized according to subsystem and/or OS +platform. The include directory contains files that are global to the +entire source set. Prototypes are found in include files that pertain to +the data they support. + +Commenting on the DAPL Reference Implementation tree: + + dapl/common + dapl/include + Contains global dapl data structures, symbols, and + prototypes + dapl/tavor + Contains tavor prototypes and symbols + dapl/torrent + Contains torrent prototypes and symbols + dapl/udapl + Contains include files to support udapl specific files + dapl/udapl/linux + Contains osd files for Linux + dapl/udapl/windows + Contains osd files for Windows + +For completeness, the dat files described by the DAT Specification are +in the tree under the dat/ subdirectory, + + dat/include/dat/ + + diff --git a/trunk/ulp/dapl2/doc/dapl_end_point_design.txt b/trunk/ulp/dapl2/doc/dapl_end_point_design.txt new file mode 100644 index 00000000..ccf11aa5 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_end_point_design.txt @@ -0,0 +1,1129 @@ +####################################################################### +# # +# DAPL End Point Management Design # +# # +# Steve Sears # +# sjs2 at users.sourceforge.net # +# # +# 10/04/2002 # +# Updates # +# 02/06/04 # +# 10/07/04 # +# # +####################################################################### + + +====================================================================== +Referenced Documents +====================================================================== + +uDAPL: User Direct Access Programming Library, Version 1.1. Published +05/08/2003. http://www.datcollaborative.org/uDAPL_050803.pdf. +Referred to in this document as the "DAT Specification". + +InfiniBand Access Application Programming Interface Specification, +Version 1.2, 4/15/2002. In DAPL SourceForge repository at +doc/api/access_api.pdf. Referred to in this document as the "IBM +Access API Specification". + +InfiniBand Architecture Specification Volume 1, Release 1.0.a Referred +to in this document at the "InfiniBand Spec". + +====================================================================== +Introduction to EndPoints +====================================================================== + +An EndPoint is the fundamental channel abstraction for the DAT API. An +application communicates and exchanges data using an EndPoint. Most of +the time EndPoints are explicitly allocated, but there is an exception +whereby a connection event can yield an EndPoint as a side effect; this +is not supported by all transports or implementations, but it is +supported in the InfiniBand reference implementation. + +Each DAT API function is implemented in a file named + + dapl_.c + +There is a simple mapping provided by the dat library that maps dat_* to +dapl_*. For example, dat_pz_create is implemented in dapl_pz_create.c. +Other examples: + + DAT DAPL Found in + ------------ --------------- ------------------ + dat_ep_create dapl_ep_create dapl_ep_create.c + dat_ep_query dapl_ep_query dapl_ep_query.c + +There are very few exceptions to this naming convention, the Reference +Implementation tried to be consistent. + +There are also dapl__util.{h,c} files for each object. For +example, there are dapl_pz_util.h and dapl_pz_util.c files which contain +common helper functions specific to the 'pz' subsystem. The use of util +files follows the convention used elsewhere in the DAPL reference +implementation. These files contain common object creation and +destruction code, link list manipulation, other helper functions. + +This implementation has a simple naming convention designed to alert +someone reading the source code to the nature and scope of a +function. The convention is in the function name, such that: + + dapl_ Primary entry from a dat_ function, e.g. + dapl_ep_create(), which mirrors dat_ep_create(). + dapls_ The 's' restricts it to the subsystem, e.g. the + 'ep' subsystem. dapls_ functions are not exposed + externally, but are internal to dapl. + dapli_ The 'i' restricts the function to the file where it + is declared. These functions are always 'static' C + functions. + +This convention is not followed as consistently as we would like, but is +common in the reference implementation. + +1. End Points (EPs) +------------------------- + +DAPL End Points provide a channel abstraction necessary to transmit and +receive data. EPs interact with Service Points, either Public Service +Points or Reserved Service Points, to establish a connection from one +provider to another. + +The primary EP entry points in the DAT API as they relate to DAPL are +listed in the following table: + + dat_ep_create + dat_ep_query + dat_ep_modify + dat_ep_connect + dat_ep_dup_connect + dat_ep_disconnect + dat_ep_post_send + dat_ep_post_recv + dat_ep_post_rdma_read + dat_ep_post_rdma_write + dat_ep_get_status + dat_ep_reset + dat_ep_free + +Additionally, the following connection functions interact with +EndPoints: + dat_psp_create + dat_psp_query + dat_psp_free + dat_rsp_create + dat_rsp_query + dat_rsp_free + dat_cr_accept + dat_cr_reject + dat_cr_query + dat_cr_handoff + +The reference implementation maps the EndPoint abstraction onto an +InfiniBand Queue Pair (QP). + +The DAPL_EP structure is used to maintain the state and components of +the EP object and the underlying QP. As will be explained below, keeping +track of the QP state is critical for successful operation. Access to +the DAPL_EP fields are done atomically. + + +====================================================================== +Goals +====================================================================== + +Initial goals +------------- +-- Implement all of the dat_ep_* calls described in the DAT + Specification. + +-- Implement connection calls described in the DAT Specification with + the following exception: + - dat_cr_handoff. This is best done with kernel mediation, and is + therefore out of scope for the reference implementation. + +-- The implementation should be as portable as possible, to facilitate + HCA Vendors efforts to implement vendor-specific versions of DAPL. + +-- The implementation must be able to work during ongoing development + of provider software agents, drivers, etc. + +Later goals +----------- +-- Examine various possible performance optimizations. This document + lists potential performance improvements, but the specific + performance improvements implemented should be guided by customer + requirements. + +============================================ +Requirements, constraints, and design inputs +============================================ + +The EndPoint is the base channel abstraction. An Endpoint must be +established before data can be exchanged with a remote node. The +EndPoint is mapped to the underlying InfiniBand QP channel abstraction. +When a connection is initiated, the InfiniBand Connection Manager will +be solicited. The implementation is constrained by the capabilities and +behavior of the underlying InfiniBand facilities. + +Note that transports other than InfiniBand may not need to rely on +Connection Managers or other infrastructure, this is an artifact of +this transport. + +An EP is not an exact match to an InfiniBand QP, the differences +introduce constraints that are not obvious. There are three primary +areas of conflict between the DAPL and InfiniBand models: + +1) EP and QP creation differences +2) Provider provided EPs on passive side of connections +3) Connection timeouts + +-- EP and QP creation + +The most obvious difference between an EP and a QP is the presence of a +memory handle when the object is created. InfiniBand requires a +Protection Domain (PD) be specified when a QP is created; in the DAPL +world, a Protection Zone (PZ) maps to an InfiniBand Protection Domain. +DAPL does not require a PZ to be present when an EP is created, and that +introduces two problems: + +1) If a PZ is NULL when an EP is created, a QP will not be bound to + the EP until dat_ep_modify() is used to assign it later. A PZ is + required before RECV requests can be posted and before a connection + can be established. + +2) If a DAPL user changes the PZ on an EP before it is connected, + DAPL must release the current QP and create a new one with a + new Protection Domain. + +-- Provider provided EPs on connection + +The second area where the DAPL and IB models conflict is a direct result +of the requirement to specify a Protection Domain when a QP is created. + +DAPL allows a PSP to be created in such a way that an EP will +automatically be provided to the user when a connection occurs. This is +not critical to the DAPL model but in fact does provide some convenience +to the user. InfiniBand provides a similar mechanism, but with an +important difference: InfiniBand requires the user to supply the +Protection Domain for the passive connection endpoint that will be +supplied to all QPs created as a result of connection requests; DAPL +mandates a NULL PZ and requires the user to change the PZ before using +the EP. + +The reference implementation creates an 'empty' EP when the user +specifies the DAT_PSP_PROVIDER flag; it is empty in the sense that a QP +is not attached to the EP. Before the user can dat_cr_accept the +connection, the EP must be modified to have a PZ bound to it, which in +turn will cause a QP to be bound to the EP. + +To keep track of the current state of the EP, the DAPL_EP structure +has a qp_state field. The type of this field is specific to the +provider and the states are provider-specified states for a particular +transport, with the addition of a single state from dapl: +DAPL_QP_STATE_UNATTACHED, indicating that no QP has been bound to the +EP. The qp_state field is an open enumerator, containing a single DAPL +state in addition states specified by the provider. +DAPL_QP_STATE_UNATTACHED is randomly defined to be 0xFFF0, a +value selected strictly because it has the property that it will not +collide with provider states; if this is not true, this value must be +changed such that it is unique. + +The common layer of DAPL only looks at this single value for qp_state, +it cannot be aware of states that are unique to the provider. However, +the provider layer is free to update this field and may use it as a +cache for current QP state. The field must be updated when a QP (or +other endpoint resource) is bound to the EP. + +DAPL 1.2 provides DAT level states that will make this obsolete, but it +exists in pre DAPL 1.2 code. + + +-- Connection Timeouts + +The third difference in the DAPL and InfiniBand models has to do with +timeouts on connections. InfiniBand does not provide a way to specify a +connection timeout, so it will wait indefinitely for a connection to +occur. dat_ep_connect supports a timeout value providing the user with +control over how long they are willing to wait for a connection to +occur. + +DAPL maintains a timer thread to watch over pending connections. A +shared timer queue has a sorted list of timeout values. If a timeout +is requested, dapl_ep_connect() will invoke dapls_timer_set(), which +will add a timer record to the sorted list of timeouts. The timeout +thread is started lazily: that is, it isn't started until a timeout is +requested. Once a timeout has been requested, the thread will continue +to exist until the application terminates. + +The timer record is actually a part of the DAPL_EP structure, so there +are no extra memory allocations required for timeouts. dapls_timer_set() +will initialize the timer record and insert it into the sorted queue at +the appropriate place. If this is the first record, or is inserted +before the first record (which will be the 'next' timeout to expire), +the timer thread will be awakened so it can recalculate how long it must +sleep until the timeout occurs. + +When a timeout does occur, the timeout code will cancel the connection +request by invoking the provider routine dapls_ib_disconnect_clean(). +This allows the software module with explicit knowledge of the provider +to take appropriate action and cancel the connection attempt. As a side +effect, the EP will be placed into the UNCONNECTED state, and the QP +will be in the ERROR state. A side effect of this state change is that +all DTOs will be flushed. The provider must support a mechanism to +completely cancel a connection request. + + +====================================================================== +DAPL EP Subsystem Design +====================================================================== + +In section 6.5.1 of the DAT Specification there is a UML state +transition diagram for an EndPoint which goes over the transitions and +states during the lifetime of an EP. It is nearly impossible to read. +The reference implementation is faithful to the DAT Spec and is +believed to be correct. + +This description of the EP will follow from creation to connection to +termination. It will also discuss the source code organization as this +is part of the design expression. + +-- EP and QP creation + +The preamble to creating an EP requires us to verify the attributes +specified by the user. If a user were to specify max_recv_dtos as 0, for +example, the EP would not be useful in any regard. If the user does not +provide EP attrs, the DAPL layer will supply a set of common defaults +resulting in a reasonable EP. The defaults are set up in +dapli_ep_default_attrs(), and the default values are given at the top of +dapl_ep_util.c. Non-InfiniBand transports will want to examine these +values to make sure they are 'reasonable'. This simplistic mechanism may +change in the future. + +A number of handles are bound to the EP, so a reference count is taken +on each of them. All reference counts in the DAPL system are incremented +or decremented using atomic operations; it is important to always use +the OS dependent atomic routines and not substitute a lock, as it will +not be observed elsewhere in the system and will have unpredictable +results. + +Reference counts are taken if there are non NULL values on any of: + pz_handle + connect_evd_handle + recv_evd_handle + request_evd_handle + +The purpose of reference counts should be obvious: to prevent premature +release of resources that are still being used. + +As has been discussed above, each EP is bound to a QP before it can be +connected. If a valid PZ is provided at creation time then a QP is bound +to the EP immediately. If the user later uses ep_modify() to change the +PZ, the QP will be destroyed and a new one created with the appropriate +Protection Domain. + +Finally, an EP is an IA resource and is linked onto the EP chain of the +superior IA. EPs linked onto an IA are assumed to be complete, so this +is the final step of EP creation. + +After an EP is created, the ep_state will be DAT_EP_STATE_UNCONNECTED +and the qp_state will either be DAPL_QP_STATE_UNATTACHED or assigned by +the provider layer (e.g.IB_QP_STATE_INIT). The qp_state indicates the QP +binding and the current state of the QP. + +A qp_state of DAPL_QP_STATE_UNATTACHED indicates there is no QP bound +to this EP. This is a result of a NULL PZ when dat_ep_create() was +invoked, and which has been explained in detail above. The user must +call dat_ep_modify() and install a valid PZ before the EP can be used. + +When an InfiniBand QP is created it is in the RESET state, which is +specified in the InfiniBand Spec, section 10.3. However, DAPL creates +the EP in the UNCONNECTED state and requires an unconnected EP to be +able to queue RECV requests before a connection occurs. The InfiniBand +spec allows RECV requests to be queued on an QP if the QP is in the INIT +state, so after creating a QP the DAPL provider code must transition it +to the INIT state. + +There is a mapping between the DAPL EP state and the InfiniBand QP +state. DAPL_QP_STATE_UNATTACHED indicates the underlying QP is in the +INIT state. This is critical: RECV DTOs can be posted on an EP in the +UNATTACHED state, so the underlying QP must be in the appropriate state +to allow this to happen. + +There is an obvious design tradeoff in transitioning the QP +state. Immediately moving the state to INIT takes extra time at creation +but allows immediate posting of RECV operations; however, it will +involve a more complex tear down procedure if the QP must be replaced as +a side effect of a dat_ep_modify operation. The alternative would be to +delay transitioning the QP to INIT until a post operation is invoked, +but that requires a run time check for every post operation. This design +assumes users will infrequently cause a QP to be replaced after it is +created and prefer to pay the state transition penalty at creation time. + +-- EP Query and Modify operations + +Because all of the ep_param data are kept up to date in the dapl_ep +structure, and because they use the complete DAT specified structure, a +query operation is trivial; a simple assignment from the internal +structure to the user parameter. uDAPL allows the implementation to +either return the fields specified by the user, or to return more than +the user requested; the reference implementation does the latter. It is +simpler and faster to copy the entire structure rather than to determine +which of all of the possible fields the user requested. + +dat_ep_query() requires the implementation to report the address of the +remote node, if the EP is connected. This is different from standard +InfiniBand, if only because of the difference in name space. InfiniBand +has the information on the remote LID, but it does not have the remote +IP address, which is what DAT specifies. The reference implementation +makes use of a lookup/name-service called ATS ( Address Translation +Service), which is built using the InfiniBand Subnet Administrator. ATS +is InfiniBand only, other transports will use a different mechanism. + +A driver will register itself and one or more IP addresses with ATS +at some point before a connection can be made. How the addresses are +provided to the driver, or how this is managed by the driver is not +specified. The ATS proposal is available from the DAT Collaborative. + +When dat_ep_query() is invoked on a connected EP, it will request the +remote address from the provider layer. The provider layer will use +whatever means are necessary to obtain the IP address of the other end of +the connection. The results are placed into a buffer that is part of the +EP structure. Finally, the address of the EP structure is placed into +the ep_param.remote_ia_address_ptr field. + +The ep_modify operation will modify the fields in the DAT_PARAM +structure. There are some fields that cannot be updated, and there are +others that can only be updated if the EP is in the correct state. The +uDAPL spec outlines the EP states permitting ep modifications, but +generally they are DAT_EP_STATE_UNCONNECTED and +DAT_EP_STATE_PASSIVE_CONNECTION_PENDING. + +When replacing EVD handles it is a simple matter of releasing a +reference on the previous handle and taking a new reference on the new +handle. The Reference Implementation manages resource tracking using +reference counts, which guarantees a particular handle will not be +released prematurely. Reference counts are checked in the free routines +of various objects. + +As has been mentioned previously, if the PZ handle is changed then the +QP must be released, if already assigned, and a new QP must be created +to bind to this EP. + +There are some fields in the DAT_PARAM structure that are related to the +underlying hardware implementation. For these values DAPL will do a +fresh query of the QP, rather than depend on stale values. Even so, the +values returned are 'best effort' as a competing thread may change +certain values before the requesting thread has the opportunity to read +them. Applications should protect against this. + +Finally, the underlying provider is invoked to update the QP with new +values, but only if some of the attributes have been changed. As is +true of most of the implementation, we only invoke the provider code +when necessary. + +====================================================================== +Connections +====================================================================== + +There are of course two sides to a connection, and in the DAPL model +there is an Active and a Passive side. For clarity, the Passive side +is a server waiting for a connection, and the Active side is a client +requesting a connection from the Passive server. We will discuss each +of these in turn. + +Connections happen in the InfiniBand world by using a Connection Manager +(CM) interface. Those unfamiliar with the IB model of addressing and +management agents may want to familiarize themselves with these aspects +of the IB spec before proceeding in this document. Be warned that the +connection section of the IB spec is the most ambiguous portion of the +spec. + +First, let's walk through a primitive diagram of a connection: + + +SERVER (passive) CLIENT (active) +--------------- --------------- +1. dapl_psp_create + or dapl_rsp_create + [ now listening ] + +2. dapl_ep_connect + <------------- +3. dapls_cr_callback + DAT_CONNECTION_REQUEST_EVENT + [ Create and post a DAT_CONNECTION_REQUEST_EVENT event ] + +4. Event code processing + +5. Create an EP if necessary + (according to the flags + when the PSP was created) + +6. dapl_cr_accept or dapl_cr_reject + -------------> +7. dapl_evd_connection_callback + DAT_CONNECTION_EVENT_ESTABLISHED + [ Create and post a + DAT_CONNECTION_EVENT_ESTABLISHED + event ] + +8. <------------- RTU + +9. dapls_cr_callback + DAT_CONNECTION_EVENT_ESTABLISHED + [ Create and post a DAT_CONNECTION_EVENT_ESTABLISHED + event ] + +10. ...processing... + +11. Either side issues a dat_ep_disconnect + +12. dapls_cr_callback + DAT_CONNECTION_EVENT_DISCONNECTED + + [ Create and post a + DAT_CONNECTION_EVENT_DISCONNECTED + event ] + +13. dapl_evd_connection_callback + DAT_CONNECTION_EVENT_DISCONNECTED + [ Create and post a + DAT_CONNECTION_EVENT_DISCONNECTED + event ] + + +In the above diagram, time is numbered in the left hand column and is +represented vertically. + +We will continue our discussion of connections using the above diagram, +following a sequential order for connection establishment. + +There are in fact two types of service points detailed in the uDAPL +specification. We will limit our discussion to PSPs for convenience, but +there are only minor differences between PSPs and RSPs. + +The reader should observe that all passive-side connection events will +be received by dapls_cr_callback(), and all active side connection +events occur through dapl_evd_connection_callback(). At one point during +the implementation these routines were combined as they are very +similar, but there are subtle differences causing them to remain +separate. + +Progressing through the series of events as outlined in the diagram +above: + +1. dapl_psp_create + + When a PSP is created, the final act will be to set it listening for + connections from remote nodes. It is important to realize that a + connection may in fact arrive from a remote node before the routine + setting up a listener has returned to dapl_psp_create; as soon as + dapls_ib_setup_conn_listener() is invoked connection callbacks may + arrive. To reduce race conditions this routine must be called as the + last practical operation when creating a PSP. + + dapls_ib_setup_conn_listener() is provider specific. The key insight + is that the DAPL connection qualifier (conn_qual) will become the + InfiniBand Service ID. The passive side of the connection is now + listening for connection requests. It should be obvious that the + conn_qual must be unique. + + InfiniBand allows a 64 bit connection qualifier, which is supported + by the DAT spec. IP based networks may be limited to 16 bits, so + provider implementations may want to return an error if it exceeds + the maximum allowable by the transport. + +2. dapl_ep_connect + + The active side initiates a connection with dapl_ep_connect, which + will transition the EP into DAT_EP_STATE_ACTIVE_CONNECTION_PENDING. + Again, connections are in the domain of the providers' Connection + Manager and the mechanics are very much provider specific. The key + points are that a DAT_IA_ADDRESS_PTR must be translated to a GID + before a connection initiation can occur. This is discussed below. + + InfiniBand supports different amounts of private data on various + connection functions. Other transports allow variable sizes of + private data with no practical limit.The DAPL connection code does + not enforce a fixed amount of private data, but rather makes + available to the user all it has available, as specified by + DAPL_MAX_PRIVATE_DATA_SIZE. + + Private data will be stored in a fixed buffer as part of the + connection record, which is the primary reason to limit the size. + + To assist development on new transports that do not have a full + connection infrastructure in place, there are a couple of compile time + flags that will include certain code: CM_BUSTED and + IBOSTS_NAMING. These are discussed below in more detail, but + essentially: + + CM_BUSTED: fakes a connection on both sides of the wire, does not + transmit any private data. + + IBHOSTS_NAMING: provides a simple IP_ADDRESS to LID translation + mechanism in a text file, which is read when the dapl library + loads. Private data is exchanged in this case, but it includes a + header that contains the remote IP address. Technically, this defines + a protocol and is in violation of the DAT spec, but it has proved + useful in development. + +3. dapls_cr_callback + + The connection sequence is entirely event driven. An operation is + posted, then an asynchronous event will occur some time later. The + event may cause other actions to occur which may result in still + more events. + + dapls_ib_setup_conn_listener() registered for a callback for + connection events, and we now receive a DAT event for a connection + request. The provider layer will translate the native event type to + a DAT event. + + An upcall is invoked on the server side of the connection with an + event of type DAT_CONNECTION_REQUEST_EVENT. This is a unique event + in the callback code as it is the only case when an EP is not + already in play; in all other cases, it is possible to look up the + relevant EP for an operation. + + Code exists to make sure the relevant connection object, the PSP or + RSP, is actually in a useful state and ready to be connected + to. One of the critical differences between a PSP and an RSP is + that an RSP is a one-shot connection object; once a connection + occurs, no other connections can be made to it. + + There is a small difference in the InfiniBand and DAPL connection + models here as well. DAPL may disable a PSP at any time without + affecting current connections. When you tear down an InfiniBand + service endpoint, all of the connections are torn down too. Because + of this difference, when a DAPL app frees a PSP, only a state + change is made. The underlying service point is still available and + technically capable of receiving connections. If a connection + request arrives when the PSP is in this state, a rejection message + is sent such that the requesting node believes no service point is + listening. + + Once the connection has been examined, it will continue with the + connection protocol. The EP will move to a CONNECTION_PENDING + state. + + The connection request will cause a CR record to be allocated, + which holds all of the important connection request + information. The CR record will be linked onto the PSP structure + for retrieval in the future when other requests arrive. + + The astute reader of the spec will observe that there is not a + dapl_cr_create call: CR records are created as part of a connection + attempt on the passive side of the connection. A CR is created now + and set up. A point that will become important later, caps for + emphasis: + + A CR WILL EXIST FOR THE LIFE OF A CONNECTION; THEY ARE DESTROYED AT + DISCONNECT TIME. + + In the connection request processing a CR and an EVENT are created, + the event will be posted along with the connection information just + received. + + Private data is also copied into the CR record. Private data + arrived with the connection request and is not a permanent + resource, so it is copied into the dapl space to be used at a later + time. Different transports have varying capabilities on the size of + private data, so a call to the provider is invoked to determine how + big it actually is. There is an upper bound on the amount of + private data the implementation will deal with, set at + DAPL_MAX_PRIVATE_DATA_SIZE (256 bytes at this writing). + +4. Event code processing + + The final stage in a connection request is to generate an event on + a connection EVD using dapls_evd_post_cr_arrival_event(). + +5. Create an EP if necessary + + When the app processes a connection event, it needs to respond. If + the PSP is configured to create an EP automatically, the callback + code has already done it; creating an EP with no attached QP. Else, + the user must provide an EP to make the connection. + + (4) and (5) are all done in user mode. The only interesting thing is + that when the user calls dat_cr_accept(), a ready EP must be + provided. If the EP was supplied by the PSP in the callback, it + must have a PZ associated with it and whatever other attributes + need to be set. + +6. dapl_cr_accept or dapl_cr_reject + + For discussion purposes, we will follow the accept + path. dapl_cr_reject says you are done and there will be no further + events to deal with. + + Assuming it accepts the connection for our example, the dapl code + will verify that an EP is in place and will deal with private data + that can be transmitted in a cr_accept call. The underlying + provider is invoked to complete this leg of the protocol. + +7. dapl_evd_connection_callback + + An EVD callback is always a response to a connection oriented + request. As such, an EP is always present, and in fact is passed + into the upcall as the 'context' argument. + + Connection requests may take an arbitrary amount of time, so the EP + is always checked for a running timer when the upcall is made. As + has been discussed above, if a timer expires before an upcall + occurs, the connection must be completely canceled such that there + is no upcall. + + The event signifying completion of the connection is + DAT_CONNECTION_EVENT_ESTABLISHED, and it will move the EP to the + CONNECTED state and post this event on the connection EVD. Private + data will be copied to an area in the EP structure, which is + persistent. + + At this point, the EP is connected and the application is free to + post DTOs. + +8i. RTU + + This item is labeled "8i" as it is internal to the InfiniBand + implementation, it is not initiated by dapl. The final leg of a + connection is an RTU sent from the initiating node to the server + node, indicating the connection has been made successfully. + + Other transports may have a different connection protocol. + +9. dapls_cr_callback + + When the RTU arrives, an upcall is invoked with a + DAT_CONNECTION_EVENT_ESTABLISHED event, which will be posted to the + connection EVD event queue. The EP is moved to the CONNECTED + state. + + There is no private data for dapl to deal with, even though some + transports may provide private data at each step of a connection. + + The connection activity is occurring on a separate channel than the + EP, so this is inherently a racy operation. The correct + application will always post RECV buffers on an EP before + initiating a connection sequence, as it is entirely possible for + DTOs to arrive *before* the final connection event arrives. + + The architecturally interesting feature of this exchange occurs + because of differences in the InfiniBand and the DAT connection + models, which are briefly outlined here. + + InfiniBand maintains the original connecting objects throughout the + life of the connection. That is, we originally get a callback event + associated with the Service (DAT PSP) that is listening for + connection events. A QP will be connected but the callback event + will still be received on the Service. Later, a callback event will + occur for a DISCONNECT, and again the Service will be the object of + the connection. In the DAPL implementation, the Service will + provide the PSP that is registered as listening on that connection + qualifier. + + The difference is that DAT has a PSP receive a connection event, + but subsequently hands all connection events off to an EP. After a + dat_cr_accept is issued, all connection/disconnection events occur + on the EP. DAT more closely follows the IP connection model. + + To support the DAT model, a CR is maintained through the life of + the connection. There is exactly one CR per connection, but any + number of CRs may exist for any given PSP. CRs are maintained on a + linked list pointed to by the PSP structure. A lookup routine will + match the cm_handle, unique for each connection, with the + appropriate CR. This allows us to find the appropriate EP which + will be used to create an event to be posted to the user. + +* dat_psp_destroy + + It should be understood that the PSP will maintain all of the CR + records, and hence the PSP must persist until the final disconnect. + In the DAT model there is no association between a PSP and a + connected QP, so there is no reason not to destroy a PSP before the + final disconnect. + + Because of the model mismatch we must preserve the PSP until the + final disconnect. If the user invokes dat_psp_destroy(), all of the + associations maintained by the PSP will be severed; but the PSP + structure itself remains as a container for the CR records. The PSP + structure maintains a simple count of CR records so we can easily + determine the final disconnect and release memory. Once a + disconnect event is received for a specific cm_handle, no further + events will be received and it is safe to discard the CR record. + +10. ...processing... + + This is just a place holder to show that applications actually do + something after making a connection. They might not too... + +11. Either side issues a dat_ep_disconnect + + dat_ep_disconnect() can be initiated by either side of a + connection. There are two kinds of disconnect flags that can be + passed in, but the final result is largely the same. + + DAT_CLOSE_ABRUPT_FLAG will cause the connection to be immediately + terminated. In InfiniBand terms, the QP is immediately moved to the + ERROR state, and after some time it will be moved to the RESET + state. + + DAT_CLOSE_GRACEFUL_FLAG will allow in-progress DTOs to complete. + The underlying implementation will first transition the QP to the + SQE state, before going to RESET. + + Both cases are handled by the underlying CM, there is no extra work + for DAPL. + +12. dapls_cr_callback + + A disconnect will arrive on the passive side of the connection + through dapls_cr_callback() with connection event + DAT_CONNECTION_EVENT_DISCONNECTED. With this event the EP lookup + code will free the CR associated with the connection, and may free + the PSP if it is no longer listening, indicating it has been freed + by the application. + + The callback will create and post a + DAT_CONNECTION_EVENT_DISCONNECTED event for the application. + +13. dapl_evd_connection_callback + + The active side of the connection will receive + DAT_CONNECTION_EVENT_DISCONNECTED as the connection event for + dapl_evd_connection_callback(), and will create and post a + DAT_CONNECTION_EVENT_DISCONNECTED event. Other than transitioning + the EP to the DISCONNECTED state, there is no further processing. + + +Observe that there are a number of exception conditions resulting in a +disconnect of the EP, most of which will generate unique DAT events +for the application to deal with. + + +* Addressing and Naming + + The DAT Spec calls for a DAT_IA_ADDRESS_PTR to be an IP address, + either IPv4 or IPv6. It is in fact a struct sockaddr in most + systems. The dapl structures typically use IPv6 data types to + accommodate the largest possible addresses, but most implementations + use IPv4 formatted addresses. + + InfiniBand uses a transport specific address known as a LID, which + typically is dynamically assigned by a Subnet Manager. Each HCA + also has a global address, similar to an Ethernet MAC address, + known as a GUID. ATS, mentioned above, is a mechanism using + InfiniBand infrastructure to map from GUID/LID to IP addresses. It + is not necessary for transports that use IP addresses natively, + such as Ethernet devices. + + If a new implementation does not yet have a name service + infrastructure, the DAPL implementation provides a simple name + service facility under the #ifdef NO_NAME_SERVICE. This depends on + two things: valid IP addresses registered and available to standard + DNS system calls such as gethostbyname(); and a name/GID mapping + file. + + IP addresses may be set up by system administrators or by a local + power-user simply by editing the values into the /etc/hosts file. + Setting IP addresses up in this manner is beyond the scope of this + document. + + A simple mapping of names to GIDs is maintained in the ibhosts + file, currently located at /etc/dapl/ibhosts. The format of + the file is: + + 0x 0x + + For example: + + dat-linux3-ib0p0 0xfe80000000000000 0x0001730000003d11 + dat-linux3-ib0p1 0xfe80000000000000 0x0001730000003d11 + dat-linux3-ib1 0xfe80000000000000 0x0001730000003d52 + dat-linux5-ib0 0xfe80000000000000 0x0001730000003d91 + + And for each hostname, there must be an entry in the /etc/hosts file + similar to: + + dat_linux3-ib0a 198.165.10.11 + dat_linux3-ib0b 198.165.10.12 + dat_linux3-ib1a 198.165.10.21 + dat_linux5-ib0a 198.165.10.31 + + + In this example we have adopted the convention of naming each + InfiniBand interface by using the form + + -ib[port_number] + + In the above example we can see that the machine dat_linux3 has three + InfiniBand interfaces, which in this case we have named two ports on + the first HCA and another port on a second. Utilizing standard DNS + naming, the conventions used for identifying individual ports is + completely up to the administrator. + + The GID Prefix and GUID are obtained from the HCA and map a port on + the HCA: together they form the GID that is required by a CM to + connect with the remote node. + + The simple name service builds an internal table after processing + the ibhosts file which contains IP addresses and GIDs. It will use + the standard getaddrinfo() function to obtain IP address + information. + + When an application invoked dat_ep_connect(), the + DAT_IA_ADDRESS_PTR will be compared in the table for a match and + the destination GID established if found. If the address is not + found then the user must first add the name to the ibhosts file. + + With a valid GID for the destination node, the underlying CM is + invoked to make a connection. + +* Connection Management + + Getting a working CM has taken some time, in fact the DAPL project + was nearly complete by the time a CM was available. In order to + make progress, a connection hack was introduced that allows + specific connections to take place. This is noted in the code by + the CM_BUSTED #def. + + CM_BUSTED takes the place of a CM and will manually transition a QP + through the various states to connect: INIT->RTR->RTS. It will also + disconnect the connection, although the Torrent implementation + simply destroys the QP and recreates a new one rather than + transitioning through the typical disconnect states (which didn't + work on early IB implementations). + + CM_BUSTED makes some assumptions about the remote end of the + connection as no real information is exchanged. The ibapi + implementation assumes both HCAs have the same LID, which implies + there is no SM running. The vapi implementation assumes the LIDs + are 0 and 1. Depending on the hardware, the LID value may in fact + not make any difference. This code does not set the Global Route + Header (GRH), which would cause the InfiniBand chip to be carefully + checking LID information. + + The QP number is assumed to be identical on both ends of the + connection, or differing by 1 if this is a loopback. There is an + environment variable that will be read at initialization time if + you are configured with a loopback, this value is checked when + setting up a QP. The obvious downside to this scheme is that + applications must stay synchronized in their QP usage or the + initial exchange will fail as they are not truly connected. + + Add to this the limitation that HCAs must be connected in + Point-to-Point topology or in a loopback. Without a GRH it will not + work in a fabric. Again, using an SM will not work when CM_BUSTED + is enabled. + + Despite these shortcomings, CM_BUSTED has proven very useful and + will remain in the code for a while in order to aid development + groups with new hardware and software. It is a hack to be sure, but + it is relatively well isolated. + + +-- Notes on Disconnecting + +An EP can only be disconnected if it is connected or unconnected; you +cannot disconnect 'in progress' connections. An 'in progress +connection may in fact time out, but the DAT Spec does not allow you +to 'kill' it. DAPL will use the CM interface to disconnect from the +remote node; this of course results in an asynchronous callback +notifying the application the disconnect is complete. + +Disconnecting an unconnected EP is currently the only way to remove +pending RECV operations from the EP. The DAPL spec notes that all +DTOs must be removed from an EP before it can be deallocated, yet +there is no explicit interface to remove pending RECV DTOs. The user +will disconnect an unconnected EP to force the pending operations off +of the queue, resulting in DTO callbacks indicating an error. The +underlying InfiniBand implementation will cause the correct behavior +to result. When doing this operation the DAT_CLOSE flag is ignored, +DAPL will instruct the provider layer to abruptly disconnect the QP. + +As has been noted previously, specifying DAT_CLOSE_ABRUPT_FLAG as the +disconnect completion flag will cause the CM implementation to +transition the QP to the ERROR state to abort all operations, and then +transition to the RESET state; if the flag is DAT_CLOSE_GRACEFUL_FLAG, +the CM will first move to the SQE state and allow all pending I/O's to +drain before moving to the RESET state. In either case, DAPL only +needs to know that the QP is now in the RESET state, as it will need +to be transitioned to the INIT state before it can be used again. + +====================================================================== +Data Transfer Operations (DTOs) +====================================================================== + +The DTO code is a straightforward translation of the DAT_LMR_TRIPLET +to an InfiniBand work request. Unfortunately, IB does not specify what +a work request looks like so this tends to be very vendor specific +code. Each provider will supply a routine for this operation. + +InfiniBand allows the DTO to attach a unique 64 bit work_req_id to +each work request. The DAPL implementation will install a pointer to a +DAPL_DTO_COOKIE in this field. Observe that a DAPL_DTO_COOKIE is not +the same as the user DAT_DTO_COOKIE; indeed, the former has a pointer +field pointing to the latter. Different values will be placed in the +cookie, according to the type of operation it is and the type of data +required by its completion event. This is a simple scheme to bind DAPL +data to the DTO and associated completion callback. Each DTO has a +unique cookie associated with it. + +Observe that an InfiniBand work_request remains under control of the +user, and when a post operation occurs the underlying implementation +will copy data out of the work_request into a hardware based +structure. Further, no application can perform a DTO operation on the +same EP at the same time according to the thread guarantees mandated +by the specification. This allows us to provide a recv_iov and a +send_iov in the EP structure for all DTO operations, eliminating any +malloc operations from this critical path. + +The underlying provider implementation will invoke +dapl_evd_dto_callback() upon completion of DTO operations. +dapl_evd_dto_callback() is the asynchronous completion for a DTO and +will create and post an event for the user. Much of this callback is +concerned with managing error completions. + + +====================================================================== +Data Structure +====================================================================== + +The main data structure for an EndPoint is the dapl_ep structure, +defined in include/dapl.h. The reference implementation uses the +InfiniBand QP to maintain hardware state, providing a relatively +simple mapping. + +/* DAPL_EP maps to DAT_EP_HANDLE */ +struct dapl_ep +{ + DAPL_HEADER header; + /* What the DAT Consumer asked for */ + DAT_EP_PARAM param; + + /* The RC Queue Pair (IBM OS API) */ + ib_qp_handle_t qp_handle; + unsigned int qpn; /* qp number */ + ib_qp_state_t qp_state; + + /* communications manager handle (IBM OS API) */ + ib_cm_handle_t cm_handle; + /* store the remote IA address here, reference from the param + * struct which only has a pointer, no storage + */ + DAT_SOCK_ADDR6 remote_ia_address; + + /* For passive connections we maintain a back pointer to the CR */ + void * cr_ptr; + + /* pointer to connection timer, if set */ + struct dapl_timer_entry *cxn_timer; + + /* private data container */ + DAPL_PRIVATE private; + + /* DTO data */ + DAPL_ATOMIC req_count; + DAPL_ATOMIC recv_count; + + DAPL_COOKIE_BUFFER req_buffer; + DAPL_COOKIE_BUFFER recv_buffer; + + ib_data_segment_t *recv_iov; + DAT_COUNT recv_iov_num; + + ib_data_segment_t *send_iov; + DAT_COUNT send_iov_num; +#ifdef DAPL_DBG_IO_TRC + int ibt_dumped; + struct io_buf_track *ibt_base; + DAPL_RING_BUFFER ibt_queue; +#endif /* DAPL_DBG_IO_TRC */ +}; + +The simple explanation of the fields in the dapl_ep structure follows: + +header: The dapl object header, common to all dapl objects. + It contains a lock field, links to appropriate lists, and + handles specifying the IA domain it is a part of. + +param: The bulk of the EP attributes called out in the DAT + specification and are maintained in the DAT_EP_PARAM + structure. All internal references to these fields + use this structure. + +qp_handle: Handle to the underlying InfiniBand provider implementation + for a QP. All EPs are mapped to an InfiniBand QP. + +qpn: Number of the QP as returned by the underlying provider + implementation. Primarily useful for debugging. + +qp_state: Current state of the QP. The values of this field indicate + if a QP is bound to the EP, and the current state of a + QP. + +cm_handle: Handle to the IB provider's CMA (Connection Manager Agent). + Used for CM operations used to connect and disconnect. + +remote_ia_address: + Remote IP address of the connection. Only valid after the user + has asked for it. + +cr_ptr: Attaches the EP to the appropriate CR. Assigned on the passive + side of a connection in cr_accept. It is used when an abrupt + disconnect is invoked by the app, and we need to 'fake' a + callback. It is also used in clean up of an EP and removing + connection elements from the associated PSP. + +cxn_timer: Pointer to a timer entry, used as a token to set and remove + timers. + +private: Local Private data area on the active side of a connection. + +req_count: Count of outstanding request DTO operations, including memory + ops. Atomically incremented/decremented. + +recv_count:Count of outstanding receive DTO operations. Atomically + incremented/decremented. + +req_buffer:Ring buffer of request cookies. + +recv_buffer: + Ring buffer of receive cookies. + +recv_iov: Storage for provider receive work request. + +recv_iov_num: + Maximum number of receive IOVs. Number is obtained from + the provider in a query. + +send_iov: Storage for provider send work request. + +send_iov_num: + Maximum number of send IOVs. Number is obtained from the + provider in a query. + +ibt_dumped:DTO debugging aid. Boolean value to control how often DTO + tracing data is printed. + +ibt_base: DTO debugging aid. Base address of DTO ring buffer containing + information on DTO processing. + +ibt_queue: Ring buffer containing information n DTO processing. + + +** Debug + +The Reference Implementation includes a trace facility that allows a +developer to see all DTO operations, specifically to catch those that +are not completing as expected. The DAPL_DBG_IO_TRC conditional will +enable this code. + +A simple ring buffer is used to account for all outstanding DTO +traffic. The buffer may be dumped when DTOs are not getting +completions, with enough data to aid the developer to determine where +things went wrong. + +It is implemented as a ring buffer as there are often bugs in this +part of a provider's implementation which do not manifest until +intensive data exchange has occurred for many hours. diff --git a/trunk/ulp/dapl2/doc/dapl_environ.txt b/trunk/ulp/dapl2/doc/dapl_environ.txt new file mode 100644 index 00000000..6cca3b27 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_environ.txt @@ -0,0 +1,42 @@ + DAPL Environment Guide v. 0.01 + ------------------------------ + +The following environment variables affect the behavior of the DAPL +provider library: + + +DAPL_DBG_TYPE +------------- + + Value specifies which parts of the registry will print debugging + information, valid values are + + DAPL_DBG_TYPE_ERR = 0x0001 + DAPL_DBG_TYPE_WARN = 0x0002 + DAPL_DBG_TYPE_EVD = 0x0004 + DAPL_DBG_TYPE_CM = 0x0008 + DAPL_DBG_TYPE_EP = 0x0010 + DAPL_DBG_TYPE_UTIL = 0x0020 + DAPL_DBG_TYPE_CALLBACK = 0x0040 + DAPL_DBG_TYPE_DTO_COMP_ERR = 0x0080 + DAPL_DBG_TYPE_API = 0x0100 + DAPL_DBG_TYPE_RTN = 0x0200 + DAPL_DBG_TYPE_EXCEPTION = 0x0400 + + or any combination of these. For example you can use 0xC to get both + EVD and CM output. + + Example setenv DAPL_DBG_TYPE 0xC + + +DAPL_DBG_DEST +------------- + + Value sets the output destination, valid values are + + DAPL_DBG_DEST_STDOUT = 0x1 + DAPL_DBG_DEST_SYSLOG = 0x2 + DAPL_DBG_DEST_ALL = 0x3 + + For example, 0x3 will output to both stdout and the syslog. + diff --git a/trunk/ulp/dapl2/doc/dapl_event_design.txt b/trunk/ulp/dapl2/doc/dapl_event_design.txt new file mode 100644 index 00000000..0b896bb7 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_event_design.txt @@ -0,0 +1,875 @@ + DAPL Event Subsystem Design v. 0.96 + ----------------------------------- + +================= +Table of Contents +================= + +* Table of Contents +* Referenced Documents +* Goals + + Initial Goals + + Later Goals +* Requirements, constraints, and design inputs + + DAT Specification Constraints + + Object and routine functionality, in outline + + Detailed object and routine specification + + Synchronization + + IBM Access API constraints + + Nature of DAPL Event Streams in IBM Access API. + + Nature of access to CQs + + Operating System (Pthread) Constraints + + Performance model + + A note on context switches +* DAPL Event Subsystem Design + + OS Proxy Wait Object + + Definition + + Suggested Usage + + Event Storage + + Synchronization + + EVD Synchronization: Locking vs. Producer/Consumer queues + + EVD Synchronization: Waiter vs. Callback + + CNO Synchronization + + Inter-Object Synchronization + + CQ -> CQEH Assignments + + CQ Callbacks + + Dynamic Resizing of EVDs + + Structure and pseudo-code + + EVD + + CNO +* Future directions + + Performance improvements: Reducing context switches + + Performance improvements: Reducing copying of event data + + Performance improvements: Reducing locking + + Performance improvements: Reducing atomic operations + + Performance improvements: Incrementing concurrency. + +==================== +Referenced Documents +==================== + +uDAPL: User Direct Access Programming Library, Version 1.0. Published +6/21/2002. http://www.datcollaborative.org/uDAPL_062102.pdf. +Referred to in this document as the "DAT Specification". + +InfiniBand Access Application Programming Interface Specification, +Version 1.2, 4/15/2002. In DAPL SourceForge repository at +doc/api/access_api.pdf. Referred to in this document as the "IBM +Access API Specification". + +===== +Goals +===== + +Initial goals +------------- +-- Implement the dat_evd_* calls described in the DAT Specification (except + for dat_evd_resize). + +-- The implementation should be as portable as possible, to facilitate + HCA Vendors efforts to implement vendor-specific versions of DAPL. + +Later goals +----------- +-- Examine various possible performance optimizations. This document + lists potential performance improvements, but the specific + performance improvements implemented should be guided by customer + requirements. + +-- Implement the dat_cno_* calls described in the DAT 1.0 spec + +-- Implement OS Proxy Wait Objects. + +-- Implement dat_evd_resize + +Non-goals +--------- +-- Thread safe implementation + +============================================ +Requirements, constraints, and design inputs +============================================ + +DAT Specification Constraints +----------------------------- + +-- Object and routine functionality, in outline + +The following section summarizes the requirements of the DAT +Specification in a form that is simpler to follow for purposes of +implementation. This section presumes the reader has read the DAT +Specification with regard to events. + +Events are delivered to DAPL through Event Streams. Each Event Stream +targets a specific Event Descriptor (EVD); multiple Event Streams may +target the same EVD. The Event Stream<->EVD association is +effectively static; it may not be changed after the time at which +events start being delivered. The DAT Consumer always retrieves +events from EVDs. EVDs are intended to be 1-to-1 associated with the +"native" event convergence object on the underlying transport. For +InfiniBand, this would imply a 1-to-1 association between EVDs and +CQs. + +EVDs may optionally have an associated Consumer Notification Object +(CNO). Multiple EVDs may target the same CNO, and the EVD<->CNO +association may be dynamically altered. The DAT Consumer may wait for +events on either EVDs or CNOs; if there is no waiter on an EVD and it +is enabled, its associated CNO is triggered on event arrival. An EVD +may have only a single waiter; a CNO may have multiple waiters. +Triggering of a CNO is "sticky"; if there is no waiter on a CNO when +it is triggered, the next CNO waiter will return immediately. + +CNOs may have an associated OS Proxy Wait Object, which is signaled +when the CNO is triggered. + +-- Detailed object and routine specification + +Individual events may be "signaling" or "non-signaling", depending +on the interaction of: + * Receive completion endpoint attributes + * Request completion endpoint attributes + * dat_ep_post_send completion flags + * dat_ep_post_recv completion flags +The nature of this interaction is outside the scope of this document; +see the DAT Specification 1.0 (or, failing that, clarifications in a +later version of the DAT Specification). + +A call to dat_evd_dequeue returns successfully if there are events on +the EVD to dequeue. A call to dat_evd_wait blocks if there are fewer +events present on the EVD than the value of the "threshold" parameter +passed in the call. Such a call to dat_evd_wait will be awoken by the +first signaling event arriving on the EVD that raises the EVD's event +count to >= the threshold value specified by dat_evd_wait(). + +If a signaling event arrives on an EVD that does not have a waiter, +and that EVD is enabled, the CNO associated with the EVD will be +triggered. + +A CNO has some number of associated waiters, and an optional +associated OS Proxy Wait Object. When a CNO is triggered, two things +happen independently: + * The OS Proxy Wait Object associated with the CNO, if any, is + signaled, given the handle of an EVD associated with the CNO + that has an event on it, and disassociated from the CNO. + * If: + * there is one or more waiters associated with the + CNO, one of the waiters is unblocked and given the + handle of an EVD associated with the CNO that has an + event on it. + * there are no waiters associated with the CNO, the + CNO is placed in the triggered state. + +When a thread waits on a CNO, if: + * The CNO is in the untriggered state, the waiter goes to + sleep pending the CNO being triggered. + * The CNO is in the triggered state, the waiter returns + immediately with the handle of an EVD associated with the + CNO that has an event on it, and the CNO is moved to the + untriggered state. + +Note specifically that the signaling of the OS Proxy Wait Object is +independent of the CNO moving into the triggered state or not; it +occurs based on the state transition from Not-Triggered to Triggered. +Signaling the OS Proxy Wait Object only occurs when a CNO is +triggered. In contrast, waiters on a CNO are unblocked whenever the +CNO is in the triggered *state*, and that state is sticky. + +Note also that which EVD is returned to the caller in a CNO wait is +not specified; it may be any EVD associated with the CNO on which an +event arrival might have triggered the CNO. This includes the +possibility that the EVD returned to the caller may not have any +events on it, if the dat_cno_wait() caller raced with a separate +thread doing a dat_evd_dequeue(). + +The DAT Specification is silent as to what behavior is to be expected +from an EVD after an overflow error has occurred on it. Thus this +design will also be silent on that issue. + +The DAT Specification has minimal requirements on inter-Event Stream +ordering of events. Specifically, any connection events must precede +(in consumption order) any DTO Events for the same endpoint. +Similarly, any successful disconnection events must follow any DTO +Events for an endpoint. + +-- Synchronization + +Our initial implementation is not thread safe. This means that we do +not need to protect against the possibility of multiple simultaneous +user calls occurring on the same object (EVD, CNO, EP, etc.); that is +the responsibility of the DAT Consumer. + +However, there are synchronization guards that we do need to protect +against because the DAT Consumer cannot. Specifically, since the user +cannot control the timing of callbacks from the IBM Access API +Implementation, we need to protect against possible collisions between +user calls and such callbacks. We also need to make sure that such +callbacks do not conflict with one another in some fashion, possibly +by assuring that they are single-threaded. + +In addition, for the sake of simplicity in the user interface, I have +defined "not thread safe" as "It is the DAT Consumer's responsibility +to make sure that all calls against an individual object do not +conflict". This does, however, suggest that the DAPL library needs to +protect against calls to different objects that may result in +collisions "under the covers" (e.g. a call on an EVD vs. a call on its +associated CNO). + +So our synchronization requirements for this implementation are: + + Protection against collisions between user calls and IBM + Access API callbacks. + + Avoidance of or protection against collisions between + different IBM Access API callbacks. + + Protection against collisions between user calls targeted at + different DAT objects. + +IBM Access API constraints +-------------------------- + +-- Nature of DAPL Event Streams in IBM Access API + +DAPL Event Streams are delivered through the IBM Access API in two fashions: + + Delivery of a completion to a CQ. + + A callback is made directly to a previously registered DAPL + function with parameters describing the event. +(Software events are not delivered through the IBM Access API). + +The delivery of a completion to a CQ may spark a call to a previously +registered callback depending on the attributes of the CQ and the +reason for the completion. Event Streams that fall into this class +are: + + Send data transport operation + + Receive data transport operation + + RMR bind + +The Event Streams that are delivered directly through a IBM Access API +callback include: + + Connection request arrival + + Connection resolution (establishment or rejection) + + Disconnection + + Asynchronous errors + +Callbacks associated with CQs are further structured by a member of a +particular CQ Event Handling (CQEH) domain (specified at CQ creation +time). All CQ callbacks within a CQEH domain are serviced by the same +thread, and hence will not collide. + +In addition, all connection-related callbacks are serviced by the same +thread, and will not collide. Similarly, all asynchronous error +callbacks are serviced by the same thread, and will not collide. +Collisions between any pair of a CQEH domain, a connection callback, +and an asynchronous error callback are possible. + +-- Nature of access to CQs + +The only probe operation the IBM Access API allows on CQs is +dequeuing. The only notification operation the IBM Access API +supports for CQs is calling a previously registered callback. + +Specifically, the IB Consumer may not query the number of completions +on the CQ; the only way to find out the number of completions on a CQ +is through dequeuing them all. It is not possible to block waiting +on a CQ for the next completion to arrive, with or without a +threshold parameter. + +Operating System Constraints +---------------------------- + +The initial platform for implementation of DAPL is RedHat Linux 7.2 on +Intel hardware. On this platform, inter-thread synchronization is +provided by a POSIX Pthreads implementation. From the viewpoint of +DAPL, the details of the Pthreads interface are platform specific. +However, Pthreads is a very widely used threading library, common on +almost all Unix variants (though not used on the different variations +of Microsoft Windows(tm)). In addition, RedHat Linux 7.2 provides +POSIX thread semaphore operations (e.g. see sem_init(3)), which are +not normally considered part of pthreads. + +Microsoft Windows(tm) provides many synchronization primitives, +including mutual exclusion locks, and semaphores. + +DAPL defines an internal API (not exposed to the consumer), though +which it accesses Operating Systems Dependent services; this is called +the OSD API. It is intended that this layer contain all operating +system dependencies, and that porting DAPL to a new operating system +should only require changes to this layer. + +We have chosen to define the synchronization interfaces established at +this layer in terms of two specific objects: mutexes and sempahores w/ +timeout on waiting. Mutexes provide mutual exclusion in a way that is +common to all known operating systems. The functionality of +semaphores also exists on most known operating systems, though the +sempahores provided by POSIX do not provide timeout capabilities. +This is for three reasons. First, in contrast to Condition Variables +(the native pthreads waiting/signalling object), operations on +sempahores do not require use of other synchronization variables +(i.e. mutexes). Second, it is fairly easy to emulate sempahores using +condition variables, and it is not simple to emulate condition +variables using semaphores. And third, there are some anticipated +platforms for DAPL that implement condition variables in relation to +some types of locks but not others, and hence constrain appropriate +implementation choices for a potential DAPL interface modeled after +condition variables. + +Implementation of the DAPL OS Wait Objects will initially be based on +condition variables (requiring the use of an internal lock) since +POSIX semaphores do not provide a needed timeout capability. However, +if improved performance is required, a helper thread could be created +that arranges to signal waiting semaphores when timeouts have +expired. This is a potential future (or vendor) optimization. + +Performance Model +----------------- +One constraint on the DAPL Event Subsystem implementation is that it +should perform as well as possible. We define "as well as possible" +by listing the characteristics of this subsystem that will affect its +performance most strongly. In approximate order of importance, these +are: + + The number of context switches on critical path + + The amount of copying on the critical path. + + The base cost of locking (assuming no contention) on the + critical path. This is proportional to the number of locks + taken. + + The amount of locking contention expected. We make a + simplifying assumption and take this as the number of cycles + for which we expect to hold locks on the critical path. + + The number of "atomic" bus operations executed (these take + more cycles than normal operations, as they require locking + the bus). + +We obviously wish to minimize all of these costs. + +-- A note on context switches + +In general, it's difficult to minimize context switches in a user +space library directly communicating with a hardware board. This is +because context switches, by their nature, have to go through the +operating system, but the information about which thread to wake up +and whether to wake it up is generally in user space. In addition, +the IBM Access API delivers all Event Streams as callbacks in user +context (as opposed to, for example, allowing a thread to block within +the API waiting for a wakeup). For this reason, the default sequence +of events for a wakeup generated from the hardware is: + * Hardware interrupts the main processor. + * Interrupt thread schedules a user-level IBM Access API + provider service thread parked in the kernel. + * Provider service thread wakes up the sleeping user-level + event DAT implementation thread. +This implies that any wakeup will involve three context switches. +This could be reduced by one if there were a way for user threads to +block in the kernel, we might skip the user-level provider thread. + +=========================== +DAPL Event Subsystem Design +=========================== + + +OS Proxy Wait Object +-------------------- + +The interface and nature of the OS Proxy Wait Object is specified in +the uDAPL v. 1.0 header files as a DAT_OS_WAIT_PROXY_AGENT via the +following defines: + +typedef void (*DAT_AGENT_FUNC) + ( + DAT_PVOID, /* instance data */ + DAT_EVD_HANDLE /* Event Dispatcher*/ + ); + +typedef struct dat_os_wait_proxy_agent + { + DAT_PVOID instance_data; + DAT_AGENT_FUNC proxy_agent_func; + } DAT_OS_WAIT_PROXY_AGENT; + +In other words, an OS Proxy Wait Object is a (function, data) pair, +and signalling the OS Proxy Wait Object is a matter of calling the +function on the data and an EVD handle associated with the CNO. +The nature of that function and its associated data is completely up +to the uDAPL consumer. + +Event Storage +------------- + +The data associated with an Event (the type, the EVD, and any type +specific data required) must be stored between event production and +event consumption. If storage is not provided by the underlying +Verbs, that data must be stored in the EVD itself. This may require +an extra copy (one at event production and one at event consumption). + +Event Streams associated purely with callbacks (i.e. IB events that +are not mediated by CQs) or user calls (i.e. software events) don't +have any storage allocated for them by the underlying verbs and hence +must store their data in the EVD. + +Event Streams that are associated with CQs have the possibility of +leaving the information associated with the CQ between the time the +event is produced and the time it is consumed. However, even in this +case, if the user calls dat_evd_wait with a threshold argument, the +events information must be copied to storage in the CQ. This is +because it is not possible to determine how many completions there are +on a CQ without dequeuing them, and that determination must be made by +the CQ notification callback in order to decide whether to wakeup a +dat_evd_wait() waiter. Note that this determination must be made +dynamically based on the arguments to dat_evd_wait(). + +Further, leaving events from Event Streams associated with the CQs "in +the CQs" until event consumption raises issues about in what order +events should be dequeued if there are multiple event streams entering +an EVD. Should the CQ events be dequeued first, or should the events +stored in the EVD be dequeued first? In general this is a complex +question; the uDAPL spec does not put many restrictions on event +order, but the major one that it does place is to restrict connection +events associated with a QP to be dequeued before DTOs associated with +that QP, and disconnection events after. Unfortunately, if we adopt +the policy of always dequeueing CQ events first, followed by EVD +events, this means that in situations where CQ events have been copied +to the EVD, CQ events may be received on the EVD out of order. + +However, leaving events from Event Streams associated with CQs allows +us to avoid enabling CQ callbacks in cases where there is no waiter +associated with the EVDs. This can be a potentially large savings of +gratuitous context switches. + +For the initial implementation, we will leave all event information +associated with CQs until dequeued by the consumer. All other event +information will be put in storage on the EVD itself. We will always +dequeue from the EVD first and the CQ second, to handle ordering among +CQ events in cases in which CQ events have been copied to the EVD. + + +Synchronization +--------------- + +-- EVD synchronization: Locking vs. Producer/Consumer queues. + +In the current code, two circular producer/consumer queues are used +for non-CQ event storage (one holds free events, one holds posted +events). Event producers "consume" events from the free queue, and +produce events onto the posted event queue. Event consumers consume +events from the posted event queue, and "produce" events onto the free +queue. In what follows, we discuss synchronization onto the posted +event queue, but since the usage of the queues is symmetric, all of +what we say also applies to the free event queue (just in the reverse +polarity). + +The reason for using these circular queues is to allow synchronization +between producer and consumer without locking in some situations. +Unfortunately, a circular queue is only an effective method of +synchronization if we can guarantee that there are only two accessors +to it at a given time: one producer, and one consumer. The model will +not work if there are multiple producers, or if there are multiple +consumers (though obviously a subsidiary lock could be used to +single-thread either the producers or the consumers). + +There are several difficulties with guaranteeing the producers and +consumers will each be single threaded in accessing the EVD: + * Constraints of the IB specification and IBM Access API + (differing sources for event streams without guarantees of + IB provider synchronization between them) make it difficult + to avoid multiple producers. + * The primitives used for the producer/consumer queue are not + as widely accepted as locks, and may render the design less + portable. + +We will take locks when needed when producing events. The details of +this plan are described below. + +This reasoning is described in more detail below to inform judgments +about future performance improvements. + +* EVD producer synchronization + +The producers fall into two classes: + * Callbacks announcing IA associated events such as connection + requests, connections, disconnections, DT ops, RMR bind, + etc. + * User calls posting a software event onto the EVD. + +It is the users responsibility to protect against simultaneous +postings of software events onto the same EVD. Similarly, the CQEH +mechanism provided by the IBM Access API allows us to avoid collisions +between IBM Access API callbacks associated with CQs. However, we +must protect against software events colliding with IBM Access API +callbacks, and against non-CQ associated IB verb callbacks (connection +events and asynchronous errors) colliding with CQ associated IBM +Access API callbacks, or with other non-CQ associated IBM Access API +callbacks (i.e. a connection callback colliding with an asynchronous +error callback). + +Note that CQ related callbacks do not act as producers on the circular +list; instead they leave the event information on the CQ until +dequeue; see "Event Storage" above. However, there are certain +situations in which it is necessary for the consumer to determine the +number of events on the EVD. The only way that IB provides to do this +is to dequeue the CQEs from the CQ and count them. In these +situations, the consumer will also act as an event producer for the +EVD event storage, copying all event information from the CQ to the +EVD. + +Based on the above, the only case in which we may do without locking +on the producer side is when all Event Streams of all of the following +types may be presumed to be single threaded: + * Software events + * Non-CQ associated callbacks + * Consumer's within dat_evd_wait + +We use a lock on the producer side of the EVD whenever we have +multiple threads of producers. + +* EVD Consumer synchronization + +It is the consumer's responsibility to avoid multiple callers into +dat_evd_wait and dat_evd_dequeue. For this reason, there is no +requirement for a lock on the consumer side. + +* CQ synchronization + +We simplify synchronization on the CQ by identifying the CQ consumer +with the EVD consumer. In other words, we prohibit any thread other +than a user thread in dat_evd_wait() or dat_evd_dequeue() from +dequeueing events from the CQ. This means that we can rely on the +uDAPL spec guarantee that only a single thread will be in the +dat_evd_wait() or dat_evd_dequeue() on a single CQ at a time. It has +the negative cost that (because there is no way to probe for the +number of entries on a CQ without dequeueing) the thread blocked in +dat_evd_wait() with a threshold argument greater than 1 will be woken +up on each notification on that CQ, in order to dequeue entries from +the CQ and determine if the threshold value has been reached. + +-- EVD Synchronization: Waiter vs. Callback + +Our decision to restrict dequeueing from the IB CQ to the user thread +(rather than the notification callback thread) means that +re-requesting notifications must also be done from that thread. This +leads to a subtle requirement for synchronization: the request for +notification (ib_completion_notify) must be atomic with the wait on +the condition variable by the user thread (atomic in the sense that +locks must be held to force the signalling from any such notification +to occur after the sleep on the condition variable). Otherwise it is +possible for the notification requested by the ib_completion_notify +call to occur before the return from that call. The signal done by +that notify will be ignored, and no further notifications will be +enabled, resulting in the thread sleep waiting forever. The CQE +associated with the notification might be noticed upon return from the +notify request, but that CQE might also have been reaped by a previous +call. + +-- CNO Synchronization + +In order to protect data items that are changed during CNO signalling +(OS Proxy Wait Object, EVD associated with triggering, CNO state), it +is necessary to use locking when triggering and waiting on a CNO. + +Note that the synchronization between trigerrer and waiter on CNO must +take into account the possibility of the waiter returning from the +wait because of a timeout. I.e. it must handle the possibility that, +even though the waiter was detected and the OS Wait Object signalled +under an atomic lock, there would be no waiter on the OS Wait Object +when it was signalled. To handle this case, we make the job of the +triggerer to be setting the state to triggered and signalling the OS +Wait Object; all other manipulation is done by the waiter. + +-- Inter-Object Synchronization + +By the requirements specified above, the DAPL implementation is +responsible for avoiding collisions between DAT Consumer calls on +different DAT objects, even in a non-thread safe implementation. +Luckily, no such collisions exist in this implementation; all exported +DAPL Event Subsystem calls involve operations only on the objects to +which they are targeted. No inter-object synchronization is +required. + +The one exception to this is the posting of a software event on an EVD +associated with a CNO; this may result in triggering the CNO. +However, this case was dealt with above in the discussion of +synchronizing between event producers and consumers; the posting of a +software event is a DAPL API call, but it's also a event producer. + +To avoid lock hierarchy issues between EVDs and CNOs and minimize lock +contention, we arrange not to hold the EVD lock when triggering the +CNO. That is the only context in which we would naturally attempt to +hold both locks. + +-- CQ -> CQEH Assignments + +For the initial implementation, we will assign all CQs to the same +CQEH. This is for simplicity and efficient use of threading +resources; we do not want to dedicate a thread per CQ (where the +number of CQs may grow arbitrarily high), and we have no way of +knowing which partitioning of CQs is best for the DAPL consumer. + +CQ Callbacks +------------ + +The responsibility of a CQ callback is to wakeup any waiters +associated with the CQ--no data needs to be dequeued/delivered, since +that is always done by the consumer. Therefore, CQ callbacks must be +enabled when: + * Any thread is in dat_evd_wait() on the EVD associated with + the CQ. + * The EVD is enabled and has a non-null CNO. (An alternative + design would be to have waiters on a CNO enable callbacks on + all CQs associated with all EVDs associated with the CNO, + but this choice does not scale well as the number of EVDs + associated with a CNO increases). + +Dynamic Resizing of EVDs +------------------------ + +dat_evd_resize() creates a special problem for the implementor, as it +requires that the storage allocated in the EVD be changed in size as +events may be arriving. If a lock is held by all operations that use +the EVD, implementation of dat_evd_resize() is trivial; it substitutes +a new storage mechanism for the old one, copying over all current +events, all under lock. + +However, we wish to avoid universal locking for the initial +implementation. This puts the implementation of dat_evd_resize() into +a tar pit. Because of the DAT Consumer requirements for a non-thread +safe DAPL Implementation, there will be no danger of conflict with +Event Consumers. However, if an Event Producer is in process of +adding an event to the circular list when the resize occurs, that +event may be lost or overwrite freed memory. + +If we are willing to make the simplifying decision that any EVD that +has non-CQ events on it will always do full producer side locking, we +can solve this problem relatively easily. Resizing of the underlying +CQ can be done via ib_cq_resize(), which we can assume available +because of the IB spec. Resizing of the EVD storage may be done under +lock, and there will be no collisions with other uses of the EVD as +all other uses of the EVD must either take the lock or are prohibitted +by the uDAPL spec. + +dat_evd_resize() has not yet been implemented in the DAPL Event +subsystem. + +Structure and pseudo-code +------------------------- + +-- EVD + +All EVDs will have associated with them: + + a lock + + A DAPL OS Wait Object + + An enabled/disabled bit + + A CNO pointer (may be null) + + A state (no_waiter, waiter, dead) + + A threshold count + + An event list + + A CQ (optional, but common) + +Posting an event to the EVD (presumably from a callback) will involve: +^ + Checking for valid state +|lock A + Putting the event on the event list +| ^lock B + Signal the DAPL OS Wait Object, if appropriate +v v (waiter & signaling event & over threshold) + + Trigger the CNO if appropriate (enabled & signaling + event & no waiter). Note that the EVD lock is not + held for this operation to avoid holding multiple locks. + +("lock A" is used if producer side locking is needed. "lock B" is +used if producer side locking is not needed. Regardless, the lock is +only held to confirm that the EVD is in the WAITED state, not for +the wakeup). + +Waiting on an EVD will include: + + Loop: + + Copy all elements from CQ to EVD + + If we have enough, break + + If we haven't enabled the CQ callback + + Enable it + + Continue + + Sleep on DAPL OS Wait Object + + Dequeue and return an event + +The CQ callback will include: + + If there's a waiter: + + Signal it + + Otherwise, if the evd is in the OPEN state, there's + a CNO, and the EVD is enabled: + + Reenable completion + + Trigger CNO + +Setting the enable/disable state of the EVD or setting the associated +CNO will simply set the bits and enable the completion if needed (if a +CNO trigger is implied); no locking is required. + +-- CNO + +All CNOs will have associated with them: + + A lock + + A DAPL OS Wait Object + + A state (triggered, untriggered, dead) + + A waiter count + + An EVD handle (last event which triggered the CNO) + + An OS Proxy Wait Object pointer (may be null) + +Triggering a CNO will involve: + ^ + If the CNO state is untriggerred: + | + Set it to triggered + | + Note the OS Proxy wait object and zero it. + | + If there are any waiters associated with the CNO, + | signal them. + v + Signal the OS proxy wait object if noted + +Waiting on a CNO will involve: + ^ + While the state is not triggered and the timeout has not occurred: + | + Increment the CNO waiter count + lock + Wait on the DAPL OS Wait Object + | + Decrement the CNO waiter count + v + If the state is trigerred, note fact&EVD and set to untrigerred. + + Return EVD and success if state was trigerred + + Return timeout otherwise + +Setting the OS Proxy Wait Object on a CNO, under lock, checks for a +valid state and sets the OS Proxy Wait Object. + + +============== +Known Problems +============== + +-- Because many event streams are actually delivered to EVDs by + callbacks, we cannot in general make any guarantees about the order + in which those event streams arrive; we are at the mercy of the + thread scheduler. Thus we cannot hold to the guarantee given by + the uDAPL 1.0 specification that within a particular EVD, + connection events on a QP will always be before successful DTO + operations on that QP. + + Because we have chosen to dequeue EVD events first and CQ events + second, we will also not be able to guarantee that all successful + DTO events will be received before a disconnect event. Ability to + probe the CQ for its number of entries would solve this problem. + + +================= +Future Directions +================= + +This section includes both functionality enhancements, and a series of +performance improvements. I mark these performance optimizations with +the following flags: + * VerbMod: Requires modifications to the IB Verbs/the IBM + Access API to be effective. + * VerbInteg: Requires integration between the DAPL + implementation and the IB Verbs implementation and IB device + driver. + +Functionality Enhancements +-------------------------- + +-- dat_evd_resize() may be implemented by forcing producer side + locking whenever an event producer may occur asynchronously with + calls to dat_evd_resize() (i.e. when there are non-CQ event streams + associated with the EVD). See the details under "Dynamic Resizing + of EVDs" above. + +-- [VerbMod] If we ahd a verbs modification allowing us to probe for + the current number of entries on a CQ, we could: + * Avoid waking up a dat_evd_wait(threshold>1) thread until + there were enough events for it. + * Avoid copying events from the CQ to the EVD to satisfy the + requirements of the "*nmore" out argument to dat_evd_wait(), + as well as the non-unary threshold argument. + * Implement the "all successful DTO operation events before + disconnect event" uDAPL guarantee (because we would no + longer have to copy CQ events to an EVD, and hence dequeue + first from the EVD and then from the CQ. + This optimization also is relevant for two of the performance + improvements cases below (Reducing context switches, and reducing + copies). + + +Performance improvements: Reducing context switches +--------------------------------------------------- +-- [VerbMod] If we had a verbs modification allowing us to probe for + the current size of a CQ, we could avoid waking up a + dat_evd_wait(threshhold>1) thread until there were enough events + for it. See the Functionality Enhancement entry covering this + possibility. + +-- [VerbMod] If we had a verbs modification allowing threads to wait + for completions to occur on CQs (presumably in the kernel in some + efficient manner), we could optimize the case of + dat_evd_wait(...,threshold=1,...) on EVDs with only a single CQ + associated Event Stream. In this case, we could avoid the extra + context switch into the user callback thread; instead, the user + thread waiting on the EVD would be woken up by the kernel directly. + +-- [VerbMod] If we had the above verbs modification with a threshold + argument on CQs, we could implement the threshold=n case. + +-- [VerbInteg] In general, It would be useful to provide ways for + threads blocked on EVDs or CNOs to sleep in the hardware driver, + and for the driver interrupt thread to determine if they should be + awoken rather than handing that determination off to another, + user-level thread. This would allow us to reduce by one the number + of context switches required for waking up the various blocked + threads. + +-- If an EVD has only a single Event Stream coming into it that is + only associated with one work queue (send or receive), it may be + possible to create thresholding by marking only ever nth WQE on + the associated send or receive WQ to signal a completion. The + difficulty with this is that the threshold is specified when + waiting on an EVD, and requesting completion signaling is + specified when posting a WQE; those two events may not in general + be synchronized enough for this strategy. It is probably + worthwhile letting the consumer implement this strategy directly if + they so choose, by specifying the correct flags on EP and DTO so + that the CQ events are only signaling on every nth completion. + They could then use dat_evd_wait() with a threshold of 1. + +Performance improvements: Reducing copying of event data +-------------------------------------------------------- +-- [VerbMod] If we had the ability to query a CQ for the number of + completions on it, we could avoid the cost of copying event data from the + CQ to the EVD. This is a duplicate of the second entry under + "Functionality Enhancements" above. + +Performance improvements: Reducing locking +------------------------------------------ +-- dat_evd_dequeue() may be modified to not take any locks. + +-- If there is no waiter associated with an EVD and there is only a + single event producer, we may avoid taking any locks in producing + events onto that EVD. This must be done carefully to handle the + case of racing with a waiter waiting on the EVD as we deliver the + event. + +-- If there is no waiter associated with an EVD, and we create a + producer/consumer queue per event stream with a central counter + modified with atomic operations, we may avoid locking on the EVD. + +-- It may be possible, though judicious use of atomic operations, to + avoid locking when triggering a CNO unless there is a waiter on the + CNO. This has not been done to keep the initial design simple. + +Performance improvements: Reducing atomic operations +---------------------------------------------------- +-- We could combine the EVD circular lists, to avoid a single atomic + operation on each production and each consumption of an event. In + this model, event structures would not move from list to list; + whether or not they had valid information on them would simply + depend on where they were on the lists. + +-- We may avoid the atomic increments on the circular queues (which + have a noticeable performance cost on the bus) if all accesses to an + EVD take locks. + + +Performance improvements: Increasing concurrency +------------------------------------------------ +-- When running on a multi-CPU platform, it may be appropriate to + assign CQs to several separate CQEHs, to increase the concurrency + of execution of CQ callbacks. However, note that consumer code is + never run within a CQ callback, so those callbacks should take very + little time per callback. This plan would only make sense in + situations where there were very many CQs, all of which were + active, and for whatever reason (high threshold, polling, etc) + user threads were usually not woken up by the execution of a + provider CQ callback. + + diff --git a/trunk/ulp/dapl2/doc/dapl_ibm_api_variations.txt b/trunk/ulp/dapl2/doc/dapl_ibm_api_variations.txt new file mode 100644 index 00000000..c31462a3 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_ibm_api_variations.txt @@ -0,0 +1,34 @@ + + DAPL Variations from IBM OS Access API + -------------------------------------- + +The DAPL reference implementation is targetted at the IBM OS Access +API (see doc/api/IBM_access_api.pdf). However, in the course of +developing the reference implementation it has become necessary to +alter or enhance this API specification in minor ways. This document +describes the ways in which the Access API has been altered to +accomodate the needs of the reference implementation. + +Note that this document is a work in progress/a place holder; it does +not yet contain all of the API variations used by the reference +implementation. It is intended that it will be brought up to date +before the final release of the DAPL reference implementation. + +The variations from the IBM OS Access API are listed below. + +-- Thread safety + +The IBM OS Access API specifies: + +"Implementation of the Access APIs should ensure that multiple threads + can call the APIs, provided they do not access the same InfiniBand + entity (such as a queue pair or a completion queue)." + +This has been extended in two ways: + * It is safe for multiple threads to call into the API + accessing the same HCA. + * Threads calling ib_post_send_req on a particular QP do not + conflict with threads calling ib_post_rcv_req on the same + QP. I.e. while there cannot be multiple threads in + ib_post_send_req or ib_post_rcv_req on the same QP, there + may be one thread in each routine simultaneously. diff --git a/trunk/ulp/dapl2/doc/dapl_memory_management_design.txt b/trunk/ulp/dapl2/doc/dapl_memory_management_design.txt new file mode 100644 index 00000000..70d41dba --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_memory_management_design.txt @@ -0,0 +1,173 @@ +####################################################################### +# # +# DAPL Memory Management Design # +# # +# James Lentini # +# jlentini at users.sourceforge.net # +# # +# Created 05/06/2002 # +# Updated 08/22/2002 # +# # +####################################################################### + + +Contents +------- +0. Introduction +1. Protection Zones (PZs) +2. Local Memory Regions (LMRs) +3. Remote Memory Regions (RMRs) + + +0. Introduction +--------------- + + The memory management subsystem allows consumers to register and +unregister memory regions. The DAT API distinguishes between local +and remote memory areas. The former server as local buffers for DTO +operations while the later are used for RDMA operations. + +Each DAT function is implemented in a file named dapl_.c. +For example, dat_pz_create is implemented in dapl_pz_create.c. There +are also dapl__util.{h,c} files for each object. For +example, there are dapl_pz_util.h and dapl_pz_util.c files. The +use of util files follows the convention used elsewhere in the DAPL +reference provider. These files contain common object creation and +destruction code. + + +1. Protection Zones (PZs) +------------------------- + + DAPL protection zones provide consumers with a means to associate +various DAPL objects with one another. The association can then be +validated before allowing these objects to be manipulated. The DAT +functions related to PZs are: + +dat_pz_create +dat_pz_free +dat_pz_query + +These are implemented in the DAPL reference provider by + +dapl_pz_create +dapl_pz_free +dapl_pz_query + +The reference implementation maps the DAPL PZ concept onto Infiniband +protections domains (PDs). + +The DAT_PZ_HANDLE value returned to DAT consumers is a pointer to a +DAPL_PZ data structure. The DAPL_PZ structure is used to represent all +PZ objects. Code that manipulates this structure should atomically +increment and decrement the ref_count member to track the number of +objects referencing the PZ. + + +2. Local Memory Regions (LMRs) +------------------------------ + + DAPL local memory regions represent a memory area on the host +system that the consumer wishes to access via local DTO operations. +The DAT functions related to LMRs are: + +dat_lmr_create +dat_lmr_free +dat_lmr_query + +These are implemented in + +dapl_lmr_create +dapl_lmr_free +dapl_lmr_query + +In the reference implementation, DAPL LMRs are mapped onto +Infiniband memory regions (MRs). + +LMR creation produces two values: a DAT_LMR_CONTEXT and a +DAT_LRM_HANDLE. + +The DAT_LMR_CONTEXT value is used to uniquely identify the LMR +when posting data transfer operations. These values map directly +to Infiniband L_KEYs. + +Since some DAT functions need to translate a DAT_LMR_CONTEXT value +into a DAT_LMR_HANDLE (ex. dat_rmr_bind), a dictionary data structure +is used to associate DAT_LMR_CONTEXT values with their corresponding +DAT_LMR_HANDLE. Each time a new LMR is created, the DAT_LMR_HANDLE +should be inserted into the dictionary with the associated +DAT_LMR_CONTEXT as the key. + +A hash table was chosen to implement this data structure. Since the +L_KEY values are being used by the CA hardware for indexing purposes, +there distribution is expected to be uniform and hence ideal for hashing. + +The DAT_LMR_HANDLE value returned to DAT consumers is a pointer to +a DAPL_LMR data structure. The DAPL_LMR structure is used to represent +all LMR objects. The ref_count member should be used to track objects +associated with a given LMR. + +The DAT API exposes the DAT_LMR_CONTEXT to consumers to allow +for sharing of memory registrations between multiple address spaces. +The mechanism by which such a feature would be implemented does not +yet exist. Consumers may be able to take advantage of this +feature on future transports. + + +3. Remote Memory Regions (RMRs) +------------------------------- + + DAPL remote memory regions represent a memory area on the host +system to which the consumer wishes to allow RMDA operations. The +related DAT functions are + +dat_rmr_create +dat_rmr_free +dat_rmr_query +dat_rmr_bind + +which are implemented in + +dapl_rmr_create +dapl_rmr_free +dapl_rmr_query +dapl_rmr_bind + +The reference provider maps RMR objects onto Infiniband memory +windows. + +The DAT_RMR_HANDLE value returned to DAT consumers is a pointer to +a DAPL_RMR data structure. The DAPL_RMR structure is used to represent +all RMR objects. + +The API for binding a LMR to a RMR has the following function +signature: + +DAT_RETURN +dapl_rmr_bind ( + IN DAT_RMR_HANDLE rmr_handle, + IN const DAT_LMR_TRIPLET *lmr_triplet, + IN DAT_MEM_PRIV_FLAGS mem_priv, + IN DAT_EP_HANDLE ep_handle, + IN DAT_RMR_COOKIE user_cookie, + IN DAT_COMPLETION_FLAGS completion_flags, + OUT DAT_RMR_CONTEXT *rmr_context ) + +where a DAT_LMR_TRIPLET is defined as: + +typedef struct dat_lmr_triplet + { + DAT_LMR_CONTEXT lmr_context; + DAT_UINT32 pad; + DAT_VADDR virtual_address; + DAT_VLEN segment_length; + } DAT_LMR_TRIPLET; + +In the case of IB, the DAT_LMR_CONTEXT value is a L_KEY. +As described in the IB spec, the Bind Memory Window verb +takes both a L_KEY and Memory Region Handle among other +parameters. Therefore a data structure must be used to +map a DAT_LMR_CONTEXT (L_KEY) value to a DAPL_LMR so +that the needed memory region handle can be retrieved. +The LMR hash table described above is used for this +purpose. diff --git a/trunk/ulp/dapl2/doc/dapl_patch.txt b/trunk/ulp/dapl2/doc/dapl_patch.txt new file mode 100644 index 00000000..640cfe19 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_patch.txt @@ -0,0 +1,83 @@ +####################################################################### +# # +# DAPL Patch Guide # +# # +# James Lentini # +# jlentini at users.sourceforge.net # +# # +# Created 03/30/2005 # +# Version 1.0 # +# # +####################################################################### + + +Overview +-------- + +The DAPL Reference Implementation (RI) Team welcomes code contributions +and bug fixes from RI users. This document describes the format for +submitting patches to the project. + +Directions +---------- + +When implementing a new feature or bug fix, please remember to: + ++ Use the project coding style, described in doc/dapl_coding_style.txt ++ Remember that the RI supports multiple platforms and transports. If + your modification is not applicable to all platforms and transports, + please ensure that the implement does not affect these other + configurations. + +When creating the patch: + ++ Create the patch using a unified diff as follows: + diff -Naur old-code new-code > patch ++ Create the patch from the root of the CVS tree. + +When submitting the patch: + ++ Compose an email message containing a brief description of the patch, + a signed-off by line, and the patch. ++ Have the text "[PATCH]" at the start of the subject line ++ Send the message to dapl-devel@lists.sourceforge.net + +Example +------- + +Here is an example patch message: + +------------------------------------------------------------ +Date: 30 Mar 2005 11:49:45 -0500 +From: Jane Doe +To: dapl-devel@lists.sourceforge.net +Subject: [PATCH] fixed status returns + +Here's a patch to fix the status return value in +dats_handle_vector_init(). + +Signed-off-by: Jane Doe + +--- dat/common/dat_api.c~ 2005-03-30 11:58:40.838968000 -0500 ++++ dat/common/dat_api.c 2005-03-28 12:33:29.502076000 -0500 +@@ -70,16 +70,15 @@ + { + DAT_RETURN dat_status; + int i; +- int status; + + dat_status = DAT_SUCCESS; + + g_hv.handle_max = DAT_HANDLE_ENTRY_STEP; + +- status = dat_os_lock_init (&g_hv.handle_lock); +- if ( DAT_SUCCESS != status ) ++ dat_status = dat_os_lock_init (&g_hv.handle_lock); ++ if ( DAT_SUCCESS != dat_status ) + { +- return status; ++ return dat_status; + } + + g_hv.handle_array = dat_os_alloc (sizeof(void *) * DAT_HANDLE_ENTRY_STEP); +------------------------------------------------------------ diff --git a/trunk/ulp/dapl2/doc/dapl_registry_design.txt b/trunk/ulp/dapl2/doc/dapl_registry_design.txt new file mode 100644 index 00000000..c4701a25 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_registry_design.txt @@ -0,0 +1,631 @@ + DAT Registry Subsystem Design v. 0.90 + ------------------------------------- + +================= +Table of Contents +================= + +* Table of Contents +* Referenced Documents +* Introduction +* Goals +* Provider API +* Consumer API +* Registry Design + + Registry Database + + Provider API pseudo-code + + Consumer API pseudo-code + + Platform Specific API pseudo-code + +==================== +Referenced Documents +==================== + +uDAPL: User Direct Access Programming Library, Version 1.0. Published +6/21/2002. http://www.datcollaborative.org/uDAPL_062102.pdf. Referred +to in this document as the "DAT Specification". + +============ +Introduction +============ + +The DAT architecture supports the use of multiple DAT providers within +a single consumer application. Consumers implicitly select a provider +using the Interface Adapter name parameter passed to dat_ia_open(). + +The subsystem that maps Interface Adapter names to provider +implementations is known as the DAT registry. When a consumer calls +dat_ia_open(), the appropriate provider is found and notified of the +consumer's request to access the IA. After this point, all DAT API +calls acting on DAT objects are automatically directed to the +appropriate provider entry points. + +A persistent, administratively configurable database is used to store +mappings from IA names to provider information. This provider +information includes: the file system path to the provider library +object, version information, and thread safety information. The +location and format of the registry is platform dependent. This +database is know as the Static Registry (SR). The process of adding a +provider entry is termed Static Registration. + +Within each DAT consumer, there is a per-process database that +maps from ia_name -> provider information. When dat_ia_open() is +called, the provider library is loaded, the ia_open_func is found, and +the ia_open_func is called. + +===== +Goals +===== + +-- Implement the registration mechanism described in the uDAPL + Specification. + +-- The DAT registry should be thread safe. + +-- On a consumer's performance critical data transfer path, the DAT + registry should not require any significant overhead. + +-- The DAT registry should not limit the number of IAs or providers + supported. + +-- The user level registry should be tolerant of arbitrary library + initialization orders and support calls from library initialization + functions. + +============ +Provider API +============ + +Provider libraries must register themselves with the DAT registry. +Along with the Interface Adapter name they wish to map, they must +provide a routines vector containing provider-specific implementations +of all DAT APIs. If a provider wishes to service multiple Interface +Adapter names with the same DAT APIs, it must register each name +separately with the DAT registry. The Provider API is not exposed to +consumers. + +The user level registry must ensure that the Provider API may be +called from a library's initialization function. Therefore the +registry must not rely on a specific library initialization order. + + DAT_RETURN + dat_registry_add_provider( + IN DAT_PROVIDER *provider ) + +Description: Allows the provider to add a mapping. It will return an +error if the Interface Adapter name already exists. + + DAT_RETURN + dat_registry_remove_provider( + IN DAT_PROVIDER *provider ) + +Description: Allows the Provider to remove a mapping. It will return +an error if the mapping does not already exist. + +============ +Consumer API +============ + +Consumers that wish to use a provider library call the DAT registry to +map Interface Adapter names to provider libraries. The consumer API is +exposed to both consumers and providers. + + DAT_RETURN + dat_ia_open ( + IN const DAT_NAME device_name, + IN DAT_COUNT async_event_qlen, + INOUT DAT_EVD_HANDLE *async_event_handle, + OUT DAT_IA_HANDLE *ia_handle ) + +Description: Upon success, this function returns an DAT_IA_HANDLE to +the consumer. This handle, while opaque to the consumer, provides +direct access to the provider supplied library. To support this +feature, all DAT_HANDLEs must be pointers to a pointer to a +DAT_PROVIDER structure. + + DAT_RETURN + dat_ia_close ( + IN DAT_IA_HANDLE ia_handle ) + +Description: Closes the Interface Adapter. + + DAT_RETURN + dat_registry_list_providers( + IN DAT_COUNT max_to_return, + OUT DAT_COUNT *entries_returned, + OUT DAT_PROVIDER_INFO *(dat_provider_list[]) ) + +Description: Lists the current mappings. + +=============== +Registry Design +=============== + +There are three separate portions of the DAT registry system: + +* Registry Database + +* Provider API + +* Consumer API + +We address each of these areas in order. The final section will +describe any necessary platform specific functions. + +Registry Database +----------------- + +Static Registry +................ + +The Static Registry is a persistent database containing provider +information keyed by Interface Adapter name. The Static Registry will +be examined once when the DAT library is loaded. + +There is no synchronization mechanism protecting access to the Static +Registry. Multiple readers and writers may concurrently access the +Static Registry and as a result there is no guarantee that the +database will be in a consistent format at any given time. DAT +consumers should be aware of this and not run DAT programs when the +registry is being modified (for example, when a new provider is being +installed). However, the DAT library must be robust enough to recognize +an inconsistent Static Registry and ignore invalid entries. + +Information in the Static Registry will be used to initialize the +registry database. The registry will refuse to load libraries for DAT +API versions different than its DAT API version. Switching API +versions will require switching versions of the registry library (the +library explicitly placed on the link line of DAPL programs) as well +as the header files included by the program. + +Set DAT_NO_STATIC_REGISTRY at compile time if you wish to compile +DAT without a static registry. + +UNIX Registry Format +..................... + +The UNIX registry will be a plain text file with the following +properties: + * All characters after # on a line are ignored (comments). + * Lines on which there are no characters other than whitespace + and comments are considered blank lines and are ignored. + * Non-blank lines must have seven whitespace separated fields. + These fields may contain whitespace if the field is quoted + with double quotes. Within fields quoated with double quotes, + the following are valid escape sequences: + + \\ backslash + \" quote + + * Each non-blank line will contain the following fields: + + - The IA Name. + - The API version of the library: + [k|u]major.minor where "major" and "minor" are both integers + in decimal format. Examples: "k1.0", "u1.0", and "u1.1". + - Whether the library is thread-safe: + [threadsafe|nonthreadsafe] + - Whether this is the default section: [default|nondefault] + - The path name for the library image to be loaded. + - The version of the driver: major.minor, for example, "5.13". + +The format of any remaining fields on the line is dependent on the API +version of the library specified on that line. For API versions 1.0 +and 1.1 (both kDAPL and uDAPL), there is only a single additional +field, which is: + + - An optional string with instance data, which will be passed to + the loaded library as its run-time arguments. + +This file format is described by the following grammar: + + -> | + -> + [|] | + [| -> string + -> [k|u]decimal.decimal + -> [threadsafe|nonthreadsafe] + -> [default|nondefault] + -> string + -> decimal.decimal + -> string + -> end of file + -> newline + +The location of this file may be specified by setting the environment +variable DAT_CONF. If DAT_CONF is not set, the default location will +be /etc/dat.conf. + +Windows Registry Format +....................... + +Standardization of the Windows registry format is not complete at this +time. + +Registry Database Data Structures +................................. + +The Registry Database is implemented as a dictionary data structure that +stores (key, value) pairs. + +Initially the dictionary will be implemented as a linked list. This +will allow for an arbitrary number of mappings within the resource +limits of a given system. Although the search algorithm will have O(n) +worst case time when n elements are stored in the data structure, we +do not anticipate this to be an issue. We believe that the number of +IA names and providers will remain relatively small (on the order of +10). If performance is found to be an issue, the dictionary can be +re-implemented using another data structure without changing the +Registry Database API. + +The dictionary uses IA name strings as keys and stores pointers to a +DAT_REGISTRY_ENTRY structure, which contains the following +information: + + - provider library path string, library_path + - DAT_OS_LIBRARY_HANDLE, library_handle + - IA parameter string, ia_params + - DAT_IA_OPEN_FUNC function pointer, ia_open_func + - thread safety indicator, is_thread_safe + - reference counter, ref_count + +The entire registry database data structure is protected by a single +lock. All threads that wish to query/modify the database must posses +this lock. Serializing access in this manner is not expected to have a +detrimental effect on performance as contention is expected to be +minimal. + +An important property of the registry is that entries may be inserted +into the registry, but no entries are ever removed. The contents of +the static registry are used to populate the initially empty registry +database. Since these mapping are by definition persistent, no +mechanism is provided to remove them from the registry database. + +NOTE: There is currently no DAT interface to set a provider's IA +specific parameters. A solution for this problem has been proposed for +uDAPL 1.1. + +Registry Database API +..................... + +The static variable Dat_Registry_Db is used to store information about +the Registry Database and has the following members: + + - lock + - dictionary + +The Registry Database is accessed via the following internal API: + +Algorithm: dat_registry_init + Input: void + Output: DAT_RETURN +{ + initialize Dat_Registry_Db + + dat_os_sr_load() +} + +Algorithm: dat_registry_insert + Input: IN const DAT_STATIC_REGISTRY_ENTRY sr_entry + Output: DAT_RETURN +{ + dat_os_lock(&Dat_Registry_Db.lock) + + create and initialize DAT_REGISTRY_ENTRY structure + + dat_dictionary_add(&Dat_Registry_Db.dictionary, &entry) + + dat_os_unlock(&Dat_Registry_Db.lock) +} + +Algorithm: dat_registry_search + Input: IN const DAT_NAME_PTR ia_name + IN DAT_REGISTRY_ENTRY **entry + Output: DAT_RETURN +{ + dat_os_lock(&Dat_Registry_Db.lock) + + entry gets dat_dictionary_search(&Dat_Registry_Db.dictionary, &ia_name) + + dat_os_unlock(&Dat_Registry_Db.lock) +} + +Algorithm: dat_registry_list + Input: IN DAT_COUNT max_to_return + OUT DAT_COUNT *entries_returned + OUT DAT_PROVIDER_INFO *(dat_provider_list[]) + Output: DAT_RETURN +{ + dat_os_lock(&Dat_Registry_Db.lock) + + size = dat_dictionary_size(Dat_Registry_Db.dictionary) + + for ( i = 0, j = 0; + (i < max_to_return) && (j < size); + i++, j++ ) + { + initialize dat_provider_list[i] w/ j-th element in dictionary + } + + dat_os_unlock(&Dat_Registry_Db.lock) + + *entries_returned = i; +} + +Provider API pseudo-code +------------------------ + ++ dat_registry_add_provider() + +Algorithm: dat_registry_add_provider + Input: IN DAT_PROVIDER *provider + Output: DAT_RETURN +{ + dat_init() + + dat_registry_search(provider->device_name, &entry) + + if IA name is not found then dat_registry_insert(new entry) + + if entry.ia_open_func is not NULL return an error + + entry.ia_open_func = provider->ia_open_func +} + ++ dat_registry_remove_provider() + +Algorithm: dat_registry_remove_provider + Input: IN DAT_PROVIDER *provider + Output: DAT_RETURN +{ + dat_init() + + dat_registry_search(provider->device_name, &entry) + + if IA name is not found return an error + + entry.ia_open_func = NULL +} + +Consumer API pseudo-code +------------------------ + +* dat_ia_open() + +This function looks up the specified IA name in the ia_dictionary, +loads the provider library, retrieves a function pointer to the +provider's IA open function from the provider_dictionary, and calls +the providers IA open function. + +Algorithm: dat_ia_open + Input: IN const DAT_NAME_PTR name + IN DAT_COUNT async_event_qlen + INOUT DAT_EVD_HANDLE *async_event_handle + OUT DAT_IA_HANDLE *ia_handle + Output: DAT_RETURN + +{ + dat_registry_search(name, &entry) + + if the name is not found return an error + + dat_os_library_load(entry.library_path, &entry.library_handle) + + if the library fails to load return an error + + if the entry's ia_open_func is invalid + { + dl_os_library_unload(entry.library_handle) + return an error + } + + (*ia_open_func) (name, + async_event_qlen, + async_event_handle, + ia_handle); +} + +* dat_ia_close() + +Algorithm: dat_ia_close + Input: IN DAT_IA_HANDLE ia_handle + IN DAT_CLOSE_FLAGS ia_flags + Output: DAT_RETURN +{ + provider = DAT_HANDLE_TO_PROVIDER(ia_handle) + + (*provider->ia_close_func) (ia_handle, ia_flags) + + dat_registry_search(provider->device_name, &entry) + + dat_os_library_unload(entry.library_handle) +} + ++ dat_registry_list_providers() + +Algorithm: dat_registry_list_providers + Input: IN DAT_COUNT max_to_return + OUT DAT_COUNT *entries_returned + OUT DAT_PROVIDER_INFO *(dat_provider_list[]) + Output: DAT_RETURN +{ + validate parameters + + dat_registry_list(max_to_return, entries_returned, dat_provider_list) +} + +Platform Specific API pseudo-code +-------------------------------- + +Below are descriptions of platform specific functions required by the +DAT Registry. These descriptions are for Linux. + +Each entry in the static registry is represented by an OS specific +structure, DAT_OS_STATIC_REGISTRY_ENTRY. On Linux, this structure will +have the following members: + + - IA name string + - API version + - thread safety + - default section + - library path string + - driver version + - IA parameter string + +The tokenizer will return a DAT_OS_SR_TOKEN structure +containing: + + - DAT_OS_SR_TOKEN_TYPE value + - string with the fields value + +The tokenizer will ignore all white space and comments. The tokenizer +will also translate any escape sequences found in a string. + +Algorithm: dat_os_sr_load + Input: n/a + Output: DAT_RETURN +{ + if DAT_CONF environment variable is set + static_registry_file = contents of DAT_CONF + else + static_registry_file = /etc/dat.conf + + sr_fd = dat_os_open(static_registry_file) + + forever + { + initialize DAT_OS_SR_ENTRY entry + + do + { + // discard blank lines + dat_os_token_next(sr_fd, &token) + } while token is newline + + if token type is EOF then break // all done + // else the token must be a string + + entry.ia_name = token.value + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not string then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + else if ( dat_os_convert_api(token.value, &entry.api) fails ) + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not string then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + else if ( dat_os_convert_thread_safety(token.value, &entry.thread_safety) fails ) + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not string then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + else if ( dat_os_convert_default(token.value, &entry.default) fails ) + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not string then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + entry.lib_path = token.value + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not string then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + else if ( dat_os_convert_driver_version(token.value, &entry.driver_version) fails ) + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not string then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + entry.ia_params = token.value + + dat_os_token_next(sr_fd, &token) + + if token type is EOF then break // all done + else if token type is not newline then + { + // an error has occurred + dat_os_token_sync(sr_fd) + continue + } + + if ( dat_os_sr_is_valid(entry) ) + { + dat_registry_insert(entry) + } + } + + dat_os_close(sr_fd) +} + +Algorithm: dat_os_library_load + Input: IN const DAT_NAME_PTR *library_path + OUT DAT_LIBRARY_HANDLE *library_handle + Output: DAT_RETURN +{ + *library_handle = dlopen(library_path); +} + +Algorithm: dat_os_library_unload + Input: IN const DAT_LIBRARY_HANDLE library_handle + Output: DAT_RETURN +{ + dlclose(library_handle) +} diff --git a/trunk/ulp/dapl2/doc/dapl_shared_memory_design.txt b/trunk/ulp/dapl2/doc/dapl_shared_memory_design.txt new file mode 100644 index 00000000..606ac26b --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_shared_memory_design.txt @@ -0,0 +1,876 @@ +####################################################################### +# # +# DAPL Shared Memory Design # +# # +# James Lentini # +# jlentini at users.sourceforge.net # +# # +# Created 09/17/2002 # +# Updated 01/21/2005 # +# Version 0.04 # +# # +####################################################################### + + +Contents +-------- +0. Introduction +1. Referenced Documents +2. Requirements +3. Interface +4. Implementation Options + + +Introduction +------------ + +This document describes the design of shared memory registration for +the DAPL reference implementation (RI). + +Implementing shared memory support completely within the DAPL RI +would not be an ideal solution. A more robust and efficient +implementation can be acheived by HCA vendors that integrate a DAT +provider into their software stack. Therefore the RI will not contain +an implementation of this feature. + + +Referenced Documents +-------------------- + +kDAPL: Kernel Direct Access Programming Library, Version 1.2. +uDAPL: User Direct Access Programming Library, Version 1.2. +Available in the DAPL SourceForge repository at +[doc/api/kDAPL_spec.pdf] and [doc/api/uDAPL_spec.pdf]. Collectively +referred to in this document as the "DAT Specification". + +InfiniBand Access Application Programming Interface Specification, +Version 1.2, 4/15/2002. Available in the DAPL SourceForge repository +at [doc/api/IBM_access_api.pdf]. Referred to in this document as the +"IBM Access API Specification". + +Mellanox IB-Verbs API (VAPI) Mellanox Software Programmers Interface +for InfiniBand Verbs. Available in the DAPL SourceForge repository +at [doc/api/MellanoxVerbsAPI.pdf]. Referred to in this document as the +"VAPI API Specification". + +InfiniBand Architecture Specification, Volumes 1 and 2, Release +1.2, Available from http://www.infinibandta.org/ +Referred to in this document as the "Infiniband Specification". + + +Requirements +------------ + +The DAT shared memory model can be characterized as a peer-to-peer +model since the order in which consumers register a region is not +dictated by the programming interface. + +The DAT API function used to register shared memory is: + +DAT_RETURN +dat_lmr_create ( + IN DAT_IA_HANDLE ia_handle, + IN DAT_MEM_TYPE mem_type, + IN DAT_REGION_DESCRIPTION region_description, + IN DAT_VLEN length, + IN DAT_PZ_HANDLE pz_handle, + IN DAT_MEM_PRIV_FLAGS mem_privileges, + OUT DAT_LMR_HANDLE *lmr_handle, + OUT DAT_LMR_CONTEXT *lmr_context, + OUT DAT_RMR_CONTEXT *rmr_context, + OUT DAT_VLEN *registered_length, + OUT DAT_VADDR *registered_address ); + +where a DAT_REGION_DESCRIPTION is defined as: + +typedef union dat_region_description +{ + DAT_PVOID for_va; + DAT_LMR_HANDLE for_lmr_handle; + DAT_SHARED_MEMORY for_shared_memory; +} DAT_REGION_DESCRIPTION; + +In the case of a shared memory registration the DAT consumer will set +the DAT_MEM_TYPE flag to DAT_MEM_TYPE_SHARED_VIRTUAL and place a +cookie in the DAT_REGION_DESCRIPTION union's DAT_SHARED_MEMORY +member. The DAT_SHARED_MEMORY type is defined as follows: + +typedef struct dat_shared_memory +{ + DAT_PVOID virtual_address; + DAT_LMR_COOKIE shared_memory_id; +} DAT_SHARED_MEMORY; + +Unlike the DAT peer-to-peer model, the Infiniband shared memory model +requires a master-slave relationship. A memory region must first be +registered using the Register Memory Region verb with subsequent +registrations made using the Register Shared Memory Region verb. + +The later is implemented in the IBM OS Access API as: + +ib_int32_t +ib_mr_shared_register_us( + ib_hca_handle_t hca_handle, + ib_mr_handle_t *mr_handle, /* IN-OUT: could be changed */ + ib_pd_handle_t pd_handle, /* IN */ + ib_uint32_t access_control, /* IN */ + ib_uint32_t *l_key, /* OUT */ + ib_uint32_t *r_key, /* OUT: if remote access needed */ + ib_uint8_t **va ); /* IN-OUT: virt. addr. to register */ + +The important parameter is the memory region handle which must be the +same as an already registered region. + +Two requirements are implied by this difference between the DAT and +Infiniband models. First, DAPL implementations need a way to determine +the first registration of a shared region. Second implementations must +map DAT_LMR_COOKIE values to memory region handles both within and +across processes. To satisfy the above requirements DAPL must maintain +this information in a system wide database. + +The difficulty of implementing such a database at the DAT provider +level is the reason the RI's shared memory code is meant to be +temporary. Such a database is much better suited as part of the HCA +vendor's software stack, specifically as part of their HCA driver. + +If DAPL was based on a master-slave model like InfiniBand, the +implementation of shared memory would be straight +forward. Specifically the complexity is a result of the consumer being +responsible for specifying the DAT_LMR_COOKIE values. If the DAPL +spec. were changed to allow the provider and not the consumer to +specify the DAT_LMR_COOKIE value, the implementation of this feature +would be greatly simplified. Since the DAPL API already requires +consumers to communicate the DAT_LMR_COOKIE values between processes, +such a change places minimal additional requirements on the +consumer. The dapl_lmr_query call could easily be adapted to allow the +consumer to query the provider for a given LMR's DAT_LMR_COOKIE +value. The only spec changes needed would be to add a DAT_LMR_COOKIE +member to the DAT_LMR_PARAM structure and a DAT_LMR_FIELD_LMR_COOKIE +constant to the DAT_LMR_PARAM_MASK enumeration. A provider could then +store the given LMR's memory region handle in this value, greatly +simplifying the implementation of shared memory in DAPL. + + +Interface +--------- + +To allow the database implementation to easily change, the RI would use +a well defined interface between the memory subsystem and the +database. Conceptually the database would contain a single table with +the following columns: + +[ LMR Cookie ][ MR Handle ][ Reference Count ][ Initialized ] + +where the LMR Cookie column is the primary key. + +The following functions would be used to access the database: + +DAT_RETURN +dapls_mrdb_init ( + void ); + + Called by dapl_init(.) to perform any necessary database + initialization. + +DAT_RETURN +dapls_mrdb_exit ( + void ); + + Called by dapl_fini(.) to perform any necessary database cleanup. + +DAT_RETURN +dapls_mrdb_record_insert ( + IN DAPL_LMR_COOKIE cookie ); + + If there is no record for the specified cookie, an empty record is + added with a reference count of 1 and the initialized field is set to + false. If a record already exists, the function returns an error. + +DAT_RETURN +dapls_mrdb_record_update ( + IN DAPL_LMR_COOKIE cookie, + IN ib_mr_handle_t mr_handle ); + + If there is a record for the specified cookie, the MR handle field is + set to the specified mr_handle value and the initialized field is set + to true. Otherwise an error is returned. + +DAT_RETURN +dapls_mrdb_record_query ( + IN DAPL_LMR_COOKIE cookie, + OUT ib_mr_handle_t *mr_handle ); + + If there is a record for the specified cookie and the initialized + field is true, the MR handle field is returned and the reference + count field is incremented. Otherwise an error is returned. + +DAT_RETURN +dapls_mrdb_record_dec ( + IN DAPL_LMR_COOKIE cookie ); + + If there is a record for the specified cookie, the reference count + field is decremented. If the reference count is zero after the + decrement, the record is removed from the database. Otherwise an + error is returned. + +The generic algorithms for creating and destroying a shared memory +region are: + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: CreateShared + Inputs: + ia_handle + pz_handle + address + length + lmr_cookie + privileges + Outputs: + lmr_handle + lmr_context + registered_address + registered_length + +forever +{ + if dapls_mrdb_record_insert(cookie) is successful + { + if dapl_lmr_create_virtual is not successful + dapls_mrdb_record_dec(cookie) + return error + + else if dapls_mrdb_record_update(cookie, lmr->mr_handle) is not successful + dapls_mrdb_record_dec(cookie) + return error + + else break + } + else if dapls_mrdb_record_query(cookie, mr_handle) is successful + { + if ib_mrdb_shared_register_us is not successful + dapls_mrdb_record_dec(cookie) + return error + + else break + } +} + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: FreeShared + Inputs: + lmr + Outputs: + +if dapls_ib_mr_deregister(lmr) is successful + dapls_mrdb_record_dec(lmr->cookie) + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +Implementation Options +---------------------- + +As described above the crucial functionality needed to implement +shared memory support is a system wide database for mapping LMR +cookies to memory region handles. The following designs represent some +of the options for implementing such a database. Adding a database +increases the complexity of DAPL from both an implementor and user's +perspective. These designs should be evaluated on the degree to which +they minimize the additional complexity while still providing a robust +solution. + + + File System Database + -------------------- + +Employing a database that is already part of the system would be +ideal. One option on Linux is to use the file system. An area of the +file system could be set aside for the creation of files to represent +each LMR cookie. The area of the file system could be specified +through a hard coded value, an environment variable, or a +configuration file. A good candidate would be a DAPL subdirectory of +/tmp. + +Exclusive file creation is available through the creat(2) system call +in Linux. The standard I/O interface (fopen(3), etc.) does not support +this feature making porting difficult. However porting to other +environments is not a goal of this design since the entire scheme is +only a temporary solution. + +Determining when to delete the files is a difficult problem. A +reference count is required to properly remove a file once all the +memory regions it represents are deregistered. The synchronization +mechanism necessary for maintaining the reference count is not easily +implemented. As an alternative, a script could be provided to clean up +the database by removing all the files. The script would need to be +run before any DAPL consumers were started to ensure a clean +database. The disadvantage of using a script is that no DAPL instances +can be running when it is used. Another option would be to store the +process ID (PID) of the process that created the file as part of the +file's contents. Upon finding a record for a given LMR cookie value, a +DAPL instance could determine if there was a process with the same PID +in the system. To accomplish this the kill(2) system call could be +used (ex. kill(pid, 0) ). This method of validating the record assumes +that all DAPL instances can signal one another and that the PID values +do not wrap before the check is made. + +Another difficulty with this solution is choosing an accessible +portion of the file system. The area must have permissions that allow +all processes using DAPL to access and modify its contents. System +administrators are typically reluctant to allow areas without any +access controls. Typically such areas are on a dedicated file system +of a minimal size to ensure that malicious or malfunctioning software +does not monopolize the system's storage capacity. Since very little +information will be stored in each file it is unlikely that DAPL would +need a large amount of storage space even if a large number of shared +memory regions were in use. However since a file is needed for each +shared region, a large number of shared registrations may lead to the +consumption of all a file system's inodes. Again since this solution +is meant to be only temporary this constraint may be acceptable. + +There is also the possibility for database corruption should a process +crash or deadlock at an inopportune time. If a process creates file x +and then crashes all other processes waiting for the memory handle to +be written to x will fail. + +The database interface could be implemented as follows: + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_init + Inputs: + + Outputs: + +return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_exit + Inputs: + + Outputs: + +return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_insert + Inputs: + cookie + Outputs: + +file_name = convert cookie to valid file name + +fd = exclusively create file_name +if fd is invalid + return failure + +if close fd fails + return failure + +return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_update + Inputs: + cookie + mr_handle + Outputs: + +file_name = convert cookie to valid file name + +fd = open file_name +if fd is invalid + return failure + +if write mr_handle to file_name fails + return failure + +if close fd fails + return failure + +return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_query + Inputs: + cookie + + Outputs: + mr_handle + +file_name = convert cookie to valid file name + +fd = open file_name +if fd is invalid + return failure + +if read mr_handle from file_name fails + return failure + +if close fd fails + return failure + +return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_dec + Inputs: + cookie + Outputs: + +return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + + Daemon Database + --------------- + +The database could be maintained by a separate daemon process. +The DAPL instances would act as clients of the daemon server and +communicate with the daemon through the various IPC mechanisms +available on Linux: Unix sockets, TCP/IP sockets, System V message +queues, FIFOs, or RPCs. + +As with the file system based database, process crashes can potentially +cause database corruption. + +While the portability of this implementation will depend on the chosen +IPC mechanism, this approach will be at best Unix centric and possibly +Linux specific. + +The database interface could be implemented as follows: + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_init + Inputs: + + Outputs: + +initialize IPC mechanism + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_exit + Inputs: + + Outputs: + +shutdown IPC mechanism + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_insert + Inputs: + cookie + Outputs: + +if send insert message for cookie fails + return error + +if receive insert response message fails + return error + +if insert success + return success +else return error + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_update + Inputs: + cookie + mr_handle + + Outputs: + +if send update message for cookie and mr_handle fails + return error +else return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_query + Inputs: + cookie + + Outputs: + mr_handle + +if send query message for cookie fails + return error + +else if receive query response message with mr_handle fails + return error + +else return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_dec + Inputs: + cookie + Outputs: + +if send decrement message for cookie fails + return error +else return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + + Shared Memory Database + ---------------------- + +The database could be maintained in an area of memory shared by all +DAPL instances running on a system. Linux supports the System V shared +memory functions shmget(2), shmctl(2), shmat(2), and shmdt(2). A hard +coded key_t value could be used so that each DAPL instance attached to +the same piece of shared memory. The size of the database would be +constrained by the size of the shared memory region. Synchronization +could be achieved by using atomic operations targeting memory in the +shared region. + +Such a design would suffer from the corruption problems described +above. If a process crashed there would be no easy way to clean up its +locks and roll back the database to a consistent state. + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_init + Inputs: + + Outputs: + +attach shared region + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_exit + Inputs: + + Outputs: + +detach shared region + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_insert + Inputs: + cookie + Outputs: + +lock database + +if db does not contain cookie + add record for cookie + +unlock database + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_update + Inputs: + cookie + mr_handle + Outputs: + +lock database + +if db contains cookie + update record's mr_handle + +unlock database + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_query + Inputs: + cookie + + Outputs: + mr_handle + +lock database + +if db contains cookie + set mr_handle to record's value + increment record's reference count + +unlock database + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_dec + Inputs: + cookie + Outputs: + +lock database + +if db contains cookie + decrement record's reference count + + if reference count is 0 + remove record + +unlock database + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + + Kernel Module Database + ---------------------- + +If the DAT library were integrated with an HCA vendor's software +stack, the database could be managed by the HCA driver. Placing the +database in the kernel would alleviate the synchronization problems +posed by multiple processes. Since memory registration operations +already involve a transition into the kernel, no extra overhead would +be incurred by this design. + +The RI could include a kernel module with this functionality as a +temporary solution. The module could identify itself as a character +device driver and communicate with user level processes through an +ioctl(2). The driver could also create an entry in the proc file +system to display the database's contents for diagnostic purposes. + +A major benefit of a kernel based implementation is that the database +can remain consistent even in the presence of application +errors. Since DAPL instances communicate with the driver by means of +ioctl(2) calls on a file, the driver can be arrange to be informed +when the file is closed and perform any necessary actions. The driver +is guaranteed to be notified of a close regardless of the manner in +which the process exits. + +The database could be implemented as a dictionary using the LMR cookie +values as keys. + +The following pseudo-code describes the functions needed by the kernel +module and the database interface. + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: KernelModuleInit + Inputs: + + Outputs: + +dictionary = create_dictionary() +create_proc_entry() +create_character_device_entry() + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: KernelModuleExit + Inputs: + + Outputs: + +remove_character_device_entry() +remove_proc_entry() +fee_dictionary(dictionary) + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: DeviceOpen + Inputs: + file + + Outputs: + +dev_data = allocate device data + +file->private_data = dev_data + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: DeviceClose + Inputs: + file + + Outputs: + +dev_data = file->private_data + +for each record in dev_data +{ + RecordDecIoctl +} + +deallocate dev_data + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: RecordInsertIoctl + Inputs: + file + cookie + + Outputs: + +lock dictionary + +if cookie is not in dictionary + insert cookie into dictionary + + +unlock dictionary + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: RecordUpdateIoctl + Inputs: + file + cookie + mr_handle + + Outputs: + +dev_data = file->private_data + +lock dictionary + +if cookie is in dictionary + add record reference to dev_data + update mr_handle + +unlock dictionary + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: RecordQueryIoctl + Inputs: + file + cookie + + Outputs: + mr_handle + +dev_data = file->private_data + +lock dictionary + +if cookie is in dictionary + add record reference to dev_data + retrieve mr_handle + +unlock dictionary + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: RecordDecIoctl + Inputs: + file + cookie + + Outputs: + +dev_data = file->private_data +remove record reference from dev_data + +lock dictionary + +if cookie is in dictionary + decrement reference count + if reference count is 0 + remove record + +unlock dictionary + + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_init + Inputs: + + Outputs: + +fd = open device file + +if fd is invalid + return error +else + return success + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_exit + Inputs: + + Outputs: + +close fd + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_insert + Inputs: + cookie + Outputs: + +ioctl on fd + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_update + Inputs: + cookie + mr_handle + Outputs: + +ioctl on fd + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_query + Inputs: + cookie + + Outputs: + mr_handle + +ioctl on fd + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Function: dapls_mrdb_record_dec + Inputs: + cookie + Outputs: + +ioctl on fd + ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ diff --git a/trunk/ulp/dapl2/doc/dapl_vendor_specific_changes.txt b/trunk/ulp/dapl2/doc/dapl_vendor_specific_changes.txt new file mode 100644 index 00000000..aa437807 --- /dev/null +++ b/trunk/ulp/dapl2/doc/dapl_vendor_specific_changes.txt @@ -0,0 +1,394 @@ + Suggested Vendor-Specific Changes v. 0.92 + ----------------------------------------- + +================= +Table of Contents +================= + +* Table of Contents +* Introduction +* Referenced documents +* Functionality Changes + + Missing Functionality + + dat_evd_resize + + Ordering guarantees on connect/disconnect. + + Shared memory + + dat_cr_handoff +* Performance optimizations + + Reduction of context switches + [Many interrelated optimizations] + + Reducing copying of data + + Avoidance of s/g list copy on posting + + Avoidance of event data copy from CQ to EVD + + Elimination of locks + + Eliminating subroutine calls + + +============ +Introduction +============ + +This document is a list of functionality enhancements and +optimizations hardware vendors porting uDAPL may want to consider as +part of their port. The functionality enhancements mentioned in this +document are situations in which HCA Vendors, with their access to +driver and verb-level source code, and their reduced portability +concerns, are in a much better position than the reference +implementation to implement portions of the uDAPL v. 1.0 +specification. (Additional areas in which the reference +implementation, because of a lack of time or resources, did not fully +implement the uDAPL 1.0 specification are not addressed in this file; +see the file doc/dapl_unimplemented_functionality.txt, forthcoming). +Vendors should be guided in their implementation of these +functionality enhancements by their customers need for the features +involved. + +The optimizations suggested in this document have been identified by +the uDAPL Reference Implementation team as areas in which performance +may be improved by "breaching" the IB Verbs API boundary. They are +inappropriate for the reference implementation (which has portability +as one of its primary goals) but may be appropriate for a HCA-specific +port of uDAPL. Note that no expected performance gain is attached to +the suggested optimizations. This is intentional. Vendors should be +guided in their performance improvements by performance evaluations +done in the context of a representative workload, and the expected +benefit from a particular optimization weighed against the cost in +code complexity and scheduling, before the improvement is implemented. +This document is intended to seed that process; it is not intended to +be a roadmap for that process. + +We divide functionality changes into two categories + * Areas in which functionality is lacking in the reference + implementation. + * Areas in which the functionality is present in the reference + implementation, but needs improvement. + +We divide performance improvements into three types: + * Reducing context switches + * Reducing copying of data (*) + * Eliminating subroutine calls + +(*) Note that the data referred to in "reducing copying of data" is +the meta data describing an operation (e.g. scatter/gather list or +event information), not the actual data to be transferred. No data +transfer copies are required within the uDAPL reference +implementation. + +==================== +Referenced Documents +==================== + +uDAPL: User Direct Access Programming Library, Version 1.0. Published +6/21/2002. http://www.datcollaborative.org/uDAPL_062102.pdf. +Referred to in this document as the "DAT Specification". + +InfiniBand Access Application Programming Interface Specification, +Version 1.2, 4/15/2002. In DAPL SourceForge repository at +doc/api/access_api.pdf. Referred to in this document as the "IBM +Access API Specification". + +uDAPL Reference Implementation Event System Design. In DAPL +SourceForge repository at doc/dapl_event_design.txt. + +uDAPL Reference Implementation Shared Memory Design. In DAPL +SourceForge repository at doc/dapl_shared_memory_design.txt. + +uDAPL list of unimplmented functionality. In DAPL SourceForge +repository at doc/dapl_unimplemented_funcitonality.txt (forthcoming). + +=========================================== +Suggested Vendor Functionality Enhancements +=========================================== + +Missing Functionality +--------------------- +-- dat_evd_resize + +The uDAPL event system does not currently implement dat_evd_resize. +The primary reason for this is that it is not currently possible to +identify EVDs with the CQs that back them. Hence uDAPL must keep a +separate list of events, and any changes to the size of that event +list would require careful synchronization with all users of that EVD +(see the uDAPL Event System design for more details). If the various +vendor specific optimizations in this document were implemented that +eliminated the requirement for the EVD to keep its own event list, +dat_evd_resize might be easily implemented by a call or calls to +ib_cq_resize. + +-- Ordering guarantees on connect/disconnect. + +The DAPL 1.1 specification specifies that if an EVD combines event +streams for connection events and DTO events for the same endpoint, +there is an ordering guarantee: the connection event on the AP occurs +before any DTO events, and the disconnection event occurs after all +successful DTO events. Since DTO events are provided by the IBM OS +Access API through ib_completion_poll (in response to consumer +request) and connection events are provided through callbacks (which +may race with consumer requests) there is no trivial way to implement +this functionality. The functionality may be implemented through +under the table synchronizations between EVD and EP; specifically: + * The first time a DTO event is seen on an endpoint, if the + connection event has not yet arrived it is created and + delivered ahead of that DTO event. + * When a connection event is seen on an endpoint, if a + connection event has already been created for that endpoint + it is silently discarded. + * When a disconnection event is seen on an endpoint, it is + "held" until either: a) all expected DTO events for that + endpoint have completed, or b) a DTO marked as "flushed by + disconnect" is received. At that point it is delivered. + +Because of the complexity and performance overhead of implementating +this feature, the DAPL 1.1 reference implementation has chosen to take +the second approach allowed by the 1.1 specification: disallowing +integration of connection and data transfer events on the same EVD. +This fineses the problem, is in accordance with the specification, and +is more closely aligned with the ITWG IT-API currently in development, +which only allows a single event stream type for each simple EVD. +However, other vendors may choose to implement the functionality +described above in order to support more integration of event streams. + +-- Shared memory implementation + +The difficulties involved in the dapl shared memory implementation are +fully described in doc/dapl_shared_memory_design.txt. To briefly +recap: + +The uDAPL spec describes a peer-to-peer shared memory model; all uDAPL +instances indicate that they want to share registration resources for +a section of memory do so by providing the same cookie. No uDAPL +instance is unique; all register their memory in the same way, and no +communication between the instances is required. + +In contrast, the IB shared memory interface requires the first process +to register the memory to do so using the standard memory registration +verbs. All other processes after that must use the shared memory +registration verb, and provide to that verb the memory region handle +returned from the initial call. This means that the first process to +register the memory must communicate the memory region handle it +receives to all the other processes who wish to share this memory. +This is a master-slave model of shared memory registration; the +initial process (the master), is unique in its role, and it must tell +the slaves how to register the memory after it. + +To translate between these two models, the uDAPL implementation +requires some mapping between the shared cookie and the memory region +handle. This mapping must be exclusive and must have inserts occur +atomically with lookups (so that only one process can set the memory +region handle; the others retrieve it). It must also track the +deregistration of the shared memory, and the exiting of the processes +registering the shared memory; when all processes have deregistered +(possibly by exitting) it must remove the mapping from cookie to +memory region handle. + +This mapping must obviously be shared between all uDAPL +implementations on a given host. Implementing such a shared mapping +is problematic in a pure user-space implementation (as the reference +implementation is) but is expected to be relatively easy in vendor +supplied uDAFS implementations, which will presumably include a +kernel/device driver component. For this reason, we have chosen to +leave this functionality unimplemented in the reference implementation. + +-- Implementation of dat_cr_handoff + +Given that the change of service point involves a change in associated +connection qualifier, which has been advertised at the underlying +Verbs/driver level, it is not clear how to implement this function +cleanly within the reference implementation. We thus choose to defer +it for implementation by the hardware vendors. + +========================= +Performance Optimizations +========================= + + +Reduction of context switches +----------------------------- +Currently, three context switches are required on the standard +uDAPL notification path. These are: + * Invocation of the hardware interrupt handler in the kernel. + Through this method the hardware notifies the CPU of + completion queue entries for operations that have requested + notification. + * Unblocking of the per-process IB provider service thread + blocked within the driver. This thread returns to + user-space within its process, where it causes + * Unblocking of the user thread blocked within the uDAPL entry + point (dat_evd_wait() or dat_cno_wait()). + +There are several reasons for the high number of context switches, +specifically: + * The fact that the IB interface delivers notifications + through callbacks rather than through unblocking waiting + threads; this does not match uDAPL's blocking interface. + * The fact that the IB interface for blocking on a CQ doesn't + have a threshhold. If it did, we could often convert a + dat_evd_wait() into a wait on that CQ. + * The lack of a parallel concept to the CNO within IB. + +These are all areas in which closer integration between the IB +verbs/driver and uDAPL could allow the user thread to wait within the +driver. This would allow the hardware interrupt thread to directly +unblock the user thread, saving a context switch. + +A specific listing of the optimizations considered here are: + * Allow blocking on an IB CQ. This would allow removal of the + excess context switch for dat_evd_wait() in cases where + there is a 1-to-1 correspondence between an EVD and a CQ and + no threshold was passed to dat_evd_wait(). + * Allow blocking on an IB CQ to take a threshold argument. + This would allow removal of the excess context switch for + dat_evd_wait() in cases where there is a 1-to-1 + correspondence between an EVD and a CQ regardless of the + threshold value. + * Give the HCA device driver knowledge of and access to the + implementation of the uDAPL EVD, and implement dat_evd_wait() + as an ioctl blocking within the device driver. This would + allow removal of the excess context switch in all cases for + a dat_evd_wait(). + * Give the HCA device driver knowledge of and access to the + implementation of the uDAPL CNO, and implement dat_cno_wait() + as an ioctl blocking within the device driver. This would + allow removal of the excess context switch in all cases for + a dat_cno_wait(), and could improve performance for blocking + on OS Proxy Wait Objects related to the uDAPL CNO. + +See the DAPL Event Subsystem Design (doc/dapl_event_design.txt) for +more details on this class of optimization. + +======================== +Reducing Copying of Data +======================== + +There are two primary places in which a closer integration between the +IB verbs/driver and the uDAPL implementation could reducing copying +costs: + +-- Avoidance of s/g list copy on posting + +Currently there are two copies involved in posting a data transfer +request in uDAPL: + * From the user context to uDAPL. This copy is required + because the scatter/gather list formats for uDAPL and IB + differ; a copy is required to change formats. + * From uDAPL to the WQE. This copy is required because IB + specifies that all user parameters are owned by the user + upon return from the IB call, and therefore IB must keep its + own copy for use during the data transfer operation. + +If the uDAPL data transfer dispatch operations were implemented +directly on the IB hardware, these copies could be combined. + +-- Avoidance of Event data copy from CQ to EVD + +Currently there are two copies of data involved in receiving an event +in a standard data transfer operation: + * From the CQ on which the IB completion occurs to an event + structure held within the uDAPL EVD. This is because the IB + verbs provide no way to discover how many elements have been + posted to a CQ. This copy is not + required for dat_evd_dequeue. However, dat_evd_wait + requires this copy in order to correctly implement the + threshhold argument; the callback must know when to wakeup + the waiting thread. In addition, copying all CQ entries + (not just the one to be returned) is necessary before + returning from dat_evd_wait in order to set the *nmore OUT + parameter. + * From the EVD into the event structure provided in the + dat_evd_wait() call. This copy is required because of the + DAT specification, which requires a user-provided event + structure to the dat_evd_wait() call in which the event + information will be returned. If dat_evd_wait() were + instead, for example, to hand back a pointer to the already + allocated event structure, that would eventually require the + event subsystem to allocate more event structures. This is + avoided in the critical path. + +A tighter integration between the IB verbs/driver and the uDAPL +implementation would allow the avoidance of the first copy. +Specifically, providing a way to get information as to the number of +completions on a CQ would allow avoidance of that copy. + +See the uDAPL Event Subsystem Design for more details on this class of +optimization. + +==================== +Elimination of Locks +==================== + +Currently there is only a single lock used on the critical path in the +reference implementation, in dat_evd_wait() and dat_evd_dequeue(). +This lock is in place because the ib_completion_poll() routine is not +defined as thread safe, and both dat_evd_wait() and dat_evd_dequeue() +are. If there was some way for a vendor to make ib_completion_poll() +thread safe without a lock (e.g. if the appropriate hardware/software +interactions were naturally safe against races), and certain other +modifications made to the code, the lock might be removed. + +The modifications required are: + * Making racing consumers from DAPL ring buffers thread safe. + This is possible, but somewhat tricky; the key is to make + the interaction with the producer occur through a count of + elements on the ring buffer (atomically incremented and + decremented), but to dequeue elements with a separate atomic + pointer increment. The atomic modification of the element + count synchronizes with the producer and acquires the right + to do an atomic pointer increment to get the actual data. + The atomic pointer increment synchronizes with the other + consumers and actually gets the buffer data. + * The optimization described above for avoiding copies from + the CQ to the DAPL EVD Event storage queue. Without this + optimization a potential race between dat_evd_dequeue() and + dat_evd_wait() exists where dat_evd_dequeue will return an + element advanced in the event stream from the one returned + from dat_evd_wait(): + + dat_evd_dequeue() called + + EVD state checked; ok for + dat_evd_dequeue() + dat_evd_wait() called + + State changed to reserve EVD + for dat_evd_wait() + + Partial copy of CQ to EVD Event store + + Dequeue of CQE from CQ + + Completion of copy of CQ to EVD Event store + + Return of first CQE copied to EVD Event store. + + Return of thie CQE from the middle + of the copied stream. + + + If no copy occurs, dat_evd_wait() and dat_evd_dequeue() may + race, but if all operations on which they may race (access + to the EVD Event Queue and access to the CQ) are thread + safe, this race will cause no problems. + +============================ +Eliminating Subroutine Calls +============================ + +This area is the simplest, as there are many DAPL calls on the +critical path that are very thin veneers on top of their IB +equivalents. All of these calls are canidates for being merged with +those IB equivalents. In cases where there are other optimizations +that may be acheived with the call described above (e.g. within the +event subsystem, the data transfer operation posting code), that call +is not mentioned here: + * dat_pz_create + * dat_pz_free + * dat_pz_query + * dat_lmr_create + * dat_lmr_free + * dat_lmr_query + * dat_rmr_create + * dat_rmr_free + * dat_rmr_query + * dat_rmr_bind + + diff --git a/trunk/ulp/dapl2/doc/dat_environ.txt b/trunk/ulp/dapl2/doc/dat_environ.txt new file mode 100644 index 00000000..638ba9dc --- /dev/null +++ b/trunk/ulp/dapl2/doc/dat_environ.txt @@ -0,0 +1,45 @@ + DAT Environment Guide v. 0.01 + ----------------------------- + +The following environment variables affect the behavior of the DAT +library: + + +DAT_OVERRIDE +------------ + Value used as the static registry configuration file, overriding the + default location, /etc/dat.conf + + Example: setenv DAT_OVERRIDE /path/to/my/private.conf + + +DAT_DBG_TYPE +------------ + + Value specifies which parts of the registry will print debugging + information, valid values are + + DAT_OS_DBG_TYPE_ERROR = 0x1 + DAT_OS_DBG_TYPE_GENERIC = 0x2 + DAT_OS_DBG_TYPE_SR = 0x4 + DAT_OS_DBG_TYPE_DR = 0x8 + DAT_OS_DBG_TYPE_PROVIDER_API = 0x10 + DAT_OS_DBG_TYPE_CONSUMER_API = 0x20 + DAT_OS_DBG_TYPE_ALL = 0xff + + or any combination of these. For example you can use 0xC to get both + static and dynamic registry output. + + Example setenv DAT_DBG_TYPE 0xC + +DAT_DBG_DEST +------------ + + Value sets the output destination, valid values are + + DAT_OS_DBG_DEST_STDOUT = 0x1 + DAT_OS_DBG_DEST_SYSLOG = 0x2 + DAT_OS_DBG_DEST_ALL = 0x3 + + For example, 0x3 will output to both stdout and the syslog. + diff --git a/trunk/ulp/dapl2/man/dapltest.1 b/trunk/ulp/dapl2/man/dapltest.1 new file mode 100644 index 00000000..07158c2a --- /dev/null +++ b/trunk/ulp/dapl2/man/dapltest.1 @@ -0,0 +1,390 @@ +." Text automatically generated by txt2man +.TH dapltest 1 "February 23, 2007" "uDAPL 1.2" "USER COMMANDS" + +.SH NAME +\fB +\fBdapltest \fP- test for the Direct Access Programming Library (DAPL) +\fB +.SH DESCRIPTION + +Dapltest is a set of tests developed to exercise, characterize, +and verify the DAPL interfaces during development and porting. +At least two instantiations of the test must be run. One acts +as the server, fielding requests and spawning server-side test +threads as needed. Other client invocations connect to the server +and issue test requests. The server side of the test, once invoked, +listens continuously for client connection requests, until quit or +killed. Upon receipt of a connection request, the connection is +established, the server and client sides swap version numbers to +verify that they are able to communicate, and the client sends +the test request to the server. If the version numbers match, +and the test request is well-formed, the server spawns the threads +needed to run the test before awaiting further connections. +.SH USAGE + +dapltest [ -f script_file_name ] +[ -T S|Q|T|P|L ] [ -D device_name ] [ -d ] [ -R HT|LL|EC|PM|BE ] +.PP +With no arguments, dapltest runs as a server using default values, +and loops accepting requests from clients. + +The -f option allows all arguments to be placed in a file, to ease +test automation. + +The following arguments are common to all tests: +.TP +.B +[ -T S|Q|T|P|L ] +Test function to be performed: +.RS +.TP +.B +S +- server loop +.TP +.B +Q +- quit, client requests that server +wait for any outstanding tests to +complete, then clean up and exit +.TP +.B +T +- transaction test, transfers data between +client and server +.TP +.B +P +- performance test, times DTO operations +.TP +.B +L +- limit test, exhausts various resources, +runs in client w/o server interaction +Default: S +.RE +.TP +.B +[ -D device_name ] +Specifies the interface adapter name as documented in +the /etc/dat.conf static configuration file. This name +corresponds to the provider library to open. +Default: none +.TP +.B +[ -d ] +Enables extra debug verbosity, primarily tracing +of the various DAPL operations as they progress. +Repeating this parameter increases debug spew. +Errors encountered result in the test spewing some +explanatory text and stopping; this flag provides +more detail about what lead up to the error. +Default: zero +.TP +.B +[ -R BE ] +Indicate the quality of service (QoS) desired. +Choices are: +.RS +.TP +.B +HT +- high throughput +.TP +.B +LL +- low latency +.TP +.B +EC +- economy (neither HT nor LL) +.TP +.B +PM +- premium +.TP +.B +BE +- best effort +Default: BE +.RE +.RE +.PP +.B +Usage - Quit test client +.PP +.nf +.fam C + dapltest [Common_Args] [ -s server_name ] + + Quit testing (-T Q) connects to the server to ask it to clean up and + exit (after it waits for any outstanding test runs to complete). + In addition to being more polite than simply killing the server, + this test exercises the DAPL object teardown code paths. + There is only one argument other than those supported by all tests: + + -s server_name Specifies the name of the server interface. + No default. + + +.fam T +.fi +.B +Usage - Transaction test client +.PP +.nf +.fam C + dapltest [Common_Args] [ -s server_name ] + [ -t threads ] [ -w endpoints ] [ -i iterations ] [ -Q ] + [ -V ] [ -P ] OPclient OPserver [ op3, + + Transaction testing (-T T) transfers a variable amount of data between + client and server. The data transfer can be described as a sequence of + individual operations; that entire sequence is transferred 'iterations' + times by each thread over all of its endpoint(s). + + The following parameters determine the behavior of the transaction test: + + -s server_name Specifies the name or IP address of the server interface. + No default. + + [ -t threads ] Specify the number of threads to be used. + Default: 1 + + [ -w endpoints ] Specify the number of connected endpoints per thread. + Default: 1 + + [ -i iterations ] Specify the number of times the entire sequence + of data transfers will be made over each endpoint. + Default: 1000 + + [ -Q ] Funnel completion events into a CNO. + Default: use EVDs + + [ -V ] Validate the data being transferred. + Default: ignore the data + + [ -P ] Turn on DTO completion polling + Default: off + + OP1 OP2 [ OP3, \.\.\. ] + A single transaction (OPx) consists of: + + server|client Indicates who initiates the + data transfer. + + SR|RR|RW Indicates the type of transfer: + SR send/recv + RR RDMA read + RW RDMA write + Defaults: none + + [ seg_size [ num_segs ] ] + Indicates the amount and format + of the data to be transferred. + Default: 4096 1 + (i.e., 1 4KB buffer) + + [ -f ] For SR transfers only, indicates + that a client's send transfer + completion should be reaped when + the next recv completion is reaped. + Sends and receives must be paired + (one client, one server, and in that + order) for this option to be used. + + Restrictions: + + Due to the flow control algorithm used by the transaction test, there + must be at least one SR OP for both the client and the server. + + Requesting data validation (-V) causes the test to automatically append + three OPs to those specified. These additional operations provide + synchronization points during each iteration, at which all user-specified + transaction buffers are checked. These three appended operations satisfy + the "one SR in each direction" requirement. + + The transaction OP list is printed out if -d is supplied. + +.fam T +.fi +.B +Usage - Performance test client +.PP +.nf +.fam C + dapltest [Common_Args] -s server_name [ -m p|b ] + [ -i iterations ] [ -p pipeline ] OP + + Performance testing (-T P) times the transfer of an operation. + The operation is posted 'iterations' times. + + The following parameters determine the behavior of the transaction test: + + -s server_name Specifies the name or IP address of the server interface. + No default. + + -m b|p Used to choose either blocking (b) or polling (p) + Default: blocking (b) + + [ -i iterations ] Specify the number of times the entire sequence + of data transfers will be made over each endpoint. + Default: 1000 + + [ -p pipeline ] Specify the pipline length, valid arguments are in + the range [0,MAX_SEND_DTOS]. If a value greater than + MAX_SEND_DTOS is requested the value will be + adjusted down to MAX_SEND_DTOS. + Default: MAX_SEND_DTOS + + OP Specifies the operation as follow: + + RR|RW Indicates the type of transfer: + RR RDMA read + RW RDMA write + Defaults: none + + [ seg_size [ num_segs ] ] + Indicates the amount and format + of the data to be transferred. + Default: 4096 1 + (i.e., 1 4KB buffer) +.fam T +.RE +.RE +.PP +.B +Usage - Limit test client +.PP +.nf +.fam C + Limit testing (-T L) neither requires nor connects to any server + instance. The client runs one or more tests which attempt to + exhaust various resources to determine DAPL limits and exercise + DAPL error paths. If no arguments are given, all tests are run. + + Limit testing creates the sequence of DAT objects needed to + move data back and forth, attempting to find the limits supported + for the DAPL object requested. For example, if the LMR creation + limit is being examined, the test will create a set of + {IA, PZ, CNO, EVD, EP} before trying to run dat_lmr_create() to + failure using that set of DAPL objects. The 'width' parameter + can be used to control how many of these parallel DAPL object + sets are created before beating upon the requested constructor. + Use of -m limits the number of dat_*_create() calls that will + be attempted, which can be helpful if the DAPL in use supports + essentailly unlimited numbers of some objects. + + The limit test arguments are: + + [ -m maximum ] Specify the maximum number of dapl_*_create() + attempts. + Default: run to object creation failure + + [ -w width ] Specify the number of DAPL object sets to + create while initializing. + Default: 1 + + [ limit_ia ] Attempt to exhaust dat_ia_open() + + [ limit_pz ] Attempt to exhaust dat_pz_create() + + [ limit_cno ] Attempt to exhaust dat_cno_create() + + [ limit_evd ] Attempt to exhaust dat_evd_create() + + [ limit_ep ] Attempt to exhaust dat_ep_create() + + [ limit_rsp ] Attempt to exhaust dat_rsp_create() + + [ limit_psp ] Attempt to exhaust dat_psp_create() + + [ limit_lmr ] Attempt to exhaust dat_lmr_create(4KB) + + [ limit_rpost ] Attempt to exhaust dat_ep_post_recv(4KB) + + [ limit_size_lmr ] Probe maximum size dat_lmr_create() + +.nf +.fam C + Default: run all tests + + +.fam T +.fi +.SH EXAMPLES + +dapltest -T S -d -D OpenIB-cma +.PP +.nf +.fam C + Starts a server process with debug verbosity. + +.fam T +.fi +dapltest -T T -d -s host1-ib0 -D OpenIB-cma -i 100 client SR 4096 2 server SR 4096 2 +.PP +.nf +.fam C + Runs a transaction test, with both sides + sending one buffer with two 4KB segments, + one hundred times. + +.fam T +.fi +dapltest -T P -d -s host1-ib0 -D OpenIB-cma -i 100 WR 4096 2 +.PP +.nf +.fam C + Runs a performance test, with the client + sending one buffer with two 4KB segments, + one hundred times. + +.fam T +.fi +dapltest -T Q -s host1-ib0 -D OpenIB-cma +.PP +.nf +.fam C + Asks the server to clean up and exit. + +.fam T +.fi +dapltest -T L -D OpenIB-cma -d -w 16 -m 1000 +.PP +.nf +.fam C + Runs all of the limit tests, setting up + 16 complete sets of DAPL objects, and + creating at most a thousand instances + when trying to exhaust resources. + +.fam T +.fi +dapltest -T T -V -d -t 2 -w 4 -i 55555 -s linux3 -D OpenIB-cma +client RW 4096 1 server RW 2048 4 +client SR 1024 4 server SR 4096 2 +client SR 1024 3 -f server SR 2048 1 -f +.PP +.nf +.fam C + Runs a more complicated transaction test, + with two thread using four EPs each, + sending a more complicated buffer pattern + for a larger number of iterations, + validating the data received. + + +.fam T +.fi +.RE +.TP +.B +BUGS +(and To Do List) +.PP +.nf +.fam C + Use of CNOs (-Q) is not yet supported. + + Further limit tests could be added. diff --git a/trunk/ulp/dapl2/man/dtest.1 b/trunk/ulp/dapl2/man/dtest.1 new file mode 100644 index 00000000..9f7b5a99 --- /dev/null +++ b/trunk/ulp/dapl2/man/dtest.1 @@ -0,0 +1,78 @@ +.TH dtest 1 "February 23, 2007" "uDAPL 1.2" "USER COMMANDS" + +.SH NAME +dtest \- simple uDAPL send/receive and RDMA test + +.SH SYNOPSIS +.B dtest +[\-P provider] [\-b buf size] [\-B burst count][\-v] [\-c] [\-p] [\-d]\fB [-s]\fR + +.B dtest +[\-P provider] [\-b buf size] [\-B burst count][\-v] [\-c] [\-p] [\-d]\fB [-h HOSTNAME]\fR + +.SH DESCRIPTION +.PP +dtest is a simple test used to exercise and verify the uDAPL interfaces. +At least two instantiations of the test must be run. One acts as the server +and the other the client. The server side of the test, once invoked listens +for connection requests, until timing out or killed. Upon receipt of a cd +connection request, the connection is established, the server and client +sides exchange information necessary to perform RDMA writes and reads. + +.SH OPTIONS + +.PP +.TP +\fB\-P\fR=\fIPROVIDER\fR +use \fIPROVIDER\fR to specify uDAPL interface using /etc/dat.conf (default OpenIB-cma) +.TP +\fB\-b\fR=\fIBUFFER_SIZE\fR +use buffer size \fIBUFFER_SIZE\fR for RDMA(default 64) +.TP +\fB\-B\fR=\fIBURST_COUNT\fR +use busrt count \fIBURST_COUNT\fR for interations (default 10) +.TP +\fB\-v\fR, verbose output(default off) +.TP +\fB\-c\fR, use consumer notification events (default off) +.TP +\fB\-p\fR, use polling (default wait for event) +.TP +\fB\-d\fR, delay in seconds before close (default off) +.TP +\fB\-s\fR, run as server (default - run as server) +.TP +\fB\-h\fR=\fIHOSTNAME\fR +use \fIHOSTNAME\fR to specify server hostname or IP address (default - none) + +.SH EXAMPLES + +dtest -P OpenIB-cma -v -s +.PP +.nf +.fam C + Starts a server process with debug verbosity using provider OpenIB-cma. + +.fam T +.fi +dtest -P OpenIB-cma -h server1-ib0 +.PP +.nf +.fam C + Starts a client process, using OpenIB-cma provider to connect to hostname server1-ib0. + +.fam T + +.SH SEE ALSO +.BR dapltest(1) + +.SH AUTHORS +.TP +Arlin Davis +.RI < ardavis@ichips.intel.com > + +.SH BUGS + + + + diff --git a/trunk/ulp/dapl2/test/dapltest/README b/trunk/ulp/dapl2/test/dapltest/README new file mode 100644 index 00000000..6af04808 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/README @@ -0,0 +1,293 @@ + +NAME + + dapltest - test for the Direct Access Programming Library (DAPL) + +DESCRIPTION + + Dapltest is a set of tests developed to exercise, characterize, + and verify the DAPL interfaces during development and porting. + At least two instantiations of the test must be run. One acts + as the server, fielding requests and spawning server-side test + threads as needed. Other client invocations connect to the + server and issue test requests. + + The server side of the test, once invoked, listens continuously + for client connection requests, until quit or killed. Upon + receipt of a connection request, the connection is established, + the server and client sides swap version numbers to verify that + they are able to communicate, and the client sends the test + request to the server. If the version numbers match, and the + test request is well-formed, the server spawns the threads + needed to run the test before awaiting further connections. + +USAGE + + dapltest [ -f script_file_name ] + [ -T S|Q|T|P|L ] [ -D device_name ] [ -d ] [ -R HT|LL|EC|PM|BE ] + + With no arguments, dapltest runs as a server using default values, + and loops accepting requests from clients. The -f option allows + all arguments to be placed in a file, to ease test automation. + The following arguments are common to all tests: + + [ -T S|Q|T|P|L ] Test function to be performed: + S - server loop + Q - quit, client requests that server + wait for any outstanding tests to + complete, then clean up and exit + T - transaction test, transfers data between + client and server + P - performance test, times DTO operations + L - limit test, exhausts various resources, + runs in client w/o server interaction + Default: S + + [ -D device_name ] Specifies the name of the device (interface adapter). + Default: host-specific, look for DT_MdepDeviceName + in dapl_mdep.h + + [ -d ] Enables extra debug verbosity, primarily tracing + of the various DAPL operations as they progress. + Repeating this parameter increases debug spew. + Errors encountered result in the test spewing some + explanatory text and stopping; this flag provides + more detail about what lead up to the error. + Default: zero + + [ -R BE ] Indicate the quality of service (QoS) desired. + Choices are: + HT - high throughput + LL - low latency + EC - economy (neither HT nor LL) + PM - premium + BE - best effort + Default: BE + +USAGE - Quit test client + + dapltest [Common_Args] [ -s server_name ] + + Quit testing (-T Q) connects to the server to ask it to clean up and + exit (after it waits for any outstanding test runs to complete). + In addition to being more polite than simply killing the server, + this test exercises the DAPL object teardown code paths. + There is only one argument other than those supported by all tests: + + -s server_name Specifies the name of the server interface. + No default. + + +USAGE - Transaction test client + + dapltest [Common_Args] [ -s server_name ] + [ -t threads ] [ -w endpoints ] [ -i iterations ] [ -Q ] + [ -V ] [ -P ] OPclient OPserver [ op3, + + Transaction testing (-T T) transfers a variable amount of data between + client and server. The data transfer can be described as a sequence of + individual operations; that entire sequence is transferred 'iterations' + times by each thread over all of its endpoint(s). + + The following parameters determine the behavior of the transaction test: + + -s server_name Specifies the name of the server interface. + No default. + + [ -t threads ] Specify the number of threads to be used. + Default: 1 + + [ -w endpoints ] Specify the number of connected endpoints per thread. + Default: 1 + + [ -i iterations ] Specify the number of times the entire sequence + of data transfers will be made over each endpoint. + Default: 1000 + + [ -Q ] Funnel completion events into a CNO. + Default: use EVDs + + [ -V ] Validate the data being transferred. + Default: ignore the data + + [ -P ] Turn on DTO completion polling + Default: off + + OP1 OP2 [ OP3, ... ] + A single transaction (OPx) consists of: + + server|client Indicates who initiates the + data transfer. + + SR|RR|RW Indicates the type of transfer: + SR send/recv + RR RDMA read + RW RDMA write + Defaults: none + + [ seg_size [ num_segs ] ] + Indicates the amount and format + of the data to be transferred. + Default: 4096 1 + (i.e., 1 4KB buffer) + + [ -f ] For SR transfers only, indicates + that a client's send transfer + completion should be reaped when + the next recv completion is reaped. + Sends and receives must be paired + (one client, one server, and in that + order) for this option to be used. + + Restrictions: + + Due to the flow control algorithm used by the transaction test, there + must be at least one SR OP for both the client and the server. + + Requesting data validation (-V) causes the test to automatically append + three OPs to those specified. These additional operations provide + synchronization points during each iteration, at which all user-specified + transaction buffers are checked. These three appended operations satisfy + the "one SR in each direction" requirement. + + The transaction OP list is printed out if -d is supplied. + +USAGE - Performance test client + + dapltest [Common_Args] -s server_name [ -m p|b ] + [ -i iterations ] [ -p pipeline ] OP + + Performance testing (-T P) times the transfer of an operation. + The operation is posted 'iterations' times. + + The following parameters determine the behavior of the transaction test: + + -s server_name Specifies the name of the server interface. + No default. + + -m b|p Used to choose either blocking (b) or polling (p) + Default: blocking (b) + + [ -i iterations ] Specify the number of times the entire sequence + of data transfers will be made over each endpoint. + Default: 1000 + + [ -p pipeline ] Specify the pipline length, valid arguments are in + the range [0,MAX_SEND_DTOS]. If a value greater than + MAX_SEND_DTOS is requested the value will be + adjusted down to MAX_SEND_DTOS. + Default: MAX_SEND_DTOS + + OP + An operation consists of: + + RR|RW Indicates the type of transfer: + RR RDMA read + RW RDMA write + Default: none + + [ seg_size [ num_segs ] ] + Indicates the amount and format + of the data to be transferred. + Default: 4096 1 + (i.e., 1 4KB buffer) + +USAGE - Limit test client + + Limit testing (-T L) neither requires nor connects to any server + instance. The client runs one or more tests which attempt to + exhaust various resources to determine DAPL limits and exercise + DAPL error paths. If no arguments are given, all tests are run. + + Limit testing creates the sequence of DAT objects needed to + move data back and forth, attempting to find the limits supported + for the DAPL object requested. For example, if the LMR creation + limit is being examined, the test will create a set of + {IA, PZ, CNO, EVD, EP} before trying to run dat_lmr_create() to + failure using that set of DAPL objects. The 'width' parameter + can be used to control how many of these parallel DAPL object + sets are created before beating upon the requested constructor. + Use of -m limits the number of dat_*_create() calls that will + be attempted, which can be helpful if the DAPL in use supports + essentailly unlimited numbers of some objects. + + The limit test arguments are: + + [ -m maximum ] Specify the maximum number of dapl_*_create() + attempts. + Default: run to object creation failure + + [ -w width ] Specify the number of DAPL object sets to + create while initializing. + Default: 1 + + [ limit_ia ] Attempt to exhaust dat_ia_open() + + [ limit_pz ] Attempt to exhaust dat_pz_create() + + [ limit_cno ] Attempt to exhaust dat_cno_create() + + [ limit_evd ] Attempt to exhaust dat_evd_create() + + [ limit_ep ] Attempt to exhaust dat_ep_create() + + [ limit_rsp ] Attempt to exhaust dat_rsp_create() + + [ limit_psp ] Attempt to exhaust dat_psp_create() + + [ limit_lmr ] Attempt to exhaust dat_lmr_create(4KB) + + [ limit_rpost ] Attempt to exhaust dat_ep_post_recv(4KB) + + [ limit_size_lmr ] Probe maximum size dat_lmr_create() + + Default: run all tests + + +EXAMPLES + + dapltest -T S -d -D jni0a + + Starts a server process with debug verbosity. + + dapltest -T T -d -s linux3 -D jni0a -i 100 \ + client SR 4096 2 server SR 4096 2 + + Runs a transaction test, with both sides + sending one buffer with two 4KB segments, + one hundred times. + + dapltest -T P -d -s linux3 -D JniIbdd0 -i 100 SR 4096 2 + + Runs a performance test, with the client + sending one buffer with two 4KB segments, + one hundred times. + + dapltest -T Q -s linux3 -D jni0a + + Asks the server to clean up and exit. + + dapltest -T L -D jni0a -d -w 16 -m 1000 + + Runs all of the limit tests, setting up + 16 complete sets of DAPL objects, and + creating at most a thousand instances + when trying to exhaust resources. + + dapltest -T T -V -d -t 2 -w 4 -i 55555 -s linux3 -D jni0a \ + client RW 4096 1 server RW 2048 4 \ + client SR 1024 4 server SR 4096 2 \ + client SR 1024 3 -f server SR 2048 1 -f + + Runs a more complicated transaction test, + with two thread using four EPs each, + sending a more complicated buffer pattern + for a larger number of iterations, + validating the data received. + + +BUGS (and To Do List) + + Use of CNOs (-Q) is not yet supported. + + Further limit tests could be added. diff --git a/trunk/ulp/dapl2/test/dapltest/SOURCES b/trunk/ulp/dapl2/test/dapltest/SOURCES new file mode 100644 index 00000000..d8e53697 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/SOURCES @@ -0,0 +1,32 @@ +TARGETNAME=dapl2test +TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=PROGRAM +UMTYPE=console +USE_CRTDLL=1 + +SOURCES=dapltest.rc \ + dt_cmd.c \ + dt_test.c \ + dt_common.c \ + dt_udapl.c \ + dt_mdep.c + +INCLUDES=.;include;mdep\windows;..\..\dat\include;%DDK_INC_PATH% +#INCLUDES=.;include;mdep\windows;C:\dat\inc\dat2;%DDK_INC_PATH% + +RCOPTIONS=/I..\..\..\..\inc; + +# Set defines particular to dapl2test. +#USER_C_FLAGS=$(USER_C_FLAGS) /DDAT_EXTENSIONS + +!if $(FREEBUILD) +DATLIB=dat2.lib +!else +DATLIB=dat2d.lib +!endif + +TARGETLIBS=$(TARGETPATH)\*\$(DATLIB) $(SDK_LIB_PATH)\ws2_32.lib + +# XXX do this ASAP - MSC_WARNING_LEVEL= /W3 +MSC_WARNING_LEVEL= /W1 + diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_fft_cmd.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_fft_cmd.c new file mode 100644 index 00000000..ba45c2fb --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_fft_cmd.c @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +//--------------------------------------------------------------------------- +void DT_FFT_Cmd_Init (FFT_Cmd_t *cmd) +{ + int i; + memset ((void *)cmd, 0, sizeof (FFT_Cmd_t)); + cmd->fft_type = NONE; + cmd->device_name[0] = '\0'; + cmd->server_name[0] = '\0'; + for (i=0; icases_flag[i] = false; + } + cmd->size = 0; + cmd->num_iter = 1000; + cmd->num_threads = 10; + cmd->num_vis = 500; + cmd->ReliabilityLevel = DAT_QOS_BEST_EFFORT; +} + +//------------------------------------------------------------------------------ +bool DT_FFT_Cmd_Parse (FFT_Cmd_t *cmd, + int my_argc, + char ** my_argv, + mygetopt_t *opts) +{ + int c; + int i, caseNum; + unsigned int len; + + for (;;) + { + c = DT_mygetopt_r (my_argc, my_argv, "D:f:s:i:t:v:R:", opts); + if (c == EOF) + { + break; + } + switch (c) + { + case 'D': //device name + { + strcpy (cmd->device_name, opts->optarg); + break; + } + case 's': //server name + { + strcpy (cmd->server_name, opts->optarg); + break; + } + case 'i': // num iterations + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -i option\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + cmd->num_iter = atoi (opts->optarg); + break; + } + case 't': // num threads + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -t option\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + cmd->num_threads = atoi (opts->optarg); + break; + } + case 'v': // num vis + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -v option\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + cmd->num_vis = atoi (opts->optarg); + break; + } + case 'f': //function feature + { + if (strcmp (opts->optarg, "hwconn")==0) + { + cmd->fft_type = HWCONN; + cmd->size = 4; //4 cases for hwconn + break; + } +#if 0 // not yet implemented + else if (strcmp (opts->optarg, "cqmgt")==0) + { + cmd->fft_type = CQMGT; + cmd->size = 10; //10 cases for cqmgt + break; + } +#endif + else if (strcmp (opts->optarg, "endpoint")==0) + { + cmd->fft_type = ENDPOINT; + cmd->size = 3; //13 cases for endpoint + break; + } + else if (strcmp (opts->optarg, "pz")==0) + { + cmd->fft_type = PTAGMGT; + cmd->size = 3; //10 cases for Ptagmgt + break; + } + else if (strcmp (opts->optarg, "mem")==0) + { + cmd->fft_type = MEMMGT; + cmd->size = 5; //12 cases for Memmgt + break; + } + else if (strcmp (opts->optarg, "connmgt")==0) + { + cmd->fft_type = CONNMGT; + cmd->size = 2; //16 cases for connmgt + break; + } +#if 0 // not yet implemented + else if (strcmp (opts->optarg, "connmgt_client")==0) + { + cmd->fft_type = CONNMGT_CLIENT; + cmd->size = 16; //16 cases for connmgt_client + break; + } +#endif + else if (strcmp (opts->optarg, "queryinfo")==0) + { + cmd->fft_type = QUERYINFO; +#ifndef __KDAPLTEST__ + cmd->size = 18; //18 cases for UDAPL queryinfo +#else + cmd->size = 16; //16 cases for KDAPL queryinfo +#endif + break; + } +#if 0 // not yet implemented + else if (strcmp (opts->optarg, "ns")==0) + { + cmd->fft_type = NS; + cmd->size = 10; //10 cases for ns + break; + } + else if (strcmp (opts->optarg, "errhand")==0) + { + cmd->fft_type = ERRHAND; + cmd->size = 2; //2 cases for errhand + break; + } + else if (strcmp (opts->optarg, "unsupp")==0) + { + cmd->fft_type = UNSUPP; + cmd->size = 2; //2 cases for unsupp + break; + } + else if (strcmp (opts->optarg, "stress")==0) + { + cmd->fft_type = STRESS; + cmd->size = 6; //6 cases for stress + break; + } + else if (strcmp (opts->optarg, "stress_client")==0) + { + cmd->fft_type = STRESS_CLIENT; + cmd->size = 6; //6 cases for stress_client + break; + } +#endif + else + { + DT_Mdep_printf ("don't know this function feature: %s\n", + opts->optarg); + DT_FFT_Cmd_Usage (); + return (false); + } + } + case 'R': // Service Reliability Level + { + cmd->ReliabilityLevel = DT_ParseQoS (opts->optarg); + if (0 == cmd->ReliabilityLevel) + { + DT_Mdep_printf ("Invalid FFT Test Parameter: %c\n", c); + DT_FFT_Cmd_Usage (); + return (false); + } + break; + } + + case '?': + default: + { + DT_Mdep_printf ("Invalid FFT Test Parameter: %c\n", c); + DT_FFT_Cmd_Usage (); + return (false); + } + } + } + if (cmd->device_name[0] == '\0') + { + if (!DT_Mdep_GetDefaultDeviceName (cmd->device_name)) + { + DT_Mdep_printf ("can't get default device name\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + } + + if (cmd->fft_type ==NONE) + { + DT_Mdep_printf ("must define the function feature with -f to test\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + if (cmd->server_name[0] =='\0' && + (cmd->fft_type==CONNMGT_CLIENT || cmd->fft_type == DATAXFER_CLIENT || + cmd->fft_type == UNSUPP || cmd->fft_type == STRESS_CLIENT)) + { + DT_Mdep_printf ("must define the server name with -s option\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + + if (cmd->server_name[0] =='\0' && cmd->fft_type==NS ) + { + DT_Mdep_printf ("\ + Must specify host name or host IP address with -s option to be tested\n"); + DT_FFT_Cmd_Usage (); + return (false); + } + + //now parse the test cases + if (opts->optind == my_argc) //default: test all cases + { + for (i=0; isize; i++) + { + cmd->cases_flag[i] = true; + } + return true; + } + + //test specified cases + i = opts->optind; + while (i=cmd->size) + { + DT_Mdep_printf ("test case number must be within range : 0 -- %d\n", + cmd->size-1); + DT_FFT_Cmd_Usage (); + return (false); + } + cmd->cases_flag[caseNum] = true; + i++; + } + return (true); +} + +//-------------------------------------------------------------- +void DT_FFT_Cmd_Usage (void) +{ + char usage[] = + { + "dapltest -T F [-D ] -f [-i ] \n" + "[-t ] [-v ] [-s ] [case0] [case1] [...]\n" + "USAGE: [-D ]\n" + "USAGE: -f \n" + "USAGE: hwconn\n" + "USAGE: endpoint\n" + "USAGE: pz\n" + "USAGE: mem\n" + "USAGE: connmgt\n" + "USAGE: queryinfo\n" +#if 0 // not yet implemented + "USAGE: connmgt_client (not yet implemented)\n" + "USAGE: cqmgt (not yet implemented)\n" + "USAGE: ns (not yet implemented)\n" + "USAGE: errhand (not yet implemented)\n" + "USAGE: unsupp (not yet implemented)\n" + "USAGE: stress (not yet implemented)\n" + "USAGE: stress_client (not yet implemented)\n" +#endif + "USAGE: -i : itreration time for stress test\n" + "USAGE: -t : number of threads for stress test\n" + "USAGE: -v : number of vis for stress test\n" + "USAGE: -s \n" + "USAGE: server host name or ip address\n" + "USAGE: [-R ]\n" + "USAGE: (BE == QOS_BEST_EFFORT - Default )\n" + "USAGE: (HT == QOS_HIGH_THROUGHPUT))\n" + "USAGE: (LL == QOS_LOW_LATENCY)\n" + "USAGE: (EC == QOS_ECONOMY)\n" + "USAGE: (PM == QOS_PREMIUM)\n" + "NOTE: iter_num is just for stress_client test, default 100000\n" + "NOTE: Server_name must be specified for connmgt_client, \n" + " NS and unsupp function feature.\n" + "NOTE: if test cases are not specified, test all cases in that function\n" + " feature. else just test the specified cases\n" + }; + + DT_Mdep_printf ("USAGE: -------FFT TEST------------\n"); + DT_Mdep_printf ("%s\n", usage); +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_getopt.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_getopt.c new file mode 100644 index 00000000..ccdc2bbd --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_getopt.c @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define GETOPT_MAGIC 0x04030201 + +#define BADCH (int)'?' +#define BADARG (int)':' +#define EMSG "" + +/** + * Initialize the getopt fields in preparation for parsing + * a command line. + */ +void +DT_mygetopt_init (mygetopt_t * opts) +{ + opts->magic = GETOPT_MAGIC; + opts->place = EMSG; + opts->opterr = 1; + opts->optind = 1; + opts->optopt = 0; + opts->optarg = 0; +} + +/** + * Parse command line arguments. + * + * Return either the option discovered, or + * (int) -1 when there are no more options + * (int) '?' when an illegal option is found + * (int) ':' when a required option argument is missing. + */ +int +DT_mygetopt_r (int argc, char *const * argv, + const char *ostr, mygetopt_t * opts) +{ + char *p; + char *oli; /* option letter list index */ + if (GETOPT_MAGIC != opts->magic) + { + DT_Mdep_printf ("%s: getopt warning: " + "option argument is not properly initialized.\n", + argc > 0 ? argv[0] : "unknown command"); + DT_mygetopt_init (opts); + } + if (!* (opts->place)) /* update scanning pointer */ + { + if ((opts->optind) >= argc || + * ((opts->place) = argv[ (opts->optind)]) != '-') + { + (opts->place) = EMSG; + return (EOF); + } + if ((opts->place)[0] != '-') + { + /* Invalid 1st argument */ + return (BADCH); + } + if ((opts->place)[1] && *++ (opts->place) == '-') + { + /* found "--" which is an invalid option */ + ++ (opts->optind); + (opts->place) = EMSG; + return (BADCH); + } + } /* option letter okay? */ + opts->optopt = (int) * (opts->place)++; + oli = strchr (ostr, (opts->optopt)); + if (opts->optopt == (int) ':' || ! oli) + { + /* + * if the user didn't specify '-' as an option, assume it means EOF. + */ + if ((opts->optopt) == (int) '-') + { + /* return (EOF); */ + return (BADCH); + } + if (!* (opts->place)) + { + ++ (opts->optind); + } + if ((opts->opterr) && *ostr != ':') + { + p = strchr (*argv, '/'); + if (!p) + { + p = *argv; + } + else + { + ++p; + } + + if (opts->optopt != '?') /* Anything but '?' needs error */ + { + DT_Mdep_printf ("%s: Illegal option -- %c\n", + p, (opts->optopt)); + } + } + return (BADCH); + } + if (*++oli != ':') /* don't need argument */ + { + (opts->optarg) = NULL; + if (!* (opts->place)) + { + ++ (opts->optind); + } + } + else /* need an argument */ + { + if (* (opts->place)) /* no white space */ + { + (opts->optarg) = (opts->place); + } + else + { + if (argc <= ++ (opts->optind)) /* no arg */ + { + (opts->place) = EMSG; + if (*ostr == ':') + { + return (BADARG); + } + p = strchr (*argv, '/'); + if (!p) + { + p = *argv; + } + else + { + ++p; + } + if ((opts->opterr)) + { + DT_Mdep_printf ( + "%s: option requires an argument -- %c\n", + p, (opts->optopt)); + } + return (BADCH); + } + else /* white space */ + { + (opts->optarg) = argv[ (opts->optind)]; + } + } + (opts->place) = EMSG; + ++ (opts->optind); + } + return (opts->optopt); /* dump back option letter */ +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_limit_cmd.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_limit_cmd.c new file mode 100644 index 00000000..cd55f50f --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_limit_cmd.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* --------------------------------------------------- */ +void +DT_Limit_Cmd_Init (Limit_Cmd_t * cmd) +{ + memset ((void *) cmd, 0, sizeof (Limit_Cmd_t)); + cmd->ReliabilityLevel = DAT_QOS_BEST_EFFORT; + cmd->width = 1; + cmd->maximum = ~0U; +} + +/* --------------------------------------------------- */ +bool +DT_Limit_Cmd_Parse ( Limit_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts) +{ + int c; + int i; + + for (;;) + { + c = DT_mygetopt_r (my_argc, my_argv, "dm:w:D:R:", opts); + if (c == EOF) + { + break; + } + switch (c) + { + case 'D': /* device name */ + { + strcpy (cmd->device_name, opts->optarg); + break; + } + case 'R': /* Service Reliability Level */ + { + cmd->ReliabilityLevel = DT_ParseQoS (opts->optarg); + break; + } + case 'd': /* print debug messages */ + { + DT_dapltest_debug++; + cmd->debug = true; + break; + } + case 'm': /* maximum for exhaustion testing */ + { + unsigned int len = strspn (opts->optarg, "0123456789"); + + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -m option\n"); + DT_Limit_Cmd_Usage (); + return (false); + } + cmd->maximum = atol (opts->optarg); + break; + } + case 'w': /* width (number of {ia,evd,ep,...} sets) */ + { + unsigned int len = strspn (opts->optarg, "0123456789"); + + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -w option\n"); + DT_Limit_Cmd_Usage (); + return (false); + } + cmd->width = atol (opts->optarg); + break; + } + case '?': + default: + { + DT_Mdep_printf ("Invalid Limit Test Parameter: %c\n", c); + DT_Limit_Cmd_Usage (); + return (false); + } + } + } + if (cmd->device_name[0] == '\0') + { + if (!DT_Mdep_GetDefaultDeviceName (cmd->device_name)) + { + DT_Mdep_printf ("can't get default device name\n"); + DT_Limit_Cmd_Usage (); + return (false); + } + } + + /* + * by default: test all limit tests + * otherwise: parse the remaining limit test arguments + */ + if (opts->optind == my_argc) + { + for (i = 0; i < LIM_NUM_TESTS; i++) + { + cmd->Test_List[i] = 1; + } + } + else + { + for (i = opts->optind; i < my_argc; i++) + { + + if (strcmp (my_argv[i], "limit_ia") == 0) + { + cmd->Test_List[LIM_IA] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_pz") == 0) + { + cmd->Test_List[LIM_PZ] = 1; + continue; + } +#ifndef __KDAPLTEST__ + if (strcmp (my_argv[i], "limit_cno") == 0) + { + cmd->Test_List[LIM_CNO] = 1; + continue; + } +#endif + if (strcmp (my_argv[i], "limit_evd") == 0) + { + cmd->Test_List[LIM_EVD] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_ep") == 0) + { + cmd->Test_List[LIM_EP] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_rsp") == 0) + { + cmd->Test_List[LIM_RSP] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_psp") == 0) + { + cmd->Test_List[LIM_PSP] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_lmr") == 0) + { + cmd->Test_List[LIM_LMR] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_rpost") == 0) + { + cmd->Test_List[LIM_RPOST] = 1; + continue; + } + if (strcmp (my_argv[i], "limit_size_lmr") == 0) + { + cmd->Test_List[LIM_SIZE_LMR] = 1; + continue; + } + + DT_Mdep_printf ("Cannot find this limit test: %s\n", my_argv[i]); + DT_Limit_Cmd_Usage (); + return (false); + + } /* end foreach remaining argv */ + } + + return (true); +} + +/* --------------------------------------------------- */ +void +DT_Limit_Cmd_Usage (void) +{ + DT_Mdep_printf ("USAGE: ---- LIMIT TEST ----\n"); + DT_Mdep_printf ("USAGE: dapltest -T L\n"); + DT_Mdep_printf ("USAGE: [-D ]\n"); + DT_Mdep_printf ("USAGE: [-d] : debug (zero)\n"); + DT_Mdep_printf ("USAGE: [-w ]\n"); + DT_Mdep_printf ("USAGE: [-m ]\n"); + DT_Mdep_printf ("USAGE: [-R ]\n"); + DT_Mdep_printf ("USAGE: (BE == QOS_BEST_EFFORT - Default)\n"); + DT_Mdep_printf ("USAGE: (HT == QOS_HIGH_THROUGHPUT)\n"); + DT_Mdep_printf ("USAGE: (LL == QOS_LOW_LATENCY)\n"); + DT_Mdep_printf ("USAGE: (EC == QOS_ECONOMY)\n"); + DT_Mdep_printf ("USAGE: (PM == QOS_PREMIUM)\n"); + DT_Mdep_printf ("USAGE: [limit_ia [limit_pz] [limit_evd] ... ]\n"); + DT_Mdep_printf ("NOTE: If test is not specified, do all the limit tests\n"); + DT_Mdep_printf ("NOTE: Else, just do the specified tests\n"); + DT_Mdep_printf ("NOTE: Each test is separated by space, the test can be:\n"); + + DT_Mdep_printf ("NOTE: [limit_ia] test max num of open IAs\n"); + DT_Mdep_printf ("NOTE: [limit_pz] test max num of PZs\n"); +#ifndef __KDAPLTEST__ + DT_Mdep_printf ("NOTE: [limit_cno] test max num of CNOs\n"); +#endif + DT_Mdep_printf ("NOTE: [limit_evd] test max num of EVDs\n"); + DT_Mdep_printf ("NOTE: [limit_rsp] test max num of RSPs\n"); + DT_Mdep_printf ("NOTE: [limit_psp] test max num of PSPs\n"); + DT_Mdep_printf ("NOTE: [limit_ep] test max num of EPs\n"); + DT_Mdep_printf ("NOTE: [limit_lmr] test max num of LMRs\n"); + DT_Mdep_printf ("NOTE: [limit_rpost] test max num of recvs posted\n"); + DT_Mdep_printf ("NOTE: [limit_size_lmr] test max size of LMR\n"); +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_main.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_main.c new file mode 100644 index 00000000..f16eb2b4 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_main.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* Main Entry Point */ +int +main (int argc, char *argv[]) +{ + return ( dapltest (argc, argv) ); +} + + +/* + * dapltest main program + */ +int +dapltest (int argc, char *argv[]) +{ + Params_t *params_ptr; + + /* check memory leaking */ + /* + * DT_Mdep_LockInit(&Alloc_Count_Lock); alloc_count = 0; + */ + +#if defined(_WIN32) || defined(_WIN64) + { + /* Cannot be done from DT_Mdep_Init as dapl_init makes some socket + * calls....So need to do this before calling dapl_init */ + WSADATA wsaData; + int i; + + i = WSAStartup (MAKEWORD(2,2), &wsaData); + if ( i != 0 ) { + printf("%s WSAStartup(2.2) failed? (0x%x)\n",argv[0],i); + exit(1); + } + } +#endif + +#ifdef GPROF + { + extern void dapl_init (void); + dapl_init (); + } +#endif + DT_dapltest_debug = 0; + + params_ptr = (Params_t *) DT_Mdep_Malloc (sizeof (Params_t)); + if (!params_ptr) + { + DT_Mdep_printf ("Cannot allocate memory for Params structure\n"); + return ( 1 ); + } + + DT_Tdep_Init (); /* init (kdapl/udapl)test-dependent code */ + DT_Endian_Init (); /* init endian of local machine */ + DT_Mdep_Init (); /* init OS, libraries, etc. */ + + params_ptr->local_is_little_endian = DT_local_is_little_endian; + + /* + * parse command line arguments + */ + if (!DT_Params_Parse (argc, argv, params_ptr)) + { + DT_Mdep_printf ("Command line syntax error\n"); + return 1; + } + params_ptr->cpu_mhz = DT_Mdep_GetCpuMhz (); + /* call the test-dependent code for invoking the actual test */ + DT_Tdep_Execute_Test (params_ptr); + + /* cleanup */ + + DT_Mdep_End (); + DT_Mdep_Free (params_ptr); + DT_Tdep_End (); +#ifdef GPROF + { + extern void dapl_fini (void); + dapl_fini (); + } +#endif + + /* + * check memory leaking DT_Mdep_printf("App allocated Memory left: %d\n", + * alloc_count); DT_Mdep_LockDestroy(&Alloc_Count_Lock); + */ + + return ( 0 ); +} + + +void +Dapltest_Main_Usage (void) +{ + DT_Mdep_printf ("USAGE:\n"); + DT_Mdep_printf ("USAGE: dapltest -T [test-specific args]\n"); + DT_Mdep_printf ("USAGE: where \n"); + DT_Mdep_printf ("USAGE: S = Run as a server\n"); + DT_Mdep_printf ("USAGE: T = Transaction Test\n"); + DT_Mdep_printf ("USAGE: Q = Quit Test\n"); + DT_Mdep_printf ("USAGE: P = Performance Test\n"); + DT_Mdep_printf ("USAGE: L = Limit Test\n"); + DT_Mdep_printf ("USAGE: F = FFT Test\n"); + DT_Mdep_printf ("USAGE:\n"); + DT_Mdep_printf ("NOTE:\tRun as server taking defaults (dapltest -T S)\n"); + DT_Mdep_printf ("NOTE: dapltest\n"); + DT_Mdep_printf ("NOTE:\n"); + DT_Mdep_printf ("NOTE:\tdapltest arguments may be supplied in a script file\n"); + DT_Mdep_printf ("NOTE:\tdapltest -f \n"); + DT_Mdep_printf ("USAGE:\n"); +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_netaddr.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_netaddr.c new file mode 100644 index 00000000..c1eadbf6 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_netaddr.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +DAT_IA_ADDRESS_PTR +DT_NetAddrAlloc (Per_Test_Data_t * pt_ptr) +{ + DAT_IA_ADDRESS_PTR netaddr; + + netaddr = (DAT_IA_ADDRESS_PTR) DT_Mdep_Malloc (sizeof (DAT_SOCK_ADDR)); + if (!netaddr) + { + DT_Mdep_printf ("dapltest: No Memory to create netaddr!\n"); + } + return netaddr; +} + + +void +DT_NetAddrFree (Per_Test_Data_t * pt_ptr, DAT_IA_ADDRESS_PTR netaddr) +{ + DT_Mdep_Free (netaddr); +} + + +bool +DT_NetAddrLookupHostAddress (DAT_IA_ADDRESS_PTR to_netaddr, + DAT_NAME_PTR hostname) +{ + struct addrinfo *target; + int rval; + + rval = getaddrinfo (hostname, NULL, NULL, &target); + if (rval != 0) + { + char *whatzit = "unknown error return"; + + switch (rval) + { + case EAI_FAMILY: + { + whatzit = "unsupported address family"; + break; + } + case EAI_SOCKTYPE: + { + whatzit = "unsupported socket type"; + break; + } + case EAI_BADFLAGS: + { + whatzit = "invalid flags"; + break; + } + case EAI_NONAME: + { + whatzit = "unknown node name"; + break; + } + case EAI_SERVICE: + { + whatzit = "service unavailable"; + break; + } +#if !defined(WIN32) + case EAI_ADDRFAMILY: + { + whatzit = "node has no address in this family"; + break; + } + case EAI_NODATA: + { + whatzit = "node has no addresses defined"; + break; + } +#endif + case EAI_MEMORY: + { + whatzit = "out of memory"; + break; + } + case EAI_FAIL: + { + whatzit = "permanent name server failure"; + break; + } + case EAI_AGAIN: + { + whatzit = "temporary name server failure"; + break; + } +#if !defined(WIN32) + case EAI_SYSTEM: + { + whatzit = "system error"; + break; + } +#endif + } + + DT_Mdep_printf ("getaddrinfo (%s) failed (%s)\n", + hostname, whatzit); + return DAT_FALSE; + } + + /* Pull out IP address and print it as a sanity check */ + rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr; + DT_Mdep_printf ("Server Name: %s \n", hostname); + DT_Mdep_printf ("Server Net Address: %d.%d.%d.%d\n", + (rval >> 0) & 0xff, + (rval >> 8) & 0xff, + (rval >> 16) & 0xff, + (rval >> 24) & 0xff); + + *to_netaddr = * ((DAT_IA_ADDRESS_PTR) target->ai_addr); + + return ( DAT_TRUE ); +} + diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_params.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_params.c new file mode 100644 index 00000000..e0df9805 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_params.c @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define MAX_ARGC 500 +#define MAX_ARG_LEN 100 + + +/* Parse command line arguments */ +bool +DT_Params_Parse (int argc, char *argv[], Params_t * params_ptr) +{ + Server_Cmd_t *Server_Cmd; + Transaction_Cmd_t *Transaction_Cmd; + Quit_Cmd_t *Quit_Cmd; + Limit_Cmd_t *Limit_Cmd; + Performance_Cmd_t *Performance_Cmd; + FFT_Cmd_t *FFT_Cmd; + + char *filename; + FILE *fd; + mygetopt_t opts; + int c; + char *cp; + char *sp; + char line[256]; + char *my_argv[MAX_ARGC]; + int my_argc; + int i; + DT_mygetopt_init (&opts); + opts.opterr = 0; /* turn off automatical error handler */ + + fd = 0; + my_argc = 0; + for (i = 0; i < MAX_ARGC; i++) + { + my_argv[i] = NULL; + } + + /* dapltest with no arguments means run as a server with default values */ + if (argc == 1) + { + params_ptr->test_type = SERVER_TEST; + params_ptr->ReliabilityLevel = DAT_QOS_BEST_EFFORT; + Server_Cmd = ¶ms_ptr->u.Server_Cmd; + DT_Server_Cmd_Init (Server_Cmd); + if (!DT_Mdep_GetDefaultDeviceName (Server_Cmd->dapl_name)) + { + DT_Mdep_printf ("can't get default device name\n"); + return false; + } + return true; + } + /* check for a script file */ + if (strncmp (argv[1], "-f", 2) == 0) + { + if (argc == 2) /* dapltest -fdata */ + { + filename = argv[1] + 2; + } + else + { + if (argc == 3 && strcmp (argv[1], "-f") == 0) /* dapltest -f data */ + { + filename = argv[2]; + } + else + { + DT_Mdep_printf ("-f allows no additional options\n"); + goto main_usage; + } + } + + if (!filename || strlen (filename) == 0) + { + DT_Mdep_printf ("Missing with -f option\n"); + goto main_usage; + } + /* read the script file and create a fake argc, argv */ + fd = fopen (filename, "r"); + if (fd == 0) + { + DT_Mdep_printf ("Cannot open script file: %s\n", filename); + goto main_usage; + } + my_argc = 1; + my_argv[0] = DT_Mdep_Malloc (MAX_ARG_LEN); + if (!my_argv[0]) + { + DT_Mdep_printf ("No Memory\n"); + goto error_return; + } + strcpy (my_argv[0], argv[0]); + while (fgets (&line[0], 256, fd)) + { + sp = &line[0]; + for (;;) + { + cp = strtok (sp, " \t\n"); + sp = 0; /* so can continue to parse this string */ + if (!cp) /* no more token found */ + { + break; + } + if (*cp == '#') /* Comment; go to next line. */ + { + break; + } + my_argv[my_argc] = DT_Mdep_Malloc (MAX_ARG_LEN); + if (!my_argv[my_argc]) + { + DT_Mdep_printf ("No Memory\n"); + goto error_return; + } + strcpy (my_argv[my_argc], cp); + my_argc++; + } + } + } + else + { + my_argc = argc; + for (i = 0; i < argc; i++) + { + my_argv[i] = argv[i]; + } + } + +#if 0 + for (i = 0; i < my_argc; i++) + { + DT_Mdep_printf ("ARG %s\n", my_argv[i]); + } + exit (1); +#endif + + /* get test type - which must be the first arg */ + c = DT_mygetopt_r (my_argc, my_argv, "T:", &opts); + if (c != 'T') + { + DT_Mdep_printf ("Must Specify Test (-T) option first\n"); + goto main_usage; + } + if ((opts.optarg == 0) || strlen (opts.optarg) == 0 || *opts.optarg == '-') + { + DT_Mdep_printf ("Must specify test type\n"); + goto main_usage; + } + switch (*opts.optarg) + { + case 'S': /* Server Test */ + { + params_ptr->test_type = SERVER_TEST; + Server_Cmd = ¶ms_ptr->u.Server_Cmd; + DT_Server_Cmd_Init (Server_Cmd); + if (!DT_Server_Cmd_Parse ( Server_Cmd, + my_argc, my_argv, &opts)) + { + goto error_return; + } + params_ptr->ReliabilityLevel = Server_Cmd->ReliabilityLevel; + params_ptr->debug = Server_Cmd->debug; + break; + } + case 'T': /* Transaction Test */ + { + params_ptr->test_type = TRANSACTION_TEST; + Transaction_Cmd = ¶ms_ptr->u.Transaction_Cmd; + DT_Transaction_Cmd_Init (Transaction_Cmd); + if (!DT_Transaction_Cmd_Parse ( Transaction_Cmd, + my_argc, my_argv, &opts)) + { + goto error_return; + } + params_ptr->ReliabilityLevel = Transaction_Cmd->ReliabilityLevel; + params_ptr->debug = Transaction_Cmd->debug; + DT_NetAddrLookupHostAddress (¶ms_ptr->server_netaddr, + Transaction_Cmd->server_name); + break; + } + case 'Q': /* Quit server Test */ + { + params_ptr->test_type = QUIT_TEST; + Quit_Cmd = ¶ms_ptr->u.Quit_Cmd; + DT_Quit_Cmd_Init (Quit_Cmd); + if (!DT_Quit_Cmd_Parse ( Quit_Cmd, + my_argc, my_argv, &opts)) + { + goto error_return; + } + params_ptr->ReliabilityLevel = Quit_Cmd->ReliabilityLevel; + params_ptr->debug = Quit_Cmd->debug; + DT_NetAddrLookupHostAddress (¶ms_ptr->server_netaddr, + Quit_Cmd->server_name); + break; + } + case 'L': /* Limit Test */ + { + params_ptr->test_type = LIMIT_TEST; + Limit_Cmd = ¶ms_ptr->u.Limit_Cmd; + DT_Limit_Cmd_Init (Limit_Cmd); + if (!DT_Limit_Cmd_Parse (Limit_Cmd,my_argc, my_argv, &opts)) + { + goto error_return; + } + params_ptr->ReliabilityLevel = Limit_Cmd->ReliabilityLevel; + params_ptr->debug = Limit_Cmd->debug; + break; + } + case 'P': /* Performance Test */ + { + params_ptr->test_type = PERFORMANCE_TEST; + Performance_Cmd = ¶ms_ptr->u.Performance_Cmd; + + if (!DT_Performance_Cmd_Init (Performance_Cmd)) + { + goto error_return; + } + + if (!DT_Performance_Cmd_Parse ( Performance_Cmd, + my_argc, my_argv, &opts)) + { + goto error_return; + } + + params_ptr->ReliabilityLevel = Performance_Cmd->qos; + params_ptr->debug = Performance_Cmd->debug; + DT_NetAddrLookupHostAddress (¶ms_ptr->server_netaddr, + Performance_Cmd->server_name); + break; + } + case 'F': + { + params_ptr->test_type = FFT_TEST; + FFT_Cmd = ¶ms_ptr->u.FFT_Cmd; + DT_FFT_Cmd_Init (FFT_Cmd); + if (!DT_FFT_Cmd_Parse (FFT_Cmd, my_argc, my_argv, &opts)) + { + goto error_return; + } + params_ptr->ReliabilityLevel = FFT_Cmd->ReliabilityLevel; + DT_NetAddrLookupHostAddress (¶ms_ptr->server_netaddr, + FFT_Cmd->server_name); + params_ptr->debug = false; + break; + } + + default: + { + DT_Mdep_printf ("Invalid Test Type\n"); + goto main_usage; + } + } + + if (fd) + { + for (i = 0; i < my_argc; i++) + { + DT_Mdep_Free (my_argv[i]); + } + fclose (fd); + } + return true; + +main_usage: + Dapltest_Main_Usage (); +error_return: + if (fd) + { + for (i = 0; i < my_argc; i++) + { + DT_Mdep_Free (my_argv[i]); + } + fclose (fd); + } + return false; +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_performance_cmd.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_performance_cmd.c new file mode 100644 index 00000000..4423d32d --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_performance_cmd.c @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +static void +DT_Performance_Cmd_Usage (void) +{ + DT_Mdep_printf ("USAGE: ---- PERFORMANCE TEST ----\n"); + DT_Mdep_printf ("USAGE: dapltest -T P\n"); + DT_Mdep_printf ("USAGE: -s \n"); + DT_Mdep_printf ("USAGE: [-m b|p]\n"); + DT_Mdep_printf ("USAGE: [-D ]\n"); + DT_Mdep_printf ("USAGE: [-d] : debug (zero)\n"); + DT_Mdep_printf ("USAGE: [-i ] : (1, 000)\n"); + DT_Mdep_printf ("USAGE: [-p ]\n"); + DT_Mdep_printf ("USAGE: [-R ]\n"); + DT_Mdep_printf ("USAGE: (BE == QOS_BEST_EFFORT - Default)\n"); + DT_Mdep_printf ("USAGE: (HT == QOS_HIGH_THROUGHPUT)\n"); + DT_Mdep_printf ("USAGE: (LL == QOS_LOW_LATENCY)\n"); + DT_Mdep_printf ("USAGE: (EC == QOS_ECONOMY)\n"); + DT_Mdep_printf ("USAGE: (PM == QOS_PREMIUM)\n"); + DT_Mdep_printf ("USAGE: \n"); + DT_Mdep_printf ("USAGE:\n"); + DT_Mdep_printf ("USAGE: Each OP consists of:\n"); + DT_Mdep_printf ("USAGE: : \"RR\" (RDMA READ)\n"); + DT_Mdep_printf ("USAGE: : \"RW\" (RDMA WRITE)\n"); + DT_Mdep_printf ("USAGE: [seg_size [num_segs] ] : (4096, 1)\n"); +} + +static bool +DT_Performance_Cmd_Parse_Op ( + Performance_Cmd_t * cmd, + int index, + int my_argc, + char **my_argv) +{ + int i; + + /* + * Op Format: [seg_size] [num_segs] + */ + + if ( index == my_argc ) + { + DT_Mdep_printf ("Operation Missing Transfer Type\n"); + return (false); + } + + for ( i = 0; index < my_argc; i++, index++ ) + { + switch ( i ) + { + case 0: + { + if ( 0 == strncmp (my_argv[index], "RR", strlen ("RR")) ) + { + cmd->op.transfer_type = RDMA_READ; + } + else if ( 0 == strncmp (my_argv[index], "RW", strlen ("RW")) ) + { + cmd->op.transfer_type = RDMA_WRITE; + } + else + { + DT_Mdep_printf ("OP type must be \n"); + return (false); + } + break; + } + case 1: + { + cmd->op.seg_size = atoi (my_argv[index]); + break; + } + case 2: + { + cmd->op.num_segs = atoi (my_argv[index]); + break; + } + default: + { + DT_Mdep_printf ("Too many OP args\n"); + return (false); + } + } + } + + return (true); +} + + +static bool +DT_Performance_Cmd_Validate ( + Performance_Cmd_t *cmd) +{ + if ( '\0' == cmd->server_name[0] ) + { + DT_Mdep_printf ("Must specify server_name in command line or scriptfile\n"); + return (false); + } + + if ( '\0' == cmd->dapl_name[0] ) + { + DT_Mdep_printf ("Must specify device_name in command line or scriptfile\n"); + return (false); + } + + if ( 0 == cmd->pipeline_len ) + { + DT_Mdep_printf ("Pipeline size must not be 0\n"); + return (false); + } + + if ( cmd->debug ) + { + DT_Performance_Cmd_Print (cmd); + } + + return true; +} + +bool +DT_Performance_Cmd_Parse ( + Performance_Cmd_t *cmd, + int my_argc, + char **my_argv, + mygetopt_t *opts) +{ + int c; + unsigned int len; + + for (;;) + { + c = DT_mygetopt_r (my_argc, my_argv, "D:dm:i:p:R:s:", opts); + + if ( EOF == c ) + { + break; + } + + switch ( c ) + { + case 'D': /* device name */ + { + strncpy (cmd->dapl_name, opts->optarg, NAME_SZ); + break; + } + case 'd': /* print debug messages */ + { + DT_dapltest_debug++; + cmd->debug = true; + break; + } + case 'm': /* mode */ + { + if ( !strncmp (opts->optarg, "b", strlen ("b")) ) + { + cmd->mode = BLOCKING_MODE; + } + else if ( !strncmp (opts->optarg, "p", strlen ("p")) ) + { + cmd->mode = POLLING_MODE; + } + else + { + DT_Mdep_printf ("Syntax Error -m option\n"); + DT_Performance_Cmd_Usage (); + return (false); + } + + break; + } + case 'i': /* num iterations */ + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -i option\n"); + DT_Performance_Cmd_Usage (); + return (false); + } + cmd->num_iterations = atol (opts->optarg); + break; + } + case 'p': /* pipline size */ + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -p option\n"); + DT_Performance_Cmd_Usage (); + return (false); + } + cmd->pipeline_len = atol (opts->optarg); + break; + } + case 'R': /* Service Reliability Level */ + { + cmd->qos = DT_ParseQoS (opts->optarg); + break; + } + case 's': /* server name */ + { + if ((opts->optarg == 0) || + strlen (opts->optarg) == 0 || + *opts->optarg == '-') + { + DT_Mdep_printf ("must specify server name\n"); + DT_Performance_Cmd_Usage (); + return (false); + } + + strncpy (cmd->server_name, opts->optarg, NAME_SZ); + break; + } + default: + { + DT_Mdep_printf ("Invalid Performance Test Parameter: %c\n", c); + DT_Performance_Cmd_Usage (); + return (false); + } + } + } + + /* + * now parse the op + */ + if ( !DT_Performance_Cmd_Parse_Op (cmd, opts->optind, my_argc, my_argv) ) + { + DT_Performance_Cmd_Usage (); + return (false); + } + + if ( !DT_Performance_Cmd_Validate (cmd) ) + { + DT_Performance_Cmd_Usage (); + return (false); + } + + return (true); +} + + +bool +DT_Performance_Cmd_Init (Performance_Cmd_t * cmd) +{ + memset (cmd, 0, sizeof (Performance_Cmd_t)); + cmd->dapltest_version = DAPLTEST_VERSION; + cmd->client_is_little_endian = DT_local_is_little_endian; + cmd->qos = DAT_QOS_BEST_EFFORT; + cmd->debug = false; + cmd->num_iterations = 1000; + cmd->pipeline_len = ~0; + + cmd->op.transfer_type = RDMA_WRITE; + cmd->op.seg_size = 4096; + cmd->op.num_segs = 1; + + if ( !DT_Mdep_GetDefaultDeviceName (cmd->dapl_name) ) + { + DT_Mdep_printf ("can't get default device name\n"); + return (false); + } + + return (true); +} + +void +DT_Performance_Cmd_Print ( + Performance_Cmd_t *cmd) +{ + DT_Mdep_printf ("-------------------------------------\n"); + DT_Mdep_printf ("PerfCmd.server_name : %s\n", + cmd->server_name); + DT_Mdep_printf ("PerfCmd.dapl_name : %s\n", + cmd->dapl_name); + DT_Mdep_printf ("PerfCmd.mode : %s\n", + (cmd->mode == BLOCKING_MODE) ? "BLOCKING" : "POLLING"); + DT_Mdep_printf ("PerfCmd.num_iterations : %d\n", + cmd->num_iterations); + DT_Mdep_printf ("PerfCmd.pipeline_len : %d\n", + cmd->pipeline_len); + DT_Mdep_printf ("PerfCmd.op.transfer_type : %s\n", + cmd->op.transfer_type == RDMA_READ ? "RDMA_READ" : + cmd->op.transfer_type == RDMA_WRITE ? "RDMA_WRITE" : + "SEND_RECV"); + DT_Mdep_printf ("PerfCmd.op.num_segs : %d\n", + cmd->op.num_segs); + DT_Mdep_printf ("PerfCmd.op.seg_size : %d\n", + cmd->op.seg_size); +} + diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_qos_util.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_qos_util.c new file mode 100644 index 00000000..677f5b6d --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_qos_util.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* + * Parse a QOS argument into a DAT_QOS. + * + * Returns no errors: defaults to best effort. + */ +DAT_QOS +DT_ParseQoS (char *arg) +{ + if (0 == strcmp (arg, "HT")) + { + return ( DAT_QOS_HIGH_THROUGHPUT ); + } + + if (0 == strcmp (arg, "LL")) + { + return ( DAT_QOS_LOW_LATENCY ); + } + + if (0 == strcmp (arg, "EC")) + { + return ( DAT_QOS_ECONOMY ); + } + + if (0 == strcmp (arg, "PM")) + { + return ( DAT_QOS_PREMIUM ); + } + /* + * Default to "BE" so no point in checking further + */ + return ( DAT_QOS_BEST_EFFORT ); +} + diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_quit_cmd.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_quit_cmd.c new file mode 100644 index 00000000..084a7b28 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_quit_cmd.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/*--------------------------------------------------------- */ +void +DT_Quit_Cmd_Init (Quit_Cmd_t * cmd) +{ + memset ((void *)cmd, 0, sizeof (Quit_Cmd_t)); + cmd->ReliabilityLevel = DAT_QOS_BEST_EFFORT; +} + +/*--------------------------------------------------------- */ +bool +DT_Quit_Cmd_Parse (Quit_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts) +{ + int c; + + for (;;) + { + c = DT_mygetopt_r (my_argc, my_argv, "ds:D:R:", opts); + if (c == EOF) + { + break; + } + switch (c) + { + case 'D': + { + strcpy (cmd->device_name, opts->optarg); + break; + } + case 's': + { + strcpy (cmd->server_name, opts->optarg); + break; + } + case 'd': /* print debug messages */ + { + DT_dapltest_debug++; + cmd->debug = true; + break; + } + case 'R': /* Service Reliability Level */ + { + cmd->ReliabilityLevel= DT_ParseQoS (opts->optarg); + break; + } + case '?': + default: + { + DT_Mdep_printf ("Invalid Quit option: %c\n", opts->optopt); + DT_Quit_Cmd_Usage (); + return (false); + } + } + } + if (cmd->device_name[0] == '\0') + { + if (!DT_Mdep_GetDefaultDeviceName (cmd->device_name)) + { + DT_Mdep_printf ("can't get default device name\n"); + DT_Quit_Cmd_Usage (); + return (false); + } + } + if (!DT_Quit_Cmd_Validate (cmd)) + { + DT_Quit_Cmd_Usage (); + return (false); + } + return (true); +} + +/*------------------------------------------------------------------------------ */ +bool +DT_Quit_Cmd_Validate (Quit_Cmd_t * cmd) +{ + if (cmd->server_name[0] == '\0') + { + DT_Mdep_printf ("Must specify server_name in command line or scriptfile\n"); + return (false); + } + return (true); +} + +/*--------------------------------------------------------- */ +void +DT_Quit_Cmd_Usage (void) +{ + DT_Mdep_printf ("USAGE: ---- QUIT TEST ----\n"); + DT_Mdep_printf ("USAGE: dapltest -T Q\n"); + DT_Mdep_printf ("USAGE: -s \n"); + DT_Mdep_printf ("USAGE: [-D ]\n"); + DT_Mdep_printf ("USAGE: [-d] : debug (zero)\n"); + DT_Mdep_printf ("USAGE: [-R ]\n"); + DT_Mdep_printf ("USAGE: (BE == QOS_BEST_EFFORT - Default)\n"); + DT_Mdep_printf ("USAGE: (HT == QOS_HIGH_THROUGHPUT)\n"); + DT_Mdep_printf ("USAGE: (LL == QOS_LOW_LATENCY)\n"); + DT_Mdep_printf ("USAGE: (EC == QOS_ECONOMY)\n"); + DT_Mdep_printf ("USAGE: (PM == QOS_PREMIUM)\n"); +} + +/*--------------------------------------------------------- */ +void +DT_Quit_Cmd_Print (Quit_Cmd_t * cmd) +{ + DT_Mdep_printf ("Quit_Cmd.server_name: %s\n", cmd->server_name); + DT_Mdep_printf ("Quit_Cmd.device_name: %s\n", cmd->device_name); +} + diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_server_cmd.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_server_cmd.c new file mode 100644 index 00000000..02799052 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_server_cmd.c @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_Server_Cmd_Init (Server_Cmd_t * Server_Cmd) +{ + DT_dapltest_debug = 0; + Server_Cmd->debug = false; + Server_Cmd->dapl_name[0] = '\0'; + Server_Cmd->ReliabilityLevel = DAT_QOS_BEST_EFFORT; +} + + +bool +DT_Server_Cmd_Parse (Server_Cmd_t * Server_Cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts) +{ + int c; + for (;;) + { + c = DT_mygetopt_r (my_argc, my_argv, "dD:R:", opts); + if (c == EOF) + { + break; + } + switch (c) + { + case 'D': + { + strcpy (Server_Cmd->dapl_name, opts->optarg); + break; + } + case 'd': + { + DT_dapltest_debug++; + Server_Cmd->debug = true; + break; + } + case 'R': /* Service Reliability Level */ + { + Server_Cmd->ReliabilityLevel = DT_ParseQoS (opts->optarg); + if (0 == Server_Cmd->ReliabilityLevel) + { + return (false); + } + break; + } + case '?': + default: + { + DT_Mdep_printf ("Invalid Server option: %c\n", opts->optopt); + DT_Server_Cmd_Usage (); + return (false); + } + } + } + if (Server_Cmd->dapl_name == '\0') + { + if (!DT_Mdep_GetDefaultDeviceName (Server_Cmd->dapl_name)) + { + DT_Mdep_printf ("can't get default device name\n"); + DT_Server_Cmd_Usage (); + return (false); + } + } + return (true); +} + +void +DT_Server_Cmd_Usage (void) +{ + DT_Mdep_printf ("USAGE: ---- SERVER MODE ----\n"); + DT_Mdep_printf ("USAGE: dapltest -T S\n"); + DT_Mdep_printf ("USAGE: [-D ]\n"); + DT_Mdep_printf ("USAGE: [-d] : debug (zero)\n"); + DT_Mdep_printf ("USAGE: [-R ]\n"); + DT_Mdep_printf ("USAGE: (BE == QOS_BEST_EFFORT - Default)\n"); + DT_Mdep_printf ("USAGE: (HT == QOS_HIGH_THROUGHPUT)\n"); + DT_Mdep_printf ("USAGE: (LL == QOS_LOW_LATENCY)\n"); + DT_Mdep_printf ("USAGE: (EC == QOS_ECONOMY)\n"); + DT_Mdep_printf ("USAGE: (PM == QOS_PREMIUM)\n"); + DT_Mdep_printf ("USAGE: Run as server using default parameters\n"); + DT_Mdep_printf ("USAGE: dapltest\n"); + DT_Mdep_printf ("USAGE:\n"); +} + +void +DT_Server_Cmd_Print (Server_Cmd_t * Server_Cmd) +{ + DT_Mdep_printf ("Server_Cmd.debug: %d\n", Server_Cmd->debug); + DT_Mdep_printf ("Server_Cmd.dapl_name: %s\n", Server_Cmd->dapl_name); +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_test_data.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_test_data.c new file mode 100644 index 00000000..5eb6bab2 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_test_data.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +DAT_COUNT DT_dapltest_debug = 0; +/* + * check memory leaking int alloc_count; DT_Mdep_LockType + * Alloc_Count_Lock; + */ + +Per_Test_Data_t * +DT_Alloc_Per_Test_Data (void) +{ + Per_Test_Data_t *pt_ptr; + pt_ptr = 0; + + pt_ptr = DT_Mdep_Malloc (sizeof (Per_Test_Data_t)); + if (!pt_ptr) + { + DT_Mdep_printf ("No Memory to create per_test_data!\n"); + } + + return (pt_ptr); +} + +void +DT_Free_Per_Test_Data (Per_Test_Data_t * pt_ptr) +{ + DT_Mdep_Free (pt_ptr); +} diff --git a/trunk/ulp/dapl2/test/dapltest/cmd/dapl_transaction_cmd.c b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_transaction_cmd.c new file mode 100644 index 00000000..6fdca06e --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/cmd/dapl_transaction_cmd.c @@ -0,0 +1,500 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +static bool +DT_Transaction_Cmd_Parse_Op (Transaction_Cmd_t * cmd, char *arg) +{ + char *c_ptr; + int op; + if (cmd->num_ops >= MAX_OPS) + { + DT_Mdep_printf ("Client: Too Many Ops - Max %d\n", MAX_OPS); + goto error_return; + } + op = cmd->num_ops; + cmd->num_ops++; + + /* set some defaults */ + cmd->op[op].seg_size = 4096; + cmd->op[op].num_segs = 1; + cmd->op[op].reap_send_on_recv = false; + + /* + * packet format: [seg_size] [num_segs] + */ + c_ptr = strtok (arg, " \t"); + if (!c_ptr) + { + DT_Mdep_printf ("OP first arg must \n"); + goto error_return; + } + /* first token is : */ + if (strcmp (c_ptr, "server") == 0) + { + cmd->op[op].server_initiated = true; + } + else + { + if (strcmp (c_ptr, "client") == 0) + { + cmd->op[op].server_initiated = false; + } + else + { + DT_Mdep_printf ("OP first arg must \n"); + goto error_return; + } + } + + c_ptr = strtok (0, " \t"); + if (!c_ptr) + { + DT_Mdep_printf ("OP Second arg must be \n"); + goto error_return; + } + /* second token is : */ + if (strcmp (c_ptr, "RR") == 0) + { + cmd->op[op].transfer_type = RDMA_READ; + } + else + { + if (strcmp (c_ptr, "RW") == 0) + { + cmd->op[op].transfer_type = RDMA_WRITE; + } + else + { + if (strcmp (c_ptr, "SR") == 0) + { + cmd->op[op].transfer_type = SEND_RECV; + } + else + { + DT_Mdep_printf ("OP Second arg must be \n"); + goto error_return; + } + } + } + /* + * there may or may not be additional parameters... [seg_size] [num_segs] + * [-f] + */ + c_ptr = strtok (0, " \t"); + if (c_ptr && strspn (c_ptr, "0123456789") != 0) + { + cmd->op[op].seg_size = atoi (c_ptr); + c_ptr = strtok (0, " \t"); + } + if (c_ptr && strspn (c_ptr, "0123456789") != 0) + { + cmd->op[op].num_segs = atoi (c_ptr); + c_ptr = strtok (0, " \t"); + } + if (c_ptr && strcmp (c_ptr, "-f") == 0) + { + cmd->op[op].reap_send_on_recv = true; + if (cmd->op[op].transfer_type != SEND_RECV) + { + DT_Mdep_printf ("OP: -f only valid on SEND_RECV\n"); + goto error_return; + } + c_ptr = strtok (0, " \t"); + } + if (c_ptr) + { + DT_Mdep_printf ("OP too many args \n"); + goto error_return; + } + return true; + +error_return: + return false; +} + +static bool +DT_Transaction_Cmd_Validate (Transaction_Cmd_t * cmd) +{ + unsigned int i; + bool has_server_send; + bool has_client_send; + unsigned int reap_count; + has_server_send = false; + has_client_send = false; + reap_count = 0; + + if (cmd->server_name[0] == '\0') + { + DT_Mdep_printf ("Must specify server_name in command line or scriptfile\n"); + return (false); + } + for (i = 0; i < cmd->num_ops; i++) + { + switch (cmd->op[i].transfer_type) + { + case SEND_RECV: + { + if (cmd->op[i].server_initiated) + { + has_server_send = true; + } + else + { + has_client_send = true; + } + if (cmd->op[i].reap_send_on_recv) + { + if (!cmd->op[i].server_initiated) + { + /* client */ + reap_count++; + } + else + { + /* server */ + if (reap_count > 0) + { + reap_count--; + } + else + { + DT_Mdep_printf ("OP: Unbalanced -f options\n"); + return false; + } + } + } + break; + } + case RDMA_READ: + { + break; + } + case RDMA_WRITE: + { + break; + } + } + } + + if (!has_server_send || !has_client_send) + { + DT_Mdep_printf ("Error: Transaction test requires \n"); + DT_Mdep_printf ("Error: At least one server SR and one client SR Operation\n"); + return false; + } + if (reap_count != 0) + { + DT_Mdep_printf ("OP: Unbalanced -f options\n"); + return false; + } + if (cmd->debug) + { + DT_Transaction_Cmd_Print (cmd); + } + return true; +} + + +static void +DT_Transaction_Cmd_Usage (void) +{ + DT_Mdep_printf ("USAGE: ---- TRANSACTION TEST ----\n"); + DT_Mdep_printf ("USAGE: dapltest -T T\n"); + DT_Mdep_printf ("USAGE: -s \n"); + DT_Mdep_printf ("USAGE: [-D ]\n"); + DT_Mdep_printf ("USAGE: [-d] : debug (zero)\n"); + DT_Mdep_printf ("USAGE: [-i ] : (1, 000)\n"); + DT_Mdep_printf ("USAGE: [-t ] : (1)\n"); + DT_Mdep_printf ("USAGE: [-w ] : (1)\n"); + DT_Mdep_printf ("USAGE: [-V ] : Validate data: (false)\n"); + DT_Mdep_printf ("USAGE: [-P ] : DTO Completion Polling: (false)\n"); + DT_Mdep_printf ("USAGE: [-r ] : Use RSPs: (false)\n"); + DT_Mdep_printf ("USAGE: [-R ]\n"); + DT_Mdep_printf ("USAGE: (BE == QOS_BEST_EFFORT - Default)\n"); + DT_Mdep_printf ("USAGE: (HT == QOS_HIGH_THROUGHPUT)\n"); + DT_Mdep_printf ("USAGE: (LL == QOS_LOW_LATENCY)\n"); + DT_Mdep_printf ("USAGE: (EC == QOS_ECONOMY)\n"); + DT_Mdep_printf ("USAGE: (PM == QOS_PREMIUM)\n"); + DT_Mdep_printf ("USAGE: : \"server\"/\"client\"\n"); + DT_Mdep_printf ("USAGE: : \"SR\" (SEND/RECV)\n"); + DT_Mdep_printf ("USAGE: : \"RR\" (RDMA READ)\n"); + DT_Mdep_printf ("USAGE: : \"RW\" (RDMA WRITE)\n"); + DT_Mdep_printf ("USAGE: [seg_size [num_segs] ] : (4096, 1)\n"); + DT_Mdep_printf ("USAGE: [-f] : Reap sends on recv\n"); + DT_Mdep_printf ("USAGE:\n"); + DT_Mdep_printf ("NOTE: -f is only allowed on \"SR\" OPs\n"); + DT_Mdep_printf ("NOTE: -f must appear in pairs (one client, one server)\n"); + DT_Mdep_printf ( + "NOTE: At least one server SR and one client SR OP are required\n"); + DT_Mdep_printf ( + "NOTE: and use of -V results in the use of three extra OPs\n"); +} + + +void +DT_Transaction_Cmd_Init (Transaction_Cmd_t * cmd) +{ + memset ((void *)cmd, 0, sizeof (Transaction_Cmd_t)); + cmd->dapltest_version = DAPLTEST_VERSION; + cmd->client_is_little_endian = DT_local_is_little_endian; + cmd->num_iterations = 1000; + cmd->num_threads = 1; + cmd->eps_per_thread = 1; + cmd->debug = false; + cmd->validate = false; + cmd->ReliabilityLevel = DAT_QOS_BEST_EFFORT; +} + + +bool +DT_Transaction_Cmd_Parse (Transaction_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts) +{ + int c; + unsigned int len; + int i; + char op[100]; + for (;;) + { + c = DT_mygetopt_r (my_argc, my_argv, "rQVPdw:s:D:i:t:v:R:", opts); + if (c == EOF) + { + break; + } + switch (c) + { + case 's': /* server name */ + { + if ((opts->optarg == 0) || strlen (opts->optarg) == 0 + || *opts->optarg == '-') + { + DT_Mdep_printf ("must specify server name\n"); + DT_Transaction_Cmd_Usage (); + return (false); + } + strcpy (cmd->server_name, opts->optarg); + break; + } + case 'D': /* device name */ + { + strcpy (cmd->dapl_name, opts->optarg); + break; + } + + case 'i': /* num iterations */ + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -i option\n"); + DT_Transaction_Cmd_Usage (); + return (false); + } + cmd->num_iterations = atol (opts->optarg); + + break; + } + case 't': /* num threads */ + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -t option\n"); + DT_Transaction_Cmd_Usage (); + return (false); + } + cmd->num_threads = atol (opts->optarg); + break; + } + case 'w': /* num EPs per thread */ + { + len = strspn (opts->optarg, "0123456789"); + if (len == 0 || len != strlen (opts->optarg)) + { + DT_Mdep_printf ("Syntax Error -w option\n"); + DT_Transaction_Cmd_Usage (); + return (false); + } + cmd->eps_per_thread = atol (opts->optarg); + break; + } + case 'd': /* print debug messages */ + { + DT_dapltest_debug++; + cmd->debug = true; + break; + } + case 'r': /* use RSP instead of PSP */ + { + cmd->use_rsp = true; + break; + } + case 'V': /* validate data being sent/received */ + { + cmd->validate = true; + break; + } + case 'P': /* use completion polling */ + { + cmd->poll = true; + break; + } + case 'R': /* Service Reliability Level */ + { + cmd->ReliabilityLevel = DT_ParseQoS (opts->optarg); + break; + } + case '?': + default: + { + DT_Mdep_printf ("Invalid Transaction Test Parameter: %c\n", c); + DT_Transaction_Cmd_Usage (); + return (false); + } + } + } + if (cmd->dapl_name[0] == '\0') + { + if (!DT_Mdep_GetDefaultDeviceName (cmd->dapl_name)) + { + DT_Mdep_printf ("can't get default device name\n"); + DT_Transaction_Cmd_Usage (); + return (false); + } + } + /* + * now parse the transaction ops this is ugly, but it's easier to gather + * each transaction into a single string + */ + for (i = opts->optind; i < my_argc; i++) + { + strcpy (&op[0], my_argv[i]); + while (i < my_argc - 1) + { + i++; + if ((strncmp (my_argv[i], "client", 6) == 0) || + strncmp (my_argv[i], "server", 6) == 0) + { + i--; + break; + } + strcat (op, " "); + strcat (op, my_argv[i]); + } + if (!DT_Transaction_Cmd_Parse_Op (cmd, op)) + { + DT_Transaction_Cmd_Usage (); + return (false); + } + } + + /* + * If we're going to validate the data, we append 3 OPs that + * serve as barriers so that both the client and server can + * validate their entire set of recv transactions without + * interference. + * + * The first op appended serves to notify the client that the + * server is at the rendezvous and will transfer nothing else, + * so the client can validate all recv buffers. The second op + * notifies the server that the client is quiescent, so the + * server can safely validate its recv buffers. The final op + * tells the client that the server is done, and both can + * proceed with the next iteration. + */ + if (cmd->validate) + { + DT_Mdep_printf ("NOTE: Adding OP \"server SR\" - for validation\n"); + memcpy (op, "server SR", strlen ("server SR") + 1); + DT_Transaction_Cmd_Parse_Op (cmd, op); + + DT_Mdep_printf ("NOTE: Adding OP \"client SR\" - for validation\n"); + memcpy (op, "client SR", strlen ("client SR") + 1); + DT_Transaction_Cmd_Parse_Op (cmd, op); + + DT_Mdep_printf ("NOTE: Adding OP \"server SR\" - for validation\n"); + memcpy (op, "server SR", strlen ("server SR") + 1); + DT_Transaction_Cmd_Parse_Op (cmd, op); + } + if (!DT_Transaction_Cmd_Validate (cmd)) + { + DT_Transaction_Cmd_Usage (); + return (false); + } + return (true); +} + +void +DT_Transaction_Cmd_Print (Transaction_Cmd_t * cmd) +{ + unsigned int i; + DT_Mdep_printf ("-------------------------------------\n"); + DT_Mdep_printf ("TransCmd.server_name : %s\n", + cmd->server_name); + DT_Mdep_printf ("TransCmd.num_iterations : %d\n", + cmd->num_iterations); + DT_Mdep_printf ("TransCmd.num_threads : %d\n", + cmd->num_threads); + DT_Mdep_printf ("TransCmd.eps_per_thread : %d\n", + cmd->eps_per_thread); + DT_Mdep_printf ("TransCmd.validate : %d\n", + cmd->validate); + DT_Mdep_printf ("TransCmd.dapl_name : %s\n", + cmd->dapl_name); + DT_Mdep_printf ("TransCmd.num_ops : %d\n", + cmd->num_ops); + + for (i = 0; i < cmd->num_ops; i++) + { + DT_Mdep_printf ("TransCmd.op[%d].transfer_type : %s %s\n", + i, + cmd->op[i].transfer_type == 0 ? "RDMA_READ" : + cmd->op[i].transfer_type == 1 ? "RDMA_WRITE" : + "SEND_RECV", + cmd->op[i].server_initiated ? " (server)" : " (client)" ); + DT_Mdep_printf ("TransCmd.op[%d].seg_size : %d\n", + i, + cmd->op[i].seg_size); + DT_Mdep_printf ("TransCmd.op[%d].num_segs : %d\n", + i, + cmd->op[i].num_segs); + DT_Mdep_printf ("TransCmd.op[%d].reap_send_on_recv : %d\n", + i, + cmd->op[i].reap_send_on_recv); + } +} + diff --git a/trunk/ulp/dapl2/test/dapltest/common/dapl_endian.c b/trunk/ulp/dapl2/test/dapltest/common/dapl_endian.c new file mode 100644 index 00000000..9c16d1d5 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/common/dapl_endian.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_Endian_Init (void) +{ + int endian; + endian = 1; + DT_local_is_little_endian = * ((unsigned char *) (&endian)) == 1; +} + +/* + * Big/Little Endian conversion functions + */ + +#define c1a32 ((DAT_UINT32)0x00FF00FF) +#define c1b32 ((DAT_UINT32)0xFF00FF00) +#define c2a32 ((DAT_UINT32)0x0000FFFF) +#define c2b32 ((DAT_UINT32)0xFFFF0000) +#define c164 ((DAT_UINT64)0x00FF00FF) +#define c1a64 (c164 | (c164 << 32)) +#define c1b64 (c1a64 << 8) +#define c264 ((DAT_UINT64)0x0000FFFF) +#define c2a64 (c264 | (c264 << 32)) +#define c2b64 (c2a64 << 16) +#define c3a64 ((DAT_UINT64)0xFFFFFFFF) +#define c3b64 (c3a64 << 32) + +DAT_UINT32 +DT_Endian32 (DAT_UINT32 val) +{ + if (DT_local_is_little_endian) + { + return val; + } + val = ((val & c1a32) << 8) | ((val & c1b32) >> 8); + val = ((val & c2a32) << 16) | ((val & c2b32) >> 16); + return (val); +} + +DAT_UINT64 +DT_Endian64 (DAT_UINT64 val) +{ + if (DT_local_is_little_endian) + { + return val; + } + val = ((val & c1a64) << 8) | ((val & c1b64) >> 8); + val = ((val & c2a64) << 16) | ((val & c2b64) >> 16); + val = ((val & c3a64) << 32) | ((val & c3b64) >> 32); + return (val); +} + +DAT_UINT32 +DT_EndianMemHandle (DAT_UINT32 val) +{ + val = ((val & c1a32) << 8) | ((val & c1b32) >> 8); + val = ((val & c2a32) << 16) | ((val & c2b32) >> 16); + return (val); +} + +DAT_UINT64 +DT_EndianMemAddress (DAT_UINT64 val) +{ + DAT_UINT64 val64; + val64 = val; + val64 = ((val64 & c1a64) << 8) | ((val64 & c1b64) >> 8); + val64 = ((val64 & c2a64) << 16) | ((val64 & c2b64) >> 16); + val64 = ((val64 & c3a64) << 32) | ((val64 & c3b64) >> 32); + return val64; +} diff --git a/trunk/ulp/dapl2/test/dapltest/common/dapl_global.c b/trunk/ulp/dapl2/test/dapltest/common/dapl_global.c new file mode 100644 index 00000000..17ccb709 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/common/dapl_global.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +bool DT_local_is_little_endian; +DAT_COUNT DT_dapltest_debug = 0; + + diff --git a/trunk/ulp/dapl2/test/dapltest/common/dapl_performance_cmd_util.c b/trunk/ulp/dapl2/test/dapltest/common/dapl_performance_cmd_util.c new file mode 100644 index 00000000..6be7827b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/common/dapl_performance_cmd_util.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_Performance_Cmd_Endian (Performance_Cmd_t * cmd) +{ + cmd->dapltest_version = DT_Endian32 (cmd->dapltest_version); + cmd->qos = DT_Endian32 (cmd->qos); + cmd->num_iterations = DT_Endian32 (cmd->num_iterations); + cmd->debug = DT_Endian32 (cmd->debug); + + cmd->op.transfer_type = DT_Endian32 (cmd->op.transfer_type); + cmd->op.seg_size = DT_Endian32 (cmd->op.seg_size); + cmd->op.num_segs = DT_Endian32 (cmd->op.num_segs); +} +/* + * * Map Performance_Mode_Type values to readable strings + * */ +const char * +DT_PerformanceModeToString (Performance_Mode_Type mode) +{ + if ( BLOCKING_MODE == mode ) + { + return "blocking"; + } + else if ( POLLING_MODE == mode ) + { + return "polling"; + } + else + { + return "error: unkown mode"; + } +} + diff --git a/trunk/ulp/dapl2/test/dapltest/common/dapl_quit_cmd_util.c b/trunk/ulp/dapl2/test/dapltest/common/dapl_quit_cmd_util.c new file mode 100644 index 00000000..91a7ad1f --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/common/dapl_quit_cmd_util.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/*--------------------------------------------------------- */ +void +DT_Quit_Cmd_Endian (Quit_Cmd_t * cmd, + bool to_wire) +{ + /* do nothing */ +} + diff --git a/trunk/ulp/dapl2/test/dapltest/common/dapl_transaction_cmd_util.c b/trunk/ulp/dapl2/test/dapltest/common/dapl_transaction_cmd_util.c new file mode 100644 index 00000000..8131794c --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/common/dapl_transaction_cmd_util.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + + +void +DT_Transaction_Cmd_Endian (Transaction_Cmd_t * cmd, bool to_wire) +{ + unsigned int i; + + cmd->dapltest_version = DT_Endian32 (cmd->dapltest_version); + cmd->num_iterations = DT_Endian32 (cmd->num_iterations); + cmd->num_threads = DT_Endian32 (cmd->num_threads); + cmd->eps_per_thread = DT_Endian32 (cmd->eps_per_thread); + cmd->use_rsp = DT_Endian32 (cmd->use_rsp); + cmd->debug = DT_Endian32 (cmd->debug); + cmd->validate = DT_Endian32 (cmd->validate); + cmd->ReliabilityLevel = DT_Endian32 (cmd->ReliabilityLevel); + + if (!to_wire) + { + cmd->num_ops = DT_Endian32 (cmd->num_ops); + } + for (i = 0; i < cmd->num_ops; i++) + { + cmd->op[i].server_initiated = DT_Endian32 (cmd->op[i].server_initiated); + cmd->op[i].transfer_type = DT_Endian32 (cmd->op[i].transfer_type); + cmd->op[i].num_segs = DT_Endian32 (cmd->op[i].num_segs); + cmd->op[i].seg_size = DT_Endian32 (cmd->op[i].seg_size); + cmd->op[i].reap_send_on_recv = + DT_Endian32 (cmd->op[i].reap_send_on_recv); + } + if (to_wire) + { + cmd->num_ops = DT_Endian32 (cmd->num_ops); + } +} diff --git a/trunk/ulp/dapl2/test/dapltest/dapltest.rc b/trunk/ulp/dapl2/test/dapltest/dapltest.rc new file mode 100644 index 00000000..c9868290 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/dapltest.rc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id$ + */ + + +#include + +#define VER_FILETYPE VFT_APP +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#if DBG +#define VER_FILEDESCRIPTION_STR "DAPL/DAT test Application (Debug)" +#else +#define VER_FILEDESCRIPTION_STR "DAPL/DAT test Application" +#endif + +#define VER_INTERNALNAME_STR "dapl2test.exe" +#define VER_ORIGINALFILENAME_STR "dapl2test.exe" + +#include diff --git a/trunk/ulp/dapl2/test/dapltest/dt_cmd.c b/trunk/ulp/dapl2/test/dapltest/dt_cmd.c new file mode 100644 index 00000000..29c86b6b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/dt_cmd.c @@ -0,0 +1,14 @@ + +#include "cmd/dapl_main.c" +#include "cmd/dapl_params.c" +#include "cmd/dapl_fft_cmd.c" +#include "cmd/dapl_getopt.c" +#include "cmd/dapl_limit_cmd.c" +#include "cmd/dapl_netaddr.c" +#include "cmd/dapl_performance_cmd.c" +#include "cmd/dapl_qos_util.c" +#include "cmd/dapl_quit_cmd.c" +#include "cmd/dapl_server_cmd.c" +#include "cmd/dapl_transaction_cmd.c" + + diff --git a/trunk/ulp/dapl2/test/dapltest/dt_common.c b/trunk/ulp/dapl2/test/dapltest/dt_common.c new file mode 100644 index 00000000..cecfba2f --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/dt_common.c @@ -0,0 +1,5 @@ +#include "common/dapl_endian.c" +#include "common/dapl_global.c" +#include "common/dapl_performance_cmd_util.c" +#include "common/dapl_quit_cmd_util.c" +#include "common/dapl_transaction_cmd_util.c" diff --git a/trunk/ulp/dapl2/test/dapltest/dt_mdep.c b/trunk/ulp/dapl2/test/dapltest/dt_mdep.c new file mode 100644 index 00000000..bc08e784 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/dt_mdep.c @@ -0,0 +1,3 @@ + +#include "mdep/windows/dapl_mdep_user.c" + diff --git a/trunk/ulp/dapl2/test/dapltest/dt_test.c b/trunk/ulp/dapl2/test/dapltest/dt_test.c new file mode 100644 index 00000000..e4aadab2 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/dt_test.c @@ -0,0 +1,32 @@ +#include "test/dapl_bpool.c" +#include "test/dapl_client.c" +#include "test/dapl_client_info.c" +#include "test/dapl_cnxn.c" +#include "test/dapl_execute.c" +#include "test/dapl_fft_connmgt.c" +#include "test/dapl_fft_endpoint.c" +#include "test/dapl_fft_hwconn.c" +#include "test/dapl_fft_mem.c" +#include "test/dapl_fft_pz.c" +#include "test/dapl_fft_queryinfo.c" +#include "test/dapl_fft_test.c" +#include "test/dapl_fft_util.c" +#include "test/dapl_limit.c" +#include "test/dapl_memlist.c" +#include "test/dapl_performance_client.c" +#include "test/dapl_performance_server.c" +#include "test/dapl_performance_stats.c" +#include "test/dapl_performance_util.c" +#include "test/dapl_quit_util.c" +#include "test/dapl_server.c" +#include "test/dapl_server_info.c" +#include "test/dapl_test_data.c" +#include "test/dapl_test_util.c" +#include "test/dapl_thread.c" +#include "test/dapl_transaction_stats.c" +#include "test/dapl_transaction_test.c" +#include "test/dapl_transaction_util.c" +#include "test/dapl_util.c" +#if defined(DAT_EXPORT_SYMBOLS) +#error DAT_EXPORT? +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/dt_udapl.c b/trunk/ulp/dapl2/test/dapltest/dt_udapl.c new file mode 100644 index 00000000..a0e1eb1a --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/dt_udapl.c @@ -0,0 +1,3 @@ + +#include "udapl/udapl_tdep.c" + diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_bpool.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_bpool.h new file mode 100644 index 00000000..724f6239 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_bpool.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_BPOOL_H__ +#define __DAPL_BPOOL_H__ + +#include "dapl_proto.h" + +typedef struct Bpool_tag Bpool; +/* + * struct Bpool + */ + +struct Bpool_tag +{ + unsigned char *alloc_ptr; + DAT_UINT32 alloc_size; + DAT_PZ_HANDLE pz_handle; + DAT_COUNT seg_size; + DAT_COUNT num_segs; /* num segments */ + unsigned char *buffer_start; /* Start of buffer area */ + DAT_VLEN buffer_size; /* Size of data buffer (rounded) */ + DAT_VADDR reg_addr; /* start of registered area */ + DAT_VLEN reg_size; /* size of registered area */ + DAT_EP_HANDLE ep_handle; /* EP area is registered to */ + DAT_LMR_HANDLE lmr_handle; /* local access */ + DAT_LMR_CONTEXT lmr_context; + DAT_LMR_TRIPLET*tripl_start; /* local IOV */ + DAT_BOOLEAN enable_rdma_write; /* remote access */ + DAT_BOOLEAN enable_rdma_read; + DAT_RMR_HANDLE rmr_handle; + DAT_RMR_CONTEXT rmr_context; + DAT_EVD_HANDLE rmr_evd_handle; +}; +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_client_info.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_client_info.h new file mode 100644 index 00000000..c5f508f0 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_client_info.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_CLIENT_INFO_H__ +#define __DAPL_CLIENT_INFO_H__ + +#include "dapl_proto.h" + +#pragma pack (2) + +typedef struct +{ + DAT_UINT32 dapltest_version; + DAT_UINT32 is_little_endian; + DAT_UINT32 test_type; + DAT_UINT32 total_threads; +} Client_Info_t; +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_common.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_common.h new file mode 100644 index 00000000..c4040357 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_common.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_COMMON_H__ +#define __DAPL_COMMON_H__ + +#include "dapl_proto.h" + +typedef enum +{ + RDMA_READ, + RDMA_WRITE, + SEND_RECV +} DT_Transfer_Type; + + +#pragma pack(1) +typedef struct +{ + DAT_RMR_CONTEXT rmr_context; + DAT_CONTEXT mem_address; +} RemoteMemoryInfo; +#pragma pack() + + +#endif /* __DAPL_COMMON_H__ */ diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_execute.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_execute.h new file mode 100644 index 00000000..c34a1070 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_execute.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef _DAPL_EXECUTE_H_ +#define _DAPL_EXECUTE_H_ + +#include "dapl_proto.h" +#include "dapl_params.h" + +void +DT_Execute_Test ( Params_t *params_ptr ) ; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_fft_cmd.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_fft_cmd.h new file mode 100644 index 00000000..900e19cf --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_fft_cmd.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_FFT_CMD_H__ +#define __DAPL_FFT_CMD_H__ + +#include "dapl_proto.h" + +#define MAXCASES 100 + +typedef enum +{ + NONE, + HWCONN, + CQMGT, + ENDPOINT, + PTAGMGT, + MEMMGT, + CONNMGT, + CONNMGT_CLIENT, + DATAXFER, + DATAXFER_CLIENT, + QUERYINFO, + NS, + ERRHAND, + UNSUPP, + STRESS, + STRESS_CLIENT, +} FFT_Type_e; + + +typedef struct +{ + FFT_Type_e fft_type; + char device_name[256]; //-D + char server_name[256]; + bool cases_flag[MAXCASES]; + int size; + int num_iter; //-i + int num_threads; //-t + int num_vis; //-v + DAT_QOS ReliabilityLevel; //-R +} FFT_Cmd_t; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_fft_util.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_fft_util.h new file mode 100644 index 00000000..34b47807 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_fft_util.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef DAPL_FFT_UTIL_H +#define DAPL_FFT_UTIL_H + +#include "dapl_proto.h" +#include "dapl_test_data.h" + +#define DT_assert_dat(test_num, x) if(x) ; \ + else { \ + DT_assert_fail(test_num, #x, __FILE__, __BASE_FILE__, __LINE__); \ + DT_Tdep_PT_Printf(test_num,"Error = %d, %s\n", rc, DT_RetToString(rc)); \ + res = 0; \ + goto cleanup; \ + } + +#define DT_assert(test_num, x) if(x) ; \ + else { \ + DT_assert_fail(test_num, #x, __FILE__, __BASE_FILE__, __LINE__); \ + res = 0; \ + goto cleanup; \ + } + +#define DT_assert_clean(test_num, x) if(x) ; \ + else { \ + DT_assert_fail(test_num, #x, __FILE__, __BASE_FILE__, __LINE__); \ + DT_Tdep_PT_Printf(test_num,"Error = %d, %s\n", rc, DT_RetToString(rc)); \ + return 0; \ + } + +typedef struct +{ + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_PSP_HANDLE psp_handle; + DAT_EP_HANDLE ep_handle; + DAT_EVD_HANDLE cr_evd, conn_evd, send_evd, recv_evd; + DAT_EVENT event; + DAT_COUNT count; + DAT_CR_HANDLE cr_handle; + Bpool *bpool; + DAT_CR_ARRIVAL_EVENT_DATA cr_stat; + DAT_EVENT_NUMBER event_num; + DAT_IA_ADDRESS_PTR remote_netaddr; + Per_Test_Data_t *pt_ptr; + bool connected; +} FFT_Connection_t; + +typedef enum +{ + QUERY_CNO, + QUERY_CR, + QUERY_EP, + QUERY_EVD, + QUERY_IA, + QUERY_LMR, + QUERY_RMR, + QUERY_PSP, + QUERY_RSP, + QUERY_PZ, +} FFT_query_enum; + +typedef struct +{ + int (*fun) (Params_t *params_ptr, FFT_Cmd_t* cmd); +} FFT_Testfunc_t; + +#endif + diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_getopt.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_getopt.h new file mode 100644 index 00000000..e6a69bcd --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_getopt.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_GETOPT_H__ +#define __DAPL_GETOPT_H__ + +typedef struct +{ + int magic; + char *place; + + int opterr; + int optind; + int optopt; + char *optarg; +} mygetopt_t; +/* function prototypes */ +void +DT_mygetopt_init (mygetopt_t * opts); +int +DT_mygetopt_r (int argc, + char *const * argv, + const char *ostr, + mygetopt_t * opts); +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_global.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_global.h new file mode 100644 index 00000000..0e91faf8 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_global.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_GLOBAL_H__ +#define __DAPL_GLOBAL_H__ + +#include "dapl_proto.h" + +extern DAT_COUNT DT_dapltest_debug; +extern bool DT_local_is_little_endian; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_limit_cmd.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_limit_cmd.h new file mode 100644 index 00000000..6f4bc82c --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_limit_cmd.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_LIMIT_CMD_H__ +#define __DAPL_LIMIT_CMD_H__ + +#include "dapl_proto.h" + +typedef enum +{ + LIM_IA, + LIM_PZ, +#ifndef __KDAPLTEST__ + LIM_CNO, +#endif + LIM_EVD, + LIM_EP, + LIM_RSP, + LIM_PSP, + LIM_LMR, + LIM_RPOST, + LIM_SIZE_LMR, + /* add further tests here */ + + LIM_NUM_TESTS /* for array size & limit checks */ +} Limit_Index; + +//------------------------------------- +#pragma pack (2) +typedef struct +{ + char device_name[256]; /* -D */ + DAT_QOS ReliabilityLevel; /* -R */ + DAT_UINT32 width; /* -w */ + DAT_UINT32 debug; /* -d */ + DAT_UINT32 maximum; /* -m */ + DAT_UINT32 Test_List[ LIM_NUM_TESTS ]; +} Limit_Cmd_t; + +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_mdep.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_mdep.h new file mode 100644 index 00000000..da92611d --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_mdep.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_MDEP_H__ +#define __DAPL_MDEP_H__ + +#ifdef __KERNEL__ +#include "dapl_mdep_kernel.h" +#else +#include "dapl_mdep_user.h" +#endif + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_memlist.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_memlist.h new file mode 100644 index 00000000..b992dc1a --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_memlist.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_MEMLIST_H__ +#define __DAPL_MEMLIST_H__ + +#include "dapl_proto.h" + +typedef enum +{ + BPOOL, + BUFF, + PERTESTDATA, + NIC, + NETADDRESS, + TRANSACTIONTEST, + THREAD, + EPCONTEXT +} mem_type_e; + +struct Mem_list_entry +{ + char filename[50]; + mem_type_e MemType; + void *mem_ptr; + struct Mem_list_entry *next; +}; + +typedef struct Mem_list_entry MemListEntry_t; +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_params.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_params.h new file mode 100644 index 00000000..35d018e4 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_params.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_PARAMS_H__ +#define __DAPL_PARAMS_H__ + +#include "dapl_proto.h" +#include "dapl_server_cmd.h" +#include "dapl_transaction_cmd.h" +#include "dapl_performance_cmd.h" +#include "dapl_limit_cmd.h" +#include "dapl_quit_cmd.h" +#include "dapl_fft_cmd.h" + +typedef enum +{ + SERVER_TEST, + TRANSACTION_TEST, + PERFORMANCE_TEST, + LIMIT_TEST, + QUIT_TEST, + FFT_TEST +} test_type_e; + +typedef struct +{ + test_type_e test_type; + + union + { + Server_Cmd_t Server_Cmd; + Transaction_Cmd_t Transaction_Cmd; + Performance_Cmd_t Performance_Cmd; + Limit_Cmd_t Limit_Cmd; + Quit_Cmd_t Quit_Cmd; + FFT_Cmd_t FFT_Cmd; + } u; + + /* Needed here due to structure of command processing */ + DAT_QOS ReliabilityLevel; + DAT_SOCK_ADDR server_netaddr; + void * phead; + bool local_is_little_endian; + bool debug; + double cpu_mhz; +} Params_t; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_cmd.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_cmd.h new file mode 100644 index 00000000..5e3036f8 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_cmd.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_PERFORMANCE_CMD_H__ +#define __DAPL_PERFORMANCE_CMD_H__ + +#include "dapl_proto.h" + +#define NAME_SZ 256 + +typedef enum +{ + BLOCKING_MODE, + POLLING_MODE +} Performance_Mode_Type; + +#pragma pack (2) +typedef struct +{ + DAT_UINT32 transfer_type; + DAT_UINT32 seg_size; + DAT_UINT32 num_segs; +} Performance_Cmd_Op_t; + +typedef struct +{ + DAT_UINT32 dapltest_version; + DAT_UINT32 client_is_little_endian; + char server_name[NAME_SZ]; /* -s */ + char dapl_name[NAME_SZ]; /* -D */ + DAT_QOS qos; + DAT_UINT32 debug; /* -d */ + Performance_Mode_Type mode; /* -m */ + DAT_UINT32 num_iterations; /* -i */ + DAT_UINT32 pipeline_len; /* -p */ + Performance_Cmd_Op_t op; + DAT_UINT32 use_rsp; /* -r */ + +} Performance_Cmd_t; +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_stats.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_stats.h new file mode 100644 index 00000000..fbb1c955 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_stats.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_STATS_H__ +#define __DAPL_STATS_H__ + +#include "dapl_proto.h" + +#define DT_min(a, b) ((a < b) ? (a) : (b)) +#define DT_max(a, b) ((a > b) ? (a) : (b)) +#define DT_whole(num) ((unsigned int)(num)) +#define DT_hundredths(num) ((unsigned int)(((num) - (unsigned int)(num)) * 100)) + +typedef struct +{ + unsigned int num; + DT_Mdep_TimeStamp total_ts; + DT_Mdep_TimeStamp max_ts; + DT_Mdep_TimeStamp min_ts; +} Performance_Stats_Data_t; + + +typedef struct +{ + unsigned int num_ops; + int64_t bytes; + unsigned int post_ctxt_switch_num; + unsigned int reap_ctxt_switch_num; + double cpu_utilization; + DT_Mdep_TimeStamp time_ts; + Performance_Stats_Data_t posts_sans_ctxt; + Performance_Stats_Data_t posts_with_ctxt; + Performance_Stats_Data_t reaps_sans_ctxt; + Performance_Stats_Data_t reaps_with_ctxt; + Performance_Stats_Data_t latency; +} Performance_Stats_t; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_test.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_test.h new file mode 100644 index 00000000..92b1ea5e --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_performance_test.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_PERFORMANCE_TEST_H__ +#define __DAPL_PERFORMANCE_TEST_H__ + +#include "dapl_proto.h" +#include "dapl_tdep.h" +#include "dapl_common.h" +#include "dapl_test_data.h" +#include "dapl_performance_cmd.h" + + +#define DT_PERF_SYNC_SEND_BUFFER_ID 0 +#define DT_PERF_SYNC_RECV_BUFFER_ID 1 +#define DT_PERF_SYNC_BUFF_SIZE sizeof (RemoteMemoryInfo) +#define DT_PERF_DFLT_EVD_LENGTH 8 + + +typedef struct +{ + DT_Transfer_Type transfer_type; + DAT_UINT32 num_segs; + DAT_UINT32 seg_size; + Bpool *bp; + + /* RDMA info */ + DAT_RMR_CONTEXT Rdma_Context; + DAT_VADDR Rdma_Address; +} Performance_Test_Op_t; + +typedef struct +{ + DAT_EP_HANDLE ep_handle; + DAT_EP_ATTR ep_attr; + DAT_CONN_QUAL port; + DAT_COUNT pipeline_len; + Bpool *bp; + Performance_Test_Op_t op; +} Performance_Ep_Context_t; + +typedef struct +{ + Per_Test_Data_t *pt_ptr; + Performance_Cmd_t *cmd; + DAT_IA_ADDRESS_PTR remote_ia_addr; + DAT_BOOLEAN is_remote_little_endian; + DAT_CONN_QUAL base_port; + DAT_IA_ATTR ia_attr; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_CNO_HANDLE cno_handle; + DAT_COUNT reqt_evd_length; + DAT_EVD_HANDLE reqt_evd_hdl; /* request+rmr */ + DAT_COUNT recv_evd_length; + DAT_EVD_HANDLE recv_evd_hdl; /* receive */ + DAT_COUNT conn_evd_length; + DAT_EVD_HANDLE conn_evd_hdl; /* connect */ + DAT_COUNT creq_evd_length; + DAT_EVD_HANDLE creq_evd_hdl; /* "" request */ + Performance_Ep_Context_t ep_context; +} Performance_Test_t; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_proto.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_proto.h new file mode 100644 index 00000000..46f1f2a3 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_proto.h @@ -0,0 +1,669 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_PROTO_H__ +#define __DAPL_PROTO_H__ + +#ifdef __KERNEL__ +#include +#else +#include + +#include +#include +#include +#include +#endif + +#include "dapl_mdep.h" +#include "dapl_tdep_print.h" +#include "dapl_bpool.h" +#include "dapl_client_info.h" +#include "dapl_common.h" +#include "dapl_client_info.h" +#include "dapl_execute.h" +#include "dapl_getopt.h" +#include "dapl_global.h" +#include "dapl_fft_cmd.h" +#include "dapl_fft_util.h" +#include "dapl_limit_cmd.h" +#include "dapl_memlist.h" +#include "dapl_params.h" +#include "dapl_performance_stats.h" +#include "dapl_performance_test.h" +#include "dapl_quit_cmd.h" +#include "dapl_server_info.h" +#include "dapl_tdep.h" +#include "dapl_test_data.h" +#include "dapl_transaction_cmd.h" +#include "dapl_transaction_test.h" +#include "dapl_transaction_stats.h" +#include "dapl_version.h" + +/* + * Prototypes + */ + +/* dapl_bpool.c */ +Bpool * DT_BpoolAlloc (Per_Test_Data_t * pt_ptr, + DT_Tdep_Print_Head* phead, + DAT_IA_HANDLE ia_handle, + DAT_PZ_HANDLE pz_handle, + DAT_EP_HANDLE ep_handle, + DAT_EVD_HANDLE rmr_evd_handle, + DAT_COUNT seg_size, + DAT_COUNT num_segs, + DAT_COUNT alignment, + DAT_BOOLEAN enable_rdma_write, + DAT_BOOLEAN enable_rdma_read); + +bool DT_Bpool_Destroy (Per_Test_Data_t * pt_ptr, + DT_Tdep_Print_Head *phead, + Bpool * bpool_ptr); + +unsigned char *DT_Bpool_GetBuffer (Bpool * bpool_ptr, int index); +DAT_COUNT DT_Bpool_GetBuffSize (Bpool * bpool_ptr, int index); +DAT_LMR_TRIPLET *DT_Bpool_GetIOV (Bpool * bpool_ptr, int index); +DAT_LMR_CONTEXT DT_Bpool_GetLMR (Bpool * bpool_ptr, int index); +DAT_RMR_CONTEXT DT_Bpool_GetRMR (Bpool * bpool_ptr, int index); + +void DT_Bpool_print (DT_Tdep_Print_Head* phead, Bpool *bpool_ptr); + +/* dapl_cnxn.c */ +int get_ep_connection_state (DT_Tdep_Print_Head* phead, + DAT_EP_HANDLE ep_handle); + +/* dapl_client.c */ +void DT_cs_Client (Params_t * params_ptr, + char *dapl_name, + char *server_name, + DAT_UINT32 total_threads); + +/* dapl_client_info.c */ +void DT_Client_Info_Endian (Client_Info_t * client_info); + +void DT_Client_Info_Print (DT_Tdep_Print_Head *phead, + Client_Info_t * client_info); + +/* dapl_transaction_stats.c */ +void DT_init_transaction_stats (Transaction_Stats_t * transaction_stats, + unsigned int nums); +void DT_transaction_stats_set_ready (DT_Tdep_Print_Head* phead, + Transaction_Stats_t* transaction_stats); + +void DT_transaction_stats2_set_ready (DT_Tdep_Print_Head* phead, + Transaction_Stats_t* transaction_stats); + +bool DT_transaction_stats_wait_for_all (DT_Tdep_Print_Head* phead, + Transaction_Stats_t* transaction_stats); + +bool DT_transaction_stats2_wait_for_all (DT_Tdep_Print_Head* phead, + Transaction_Stats_t* transaction_stats); + +void DT_update_transaction_stats (Transaction_Stats_t * transaction_stats, + unsigned int num_ops, + unsigned int time_ms, + unsigned int bytes_send, + unsigned int bytes_recv, + unsigned int bytes_rdma_read, + unsigned int bytes_rdma_write); + +void DT_print_transaction_stats (DT_Tdep_Print_Head* phead, + Transaction_Stats_t* transaction_stats, + unsigned int num_threads, + unsigned int num_EPs); + +/* dapl_endian.c */ +void DT_Endian_Init (void); +DAT_UINT32 DT_Endian32 (DAT_UINT32 val); +DAT_UINT64 DT_Endian64 (DAT_UINT64 val); +DAT_UINT32 DT_EndianMemHandle (DAT_UINT32 val); +DAT_UINT64 DT_EndianMemAddress (DAT_UINT64 val); + +/* dapl_getopt.c */ +void DT_mygetopt_init (mygetopt_t * opts); + +int DT_mygetopt_r (int argc, + char *const * argv, + const char *ostr, + mygetopt_t * opts); + +/* dapl_main.c */ +int main (int argc, char *argv[]); + +int dapltest (int argc, char *argv[]); + +void Dapltest_Main_Usage (void); + +/* dapl_mdep.c */ +void DT_Mdep_Init (void); +void DT_Mdep_End (void); +bool DT_Mdep_GetDefaultDeviceName (char *dapl_name); +void DT_Mdep_Sleep (int msec); +void DT_Mdep_Schedule (void); +bool DT_Mdep_GetCpuStat (DT_CpuStat *sys_stat); +unsigned long DT_Mdep_GetTime (void); +double DT_Mdep_GetCpuMhz (void); +unsigned long DT_Mdep_GetContextSwitchNum (void); +void *DT_Mdep_Malloc (size_t l_); +void DT_Mdep_Free (void *a_); +bool DT_Mdep_LockInit (DT_Mdep_LockType * lock_ptr); +void DT_Mdep_LockDestroy (DT_Mdep_LockType * lock_ptr); +void DT_Mdep_Lock (DT_Mdep_LockType * lock_ptr); +void DT_Mdep_Unlock (DT_Mdep_LockType * lock_ptr); +void DT_Mdep_Thread_Init_Attributes (Thread * thread_ptr); +void DT_Mdep_Thread_Destroy_Attributes (Thread * thread_ptr); +bool DT_Mdep_Thread_Start (Thread * thread_ptr); + +void DT_Mdep_Thread_Detach (DT_Mdep_ThreadHandleType thread_id); +DT_Mdep_ThreadHandleType DT_Mdep_Thread_SELF ( void ); +void DT_Mdep_Thread_EXIT ( void * thread_handle ); +int DT_Mdep_wait_object_init ( IN DT_WAIT_OBJECT *wait_obj); +int DT_Mdep_wait_object_wait ( + IN DT_WAIT_OBJECT *wait_obj, + IN int timeout_val); +int DT_Mdep_wait_object_wakeup ( IN DT_WAIT_OBJECT *wait_obj); +int DT_Mdep_wait_object_destroy ( IN DT_WAIT_OBJECT *wait_obj); + + +DT_Mdep_Thread_Start_Routine_Return_Type + DT_Mdep_Thread_Start_Routine (void *thread_handle); + +/* dapl_memlist.c */ +void DT_MemListInit (Per_Test_Data_t * pt_ptr); +void *DT_MemListAlloc (Per_Test_Data_t * pt_ptr, + char *file, + mem_type_e t, + int size); +void DT_MemListFree (Per_Test_Data_t * pt_ptr, + void *ptr); +void DT_PrintMemList (Per_Test_Data_t * pt_ptr); + +/* dapl_netaddr.c */ +bool DT_NetAddrLookupHostAddress (DAT_IA_ADDRESS_PTR to_netaddr, + char *hostname); + +DAT_IA_ADDRESS_PTR DT_NetAddrAlloc (Per_Test_Data_t * pt_ptr); + +void DT_NetAddrFree (Per_Test_Data_t * pt_ptr, + DAT_IA_ADDRESS_PTR netaddr); + +/* dapl_params.c */ +bool DT_Params_Parse (int argc, + char *argv[], + Params_t * params_ptr); + +/* dapl_performance_cmd.c */ +const char * DT_PerformanceModeToString (Performance_Mode_Type mode); + +bool DT_Performance_Cmd_Init (Performance_Cmd_t * cmd); + +bool DT_Performance_Cmd_Parse (Performance_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts); + +void DT_Performance_Cmd_Print (Performance_Cmd_t * cmd); +void DT_Performance_Cmd_PT_Print (DT_Tdep_Print_Head* phead, + Performance_Cmd_t * cmd); + +void DT_Performance_Cmd_Endian (Performance_Cmd_t * cmd); + +/* dapl_performance_client.c */ +void DT_Performance_Test_Client ( Params_t *params_ptr, + Per_Test_Data_t * pt_ptr, + DAT_IA_HANDLE * ia_handle, + DAT_IA_ADDRESS_PTR remote); + +bool DT_Performance_Test_Client_Connect ( + DT_Tdep_Print_Head *phead, + Performance_Test_t * test_ptr); + +bool DT_Performance_Test_Client_Exchange ( + Params_t *params_ptr, + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr); + +/* dapl_performance_server.c */ +void DT_Performance_Test_Server (void * pt_ptr); + +bool DT_Performance_Test_Server_Connect ( + DT_Tdep_Print_Head *phead, + Performance_Test_t * test_ptr); + +bool DT_Performance_Test_Server_Exchange ( + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr); + +/* dapl_performance_util.c */ +bool DT_Performance_Test_Create (Per_Test_Data_t * pt_ptr, + DAT_IA_HANDLE * ia_handle, + DAT_IA_ADDRESS_PTR remote_ia_addr, + DAT_BOOLEAN is_server, + DAT_BOOLEAN is_remote_little_endian, + Performance_Test_t **perf_test); + +void DT_Performance_Test_Destroy (Per_Test_Data_t * pt_ptr, + Performance_Test_t *test_ptr, + DAT_BOOLEAN is_server); + +bool DT_performance_post_rdma_op (Performance_Ep_Context_t *ep_context, + DAT_EVD_HANDLE reqt_evd_hdl, + Performance_Stats_t *stats); + +unsigned int DT_performance_reap (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + Performance_Mode_Type mode, + Performance_Stats_t *stats); + +unsigned int DT_performance_wait (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + Performance_Stats_t *stats); + +unsigned int DT_performance_poll (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + Performance_Stats_t *stats); + +/* dapl_performance_stats.c */ +void DT_performance_stats_init (Performance_Stats_t * stats); + +void DT_performance_stats_record_post (Performance_Stats_t *stats, + unsigned long ctxt_switch_num, + DT_Mdep_TimeStamp ts); + +void DT_performance_stats_record_reap (Performance_Stats_t *stats, + unsigned long ctxt_switch_num, + DT_Mdep_TimeStamp ts); + +void DT_performance_stats_record_latency (Performance_Stats_t *stats, + DT_Mdep_TimeStamp ts); + +void DT_performance_stats_data_combine (Performance_Stats_Data_t * dest, + Performance_Stats_Data_t * src_a, + Performance_Stats_Data_t * src_b); + +void DT_performance_stats_combine (Performance_Stats_t * dest, + Performance_Stats_t * src_a, + Performance_Stats_t * src_b); + +double DT_performance_stats_data_print (DT_Tdep_Print_Head* phead, + Performance_Stats_Data_t* data, + double cpu_mhz); + +void DT_performance_stats_print (Params_t * params_ptr, + DT_Tdep_Print_Head* phead, + Performance_Stats_t * stats, + Performance_Cmd_t * cmd, + Performance_Test_t * test); + + +/* dapl_server.c */ +void DT_cs_Server (Params_t * params_ptr); + +/* dapl_server_cmd.c */ +void DT_Server_Cmd_Init (Server_Cmd_t * Server_Cmd); + +bool DT_Server_Cmd_Parse (Server_Cmd_t * Server_Cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts); + +void DT_Server_Cmd_Print (Server_Cmd_t * Server_Cmd); +void DT_Server_Cmd_PT_Print (DT_Tdep_Print_Head* phead, + Server_Cmd_t * Server_Cmd); + +void DT_Server_Cmd_Usage (void); + +/* dapl_server_info.c */ +void DT_Server_Info_Endian (Server_Info_t * server_info); + +void DT_Server_Info_Print (DT_Tdep_Print_Head* phead, + Server_Info_t * server_info); + +/* dapl_test_data.c */ +Per_Test_Data_t *DT_Alloc_Per_Test_Data (DT_Tdep_Print_Head* phead); + +void DT_Free_Per_Test_Data (Per_Test_Data_t * pt_ptr); + +/* dapl_test_util.c */ +DAT_BOOLEAN DT_query (Per_Test_Data_t *pt_ptr, + DAT_IA_HANDLE ia_handle, + DAT_EP_HANDLE ep_handle); + +DAT_BOOLEAN DT_post_recv_buffer (DT_Tdep_Print_Head* phead, + DAT_EP_HANDLE ep_handle, + Bpool * bp, + int index, + int size); + +DAT_BOOLEAN DT_post_send_buffer (DT_Tdep_Print_Head* phead, + DAT_EP_HANDLE ep_handle, + Bpool * bp, + int index, + int size); + +bool DT_conn_event_wait (DT_Tdep_Print_Head* phead, + DAT_EP_HANDLE ep_handle, + DAT_EVD_HANDLE evd_handle, + DAT_EVENT_NUMBER *event_number); + +bool DT_disco_event_wait ( DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + DAT_EP_HANDLE *ep_handle ); + +bool DT_cr_event_wait (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + DAT_CR_ARRIVAL_EVENT_DATA *cr_stat_p); + +bool DT_dto_event_reap (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + bool poll, + DAT_DTO_COMPLETION_EVENT_DATA *dtop); + +bool DT_dto_event_wait (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + DAT_DTO_COMPLETION_EVENT_DATA *dtop); + +bool DT_dto_event_poll (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + DAT_DTO_COMPLETION_EVENT_DATA *dtop); + +bool DT_rmr_event_wait (DT_Tdep_Print_Head* phead, + DAT_EVD_HANDLE evd_handle, + DAT_RMR_BIND_COMPLETION_EVENT_DATA *rmr_ptr); + +bool DT_dto_check ( DT_Tdep_Print_Head* phead, + DAT_DTO_COMPLETION_EVENT_DATA *dto_p, + DAT_EP_HANDLE ep_expected, + DAT_COUNT len_expected, + DAT_DTO_COOKIE cookie_expected, + char *message); + +bool DT_rmr_check ( DT_Tdep_Print_Head* phead, + DAT_RMR_BIND_COMPLETION_EVENT_DATA*rmr_p, + DAT_RMR_HANDLE rmr_expected, + DAT_PVOID cookie_expected, + char *message); + +bool DT_cr_check (DT_Tdep_Print_Head* phead, + DAT_CR_ARRIVAL_EVENT_DATA *cr_stat_p, + DAT_PSP_HANDLE psp_handle_expected, + DAT_CONN_QUAL port_expected, + DAT_CR_HANDLE *cr_handlep, + char *message); + +/* dapl_thread.c */ +void DT_Thread_Init (Per_Test_Data_t * pt_ptr); + +void DT_Thread_End (Per_Test_Data_t * pt_ptr); + +Thread *DT_Thread_Create (Per_Test_Data_t * pt_ptr, + void (*fn) (void *), + void *param, + unsigned int stacksize); + +void DT_Thread_Destroy (Thread * thread_ptr, + Per_Test_Data_t * pt_ptr); + +bool DT_Thread_Start (Thread * thread_ptr); + +/* dapl_quit_cmd.c */ +void DT_Quit_Cmd_Init (Quit_Cmd_t * cmd); + +bool DT_Quit_Cmd_Parse (Quit_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts); + +bool DT_Quit_Cmd_Validate (Quit_Cmd_t * cmd); + +void DT_Quit_Cmd_Endian (Quit_Cmd_t * cmd, + bool to_wire); + +void DT_Quit_Cmd_Print (Quit_Cmd_t * cmd); +void DT_Quit_Cmd_PT_Print (DT_Tdep_Print_Head *phead, Quit_Cmd_t * cmd); + +void DT_Quit_Cmd_Usage (void); + +/* dapl_transaction_cmd.c */ +void DT_Transaction_Cmd_Init (Transaction_Cmd_t * cmd); + +bool DT_Transaction_Cmd_Parse (Transaction_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts); + +void DT_Transaction_Cmd_Print (Transaction_Cmd_t * cmd); +void DT_Transaction_Cmd_PT_Print (DT_Tdep_Print_Head* phead, + Transaction_Cmd_t * cmd); + +void DT_Transaction_Cmd_Endian (Transaction_Cmd_t * cmd, + bool to_wire); +/* dapl_transaction_test.c */ +void DT_Transaction_Test_Client (Per_Test_Data_t * pt_ptr, + DAT_IA_HANDLE ia_handle, + DAT_IA_ADDRESS_PTR remote); + +void DT_Transaction_Test_Server (void *params); + +bool DT_Transaction_Create_Test (Per_Test_Data_t * pt_ptr, + DAT_IA_HANDLE * ia_handle, + DAT_BOOLEAN is_server, + unsigned int port_num, + DAT_BOOLEAN remote_is_little_endian, + DAT_IA_ADDRESS_PTR remote_ia_addr); + +void DT_Transaction_Main (void *param); +bool DT_Transaction_Run (DT_Tdep_Print_Head* phead, + Transaction_Test_t * test_ptr); +void DT_Transaction_Validation_Fill (DT_Tdep_Print_Head* phead, + Transaction_Test_t * test_ptr, + unsigned int iteration); +bool DT_Transaction_Validation_Check (DT_Tdep_Print_Head* phead, + Transaction_Test_t * test_ptr, + int iteration); +void DT_Print_Transaction_Test (DT_Tdep_Print_Head* phead, + Transaction_Test_t* test_ptr); +void DT_Print_Transaction_Stats (DT_Tdep_Print_Head* phead, + Transaction_Test_t* test_ptr); + +/* dapl_transaction_util.c */ +bool DT_handle_post_recv_buf (DT_Tdep_Print_Head* phead, + Ep_Context_t * ep_context, + unsigned int num_eps, + int op_indx); + +bool DT_handle_send_op (DT_Tdep_Print_Head* phead, + Ep_Context_t * ep_context, + DAT_EVD_HANDLE reqt_evd_hdl, + unsigned int num_eps, + int op_indx, + bool poll); + +bool DT_handle_recv_op (DT_Tdep_Print_Head* phead, + Ep_Context_t * ep_context, + DAT_EVD_HANDLE recv_evd_hdl, + DAT_EVD_HANDLE reqt_evd_hdl, + unsigned int num_eps, + int op_indx, + bool poll, + bool repost_recv); + +bool DT_handle_rdma_op (DT_Tdep_Print_Head* phead, + Ep_Context_t * ep_context, + DAT_EVD_HANDLE reqt_evd_hdl, + unsigned int num_eps, + DT_Transfer_Type opcode, + int op_indx, + bool poll); + +bool DT_check_params (Per_Test_Data_t *pt_ptr, + char *module); + +void DT_Test_Error (void); + +/* dapl_util.c */ +const char *DT_RetToString (DAT_RETURN ret_value); + +const char *DT_TransferTypeToString (DT_Transfer_Type type); + +const char *DT_AsyncErr2Str (DAT_EVENT_NUMBER error_code); + +const char *DT_EventToSTr (DAT_EVENT_NUMBER event_code); + +const char *DT_State2Str (DAT_EP_STATE state_code); + +DAT_QOS DT_ParseQoS (char *arg); + +unsigned char *DT_AlignPtr (void * val, DAT_COUNT align); + +DAT_COUNT DT_RoundSize (DAT_COUNT val, DAT_COUNT align); + +/* dapl_limit_cmd.c */ +void DT_Limit_Cmd_Init ( Limit_Cmd_t * cmd); + +bool DT_Limit_Cmd_Parse ( Limit_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts); + +void DT_Limit_Cmd_Usage (void); + +/* dapl_limit.c */ +void DT_cs_Limit (Params_t *params, Limit_Cmd_t * cmd); + +/* dapl_fft_cmd.c */ +void DT_FFT_Cmd_Init ( FFT_Cmd_t * cmd); + +bool DT_FFT_Cmd_Parse ( FFT_Cmd_t * cmd, + int my_argc, + char **my_argv, + mygetopt_t * opts); + +void DT_FFT_Cmd_Usage (void); + +/* dapl_fft_test.c */ +void DT_cs_FFT (Params_t *params, FFT_Cmd_t * cmd); + +/* dapl_fft_hwconn.c */ +void DT_hwconn_test (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case4 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case5 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case6 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_hwconn_case7 (Params_t *params_ptr, FFT_Cmd_t *cmd); + +/* dapl_fft_endpoint.c */ +void DT_endpoint_test (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_endpoint_generic (Params_t *params_ptr, FFT_Cmd_t *cmd, + bool destroy_pz_early); +int DT_endpoint_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_endpoint_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_endpoint_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_endpoint_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_endpoint_case4 (Params_t *params_ptr, FFT_Cmd_t *cmd); + +/* dapl_fft_pz.c */ +void DT_pz_test (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case4 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case5 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_pz_case6 (Params_t *params_ptr, FFT_Cmd_t *cmd); + +/* dapl_fft_util.c */ +void DT_assert_fail (DT_Tdep_Print_Head* phead, + char *exp, + char *file, + char *baseFile, + int line); +int DT_ia_open (DAT_NAME_PTR dev_name, DAT_IA_HANDLE *ia_handle); +int DT_ep_create (Params_t *params_ptr, + DAT_IA_HANDLE ia_handle, + DAT_PZ_HANDLE pz_handle, + DAT_EVD_HANDLE *cr_evd, + DAT_EVD_HANDLE *conn_evd, + DAT_EVD_HANDLE *send_evd, + DAT_EVD_HANDLE *recv_evd, + DAT_EP_HANDLE *ep_handle); +void DT_fft_init_conn_struct (FFT_Connection_t *conn); +void DT_fft_init_client (Params_t *params_ptr, + FFT_Cmd_t *cmd, + FFT_Connection_t *conn); +int DT_fft_destroy_conn_struct (Params_t *params_ptr, FFT_Connection_t *conn); +void DT_fft_init_server (Params_t *params_ptr, FFT_Cmd_t *cmd, + FFT_Connection_t *conn); +void DT_fft_listen (Params_t *params_ptr, FFT_Connection_t *conn); +int DT_fft_connect (Params_t *params_ptr, FFT_Connection_t *conn); + +/* dapl_fft_connmgt.c */ +int DT_connmgt_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_connmgt_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd); +void DT_connmgt_test (Params_t *params_ptr, FFT_Cmd_t *cmd); + +/* dapl_fft_mem.c */ +int DT_mem_generic (Params_t *params_ptr, FFT_Cmd_t *cmd, int flag); +int DT_mem_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_mem_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_mem_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_mem_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_mem_case4 (Params_t *params_ptr, FFT_Cmd_t *cmd); +void DT_mem_test (Params_t *params_ptr, FFT_Cmd_t *cmd); + +/* dapl_fft_queryinfo.c */ +int DT_queryinfo_basic (Params_t *params_ptr, FFT_Cmd_t *cmd, + FFT_query_enum object_to_query, + DAT_RETURN result_wanted); +int DT_queryinfo_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case4 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case5 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case6 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case7 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case8 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case9 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case10 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case11 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case12 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case13 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case14 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case15 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case16 (Params_t *params_ptr, FFT_Cmd_t *cmd); +int DT_queryinfo_case17 (Params_t *params_ptr, FFT_Cmd_t *cmd); +void DT_queryinfo_test (Params_t *params_ptr, FFT_Cmd_t *cmd); + +#endif /* __DAPL_PROTO_H__ */ diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_quit_cmd.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_quit_cmd.h new file mode 100644 index 00000000..15a7b426 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_quit_cmd.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_QUIT_CMD_H__ +#define __DAPL_QUIT_CMD_H__ + +#pragma pack (2) +typedef struct +{ + char server_name[256]; /* -s */ + char device_name[256]; /* -D */ + DAT_UINT32 debug; /* -d */ + DAT_QOS ReliabilityLevel; /* -R */ +} Quit_Cmd_t; +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_server_cmd.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_server_cmd.h new file mode 100644 index 00000000..722eaaaa --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_server_cmd.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_SERVER_CMD_H__ +#define __DAPL_SERVER_CMD_H__ + +#pragma pack (2) + +typedef struct +{ + bool debug; /* -d */ + char dapl_name[256]; /* -D device name */ + DAT_QOS ReliabilityLevel; /* -R */ +} Server_Cmd_t; +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_server_info.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_server_info.h new file mode 100644 index 00000000..6538153b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_server_info.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_SERVER_INFO_H__ +#define __DAPL_SERVER_INFO_H__ + +#include "dapl_proto.h" + +#pragma pack (2) + +struct started_server +{ + char devicename[80]; + struct started_server *next; +}; + +typedef struct started_server Started_server_t; + +extern Started_server_t *DT_started_server_list; + +#define SERVER_PORT_NUMBER ((DAT_CONN_QUAL)0xB0de) + +typedef struct +{ + DAT_UINT32 dapltest_version; + DAT_UINT32 is_little_endian; + DAT_UINT32 first_port_number; +} Server_Info_t; +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_tdep.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_tdep.h new file mode 100644 index 00000000..20cbe061 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_tdep.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef _DAPL_TDEP_H_ +#define _DAPL_TDEP_H_ + +#include "dapl_proto.h" + +#ifdef __KDAPL__ +typedef DAT_HANDLE DAT_CNO_HANDLE; +#endif + +/* function prototypes */ +void +DT_Tdep_Init ( void ) ; + +void +DT_Tdep_End ( void ) ; + +void +DT_Tdep_Execute_Test ( Params_t *params_ptr ) ; + +DAT_RETURN +DT_Tdep_lmr_create (DAT_IA_HANDLE ia_handle, + DAT_MEM_TYPE mem_type, + DAT_REGION_DESCRIPTION region, + DAT_VLEN len, + DAT_PZ_HANDLE pz_handle, + DAT_MEM_PRIV_FLAGS priv_flag, + DAT_LMR_HANDLE *lmr_handle_ptr, + DAT_LMR_CONTEXT *lmr_context_ptr, + DAT_RMR_CONTEXT *rmr_context_ptr, + DAT_VLEN *reg_size_ptr, + DAT_VADDR *reg_addr_ptr); + +DAT_RETURN +DT_Tdep_evd_create (DAT_IA_HANDLE ia_handle, + DAT_COUNT evd_min_qlen, + DAT_CNO_HANDLE cno_handle, + DAT_EVD_FLAGS evd_flags, + DAT_EVD_HANDLE *evd_handle_ptr); + +DAT_RETURN +DT_Tdep_evd_free (DAT_EVD_HANDLE evd_handle); + +DAT_RETURN +DT_Tdep_evd_wait (DAT_EVD_HANDLE evd_handle, + DAT_TIMEOUT timeout, + DAT_EVENT *event); +DAT_RETURN +DT_Tdep_evd_dequeue (DAT_EVD_HANDLE evd_handle, + DAT_EVENT *event); + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_tdep_print.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_tdep_print.h new file mode 100644 index 00000000..d2d565fd --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_tdep_print.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef _DAPL_TDEP_PRINT_H_ +#define _DAPL_TDEP_PRINT_H_ + +#define DT_Tdep_PT_Debug(N, _X_) \ +do { \ + if (DT_dapltest_debug >= (N)) \ + { \ + DT_Tdep_PT_Printf _X_; \ + } \ +} while (0) + +#ifdef __KERNEL__ +typedef struct Tdep_Print_Entry_Tag +{ + struct Tdep_Print_Entry_Tag *next; + char buffer[PRINT_MAX]; +} Tdep_Print_Entry; + +typedef struct DT_Tdep_Print_Head_Tag +{ + int instance; + struct DT_Tdep_Print_Head_Tag *next; + Tdep_Print_Entry *head; + Tdep_Print_Entry *tail; + DT_Mdep_LockType lock; + DT_WAIT_OBJECT wait_object; +} DT_Tdep_Print_Head; + + +void DT_Tdep_PT_Printf (DT_Tdep_Print_Head *phead, const char * fmt, ...); + +#else + +typedef void * DT_Tdep_Print_Head; +void DT_Tdep_PT_Printf (DT_Tdep_Print_Head *phead, const char * fmt, ...); + +#endif +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_test_data.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_test_data.h new file mode 100644 index 00000000..927792fc --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_test_data.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_TEST_DATA_H__ +#define __DAPL_TEST_DATA_H__ + +#include "dapl_proto.h" +#include "dapl_bpool.h" +#include "dapl_client_info.h" +#include "dapl_transaction_stats.h" +#include "dapl_memlist.h" +#include "dapl_params.h" +#include "dapl_server_info.h" + +/* This lock allows the client side to run + * in a shell script loop without breaking + * connections. Remove it and due to timing + * problems on the server side occasionally + * the server will reject connections. + */ +extern DT_Mdep_LockType g_PerfTestLock; + +/* + * check memory leaking extern int alloc_count ; extern + * DT_Mdep_LockType Alloc_Count_Lock; + */ + +typedef struct +{ + int NextPortNumber; + int num_clients; + DT_Mdep_LockType num_clients_lock; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_EVD_HANDLE recv_evd_hdl; + DAT_EVD_HANDLE reqt_evd_hdl; + DAT_EVD_HANDLE conn_evd_hdl; + DAT_EVD_HANDLE creq_evd_hdl; + DAT_EVD_HANDLE async_evd_hdl; + DAT_EVD_HANDLE rmr_evd_hdl; + DAT_EP_HANDLE ep_handle; + DAT_PSP_HANDLE psp_handle; + Bpool *bpool; +} Per_Server_Data_t; + +typedef struct +{ + DT_Mdep_LockType MemListLock; + MemListEntry_t *MemListHead; + + DT_Mdep_LockType Thread_counter_lock; + int Thread_counter; + Thread *thread; + + bool local_is_server; + Server_Info_t Server_Info; + Client_Info_t Client_Info; + Params_t Params; + DAT_IA_ATTR ia_attr; + DAT_PROVIDER_ATTR provider_attr; + DAT_EP_ATTR ep_attr; + Per_Server_Data_t *ps_ptr; + Transaction_Stats_t Client_Stats; + + /* synchronize the server with the server's spawned test thread. + * That test thread uses a PSP that only one test at a time can + * use. If we don't synchronize access between the teardown and + * creation of that PSP then the client will fail to connect + * randomly, a symptom that the server is not coordinated with + * its test threads. Remove this at your own peril, or if you + * really want your test client to experience rejection on a + * random but regular basis. + */ + DT_WAIT_OBJECT synch_wait_object; + int Countdown_Counter; + +} Per_Test_Data_t; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_cmd.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_cmd.h new file mode 100644 index 00000000..0fbd86d6 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_cmd.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_TRANSACTION_CMD_H__ +#define __DAPL_TRANSACTION_CMD_H__ + +#include "dapl_proto.h" + +#define MAX_OPS 100 +#define NAME_SZ 256 + +#pragma pack (2) +typedef struct +{ + DAT_UINT32 server_initiated; + DAT_UINT32 transfer_type; + DAT_UINT32 num_segs; + DAT_UINT32 seg_size; + DAT_UINT32 reap_send_on_recv; +} Transaction_Cmd_Op_t; + +typedef struct +{ + DAT_UINT32 dapltest_version; + DAT_UINT32 client_is_little_endian; + char server_name[NAME_SZ]; /* -s */ + DAT_UINT32 num_iterations; /* -i */ + DAT_UINT32 num_threads; /* -t */ + DAT_UINT32 eps_per_thread; /* -w */ + DAT_UINT32 use_cno; /* NOT USED - remove and bump version*/ + DAT_UINT32 use_rsp; /* -r */ + DAT_UINT32 debug; /* -d */ + DAT_UINT32 validate; /* -V */ + DAT_UINT32 poll; /* -P */ + char dapl_name[NAME_SZ]; /* -D */ + DAT_QOS ReliabilityLevel; + DAT_UINT32 num_ops; + Transaction_Cmd_Op_t op[MAX_OPS]; +} Transaction_Cmd_t; +#pragma pack () + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_stats.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_stats.h new file mode 100644 index 00000000..6e13ae5e --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_stats.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_TRANSACTION_STATS_H__ +#define __DAPL_TRANSACTION_STATS_H__ + +#include "dapl_proto.h" + +#define whole(num) ((unsigned int)(num)) +#define hundredths(num) ((unsigned int)(((num) - (unsigned int)(num)) * 100)) + +typedef struct +{ + DT_Mdep_LockType lock; + unsigned int wait_count; + unsigned int num_ops; + unsigned int time_ms; + unsigned int bytes_send; + unsigned int bytes_recv; + unsigned int bytes_rdma_read; + unsigned int bytes_rdma_write; +} Transaction_Stats_t; +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_test.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_test.h new file mode 100644 index 00000000..7dc9d68b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_transaction_test.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_TRANSACTION_TEST_H__ +#define __DAPL_TRANSACTION_TEST_H__ + +#include "dapl_proto.h" +#include "dapl_common.h" +#include "dapl_test_data.h" +#include "dapl_transaction_cmd.h" + +typedef struct +{ + DAT_BOOLEAN server_initiated; + DT_Transfer_Type transfer_type; + DAT_UINT32 num_segs; + DAT_UINT32 seg_size; + DAT_BOOLEAN reap_send_on_recv; + Bpool *bp; + + /* RDMA info */ + DAT_RMR_CONTEXT Rdma_Context; + DAT_VADDR Rdma_Address; +} Transaction_Test_Op_t; + +typedef struct +{ + DAT_EP_HANDLE ep_handle; + DAT_EP_ATTR ep_attr; + DAT_CONN_QUAL ia_port; + Bpool *bp; + Transaction_Test_Op_t op[ MAX_OPS ]; + DAT_RSP_HANDLE rsp_handle; + DAT_PSP_HANDLE psp_handle; + +} Ep_Context_t; + +typedef struct +{ + unsigned int stat_bytes_send; + unsigned int stat_bytes_recv; + unsigned int stat_bytes_rdma_read; + unsigned int stat_bytes_rdma_write; + unsigned int start_time; + unsigned int end_time; +} Transaction_Test_Stats_t; + +typedef struct +{ + /* This group set up by DT_Transaction_Create_Test() */ + DAT_BOOLEAN is_server; + DAT_BOOLEAN remote_is_little_endian; + Per_Test_Data_t *pt_ptr; + DAT_IA_HANDLE ia_handle; + Transaction_Cmd_t *cmd; + DAT_IA_ADDRESS_PTR remote_ia_addr; + DAT_CONN_QUAL base_port; + DAT_TIMEOUT time_out; + DAT_COUNT evd_length; + Thread *thread; + + /* This group set up by each thread in DT_Transaction_Main() */ + DAT_PZ_HANDLE pz_handle; + DAT_EVD_HANDLE recv_evd_hdl; /* receive */ + DAT_EVD_HANDLE reqt_evd_hdl; /* request+rmr */ + DAT_EVD_HANDLE conn_evd_hdl; /* connect */ + DAT_EVD_HANDLE creq_evd_hdl; /* "" request */ + Ep_Context_t *ep_context; + + /* Statistics set by DT_Transaction_Run() */ + Transaction_Test_Stats_t stats; +} Transaction_Test_t; + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/include/dapl_version.h b/trunk/ulp/dapl2/test/dapltest/include/dapl_version.h new file mode 100644 index 00000000..9870b0b5 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/include/dapl_version.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_VERSION_H +#define __DAPL_VERSION_H +/* + * Dapltest version number + * + * This should be bumped everytime the "cross-the-wire" behavior changes. + */ + +#define DAPLTEST_VERSION 0x00000006 + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/makefile b/trunk/ulp/dapl2/test/dapltest/makefile new file mode 100644 index 00000000..a0c06273 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\inc\openib.def diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.c b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.c new file mode 100644 index 00000000..d6f66167 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.c @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" +#include "kdapl_tdep.h" +#include +#ifndef UPCALL_FROM_IRQ +#include /* for spin_lock_bh */ +#endif /* UPCALL_FROM_IRQ */ + +#include /* Getting Statistics */ +#include /* Definitions of smp_num_cpus and cpu_logical_map(cpu) */ +#include /* jiffies */ +#include + +/* + * Sleep specified number of milliseconds + */ + +void +DT_Mdep_Sleep (int msec) +{ + set_current_state (TASK_INTERRUPTIBLE); + schedule_timeout (msec*HZ/1000); +} + +void +DT_Mdep_Schedule (void) +{ + schedule(); +} +/* + * Get system statistics including uptime and idle time + */ + +bool +DT_Mdep_GetCpuStat ( + DT_CpuStat *cpu_stat ) +{ + unsigned long i, nice = 0; + unsigned long jif = jiffies; + int cpu , num_of_cpus; + cpu_stat->user = 0; + cpu_stat->system = 0; + cpu_stat->idle = 0; + num_of_cpus=0; + cpu =0; + + // code for vanila 2.4 kernel +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)) && !defined RHAS3 + for (i = 0 ; i < smp_num_cpus; i++) + { + cpu = cpu_logical_map(i); + cpu_stat->user += (unsigned long)kstat.per_cpu_user[cpu]; + nice += (unsigned long)kstat.per_cpu_nice[cpu]; + cpu_stat->system += (unsigned long)kstat.per_cpu_system[cpu]; + } + // kernel 2.4 AS3.0 +#elif (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)) && defined RHAS3 + cpu_stat->user= kstat_sum(accumulated_time.u_usec); + cpu_stat->system = kstat_sum(accumulated_time.s_usec); + nice = kstat_sum(accumulated_time.n_usec); + // kernel 2.6 +#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + for (i = 0; i < NR_CPUS; i++) + { + if (cpu_possible(i)) + { + cpu_stat->system += kstat_cpu(i).cpustat.system; + cpu_stat->user += kstat_cpu(i).cpustat.user; + nice += kstat_cpu(i).cpustat.nice; + num_of_cpus=+1; + } + } +#endif + cpu_stat->idle = (unsigned long)(jif * num_of_cpus - (cpu_stat->user + nice + cpu_stat->system)); + return true; +} + +/* + * Get current time in milliseconds (relative to some fixed point) + */ +unsigned long +DT_Mdep_GetTime (void) +{ + struct timeval tv; + + do_gettimeofday (&tv); + return ((unsigned long) (tv.tv_sec) * 1000L) + + (unsigned long) (tv.tv_usec) / 1000; +} + +#if 0 +unsigned long +DT_Mdep_GetContextSwitchNum (void ) +{ + return nr_context_switches(); +} +#endif + +/* + * Memory allocate and free routines for control blocks (objects) - regular + * memory, always zeroed. + */ +void * +DT_Mdep_Malloc (size_t l_) +{ + void *rval; + + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count++; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + rval = kmalloc (l_, GFP_ATOMIC); + + if (rval) + { + memset (rval, 0, l_); + } + return ( rval ); +} + +void +DT_Mdep_Free (void *a_) +{ + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count--; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + kfree (a_); +} + +/* + * Lock support + * + * Lock object constructor + */ +bool +DT_Mdep_LockInit (DT_Mdep_LockType * lock_ptr) +{ + spin_lock_init (lock_ptr); + return true; +} + +/* + * Lock object destructor + */ +void +DT_Mdep_LockDestroy (DT_Mdep_LockType * lock_ptr) +{ + /* nothing */ +} + +/* + * Locks + * + */ +void +DT_Mdep_Lock (DT_Mdep_LockType * lock_ptr) +{ + SPINLOCK (lock_ptr); +} + +/* + * unlock + */ +void +DT_Mdep_Unlock (DT_Mdep_LockType * lock_ptr) +{ + SPINUNLOCK (lock_ptr); +} + +/* + * Init Thread Attributes + */ +void +DT_Mdep_Thread_Init_Attributes (Thread * thread_ptr) +{ + /* nothing */ +} + +/* + * Destroy Thread Attributes + */ +void +DT_Mdep_Thread_Destroy_Attributes (Thread * thread_ptr) +{ + /* nothing */ +} + +/* + * Start the thread + */ +bool +DT_Mdep_Thread_Start (Thread * thread_ptr) +{ + thread_ptr->thread_handle = kernel_thread ( + (void *)DT_Mdep_Thread_Start_Routine, + (void *)thread_ptr, + 0); + return (thread_ptr->thread_handle); +} + +/* + * Thread execution entry point function + */ +DT_Mdep_Thread_Start_Routine_Return_Type +DT_Mdep_Thread_Start_Routine (void *thread_handle) +{ + Thread *thread_ptr; + thread_ptr = (Thread *) thread_handle; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) + reparent_to_init(); +#else + daemonize(__func__); +#endif + thread_ptr->function (thread_ptr->param); + return 0; +} + +/* + * Thread detach routine. Allows the pthreads + * interface to clean up resources properly at + * thread's end. + */ +void DT_Mdep_Thread_Detach ( int thread_id ) /* AMM */ +{ + /* nothing */ +} + +/* + * Allows a thread to get its own ID so it + * can pass it to routines wanting to act + * upon themselves. + */ + +int DT_Mdep_Thread_SELF (void) /* AMM */ +{ + /* nothing */ + return (0); +} + + +/* + * Allow a thread to exit and cleanup resources. + */ + +void DT_Mdep_Thread_EXIT ( void * thread_handle ) /* AMM */ +{ + /* nothing */ +} + +/* + * DT_Mdep_wait_object_init + * + * Initialize a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if unsuccessful + */ +int +DT_Mdep_wait_object_init ( + IN DT_WAIT_OBJECT *wait_obj) +{ + init_waitqueue_head (wait_obj); + return 0; +} + + +/* Wait on the supplied wait object, up to the specified time_out. + * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. + * Timeout should be specified in micro seconds. + * + * Functional returns: + * 0 -- another thread invoked dapl_os_wait object_wakeup + * -1 -- someone else is already waiting in this wait + * object. + * only one waiter is allowed at a time. + * -1 -- another thread invoked dapl_os_wait_object_destroy + * -1 -- the specified time limit was reached. + */ + +int +DT_Mdep_wait_object_wait ( + IN DT_WAIT_OBJECT *wait_obj, + IN int timeout_val) +{ + int expire; + int dat_status; + + dat_status = DAT_SUCCESS; + if ( DAT_TIMEOUT_INFINITE == timeout_val) + { + interruptible_sleep_on (wait_obj); + } + else + { + expire = timeout_val * HZ / 1000000; + while ( expire ) + { + current->state = TASK_INTERRUPTIBLE; + expire = schedule_timeout (expire); + } + dat_status = DAT_TIMEOUT_EXPIRED; + } + return dat_status; +} + + +/* + * DT_Mdep_wait_object_wakeup + * + * Wakeup a thread waiting on a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_wakeup ( + DT_WAIT_OBJECT *wait_obj ) +{ + wake_up_interruptible (wait_obj); + return 0; +} + +/* + * DT_Mdep_wait_object_destroy + * + * Destroy a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_destroy ( + IN DT_WAIT_OBJECT *wait_obj) +{ + return 0; +} + diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.h b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.h new file mode 100644 index 00000000..1e341180 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_kernel.h @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_KERNEL_MDEP_H__ +#define __DAPL_KERNEL_MDEP_H__ + +/* include files */ + +# include +# include +# include +# include +# include +# include +# include +# include +# include "kdapl_ioctl.h" +#ifndef UPCALL_FROM_IRQ +#include /* for spin_lock_bh */ +#endif /* UPCALL_FROM_IRQ */ + +/* Default Device Name */ +#define DT_MdepDeviceName "ia0a" + +/* Boolean */ +typedef int bool; + +#define true (1) +#define false (0) + +#ifndef __BASE_FILE__ +#define __BASE_FILE__ __FILE__ +#endif + +#ifndef _INLINE_ +#define _INLINE_ __inline__ +#endif + +/* + * Locks + */ + +typedef spinlock_t DT_Mdep_LockType; + +/* + * If UPCALL_FROM_IRQ is defined, the provider will invoke upcalls + * directly from IRQ. Most providers invoke upcalls from the Linux + * BH handler, the default here. + * + * Locks must be adapted to match the actual upcall context! + */ +#ifdef UPCALL_FROM_IRQ +#define SPINLOCK(lp) spin_lock_irq (lp); +#define SPINUNLOCK(lp) spin_unlock_irq (lp); +#else +#define SPINLOCK(lp) spin_lock_bh (lp); +#define SPINUNLOCK(lp) spin_unlock_bh (lp); +#endif + + +/* Wait object used for inter thread communication */ + +typedef wait_queue_head_t DT_WAIT_OBJECT; + +/* + * Thread types + */ +typedef int DT_Mdep_ThreadHandleType; +typedef void (*DT_Mdep_ThreadFunction) (void *param); +typedef void * DT_Mdep_Thread_Start_Routine_Return_Type; +#define DT_MDEP_DEFAULT_STACK_SIZE 65536 + +typedef struct +{ + void (*function) (void *); + void *param; + DT_Mdep_ThreadHandleType thread_handle; + unsigned int stacksize; +} Thread; + +/* + * System information + * + */ + +typedef struct +{ + unsigned long int system; + unsigned long int user; + unsigned long int idle; +} DT_CpuStat; + +/* + * Timing + */ + +typedef unsigned long long int DT_Mdep_TimeStamp; + +static _INLINE_ DT_Mdep_TimeStamp +DT_Mdep_GetTimeStamp ( void ) +{ +#if defined(__GNUC__) && defined(__PENTIUM__) + DT_Mdep_TimeStamp x; + __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); + return x; +#else + +#ifdef __ia64__ + unsigned long x; + + x = get_cycles (); + return x; +#else +#error "Non-Pentium Linux - unimplemented" +#endif +#endif + +} + +#ifndef __ia64__ +typedef int * intptr_t; +typedef unsigned long uintptr_t; +#endif + +#define bzero(x, y) memset(x, 0, y) + +/* + * Define long format types to be used in *printf format strings. We + * use the C string constant concatenation ability to define 64 bit + * formats, which unfortunatly are non standard in the C compiler + * world. E.g. %llx for gcc, %I64x for Windows + */ +#ifdef __x86_64__ +#define F64d "%ld" +#define F64u "%lu" +#define F64x "%lx" +#define F64X "%lX" +#else +#define F64d "%lld" +#define F64u "%llu" +#define F64x "%llx" +#define F64X "%llX" +#endif + +/* + * Define notion of a LONG LONG 0 + */ +#define LZERO 0ULL + +/* Mdep function defines */ + +#define DT_Mdep_flush() +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.c b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.c new file mode 100644 index 00000000..b23c86c0 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.c @@ -0,0 +1,603 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_mdep.h" +#include "dapl_proto.h" + +#include /* needed for pthread_atfork() */ +#include +#include /* needed for getenv() */ +#include /* needed for thread setup */ + +#include +#include +#include +#include +#include +#include +#include +#include /* for printf */ +#include +#include +#include /* for getaddrinfo */ + +/* + * Include files for setting up a network name + */ +#include +#include +#include +#include + +static FILE *Stat_Fp = NULL; +# define DT_STAT_FILE "/proc/stat" + +#include "dapl_test_data.h" /* for alloc_count */ + + +/* + * Machine dependant initialization + */ + +void +DT_Mdep_Init (void) +{ + Stat_Fp = fopen (DT_STAT_FILE, "r"); + if ( NULL == Stat_Fp ) + { + perror ("fopen of " DT_STAT_FILE " failed"); + exit (1); + } +} + +/* + * Machine dependant deinitialization + */ + +void +DT_Mdep_End (void) +{ + if ( 0 != fclose (Stat_Fp) ) + { + perror ("fclose of " DT_STAT_FILE " failed"); + exit (1); + } +} + +/* + * Generate name of IB device + */ + +bool +DT_Mdep_GetDefaultDeviceName (char *dapl_name) +{ + strcpy (dapl_name, DT_MdepDeviceName); + return true; +} + +/* + * Sleep specified number of milliseconds + */ + +void +DT_Mdep_Sleep (int msec) +{ + struct timespec t; + t.tv_sec = msec / 1000; /* Whole seconds */ + t.tv_nsec = (msec % 1000) * 1000 * 1000; + nanosleep (&t, 0); +} + +void +DT_Mdep_Schedule (void) +{ + /* nothing here */ +} +/* + * Get system statistics including uptime and idle time + */ + +bool +DT_Mdep_GetCpuStat ( + DT_CpuStat *cpu_stat ) +{ + +#define DT_CPU_STAT_STR "cpu" +#define DT_CPU_STAT_BUFFER_SIZE 1024 +#define DT_CPU_STAT_DELIMITER " " + + static char buffer[DT_CPU_STAT_BUFFER_SIZE]; + + cpu_stat->user = 0; + cpu_stat->system = 0; + cpu_stat->idle = 0; + + if ( 0 != fflush (Stat_Fp) ) + { + perror ("fflush of " DT_STAT_FILE " failed"); + exit (1); + } + + for (;;) + { + if ( NULL == fgets (buffer, DT_CPU_STAT_BUFFER_SIZE, Stat_Fp) ) + { + printf (DT_CPU_STAT_STR " not found\n"); + exit (1); + } + + if ( !strncmp (buffer, DT_CPU_STAT_STR, strlen (DT_CPU_STAT_STR) ) ) + { + break; + } + } + + (void) strtok (buffer, DT_CPU_STAT_DELIMITER); + cpu_stat->user = strtoul (strtok (NULL, DT_CPU_STAT_DELIMITER), NULL, 0); + cpu_stat->user += strtoul (strtok (NULL, DT_CPU_STAT_DELIMITER), NULL, 0); + cpu_stat->system = strtoul (strtok (NULL, DT_CPU_STAT_DELIMITER), NULL, 0); + cpu_stat->idle = strtoul (strtok (NULL, DT_CPU_STAT_DELIMITER), NULL, 0); + + rewind (Stat_Fp); + + return true; +} + +/* + * Get current time in milliseconds (relative to some fixed point) + */ +unsigned long +DT_Mdep_GetTime (void) +{ + struct tms ts; + clock_t t = times (&ts); + return (unsigned long) ((DAT_UINT64) t * 1000 / CLOCKS_PER_SEC); +} + +double +DT_Mdep_GetCpuMhz ( + void ) +{ +#define DT_CPU_MHZ_BUFFER_SIZE 128 +#if defined (__PPC64__) +#define DT_CPU_MHZ_MHZ "clock" +#else +#define DT_CPU_MHZ_MHZ "cpu MHz" +#endif +#define DT_CPU_MHZ_DELIMITER ":" + + FILE *fp; + char buffer[DT_CPU_MHZ_BUFFER_SIZE]; + char *mhz_str; + + fp = fopen ("/proc/cpuinfo", "r"); + if ( NULL == fp ) + { + perror ("fopen of /proc/cpuinfo failed"); + exit (1); + } + + for (;;) + { + if ( NULL == fgets (buffer, DT_CPU_MHZ_BUFFER_SIZE, fp) ) + { + printf ("cpu MHZ not found\n"); + exit (1); + } + + if ( !strncmp (buffer, DT_CPU_MHZ_MHZ, strlen (DT_CPU_MHZ_MHZ) ) ) + { + (void) strtok (buffer, DT_CPU_MHZ_DELIMITER); + mhz_str = strtok (NULL, DT_CPU_MHZ_DELIMITER); + + break; + } + } + + if ( 0 != fclose (fp) ) + { + perror ("fclose of /proc/cpuinfo failed"); + exit (1); + } + + return strtod (mhz_str, NULL); +} + + +unsigned long +DT_Mdep_GetContextSwitchNum (void ) +{ + +#define DT_CTXT_STR "ctxt" +#define DT_CTXT_BUFFER_SIZE 1024 +#define DT_CTXT_DELIMITER " " + + static char buffer[DT_CTXT_BUFFER_SIZE]; + char *ctxt_str; + + if ( 0 != fflush (Stat_Fp) ) + { + perror ("fflush of " DT_STAT_FILE " failed"); + exit (1); + } + + for (;;) + { + if ( NULL == fgets (buffer, DT_CTXT_BUFFER_SIZE, Stat_Fp) ) + { + printf (DT_CTXT_STR " not found\n"); + exit (1); + } + + if ( !strncmp (buffer, DT_CTXT_STR, strlen (DT_CTXT_STR) ) ) + { + (void) strtok (buffer, DT_CTXT_DELIMITER); + ctxt_str = strtok (NULL, DT_CTXT_DELIMITER); + + break; + } + } + + rewind (Stat_Fp); + + return strtoul (ctxt_str, NULL, 0); +} + +/* + * Memory allocate and free routines for control blocks (objects) - regular + * memory, always zeroed. + */ +void * +DT_Mdep_Malloc (size_t l_) +{ + void *rval; + + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count++; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + rval = malloc (l_); + + if (rval) + { + memset (rval, 0, l_); + } + return ( rval ); +} + +void +DT_Mdep_Free (void *a_) +{ + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count--; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + free (a_); +} + +/* + * Lock support + * + * Lock object constructor + */ +bool +DT_Mdep_LockInit (DT_Mdep_LockType * lock_ptr) +{ + return pthread_mutex_init (lock_ptr, 0) ? false : true; +} + +/* + * Lock object destructor + */ +void +DT_Mdep_LockDestroy (DT_Mdep_LockType * lock_ptr) +{ + pthread_mutex_destroy (lock_ptr); +} + +/* + * Lock + */ +void +DT_Mdep_Lock (DT_Mdep_LockType * lock_ptr) +{ + pthread_mutex_lock (lock_ptr); +} + +/* + * unlock + */ +void +DT_Mdep_Unlock (DT_Mdep_LockType * lock_ptr) +{ + pthread_mutex_unlock (lock_ptr); +} + +/* + * Init Thread Attributes + */ +void +DT_Mdep_Thread_Init_Attributes (Thread * thread_ptr) +{ + pthread_attr_init (&thread_ptr->attr); + pthread_attr_setstacksize (&thread_ptr->attr, thread_ptr->stacksize); + /* Create thread in detached state to free resources on termination; + * this precludes doing a pthread_join, but we don't do it + */ + pthread_attr_setdetachstate (&thread_ptr->attr, PTHREAD_CREATE_DETACHED); +} + +/* + * Destroy Thread Attributes + */ +void +DT_Mdep_Thread_Destroy_Attributes (Thread * thread_ptr) +{ + pthread_attr_destroy (&thread_ptr->attr); +} + +/* + * Start the thread + */ +bool +DT_Mdep_Thread_Start (Thread * thread_ptr) +{ + return pthread_create (&thread_ptr->thread_handle, + &thread_ptr->attr, + DT_Mdep_Thread_Start_Routine, + thread_ptr) == 0; +} + +/* + * Thread execution entry point function + */ +DT_Mdep_Thread_Start_Routine_Return_Type +DT_Mdep_Thread_Start_Routine (void *thread_handle) +{ + Thread *thread_ptr; + thread_ptr = (Thread *) thread_handle; + + thread_ptr->function (thread_ptr->param); + return 0; +} + +/* + * Thread detach routine. Allows the pthreads + * interface to clean up resources properly at + * thread's end. + */ +void DT_Mdep_Thread_Detach ( DT_Mdep_ThreadHandleType thread_id ) /* AMM */ +{ + pthread_detach (thread_id); +} + +/* + * Allows a thread to get its own ID so it + * can pass it to routines wanting to act + * upon themselves. + */ + +DT_Mdep_ThreadHandleType DT_Mdep_Thread_SELF (void) /* AMM */ +{ + return (pthread_self ()); +} + + +/* + * Allow a thread to exit and cleanup resources. + */ + +void DT_Mdep_Thread_EXIT ( void * thread_handle ) /* AMM */ +{ + pthread_exit ( thread_handle ); +} + +/* + * DT_Mdep_wait_object_init + * + * Initialize a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if unsuccessful + */ +int +DT_Mdep_wait_object_init ( + IN DT_WAIT_OBJECT *wait_obj) +{ + + wait_obj->signaled = DAT_FALSE; + if ( 0 != pthread_cond_init ( &wait_obj->cv, NULL ) ) + { + return (-1); + } + + /* Always returns 0. */ + pthread_mutex_init ( &wait_obj->lock, NULL ); + return 0; +} + + +/* Wait on the supplied wait object, up to the specified time_out. + * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. + * Timeout should be specified in micro seconds. + * + * Functional returns: + * 0 -- another thread invoked dapl_os_wait object_wakeup + * -1 -- someone else is already waiting in this wait + * object. + * only one waiter is allowed at a time. + * -1 -- another thread invoked dapl_os_wait_object_destroy + * -1 -- the specified time limit was reached. + */ + +int +DT_Mdep_wait_object_wait ( + IN DT_WAIT_OBJECT *wait_obj, + IN int timeout_val) +{ + int dat_status; + int pthread_status; + struct timespec future; + + dat_status = 0; + pthread_status = 0; + + if ( timeout_val != DAT_TIMEOUT_INFINITE ) + { + struct timeval now; + struct timezone tz; + unsigned int microsecs; + + gettimeofday (&now, &tz); + microsecs = now.tv_usec + (timeout_val % 1000000); + if (microsecs > 1000000) + { + now.tv_sec = now.tv_sec + timeout_val / 1000000 + 1; + now.tv_usec = microsecs - 1000000; + } + else + { + now.tv_sec = now.tv_sec + timeout_val / 1000000; + now.tv_usec = microsecs; + } + + /* Convert timeval to timespec */ + future.tv_sec = now.tv_sec; + future.tv_nsec = now.tv_usec * 1000; + + pthread_mutex_lock (&wait_obj->lock); + while ( wait_obj->signaled == DAT_FALSE && pthread_status == 0) + { + pthread_status = pthread_cond_timedwait ( + &wait_obj->cv, &wait_obj->lock, &future ); + + /* + * No need to reset &future if we go around the loop; + * It's an absolute time. + */ + } + /* Reset the signaled status if we were woken up. */ + if (pthread_status == 0) + { + wait_obj->signaled = false; + } + pthread_mutex_unlock (&wait_obj->lock); + } + else + { + pthread_mutex_lock (&wait_obj->lock); + while ( wait_obj->signaled == DAT_FALSE && pthread_status == 0) + { + pthread_status = pthread_cond_wait ( + &wait_obj->cv, &wait_obj->lock ); + } + /* Reset the signaled status if we were woken up. */ + if (pthread_status == 0) + { + wait_obj->signaled = false; + } + pthread_mutex_unlock (&wait_obj->lock); + } + + if (ETIMEDOUT == pthread_status) + { + return (-1); + } + else if ( 0 != pthread_status) + { + return (-1); + } + + return 0; +} + + +/* + * DT_Mdep_wait_object_wakeup + * + * Wakeup a thread waiting on a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_wakeup ( + DT_WAIT_OBJECT *wait_obj ) +{ + pthread_mutex_lock ( &wait_obj->lock ); + wait_obj->signaled = true; + pthread_mutex_unlock ( &wait_obj->lock ); + if ( 0 != pthread_cond_signal ( &wait_obj->cv ) ) + { + return (-1); + } + + return 0; +} + +/* + * DT_Mdep_wait_object_destroy + * + * Destroy a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_destroy ( + IN DT_WAIT_OBJECT *wait_obj) +{ + if ( 0 != pthread_cond_destroy ( &wait_obj->cv ) ) + { + return (-1); + } + if ( 0 != pthread_mutex_destroy ( &wait_obj->lock ) ) + { + return (-1); + } + + return 0; +} + + diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.h b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.h new file mode 100644 index 00000000..2b6e06cf --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/linux/dapl_mdep_user.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_MDEP_USER_H__ +#define __DAPL_MDEP_USER_H__ + +/* include files */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __ia64__ +#include +#endif + +/* Default Device Name */ +#define DT_MdepDeviceName "OpenIB-cma" + +/* Boolean */ +typedef int bool; + +#define true (1) +#define false (0) + +#ifndef __BASE_FILE__ +#define __BASE_FILE__ __FILE__ +#endif + +#ifndef _INLINE_ +#define _INLINE_ __inline__ +#endif + +/* + * Locks + */ + +typedef pthread_mutex_t DT_Mdep_LockType; + +/* Wait object used for inter thread communication */ + +typedef struct +{ + bool signaled; + pthread_cond_t cv; + pthread_mutex_t lock; +} DT_WAIT_OBJECT; + +/* + * Thread types + */ +typedef pthread_t DT_Mdep_ThreadHandleType; +typedef void (*DT_Mdep_ThreadFunction) (void *param); +typedef void * DT_Mdep_Thread_Start_Routine_Return_Type; +#define DT_MDEP_DEFAULT_STACK_SIZE 65536 + +typedef struct +{ + void (*function) (void *); + void *param; + DT_Mdep_ThreadHandleType thread_handle; + unsigned int stacksize; + pthread_attr_t attr; /* Thread attributes */ +} Thread; + +/* + * System information + * + */ + +typedef struct +{ + unsigned long int system; + unsigned long int user; + unsigned long int idle; +} DT_CpuStat; + +/* + * Timing + */ + +typedef unsigned long long int DT_Mdep_TimeStamp; + +static _INLINE_ DT_Mdep_TimeStamp +DT_Mdep_GetTimeStamp ( void ) +{ +#if defined(__GNUC__) && defined(__i386__) + DT_Mdep_TimeStamp x; + __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); + return x; +#else + +#ifdef __ia64__ + unsigned long x; + + x = get_cycles (); + return x; +#else +#if defined(__PPC64__) + unsigned int tbl, tbu0, tbu1; + do { + __asm__ __volatile__ ("mftbu %0" : "=r"(tbu0)); + __asm__ __volatile__ ("mftb %0" : "=r"(tbl)); + __asm__ __volatile__ ("mftbu %0" : "=r"(tbu1)); + } while (tbu0 != tbu1); + return (((unsigned long long)tbu0) << 32) | tbl; +#else +#if defined(__x86_64__) + unsigned int __a,__d; + asm volatile("rdtsc" : "=a" (__a), "=d" (__d)); + return ((unsigned long)__a) | (((unsigned long)__d)<<32); +#else +#error "Linux CPU architecture - unimplemented" +#endif +#endif +#endif +#endif +} + +/* + * Define long format types to be used in *printf format strings. We + * use the C string constant concatenation ability to define 64 bit + * formats, which unfortunatly are non standard in the C compiler + * world. E.g. %llx for gcc, %I64x for Windows + */ + +#if defined(__x86_64__) || defined(__ia64__) +#define F64d "%ld" +#define F64u "%lu" +#define F64x "%lx" +#define F64X "%lX" +#else +#define F64d "%lld" +#define F64u "%llu" +#define F64x "%llx" +#define F64X "%llX" +#endif +/* + * Define notion of a LONG LONG 0 + */ +#define LZERO 0ULL + +/* Mdep function defines */ +#define DT_Mdep_Debug(N, _X_) \ +do { \ + if (DT_dapltest_debug >= (N)) \ + { \ + DT_Mdep_printf _X_; \ + } \ +} while (0) +#define DT_Mdep_printf printf + +#define DT_Mdep_flush() fflush(NULL) + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.c b/trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.c new file mode 100644 index 00000000..504246b8 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.c @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_mdep.h" +#include "dapl_proto.h" + +#include +#include +#include +#include /* needed for getenv() */ +#include /* needed for pthread_atfork() */ +#include /* needed for thread setup */ + +#include "dapl_test_data.h" /* for alloc_count */ + + +/* + * Machine dependant initialization + */ + +void +DT_Mdep_Init (void) +{ + /* nothing */ +} + +/* + * Machine dependant deinitialization + */ + +void +DT_Mdep_End (void) +{ + /* nothing */ +} + +/* + * Generate name of IB device + */ + +bool +DT_Mdep_GetDefaultDeviceName (char *dapl_name) +{ + strcpy (dapl_name, DT_MdepDeviceName); + return true; +} + +/* + * Sleep specified number of milliseconds + */ + +void +DT_Mdep_Sleep (int msec) +{ + struct timespec t; + t.tv_sec = msec / 1000; /* Whole seconds */ + t.tv_nsec = (msec % 1000) * 1000 * 1000; + nanosleep (&t, 0); +} + +void +DT_Mdep_Schedule (void) +{ + /* void */ +} +/* + * Get system statistics including uptime and idle time + */ + +bool +DT_Mdep_GetCpuStat ( + DT_CpuStat *cpu_stat ) +{ + /* FIXME not implemented */ + return true; +} + +/* + * Get current time in milliseconds (relative to some fixed point) + */ +unsigned long +DT_Mdep_GetTime (void) +{ + struct tms ts; + clock_t t = times (&ts); + return (unsigned long) ((DAT_UINT64) t * 1000 / CLK_TCK); +} + +double +DT_Mdep_GetCpuMhz ( + void ) +{ + #error "Undefined Platform" +} + + +unsigned long +DT_Mdep_GetContextSwitchNum (void ) +{ + #error "Undefined Platform" +} + +/* + * Memory allocate and free routines for control blocks (objects) - regular + * memory, always zeroed. + */ +void * +DT_Mdep_Malloc (size_t l_) +{ + void *rval; + + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count++; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + rval = malloc (l_); + + if (rval) + { + memset (rval, 0, l_); + } + return ( rval ); +} + +void +DT_Mdep_Free (void *a_) +{ + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count--; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + free (a_); +} + +/* + * Lock support + * + * Lock object constructor + */ +bool +DT_Mdep_LockInit (DT_Mdep_LockType * lock_ptr) +{ + return pthread_mutex_init (lock_ptr, 0) ? false : true; +} + +/* + * Lock object destructor + */ +void +DT_Mdep_LockDestroy (DT_Mdep_LockType * lock_ptr) +{ + pthread_mutex_destroy (lock_ptr); +} + +/* + * Lock + */ +void +DT_Mdep_Lock (DT_Mdep_LockType * lock_ptr) +{ + pthread_mutex_lock (lock_ptr); +} + +/* + * unlock + */ +void +DT_Mdep_Unlock (DT_Mdep_LockType * lock_ptr) +{ + pthread_mutex_unlock (lock_ptr); +} + +/* + * Init Thread Attributes + */ +void +DT_Mdep_Thread_Init_Attributes (Thread * thread_ptr) +{ + pthread_attr_init (&thread_ptr->attr); + pthread_attr_setstacksize (&thread_ptr->attr, thread_ptr->stacksize); + /* Create thread in detached state to free resources on termination; + * this precludes doing a pthread_join, but we don't do it + */ + pthread_attr_setdetachstate (&thread_ptr->attr, PTHREAD_CREATE_DETACHED); +} + +/* + * Destroy Thread Attributes + */ +void +DT_Mdep_Thread_Destroy_Attributes (Thread * thread_ptr) +{ + pthread_attr_destroy (&thread_ptr->attr); +} + +/* + * Start the thread + */ +bool +DT_Mdep_Thread_Start (Thread * thread_ptr) +{ + return pthread_create (&thread_ptr->thread_handle, + &thread_ptr->attr, + DT_Mdep_Thread_Start_Routine, + thread_ptr) == 0; +} + +/* + * Thread execution entry point function + */ +DT_Mdep_Thread_Start_Routine_Return_Type +DT_Mdep_Thread_Start_Routine (void *thread_handle) +{ + Thread *thread_ptr; + thread_ptr = (Thread *) thread_handle; + + thread_ptr->function (thread_ptr->param); + return 0; +} + +/* + * Thread detach routine. Allows the pthreads + * interface to clean up resources properly at + * thread's end. + */ +void DT_Mdep_Thread_Detach ( int thread_id ) /* AMM */ +{ + pthread_detach ( thread_id); +} + +/* + * Allows a thread to get its own ID so it + * can pass it to routines wanting to act + * upon themselves. + */ + +int DT_Mdep_Thread_SELF (void) /* AMM */ +{ + + return (pthread_self ()); +} + + +/* + * Allow a thread to exit and cleanup resources. + */ + +void DT_Mdep_Thread_EXIT ( void * thread_handle ) /* AMM */ +{ + pthread_exit ( thread_handle ); +} + +/* + * DT_Mdep_wait_object_init + * + * Initialize a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if unsuccessful + */ +int +DT_Mdep_wait_object_init ( + IN DT_WAIT_OBJECT *wait_obj) +{ + + wait_obj->signaled = DAT_FALSE; + if ( 0 != pthread_cond_init ( &wait_obj->cv, NULL ) ) + { + return (-1); + } + + /* Always returns 0. */ + pthread_mutex_init ( &wait_obj->lock, NULL ); + return 0; +} + + +/* Wait on the supplied wait object, up to the specified time_out. + * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. + * Timeout should be specified in micro seconds. + * + * Functional returns: + * 0 -- another thread invoked dapl_os_wait object_wakeup + * -1 -- someone else is already waiting in this wait + * object. + * only one waiter is allowed at a time. + * -1 -- another thread invoked dapl_os_wait_object_destroy + * -1 -- the specified time limit was reached. + */ + +int +DT_Mdep_wait_object_wait ( + IN DT_WAIT_OBJECT *wait_obj, + IN int timeout_val) +{ + + int dat_status; + int pthread_status; + struct timespec future; + + dat_status = 0; + pthread_status = 0; + + if ( timeout_val != DAT_TIMEOUT_INFINITE ) + { + struct timeval now; + struct timezone tz; + unsigned int microsecs; + + gettimeofday (&now, &tz); + microsecs = now.tv_usec + (timeout_val % 1000000); + if (microsecs > 1000000) + { + now.tv_sec = now.tv_sec + timeout_val / 1000000 + 1; + now.tv_usec = microsecs - 1000000; + } + else + { + now.tv_sec = now.tv_sec + timeout_val / 1000000; + now.tv_usec = microsecs; + } + + /* Convert timeval to timespec */ + future.tv_sec = now.tv_sec; + future.tv_nsec = now.tv_usec * 1000; + + pthread_mutex_lock (&wait_obj->lock); + while ( wait_obj->signaled == DAT_FALSE && pthread_status == 0) + { + pthread_status = pthread_cond_timedwait ( + &wait_obj->cv, &wait_obj->lock, &future ); + + /* + * No need to reset &future if we go around the loop; + * It's an absolute time. + */ + } + /* Reset the signaled status if we were woken up. */ + if (pthread_status == 0) + { + wait_obj->signaled = false; + } + pthread_mutex_unlock (&wait_obj->lock); + } + else + { + pthread_mutex_lock (&wait_obj->lock); + while ( wait_obj->signaled == DAT_FALSE && pthread_status == 0) + { + pthread_status = pthread_cond_wait ( + &wait_obj->cv, &wait_obj->lock ); + } + /* Reset the signaled status if we were woken up. */ + if (pthread_status == 0) + { + wait_obj->signaled = false; + } + pthread_mutex_unlock (&wait_obj->lock); + } + + if (ETIMEDOUT == pthread_status) + { + return (-1); + } + else if ( 0 != pthread_status) + { + return (-1); + } + + return 0; +} + + +/* + * DT_Mdep_wait_object_wakeup + * + * Wakeup a thread waiting on a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_wakeup ( + DT_WAIT_OBJECT *wait_obj ) +{ + pthread_mutex_lock ( &wait_obj->lock ); + wait_obj->signaled = true; + pthread_mutex_unlock ( &wait_obj->lock ); + if ( 0 != pthread_cond_signal ( &wait_obj->cv ) ) + { + return (-1); + } + + return 0; + +} + + +/* + * DT_Mdep_wait_object_destroy + * + * Destroy a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_destroy ( + IN DT_WAIT_OBJECT *wait_obj) +{ + + if ( 0 != pthread_cond_destroy ( &wait_obj->cv ) ) + { + return (-1); + } + if ( 0 != pthread_mutex_destroy ( &wait_obj->lock ) ) + { + return (-1); + } + + return 0; + + +} + diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.h b/trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.h new file mode 100644 index 00000000..fdbb5ed8 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/solaris/dapl_mdep_user.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_MDEP_H__ +#define __DAPL_MDEP_H__ + +/* include files */ + +#include + +# include +# include +# include +# include +# include +# include + +/* Default Device Name */ +#define DT_MdepDeviceName "ia0a" + +/* Boolean */ +typedef int bool; + +#define true (1) +#define false (0) + +#ifndef __BASE_FILE__ +#define __BASE_FILE__ __FILE__ +#endif + +#ifndef _INLINE_ +#error "Must Define _INLINE_ +#endif + +/* Mdep function defines */ + +#define DT_Mdep_spew(N, _X_) \ +do { \ + if (DT_dapltest_debug >= (N)) \ + { \ + DT_Mdep_printf _X_; \ + } \ +} while (0) + +#define DT_Mdep_debug(_X_) DT_Mdep_spew(1, _X_) + +#define DT_Mdep_printf printf +#define DT_Mdep_flush() fflush(NULL) + + +/* + * Locks + */ + +typedef pthread_mutex_t DT_Mdep_LockType; + +/* Wait object used for inter thread communication */ + +typedef struct +{ + bool signaled; + pthread_cond_t cv; + pthread_mutex_t lock; +} DT_WAIT_OBJECT; + +/* + * Thread types + */ +typedef pthread_t DT_Mdep_ThreadHandleType; +typedef void (*DT_Mdep_ThreadFunction) (void *param); +typedef void * DT_Mdep_Thread_Start_Routine_Return_Type; +#define DT_MDEP_DEFAULT_STACK_SIZE 65536 + +typedef struct +{ + void (*function) (void *); + void *param; + DT_Mdep_ThreadHandleType thread_handle; + unsigned int stacksize; + pthread_attr_t attr; /* Thread attributes */ +} Thread; + +/* + * System information + * + */ + +typedef struct +{ + unsigned long int system; + unsigned long int user; + unsigned long int idle; +} DT_CpuStat; + +/* + * Timing + */ + +# error "Must Define DT_Mdep_TimeStamp" + +static _INLINE_ DT_Mdep_TimeStamp +DT_Mdep_GetTimeStamp ( void ) +{ +#error "Must implement DT_Mdep_TimeStamp" +} + +/* + * Define long format types to be used in *printf format strings. We + * use the C string constant concatenation ability to define 64 bit + * formats, which unfortunatly are non standard in the C compiler + * world. E.g. %llx for gcc, %I64x for Windows + */ + +#error "Must Define F64d" +#error "Must Define F64u" +#error "Must Define F64x" +#error "Must Define F64X" + +/* + * Define notion of a LONG LONG 0 + */ +#define LZERO 0 + +#endif diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.c b/trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.c new file mode 100644 index 00000000..03a9d37e --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.c @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_mdep.h" +#include "dapl_proto.h" + +#include + +#include "dapl_test_data.h" /* for alloc_count */ + +/* + * Machine dependant initialization + */ + +void +DT_Mdep_Init (void) +{ + /* Initialize winsic2 */ + /* + * Needs to be done in main(), it's too late here. + */ + // WORD wv = MAKEWORD (1, 1); + // WSADATA wsaData; + // WSAStartup (wv, &wsaData); +} + +/* + * Machine dependant deinitialization + */ + +void +DT_Mdep_End (void) +{ + WSACleanup (); +} + +/* + * Generate name of IB device + */ + +bool +DT_Mdep_GetDefaultDeviceName (char *dapl_name) +{ + strcpy (dapl_name, DT_MdepDeviceName); + return true; +} + +/* + * Sleep specified number of milliseconds + */ + +void +DT_Mdep_Sleep (int msec) +{ + Sleep (msec); +} + +/* + * Get system statistics including uptime and idle time + */ +void +DT_Mdep_Schedule (void) +{ + /* nothing here */ +} + +bool +DT_Mdep_GetCpuStat ( + DT_CpuStat *cpu_stat ) +{ + /* FIXME not implemented */ + return true; +} + +/* + * Get current time in milliseconds (relative to some fixed point) + */ +unsigned long +DT_Mdep_GetTime (void) +{ + return GetTickCount (); +} + +double +DT_Mdep_GetCpuMhz ( + void ) +{ + LONG retVal; + HKEY hKey; + DWORD cpuSpeed = 0; + DWORD dataSize = sizeof (DWORD); + + /* For windows need to query the registry to get the CPU + * Information...-SVSV */ + retVal = RegOpenKeyEx (HKEY_LOCAL_MACHINE, + TEXT ("Hardware\\Description\\System\\CentralProcessor\\0"), + 0, + KEY_QUERY_VALUE, + &hKey); + + if (retVal == ERROR_SUCCESS) + { + retVal = RegQueryValueEx (hKey, + TEXT ("~MHz"), NULL, NULL, + (LPBYTE)&cpuSpeed, &dataSize); + + } + + RegCloseKey (hKey); + + return cpuSpeed; +} + + +unsigned long +DT_Mdep_GetContextSwitchNum (void) +{ + return 0; +} + +/* + * Memory allocate and free routines for control blocks (objects) - regular + * memory, always zeroed. + */ +void * +DT_Mdep_Malloc (size_t l_) +{ + void *rval; + + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count++; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + rval = malloc (l_); + + if (rval) + { + memset (rval, 0, l_); + } + return ( rval ); +} + +void +DT_Mdep_Free (void *a_) +{ + /* + * check memory leaking DT_Mdep_Lock(&Alloc_Count_Lock); alloc_count--; + * DT_Mdep_Unlock(&Alloc_Count_Lock); + */ + + free (a_); +} + +/* + * Lock support + * + * Lock object constructor + */ +bool +DT_Mdep_LockInit (DT_Mdep_LockType * lock_ptr) +{ + *lock_ptr = CreateMutex (0, FALSE, 0); + return *lock_ptr ? true : false; +} + +/* + * Lock object destructor + */ +void +DT_Mdep_LockDestroy (DT_Mdep_LockType * lock_ptr) +{ + CloseHandle (*lock_ptr); +} + +/* + * Lock + */ +void +DT_Mdep_Lock (DT_Mdep_LockType * lock_ptr) +{ + WaitForSingleObject (*lock_ptr, INFINITE); +} + +/* + * unlock + */ +void +DT_Mdep_Unlock (DT_Mdep_LockType * lock_ptr) +{ + ReleaseMutex (*lock_ptr); +} + +/* + * Init Thread Attributes + */ +void +DT_Mdep_Thread_Init_Attributes (Thread * thread_ptr) +{ + /* nothing */ +} + +/* + * Destroy Thread Attributes + */ +void +DT_Mdep_Thread_Destroy_Attributes (Thread * thread_ptr) +{ + /* nothing */ +} + +/* + * Start the thread + */ +bool +DT_Mdep_Thread_Start (Thread * thread_ptr) +{ + thread_ptr->thread_handle = + CreateThread (NULL, + 0, + (LPTHREAD_START_ROUTINE)DT_Mdep_Thread_Start_Routine, + thread_ptr, + 0, + NULL); + if (thread_ptr->thread_handle == NULL) + { + return false; + } + return true; +} + +/* + * Thread execution entry point function + */ +DT_Mdep_Thread_Start_Routine_Return_Type +DT_Mdep_Thread_Start_Routine (void *thread_handle) +{ + Thread *thread_ptr; + thread_ptr = (Thread *) thread_handle; + + thread_ptr->function (thread_ptr->param); +} + +/* + * Thread detach routine. Allows the pthreads + * interface to clean up resources properly at + * thread's end. + */ +void DT_Mdep_Thread_Detach (DT_Mdep_ThreadHandleType thread_id ) /* AMM */ +{ +} + +/* + * Allows a thread to get its own ID so it + * can pass it to routines wanting to act + * upon themselves. + */ + +DT_Mdep_ThreadHandleType DT_Mdep_Thread_SELF (void) /* AMM */ +{ + return 0; +} + + +/* + * Allow a thread to exit and cleanup resources. + */ + +void DT_Mdep_Thread_EXIT ( void * thread_handle ) /* AMM */ +{ + + /* nothing */ +} + +/* + * DT_Mdep_wait_object_init + * + * Initialize a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if unsuccessful + */ +int +DT_Mdep_wait_object_init ( + IN DT_WAIT_OBJECT *wait_obj) +{ + + *wait_obj = CreateEvent (NULL, FALSE, FALSE, NULL); + + if ( *wait_obj == NULL ) + { + return -1; + } + + return 0; + + +} + + +/* Wait on the supplied wait object, up to the specified time_out. + * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. + * Timeout should be specified in micro seconds. + * + * Functional returns: + * 0 -- another thread invoked dapl_os_wait object_wakeup + * -1 -- someone else is already waiting in this wait + * object. + * only one waiter is allowed at a time. + * -1 -- another thread invoked dapl_os_wait_object_destroy + * -1 -- the specified time limit was reached. + */ + +int +DT_Mdep_wait_object_wait ( + IN DT_WAIT_OBJECT *wait_obj, + IN int timeout_val) +{ + + DAT_RETURN status; + DWORD op_status; + + status = DAT_SUCCESS; + + if ( DAT_TIMEOUT_INFINITE == timeout_val ) + { + op_status = WaitForSingleObject (*wait_obj, INFINITE); + } + else + { + /* convert to milliseconds */ + op_status = WaitForSingleObject (*wait_obj, timeout_val/1000); + } + + if (op_status == WAIT_TIMEOUT) + { + status = DAT_CLASS_ERROR | DAT_TIMEOUT_EXPIRED; + } + else if ( op_status == WAIT_FAILED) + { + status = DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return status; +} + + +/* + * DT_Mdep_wait_object_wakeup + * + * Wakeup a thread waiting on a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_wakeup ( + DT_WAIT_OBJECT *wait_obj ) +{ + DWORD op_status; + + op_status = SetEvent (*wait_obj); + if ( op_status == 0 ) + { + return DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return DAT_SUCCESS; +} + + +/* + * DT_Mdep_wait_object_destroy + * + * Destroy a wait object + * + * Input: + * wait_obj + * + * Returns: + * 0 if successful + * -1 if not successful + */ +int +DT_Mdep_wait_object_destroy ( + IN DT_WAIT_OBJECT *wait_obj) +{ + + DWORD op_status; + DAT_RETURN status = DAT_SUCCESS; + + op_status = CloseHandle (*wait_obj); + + if ( op_status == 0 ) + { + status = DAT_CLASS_ERROR | DAT_INTERNAL_ERROR; + } + + return status; +} + diff --git a/trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.h b/trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.h new file mode 100644 index 00000000..451bef8f --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/mdep/windows/dapl_mdep_user.h @@ -0,0 +1,185 @@ + +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#ifndef __DAPL_MDEP_USER_H__ +#define __DAPL_MDEP_USER_H__ + +/* include files */ + +#include + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +# include +# include +# include +# include +# include +# include + +/* Default Device Name */ +#define DT_MdepDeviceName "ibnic0" + +/* Boolean */ +typedef int bool; + +#define true (1) +#define false (0) + +#ifndef __BASE_FILE__ +#define __BASE_FILE__ __FILE__ +#endif + +#ifndef _INLINE_ +#define _INLINE_ __inline +#endif + +/* Mdep function defines */ + +#define DT_Mdep_spew(N, _X_) \ +do { \ + if (DT_dapltest_debug >= (N)) \ + { \ + DT_Mdep_printf _X_; \ + } \ +} while (0) + +#define DT_Mdep_debug(_X_) DT_Mdep_spew(1, _X_) + +#define DT_Mdep_printf printf +#define DT_Mdep_flush() fflush(NULL) + +/* + * Locks + */ + +typedef HANDLE DT_Mdep_LockType; +/* Wait object used for inter thread communication */ + +typedef HANDLE DT_WAIT_OBJECT; + +/* + * Thread types + */ +typedef HANDLE DT_Mdep_ThreadHandleType; +typedef void (*DT_Mdep_ThreadFunction) (void *param); +typedef void DT_Mdep_Thread_Start_Routine_Return_Type; + +#define DT_MDEP_DEFAULT_STACK_SIZE 65536 + +typedef struct +{ + void (*function) (void *); + void *param; + DT_Mdep_ThreadHandleType thread_handle; + unsigned int stacksize; +} Thread; + +/* + * System information + * + */ + +typedef struct +{ + unsigned long int system; + unsigned long int user; + unsigned long int idle; +} DT_CpuStat; + +/* + * Timing + */ +#if defined(_WIN64) && !defined(ReadTimeStampCounter) + +static _INLINE_ _ReadTimeStampCounter(void) +{ + LARGE_INTEGER val; + QueryPerformanceCounter( &val ); + return val.QuadPart; +} +#define ReadTimeStampCounter _ReadTimeStampCounter +#endif + +typedef unsigned __int64 DT_Mdep_TimeStamp; + +static _INLINE_ DT_Mdep_TimeStamp +DT_Mdep_GetTimeStamp ( void ) +{ +#if !defined (_WIN64) && !defined (IA64) + _asm rdtsc +#else + +#ifndef ReadTimeStampCounter + +#define ReadTimeStampCounter() __rdtsc() + + DWORD64 + __rdtsc (VOID); + +#endif + + return (DT_Mdep_TimeStamp) ReadTimeStampCounter(); + +#endif //endif !_WIN64, and !IA64 +} + +/* + * Define types for Window compatibility + */ + +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; + + +#define bzero(x, y) memset(x, 0, y) + +/* + * Define long format types to be used in *printf format strings. We + * use the C string constant concatenation ability to define 64 bit + * formats, which unfortunatly are non standard in the C compiler + * world. E.g. %llx for gcc, %I64x for Windows + */ +#define F64d "%I64d" +#define F64u "%I64u" +#define F64x "%I64x" +#define F64X "%I64X" + +/* + * Define notion of a LONG LONG 0 + */ +#define LZERO 0UL + +#endif /* __DAPL_MDEP_USER_H__ */ diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_bpool.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_bpool.c new file mode 100644 index 00000000..2c7de417 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_bpool.c @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* flag to prevent usage of Memory Windows as they are not supported in mthca */ +static DAT_BOOLEAN enable_memory_windows = DAT_FALSE; + +/*****************************************************************************/ +/* + * Allocate buffer pool (data buffers) + * + * Caller wants to allocate buffers of bytes, + * with each buffer aligned as requested. The caller is free to + * use the buffers separately, or as one contiguous segment, so + * we allocate IOV entries enough to support either usage. + */ +Bpool * +DT_BpoolAlloc ( + Per_Test_Data_t *pt_ptr, + DT_Tdep_Print_Head *phead, + DAT_IA_HANDLE ia_handle, + DAT_PZ_HANDLE pz_handle, + DAT_EP_HANDLE ep_handle, + DAT_EVD_HANDLE rmr_evd_handle, + DAT_COUNT seg_size, + DAT_COUNT num_segs, + DAT_COUNT alignment, + DAT_BOOLEAN enable_rdma_write, + DAT_BOOLEAN enable_rdma_read) +{ + char *module = "DT_BpoolAlloc"; + unsigned char *alloc_ptr = 0; + Bpool *bpool_ptr = 0; + DAT_COUNT alloc_size; + DAT_REGION_DESCRIPTION region; + DAT_RETURN ret; + + /* We'll hand out aligned buffers, compensate here */ + seg_size = DT_RoundSize (seg_size, alignment); + alloc_size = seg_size * num_segs + alignment; + + alloc_ptr = (unsigned char *) DT_MemListAlloc ( pt_ptr, "bpool", BUFF, + alloc_size); + if (!alloc_ptr) + { + DT_Tdep_PT_Printf (phead, "No Memory to create bpool buffer!\n"); + goto err; + } + + bpool_ptr = (Bpool *) DT_MemListAlloc (pt_ptr, "bpool", BPOOL, sizeof (Bpool) + + num_segs * sizeof (DAT_LMR_TRIPLET)); + if (!bpool_ptr) + { + DT_Tdep_PT_Printf (phead,"No Memory to create Bpool!\n"); + goto err; + } + + bpool_ptr->alloc_ptr = alloc_ptr; + bpool_ptr->alloc_size = alloc_size; + bpool_ptr->pz_handle = pz_handle; + bpool_ptr->num_segs = num_segs; + bpool_ptr->ep_handle = ep_handle; + bpool_ptr->buffer_size = seg_size * num_segs; + bpool_ptr->buffer_start = DT_AlignPtr (alloc_ptr, alignment); + bpool_ptr->tripl_start = (DAT_LMR_TRIPLET *) (bpool_ptr + 1); + bpool_ptr->seg_size = seg_size; + bpool_ptr->enable_rdma_write = enable_rdma_write; + bpool_ptr->enable_rdma_read = enable_rdma_read; + bpool_ptr->rmr_evd_handle = rmr_evd_handle; + + DT_Tdep_PT_Debug (3, + (phead, + "lmr_create [%p, " F64x "]\n", + bpool_ptr->buffer_start, + bpool_ptr->buffer_size)); + + memset (®ion, 0, sizeof (region)); + region.for_va = bpool_ptr->buffer_start; + ret = DT_Tdep_lmr_create (ia_handle, + DAT_MEM_TYPE_VIRTUAL, + region, + bpool_ptr->buffer_size, + pz_handle, + DAT_MEM_PRIV_ALL_FLAG, + &bpool_ptr->lmr_handle, + &bpool_ptr->lmr_context, + &bpool_ptr->rmr_context, + &bpool_ptr->reg_size, + &bpool_ptr->reg_addr); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_lmr_create failed %s\n", + module, DT_RetToString (ret)); + goto err; + } + /* verify that the outputs are reasonable */ + if (((uintptr_t)bpool_ptr->reg_addr > (uintptr_t)bpool_ptr->buffer_start) + || (bpool_ptr->reg_size < bpool_ptr->buffer_size + + ((uintptr_t)bpool_ptr->buffer_start - (uintptr_t)bpool_ptr->reg_addr))) + { + DT_Tdep_PT_Printf (phead, "%s: dat_lmr_create bogus" + "in: 0x%p, " F64x " out 0x" F64x ", " F64x "\n", + module, + bpool_ptr->buffer_start, + bpool_ptr->buffer_size, + bpool_ptr->reg_addr, + bpool_ptr->reg_size); + goto err; + } + + DT_Tdep_PT_Debug (3, (phead, + "lmr_create OK [0x" F64x ", " F64x ", lctx=%x]\n", + bpool_ptr->reg_addr, + bpool_ptr->reg_size, + bpool_ptr->lmr_context)); + + /* Enable RDMA if requested */ + if (enable_memory_windows && (enable_rdma_write || enable_rdma_read)) + { + DAT_LMR_TRIPLET iov; + DAT_RMR_COOKIE cookie; + DAT_MEM_PRIV_FLAGS mflags; + DAT_RMR_BIND_COMPLETION_EVENT_DATA rmr_stat; + + /* create the RMR */ + ret = dat_rmr_create (pz_handle, &bpool_ptr->rmr_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_rmr_create failed %s\n", + module, DT_RetToString (ret)); + goto err; + } + + /* bind the RMR */ + iov.virtual_address = bpool_ptr->reg_addr; + iov.segment_length = bpool_ptr->reg_size; + iov.lmr_context = bpool_ptr->lmr_context; + cookie.as_64 = (DAT_UINT64)0UL; + cookie.as_ptr = (DAT_PVOID) (uintptr_t) bpool_ptr->reg_addr; + mflags = (enable_rdma_write && enable_rdma_read ? DAT_MEM_PRIV_ALL_FLAG + : (enable_rdma_write ? DAT_MEM_PRIV_WRITE_FLAG + : (enable_rdma_read ? DAT_MEM_PRIV_READ_FLAG : 0))); + + DT_Tdep_PT_Debug (3, (phead, "rmr_bind [" F64x ", " F64x "]\n", + bpool_ptr->reg_addr, bpool_ptr->reg_size)); + + ret = dat_rmr_bind ( bpool_ptr->rmr_handle, + bpool_ptr->lmr_handle, + &iov, + mflags, + DAT_VA_TYPE_VA, + bpool_ptr->ep_handle, + cookie, + DAT_COMPLETION_DEFAULT_FLAG, + &bpool_ptr->rmr_context); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead,"%s: dat_rmr_bind failed %s\n", + module, DT_RetToString (ret)); + goto err; + } + + DT_Tdep_PT_Debug (3, (phead, "rmr_bind-wait\n")); + + /* await the bind result */ + if (!DT_rmr_event_wait (phead, + bpool_ptr->rmr_evd_handle, + &rmr_stat) || + !DT_rmr_check (phead, + &rmr_stat, + bpool_ptr->rmr_handle, + (DAT_PVOID) (uintptr_t) bpool_ptr->reg_addr, + "Bpool")) + { + goto err; + } + + DT_Tdep_PT_Debug (3, (phead, "rmr_bound [OK Rctx=%x]\n", bpool_ptr->rmr_context)); + } + + /* + * Finally! Return the newly created Bpool. + */ + return ( bpool_ptr ); + + + /* ********************************* + * Whoops - clean up and return NULL + */ +err: + if (bpool_ptr) + { + if (bpool_ptr->rmr_handle) + { + ret = dat_rmr_free (bpool_ptr->rmr_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_rmr_free failed %s\n", + module, + DT_RetToString (ret)); + } + } + if (bpool_ptr->lmr_handle) + { + ret = dat_lmr_free (bpool_ptr->lmr_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_lmr_free failed %s\n", + module, + DT_RetToString (ret)); + } + } + DT_MemListFree (pt_ptr, bpool_ptr); + } + if (alloc_ptr) + { + DT_MemListFree (pt_ptr, alloc_ptr); + } + + return ( 0 ); +} + +/*****************************************************************************/ +bool +DT_Bpool_Destroy (Per_Test_Data_t * pt_ptr, + DT_Tdep_Print_Head *phead, + Bpool * bpool_ptr) +{ + char *module = "DT_Bpool_Destroy"; + bool rval = true; + + if (bpool_ptr) + { + if (bpool_ptr->alloc_ptr) + { + if (bpool_ptr->rmr_handle) + { + DAT_LMR_TRIPLET iov; + DAT_RMR_COOKIE cookie; + DAT_RETURN ret; + + iov.virtual_address = bpool_ptr->reg_addr; + iov.segment_length = 0; /* un-bind */ + iov.lmr_context = bpool_ptr->lmr_context; + cookie.as_64 = (DAT_UINT64)0UL; + cookie.as_ptr = (DAT_PVOID) (uintptr_t)bpool_ptr->reg_addr; + + /* + * Do not attempt to unbind here. The remote node + * is going through the same logic and may disconnect + * before an unbind completes. Any bind/unbind + * operation requires a CONNECTED QP to complete, + * a disconnect will cause problems. Unbind is + * a simple optimization to allow rebinding of + * an RMR, doing an rmr_free will pull the plug + * and cleanup properly. + */ + ret = dat_rmr_free (bpool_ptr->rmr_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_rmr_free failed %s\n", + module, + DT_RetToString (ret)); + rval = false; + } + } + + if (bpool_ptr->lmr_handle) + { + DAT_RETURN ret = dat_lmr_free (bpool_ptr->lmr_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_lmr_free failed %s\n", + module, + DT_RetToString (ret)); + rval = false; + } + } + DT_MemListFree (pt_ptr, bpool_ptr->alloc_ptr); + } + DT_MemListFree (pt_ptr, bpool_ptr); + } + + return (rval); +} + +/*****************************************************************************/ +unsigned char * +DT_Bpool_GetBuffer (Bpool * bpool_ptr, int index) +{ + return ( bpool_ptr->buffer_start + index * bpool_ptr->seg_size ); +} + +/*****************************************************************************/ +DAT_COUNT +DT_Bpool_GetBuffSize (Bpool * bpool_ptr, int index) +{ + return ( bpool_ptr->seg_size ); +} + +/*****************************************************************************/ +DAT_LMR_TRIPLET * +DT_Bpool_GetIOV (Bpool * bpool_ptr, int index) +{ + return ( bpool_ptr->tripl_start + index ); +} + +/*****************************************************************************/ +DAT_LMR_CONTEXT +DT_Bpool_GetLMR (Bpool * bpool_ptr, int index) +{ + return ( bpool_ptr->lmr_context ); +} + +/*****************************************************************************/ +DAT_RMR_CONTEXT +DT_Bpool_GetRMR (Bpool * bpool_ptr, int index) +{ + return ( bpool_ptr->rmr_context ); +} + +/*****************************************************************************/ +void +DT_Bpool_print (DT_Tdep_Print_Head *phead, Bpool * bpool_ptr) +{ + DT_Tdep_PT_Printf (phead, + "BPOOL %p\n", + bpool_ptr); + DT_Tdep_PT_Printf (phead, + "BPOOL alloc_ptr %p\n", + bpool_ptr->alloc_ptr); + DT_Tdep_PT_Printf (phead, + "BPOOL alloc_size %x\n", + (int) bpool_ptr->alloc_size); + DT_Tdep_PT_Printf (phead, + "BPOOL pz_handle %p\n", + bpool_ptr->pz_handle); + DT_Tdep_PT_Printf (phead, + "BPOOL num_segs %x\n", + (int) bpool_ptr->num_segs); + DT_Tdep_PT_Printf (phead, + "BPOOL seg_size %x\n", + (int) bpool_ptr->seg_size); + DT_Tdep_PT_Printf (phead, + "BPOOL tripl_start %p\n", + bpool_ptr->tripl_start); + DT_Tdep_PT_Printf (phead, + "BPOOL buffer_start %p\n", + bpool_ptr->buffer_start); + DT_Tdep_PT_Printf (phead, + "BPOOL buffer_size %x\n", + (int) bpool_ptr->buffer_size); + DT_Tdep_PT_Printf (phead, + "BPOOL rdma_write %x\n", + (int) bpool_ptr->enable_rdma_write); + DT_Tdep_PT_Printf (phead, + "BPOOL rdmaread %x\n", + (int) bpool_ptr->enable_rdma_read); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_client.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_client.c new file mode 100644 index 00000000..df44a5f5 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_client.c @@ -0,0 +1,641 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#ifdef DFLT_QLEN +#undef DFLT_QLEN +#endif + +#define DFLT_QLEN 8 +#define MAX_CONN_RETRY 8 + +/* + * Client control routine Connect to the server, send the command across. + * Then start the client-side of the test - creating threads as needed + */ +void +DT_cs_Client (Params_t * params_ptr, + char *dapl_name, + char *server_name, + DAT_UINT32 total_threads) +{ + Per_Test_Data_t *pt_ptr = NULL; + DAT_IA_HANDLE ia_handle = DAT_HANDLE_NULL; + DAT_PZ_HANDLE pz_handle = DAT_HANDLE_NULL; + DAT_EVD_HANDLE recv_evd_hdl = DAT_HANDLE_NULL; + DAT_EVD_HANDLE reqt_evd_hdl = DAT_HANDLE_NULL; + DAT_EVD_HANDLE conn_evd_hdl = DAT_HANDLE_NULL; + DAT_EVD_HANDLE async_evd_hdl = DAT_HANDLE_NULL; + DAT_EP_HANDLE ep_handle = DAT_HANDLE_NULL; + Server_Info_t *sinfo = NULL; + Transaction_Cmd_t *Transaction_Cmd = NULL; + Quit_Cmd_t *Quit_Cmd = NULL; + Performance_Cmd_t *Performance_Cmd = NULL; + Bpool *bpool = NULL; + DAT_IA_ADDRESS_PTR server_netaddr = NULL; + char *module = "DT_cs_Client"; + unsigned int did_connect = 0; + unsigned int retry_cnt = 0; + DAT_DTO_COOKIE dto_cookie; + + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_EVENT_NUMBER event_num; + unsigned char * buffp; + DAT_RETURN ret; + DT_Tdep_Print_Head *phead; + + phead = params_ptr->phead; + + dto_cookie.as_64 = LZERO; + + DT_Tdep_PT_Printf (phead, "%s: Starting Test ... \n", module); + DT_Mdep_Schedule (); + /* Set up the Per_Test_Data */ + pt_ptr = DT_Alloc_Per_Test_Data (phead); + if (!pt_ptr) + { + DT_Tdep_PT_Printf (phead, "%s: no memory for Per_Test_Data\n", module); + return; + } + DT_MemListInit (pt_ptr); /* init MemlistLock and memListHead */ + DT_Thread_Init (pt_ptr); /* init ThreadLock and threadcount */ + pt_ptr->local_is_server = false; + pt_ptr->Client_Info.dapltest_version = DAPLTEST_VERSION; + pt_ptr->Client_Info.is_little_endian = DT_local_is_little_endian; + pt_ptr->Client_Info.test_type = params_ptr->test_type; + pt_ptr->Client_Info.total_threads = total_threads; + memcpy ( (void *) (uintptr_t) &pt_ptr->Params, + (const void *) params_ptr, + sizeof (Params_t)); + + /* Allocate and fill in the Server's address */ + server_netaddr = ¶ms_ptr->server_netaddr; + + /* Open the IA */ + ret = dat_ia_open (dapl_name, + DFLT_QLEN, + &async_evd_hdl, + &ia_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: Could not open %s (%s)\n", + module, + dapl_name, + DT_RetToString (ret)); + ia_handle = DAT_HANDLE_NULL; + goto client_exit; + } + DT_Tdep_PT_Debug (1,(phead, "%s: IA %s opened\n", module, dapl_name)); + + /* Create a PZ */ + ret = dat_pz_create (ia_handle, &pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_pz_create error: %s\n", + module, + DT_RetToString (ret)); + pz_handle = DAT_HANDLE_NULL; + goto client_exit; + } + + /* Create 3 events - recv, request, connect */ + + ret = DT_Tdep_evd_create (ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_DTO_FLAG, + &recv_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (recv) failed %s\n", + module, + DT_RetToString (ret)); + recv_evd_hdl = DAT_HANDLE_NULL; + goto client_exit; + } + ret = DT_Tdep_evd_create (ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + &reqt_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (send) failed %s\n", + module, + DT_RetToString (ret)); + reqt_evd_hdl = DAT_HANDLE_NULL; + goto client_exit; + } + ret = DT_Tdep_evd_create (ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_CONNECTION_FLAG, + &conn_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (conn) failed %s\n", + module, + DT_RetToString (ret)); + conn_evd_hdl = DAT_HANDLE_NULL; + goto client_exit; + } + + /* Create an EP */ + ret = dat_ep_create (ia_handle, /* IA */ + pz_handle, /* PZ */ + recv_evd_hdl, /* recv */ + reqt_evd_hdl, /* request */ + conn_evd_hdl, /* connect */ + (DAT_EP_ATTR *) NULL, + &ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ep_create error: %s\n", + module, + DT_RetToString (ret)); + ep_handle = DAT_HANDLE_NULL; + goto client_exit; + } + DT_Tdep_PT_Debug (1, (phead, "%s: EP created\n", module)); + + /* + * Gather whatever info we want about defaults, + * and check that we can handle the requested parameters. + */ + if (!DT_query (pt_ptr, ia_handle, ep_handle) || + !DT_check_params (pt_ptr, module)) + { + goto client_exit; + } + + bpool = DT_BpoolAlloc (pt_ptr, + phead, + ia_handle, + pz_handle, + ep_handle, + DAT_HANDLE_NULL, /* no RMR */ + DT_RoundSize (sizeof (Transaction_Cmd_t), 8192), + 3, /* num_buffers */ + DAT_OPTIMAL_ALIGNMENT, + false, + false); + if (bpool == 0) + { + DT_Tdep_PT_Printf (phead, + "%s: no memory for command buffer pool.\n", + module); + goto client_exit; + } + + DT_Tdep_PT_Debug (3, (phead, + "RecvSrvInfo 0 %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (bpool, 0))); + DT_Tdep_PT_Debug (3, (phead, + "SndCliInfo 1 %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (bpool, 1))); + DT_Tdep_PT_Debug (3, (phead, + "SndCommand 2 %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (bpool, 2))); + + /* Post recv buffer for Server_Info (1st buffer in pool) */ + DT_Tdep_PT_Debug (1,(phead, "%s: Posting 1 recv buffer\n", module)); +retry_repost: + if (!DT_post_recv_buffer (phead, + ep_handle, + bpool, + 0, + DT_Bpool_GetBuffSize (bpool, 0))) + { + DT_Tdep_PT_Printf (phead, + "%s: cannot post Server_Info recv buffer.\n", + module); + goto client_exit; + } + + DT_Tdep_PT_Debug (1,(phead, "%s: Connect Endpoint\n", module)); +retry: + ret = dat_ep_connect (ep_handle, + server_netaddr, + SERVER_PORT_NUMBER, + DAT_TIMEOUT_INFINITE, + 0, (DAT_PVOID) 0, /* no private data */ + params_ptr->ReliabilityLevel, + DAT_CONNECT_DEFAULT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: Cannot connect Endpoint %s\n", + module, + DT_RetToString (ret)); + goto client_exit; + } + + DT_Tdep_PT_Debug (1,(phead, "%s: Await connection ...\n", module)); + if (!DT_conn_event_wait (phead, ep_handle, conn_evd_hdl, &event_num)) + { + if ( event_num == DAT_CONNECTION_EVENT_PEER_REJECTED ) + { + DAT_EVENT event; + DAT_COUNT drained = 0; + + DT_Mdep_Sleep (1000); + DT_Tdep_PT_Printf (phead, + "%s: retrying connection...\n", + module); + retry_cnt++; + /* + * See if any buffers were flushed as a result of + * the REJECT; clean them up and repost if so + */ + dat_ep_reset (ep_handle); + do + { + + ret = DT_Tdep_evd_dequeue ( recv_evd_hdl, + &event); + drained++; + } while (DAT_GET_TYPE(ret) != DAT_QUEUE_EMPTY); + + if (drained > 1 && retry_cnt < MAX_CONN_RETRY) + { + DT_Tdep_PT_Printf (phead, "Reposting!!! %d\n", drained); + goto retry_repost; + } + else if (retry_cnt < MAX_CONN_RETRY) + { + goto retry; + } + } + DT_Tdep_PT_Printf (phead, "%s: bad connection event\n", module); + goto client_exit; + } + + did_connect++; + if (DT_dapltest_debug) + { + DT_Tdep_PT_Debug (1,(phead, "%s: Connected!\n", module)); + get_ep_connection_state (phead, ep_handle); + } + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (1000); +#endif + + + /* Send Client_Info (using 2nd buffer in the pool) */ + DT_Tdep_PT_Debug (1,(phead, "%s: Sending Client_Info\n", module)); + buffp = DT_Bpool_GetBuffer (bpool, 1); + memcpy ( (void *)buffp, + (const void *) &pt_ptr->Client_Info, + sizeof (Client_Info_t)); + DT_Client_Info_Endian ((Client_Info_t *) buffp); + if (!DT_post_send_buffer ( phead, + ep_handle, + bpool, + 1, + DT_Bpool_GetBuffSize (bpool, 1))) + { + DT_Tdep_PT_Printf (phead, "%s: cannot send Client_Info\n", module); + goto client_exit; + } + /* reap the send and verify it */ + dto_cookie.as_ptr = (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( bpool, 1); + DT_Tdep_PT_Debug (1,(phead, + "%s: Sent Client_Info - awaiting completion\n", + module)); + if (!DT_dto_event_wait (phead, reqt_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + ep_handle, + DT_Bpool_GetBuffSize (bpool, 1), + dto_cookie, + "Client_Info_Send")) + { + goto client_exit; + } + + /* Set up the Command (using 3rd buffer in pool) */ + DT_Tdep_PT_Debug (1,(phead, "%s: Sending Command\n", module)); + buffp = DT_Bpool_GetBuffer (bpool, 2); + switch (pt_ptr->Client_Info.test_type) + { + case TRANSACTION_TEST: + { + Transaction_Cmd = &pt_ptr->Params.u.Transaction_Cmd; + memcpy ( (void *)buffp, + (const void *)Transaction_Cmd, + sizeof (Transaction_Cmd_t)); + DT_Transaction_Cmd_Endian ((Transaction_Cmd_t *)buffp, true); + break; + } + + case QUIT_TEST: + { + Quit_Cmd = &pt_ptr->Params.u.Quit_Cmd; + memcpy ( (void *)buffp, + (const void *)Quit_Cmd, + sizeof (Quit_Cmd_t)); + DT_Quit_Cmd_Endian ((Quit_Cmd_t *) buffp, true); + break; + } + + case PERFORMANCE_TEST: + { + Performance_Cmd = &pt_ptr->Params.u.Performance_Cmd; + memcpy ( (void *)buffp, + (const void *)Performance_Cmd, + sizeof (Performance_Cmd_t)); + DT_Performance_Cmd_Endian ((Performance_Cmd_t *)buffp); + break; + } + default: + { + DT_Tdep_PT_Printf (phead, "Unknown Test Type\n"); + goto client_exit; + } + } + + /* Send the Command buffer */ + if (!DT_post_send_buffer ( phead, + ep_handle, + bpool, + 2, + DT_Bpool_GetBuffSize (bpool, 2))) + { + DT_Tdep_PT_Printf (phead, "%s: cannot send Command\n", module); + goto client_exit; + } + /* reap the send and verify it */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( bpool, 2); + DT_Tdep_PT_Debug (1,(phead, + "%s: Sent Command - awaiting completion\n", + module)); + if (!DT_dto_event_wait (phead, reqt_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + ep_handle, + DT_Bpool_GetBuffSize (bpool, 2), + dto_cookie, + "Client_Cmd_Send")) + { + goto client_exit; + } + + /************************************************************************/ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( bpool, 0); + DT_Tdep_PT_Debug (1,(phead, "%s: Waiting for Server_Info\n", module)); + if (!DT_dto_event_wait (phead, recv_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + ep_handle, + DT_Bpool_GetBuffSize (bpool, 0), + dto_cookie, + "Server_Info_Recv")) + { + goto client_exit; + } + + DT_Tdep_PT_Debug (1,(phead, "%s: Server_Info Received\n", module)); + sinfo = (Server_Info_t*) DT_Bpool_GetBuffer (bpool, 0); + DT_Server_Info_Endian (sinfo); + memcpy ( (void *) (uintptr_t)&pt_ptr->Server_Info, + (const void *)sinfo, + sizeof (Server_Info_t)); + + /* Perform obligatory version check */ + if (pt_ptr->Server_Info.dapltest_version != DAPLTEST_VERSION) + { + DT_Tdep_PT_Printf (phead, + "%s: DAPLTEST VERSION MISMATCH: Server %d, Client %d\n", + module, + pt_ptr->Server_Info.dapltest_version, + DAPLTEST_VERSION); + goto client_exit; + } + DT_Tdep_PT_Debug (1,(phead, "%s: Version OK!\n", module)); + + /* Dump out what we know, if requested */ + if (DT_dapltest_debug) + { + DT_Server_Info_Print (phead, &pt_ptr->Server_Info); + DT_Client_Info_Print (phead, &pt_ptr->Client_Info); + } + + /* Onward to running the actual test requested */ + switch (pt_ptr->Client_Info.test_type) + { + case TRANSACTION_TEST: + { + if (Transaction_Cmd->debug) + { + DT_Transaction_Cmd_PT_Print (phead, Transaction_Cmd); + } + DT_Transaction_Test_Client (pt_ptr, + ia_handle, + server_netaddr); + break; + } + + case QUIT_TEST: + { + DT_Quit_Cmd_PT_Print (phead, Quit_Cmd); + break; + } + + case PERFORMANCE_TEST: + { + if (Performance_Cmd->debug) + { + DT_Performance_Cmd_PT_Print (phead, Performance_Cmd); + } + + DT_Performance_Test_Client (params_ptr, + pt_ptr, + ia_handle, + server_netaddr); + break; + } + } + + /********************************************************************* + * Done - clean up and go home + */ +client_exit: + DT_Tdep_PT_Debug (1,(phead, "%s: Cleaning Up ...\n", module)); + + /* Disconnect the EP */ + if (ep_handle) + { + /* + * graceful attempt might fail because we got here due to + * some error above, so we may as well try harder. + */ + ret = dat_ep_disconnect (ep_handle, DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ep_disconnect (abrupt) error: %s\n", + module, + DT_RetToString (ret)); + } + else if (did_connect && + !DT_disco_event_wait (phead, conn_evd_hdl, NULL)) + { + DT_Tdep_PT_Printf (phead, "%s: bad disconnect event\n", module); + } + } + + /* Free the bpool (if any) */ + DT_Bpool_Destroy (pt_ptr, phead, bpool); + + /* Free the EP */ + if (ep_handle) + { + DAT_EVENT event; + /* + * Drain off outstanding DTOs that may have been + * generated by racing disconnects + */ + do + { + ret = DT_Tdep_evd_dequeue ( recv_evd_hdl, + &event); + } while (ret == DAT_SUCCESS); + + ret = dat_ep_free (ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ep_free error: %s\n", + module, + DT_RetToString (ret)); + /* keep going */ + } + } + + /* Free the 3 EVDs */ + if (conn_evd_hdl) + { + ret = DT_Tdep_evd_free (conn_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (conn) error: %s\n", + module, + DT_RetToString (ret)); + /* keep going */ + } + } + if (reqt_evd_hdl) + { + ret = DT_Tdep_evd_free (reqt_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (reqt) error: %s\n", + module, + DT_RetToString (ret)); + /* keep going */ + } + } + if (recv_evd_hdl) + { + ret = DT_Tdep_evd_free (recv_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (recv) error: %s\n", + module, + DT_RetToString (ret)); + /* keep going */ + } + } + + /* Free the PZ */ + if (pz_handle) + { + ret = dat_pz_free (pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_pz_free error: %s\n", + module, + DT_RetToString (ret)); + /* keep going */ + } + } + + /* Close the IA */ + if (ia_handle) + { + /* dat_ia_close cleans up async evd handle, too */ + ret = dat_ia_close (ia_handle, DAT_CLOSE_GRACEFUL_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ia_close (graceful) error: %s\n", + module, + DT_RetToString (ret)); + ret = dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ia_close (abrupt) error: %s\n", + module, + DT_RetToString (ret)); + } + /* keep going */ + } + else + { + DT_Tdep_PT_Debug (1,(phead, "%s: IA %s closed\n", module, dapl_name)); + } + } + + /* Free the Per_Test_Data */ + DT_Mdep_LockDestroy (&pt_ptr->Thread_counter_lock); + DT_PrintMemList (pt_ptr); /* check if we return all space allocated */ + DT_Mdep_LockDestroy (&pt_ptr->MemListLock); + DT_Free_Per_Test_Data (pt_ptr); + + DT_Tdep_PT_Printf (phead, + "%s: ========== End of Work -- Client Exiting\n", + module); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_client_info.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_client_info.c new file mode 100644 index 00000000..57f6f206 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_client_info.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_Client_Info_Endian (Client_Info_t * client_info) +{ + client_info->dapltest_version = DT_Endian32 (client_info->dapltest_version); + client_info->is_little_endian = DT_Endian32 (client_info->is_little_endian); + client_info->test_type = DT_Endian32 (client_info->test_type); + client_info->total_threads = DT_Endian32 (client_info->total_threads); +} + + +void +DT_Client_Info_Print (DT_Tdep_Print_Head *phead, Client_Info_t * client_info) +{ + DT_Tdep_PT_Printf (phead, + "-------------------------------------\n"); + DT_Tdep_PT_Printf (phead, + "Client_Info.dapltest_version : %d\n", + client_info->dapltest_version); + DT_Tdep_PT_Printf (phead, + "Client_Info.is_little_endian : %d\n", + client_info->is_little_endian); + DT_Tdep_PT_Printf (phead, + "Client_Info.test_type : %d\n", + client_info->test_type); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_cnxn.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_cnxn.c new file mode 100644 index 00000000..e035c4c9 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_cnxn.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/****************************************************************************/ +int +get_ep_connection_state (DT_Tdep_Print_Head *phead, DAT_EP_HANDLE ep_handle) +{ + DAT_EP_STATE ep_state; + DAT_BOOLEAN in_dto_idle; + DAT_BOOLEAN out_dto_idle; + DAT_RETURN ret; + char *recv_status = "Idle"; + char *req_status = "Idle"; + + + ret = dat_ep_get_status (ep_handle, &ep_state, &in_dto_idle, + &out_dto_idle); + if (ret != 0) + { + DT_Tdep_PT_Printf (phead, + "DAT_ERROR: Can't get Connection State %s\n", + DT_RetToString (ret)); + } + else + { + if (in_dto_idle == 0) + { + recv_status = "Active"; + } + if (out_dto_idle == 0) + { + req_status = "Active"; + } + + DT_Tdep_PT_Printf (phead, + "DAT_STATE: %s\n", + DT_State2Str (ep_state)); + DT_Tdep_PT_Printf (phead, + "DAT_STATE: Inbound DTO Status: %s \n", + recv_status); + DT_Tdep_PT_Printf (phead, + "DAT_STATE: Outbound DTO Status: %s\n", + req_status); + } + + return 0; +} + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_execute.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_execute.c new file mode 100644 index 00000000..6ab5aec4 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_execute.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#include "dapl_transaction_cmd.h" +#include "dapl_performance_cmd.h" +#include "dapl_quit_cmd.h" +#include "dapl_limit_cmd.h" + +void +DT_Execute_Test (Params_t *params_ptr) +{ + Transaction_Cmd_t *Transaction_Cmd; + Quit_Cmd_t *Quit_Cmd; + Limit_Cmd_t *Limit_Cmd; + Performance_Cmd_t *Performance_Cmd; + FFT_Cmd_t *FFT_Cmd; + + /* re init global data - for kdapltest, we are now in the kernel */ + DT_local_is_little_endian = params_ptr->local_is_little_endian; + DT_dapltest_debug = params_ptr->debug; + + switch (params_ptr->test_type) + { + case SERVER_TEST: + { + DT_cs_Server (params_ptr); + break; + } + case TRANSACTION_TEST: + { + Transaction_Cmd = ¶ms_ptr->u.Transaction_Cmd; + DT_cs_Client ( params_ptr, + Transaction_Cmd->dapl_name, + Transaction_Cmd->server_name, + Transaction_Cmd->num_threads * + Transaction_Cmd->eps_per_thread); + break; + } + case QUIT_TEST: + { + Quit_Cmd = ¶ms_ptr->u.Quit_Cmd; + DT_cs_Client ( params_ptr, + Quit_Cmd->device_name, + Quit_Cmd->server_name, + 0); + break; + } + case LIMIT_TEST: + { + Limit_Cmd = ¶ms_ptr->u.Limit_Cmd; + DT_cs_Limit (params_ptr, + Limit_Cmd); + break; + } + case PERFORMANCE_TEST: + { + Performance_Cmd = ¶ms_ptr->u.Performance_Cmd; + DT_cs_Client ( params_ptr, + Performance_Cmd->dapl_name, + Performance_Cmd->server_name, + 1); + break; + } + + case FFT_TEST: + { + FFT_Cmd = ¶ms_ptr->u.FFT_Cmd; + DT_cs_FFT (params_ptr, + FFT_Cmd); + break; + } + } +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_connmgt.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_connmgt.c new file mode 100644 index 00000000..f2ada53b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_connmgt.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +int DT_connmgt_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + FFT_Connection_t conn; + int res = 1; + DAT_RETURN rc = 0; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: Ensure time in dat_evd_wait works correctly\n"); + + DT_fft_init_server (params_ptr, cmd, &conn); + DT_assert (phead, NULL != conn.ia_handle); + rc = DT_Tdep_evd_wait (conn.cr_evd, 10000, &conn.event); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_TIMEOUT_EXPIRED); + +cleanup: + rc = DT_fft_destroy_conn_struct (params_ptr, &conn); + DT_assert_clean (phead, rc == DAT_SUCCESS); + + return res; +} + +int DT_connmgt_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + FFT_Connection_t conn; + int res = 1; + DAT_RETURN rc; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: Attempt to use timeout of 0 in dat_evd_wait\n"); + + DT_fft_init_server (params_ptr, cmd, &conn); + DT_assert (phead, NULL != conn.ia_handle); + + rc = DT_Tdep_evd_wait (conn.cr_evd, 0, &conn.event); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_TIMEOUT_EXPIRED); + +cleanup: + rc = DT_fft_destroy_conn_struct (params_ptr, &conn); + DT_assert_clean (phead, rc == DAT_SUCCESS); + return res; + +} + + +void DT_connmgt_test (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + int i; + int res; + DT_Tdep_Print_Head *phead; + FFT_Testfunc_t cases_func[] = + { + { DT_connmgt_case0 }, + { DT_connmgt_case1 }, + }; + phead = params_ptr->phead; + for (i = 0; i < cmd->size; i++) + { + if (cmd->cases_flag[i]) + { + if (cmd->cases_flag[i]) + { + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: Connect Management (Server side) case: %d\n", i); + res = cases_func[i].fun (params_ptr, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: not support or next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + } +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer.c new file mode 100644 index 00000000..1124dcb3 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer.c @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/*--------------------------------------------------------*/ +int DT_dataxfer_generic ( DT_Tdep_Print_Head * phead, FFT_Cmd_t *cmd, int test_case) +{ + FFT_Connection_t conn; + DAT_RETURN rc=0; + int res=1; + DT_fft_init_server (phead, cmd, &conn); + DT_assert (phead, NULL != conn.ia_handle); + + DT_fft_listen (phead, &conn); + + switch (test_case) + { + case 0: + { + DT_Tdep_PT_Printf (phead, "Posting null send buffer\n"); + rc = DT_post_send_buffer (phead, 0, conn.bpool, 0, + DT_Bpool_GetBuffSize (conn.bpool, 0)); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_INVALID_HANDLE); + break; + } + case 1: + { + DT_Tdep_PT_Printf (phead, "Call evd wait with null evd\n"); + rc =dat_evd_wait (0, DAT_TIMEOUT_INFINITE, 1, &conn.event, + &conn.count); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_INVALID_HANDLE); + break; + } + case 2: + { + DT_Tdep_PT_Printf (phead, "Call evd wait with empty send queue\n"); + rc =dat_evd_wait (conn.send_evd, 10*1000000, 1, &conn.event, + &conn.count); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_TIMEOUT_EXPIRED); + break; + } + case 3: + { + DT_Tdep_PT_Printf (phead, "Posting null recv buffer\n"); + rc =DT_post_recv_buffer (phead, 0, conn.bpool, 0, + DT_Bpool_GetBuffSize (conn.bpool, 0)); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_INVALID_HANDLE); + break; + } + } +cleanup: + DT_assert_clean (phead, DT_fft_destroy_conn_struct (phead, &conn)); + return res; +} + +int DT_dataxfer_case0 ( DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + DT_Tdep_PT_Printf (phead, "\ + Description: Call dat_ep_post_send with null ep_handle.\n"); + return DT_dataxfer_generic (phead, cmd, 0); +} + +int DT_dataxfer_case1 ( DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + DT_Tdep_PT_Printf (phead, "\ + Description: Call dat_evd_wait with null evd.\n"); + return DT_dataxfer_generic (phead, cmd, 1); +} + +int DT_dataxfer_case2 ( DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + DT_Tdep_PT_Printf (phead, "\ + Description: Call dat_evd_wait with null evd.\n"); + return DT_dataxfer_generic (phead, cmd, 2); +} + +int DT_dataxfer_case3 ( DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + DT_Tdep_PT_Printf (phead, "\ + Description: Call dat_evd_wait with null evd.\n"); + return DT_dataxfer_generic (phead, cmd, 3); +} + +/*-------------------------------------------------------------*/ +void DT_dataxfer_test (DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + int i; + int res; + FFT_Testfunc_t cases_func[] = + { + { DT_dataxfer_case0 }, + { DT_dataxfer_case1 }, + { DT_dataxfer_case2 }, + { DT_dataxfer_case3 }, + }; + + for (i=0; isize; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: Protection Zone management case: %d\n", i); + res = cases_func[i].fun (phead, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: not support or next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer_client.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer_client.c new file mode 100644 index 00000000..fc47720c --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_dataxfer_client.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define CONN_STATE 1 +#define TIMEOUT_TEST 2 +#define DATAXFER_TEST 3 + +int DT_dataxfer_client_generic (DT_Tdep_Print_Head *phead, + FFT_Cmd_t *cmd, + int flag) +{ + int res=1; + FFT_Connection_t conn; + DAT_RETURN rc=0; + + DT_fft_init_client (phead, cmd, &conn); + DT_assert_dat (phead, conn.ia_handle != NULL) + + DT_assert (phead, DT_fft_connect (phead, &conn)); + + if (flag == CONN_STATE) + { + res = 1; + goto cleanup; + } + else if (flag == TIMEOUT_TEST) + { + + + } + else if (flag == DATAXFER_TEST) + { + conn.bpool = DT_BpoolAlloc (0, + phead, + conn.ia_handle, + conn.pz_handle, + NULL, + NULL, + 4096, + 2, + DAT_OPTIMAL_ALIGNMENT, + false, + false); + DT_assert (phead, conn.bpool != 0); + rc = DT_post_send_buffer (phead, conn.ep_handle, conn.bpool, 0, + DT_Bpool_GetBuffSize (conn.bpool, 0)); + DT_assert_dat (phead, rc == DAT_SUCCESS); + rc = dat_evd_wait (conn.send_evd, 10*1000000, 1, &conn.event, + &conn.count); + DT_assert_dat (phead, rc == DAT_SUCCESS); + res = 1; + goto cleanup; + } + // cleanup +cleanup: + + if (conn.ep_handle) + { + // disconnect + DT_Tdep_PT_Printf (phead, "Disconnect\n"); + rc = dat_ep_disconnect (conn.ep_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + rc = DT_fft_destroy_conn_struct (phead, &conn); + DT_assert_clean (phead, rc == DAT_SUCCESS); + + return res; +} + +int DT_dataxfer_client_case0 (DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + DT_Tdep_PT_Printf (phead, "\ + Description: This is a helper case on the client side for dataxfer case0.\n"); + return DT_dataxfer_client_generic (phead, cmd, CONN_STATE); +} + +void DT_dataxfer_client_test (DT_Tdep_Print_Head *phead, FFT_Cmd_t *cmd) +{ + int i; + int res; + FFT_Testfunc_t cases_func[] = + { + { DT_dataxfer_client_case0 }, + }; + + for (i=0; isize; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: Dataxfer client case: %d\n", i); + res = cases_func[i].fun (phead, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: not support or next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_endpoint.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_endpoint.c new file mode 100644 index 00000000..95f53193 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_endpoint.c @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define CQENTRYCOUNT 100 +#define BUFFSIZE 1024 +#define DEFAULT_QUEUE_LEN 10 + +int DT_endpoint_generic (Params_t *params_ptr, + FFT_Cmd_t *cmd, + bool destroy_pz_early) +{ + char *dev_name; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_EP_HANDLE ep_handle; + DAT_EVD_HANDLE evd_handle; + DAT_EVD_HANDLE conn_evd_handle; + DAT_EVD_HANDLE send_evd_handle; + DAT_EVD_HANDLE recv_evd_handle; + DAT_RETURN rc, wanted; + int res; + DT_Tdep_Print_Head *phead; + + res = 1; + ia_handle = NULL; + pz_handle = NULL; + ep_handle = NULL; + evd_handle = NULL; + conn_evd_handle = NULL; + send_evd_handle = NULL; + recv_evd_handle = NULL; + dev_name = cmd->device_name; + evd_handle = DAT_HANDLE_NULL; + phead = params_ptr->phead; + + rc = dat_ia_open ((const DAT_NAME_PTR)dev_name, + DEFAULT_QUEUE_LEN, &evd_handle, &ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = dat_pz_create (ia_handle, &pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + if (destroy_pz_early) + { + if (pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + } + + rc = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + &send_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, NULL, DAT_EVD_DTO_FLAG, + &recv_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, NULL, + DAT_EVD_CONNECTION_FLAG, &conn_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = dat_ep_create (ia_handle, pz_handle, recv_evd_handle, send_evd_handle, + conn_evd_handle, NULL, &ep_handle); + if (destroy_pz_early) + { + wanted = DAT_INVALID_HANDLE; + } + else + { + wanted = DAT_SUCCESS; + } + DT_assert_dat (phead, DAT_GET_TYPE (rc) == wanted); + +cleanup: + if (ep_handle) + { + rc = dat_ep_free (ep_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (send_evd_handle) + { + rc = DT_Tdep_evd_free (send_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (recv_evd_handle) + { + rc = DT_Tdep_evd_free (recv_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (conn_evd_handle) + { + rc = DT_Tdep_evd_free (conn_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (!destroy_pz_early && pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (ia_handle) + { + rc = dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + return res; +} + +int DT_endpoint_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: Test if we can normally create endpoint and destory it.\n"); + DT_Tdep_PT_Printf (phead, "\ + The endpoint is not associated with a CQ\n"); + return DT_endpoint_generic (params_ptr, cmd, false); /* destroy pz early */ +} + +int DT_endpoint_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: try to create endpoint with pz already destroyed\n"); + return DT_endpoint_generic (params_ptr, cmd, true); /* destroy pz early */ +} + +int DT_endpoint_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + char *dev_name; + DAT_IA_HANDLE ia_handle; + DAT_EP_HANDLE ep_handle; + DAT_EVD_HANDLE send_evd, conn_evd, recv_evd, cr_evd; + DAT_PZ_HANDLE pz_handle; + DAT_EVENT event; + Bpool *bpool; + int res; + DAT_RETURN rc; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: try to destroy ep with descriptor still in working queue\n"); + res = 1; + bpool = 0; + pz_handle = 0; + ia_handle = 0; + ep_handle = 0; + send_evd = 0; + conn_evd = 0; + recv_evd = 0; + cr_evd = 0; + dev_name = cmd->device_name; + + rc = DT_ia_open (dev_name, &ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + rc = dat_pz_create (ia_handle, &pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + rc = DT_ep_create (params_ptr, + ia_handle, + pz_handle, + &cr_evd, + &conn_evd, + &send_evd, + &recv_evd, + &ep_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + bpool = DT_BpoolAlloc (0, phead, ia_handle, pz_handle, NULL, NULL, 4096, 1, + DAT_OPTIMAL_ALIGNMENT, false, false); + DT_assert (phead, bpool != 0); + DT_assert (phead, DT_post_recv_buffer (phead, + ep_handle, + bpool, + 0, + 4096) == true); + if (ep_handle) + { + rc = dat_ep_free (ep_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + + /* + * Remove all DTOs. The disconnect above may have + * flushed all posted operations, so this is just a + * clean up. + */ + do + { + rc = DT_Tdep_evd_dequeue ( recv_evd, + &event); + } while (rc == DAT_SUCCESS); +cleanup: + if (bpool) + { + rc = DT_Bpool_Destroy (0, phead, bpool); + DT_assert_clean (phead, rc != false); + } + if (pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (ia_handle) + { + rc = dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + return res; + +} + +/*-------------------------------------------------------------*/ +void DT_endpoint_test (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + int i; + int res; + DT_Tdep_Print_Head *phead; + FFT_Testfunc_t cases_func[] = + { + { DT_endpoint_case0 }, + { DT_endpoint_case1 }, + { DT_endpoint_case2 }, + }; + + phead = params_ptr->phead; + for (i=0; isize; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: EndPoint management case: %d\n", i + ); + res = cases_func[i].fun (params_ptr, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: not support or next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_hwconn.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_hwconn.c new file mode 100644 index 00000000..52b9a01b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_hwconn.c @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/*--------------------------------------------------------*/ +int DT_hwconn_case0 ( Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + char* dev_name; + DAT_IA_HANDLE nic_handle; + DAT_EVD_HANDLE evd_handle; + DAT_RETURN rc; + int res = 1; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: Test if we can normally Open NIC and then close it\n"); + + dev_name= cmd->device_name; + nic_handle=0; + evd_handle = DAT_HANDLE_NULL; + + rc=dat_ia_open ((const DAT_NAME_PTR)dev_name, 10, &evd_handle, &nic_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc=dat_ia_close (nic_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_dat (phead, rc == DAT_SUCCESS); +cleanup: + + return res; +} + +/*--------------------------------------------------------*/ +int DT_hwconn_case1 ( Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DAT_IA_HANDLE nic_handle; + DAT_RETURN rc; + DAT_EVD_HANDLE evd_handle; + char dev_name[100]; + int i; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "Description: try to open NIC with incorrect device name\n"); + DT_Tdep_PT_Printf (phead, " (just num, one letter, multiple letter, num_letter\n"); + DT_Tdep_PT_Printf (phead, "letter_num). You alse can do this test manually\n"); + DT_Tdep_PT_Printf (phead, "dapltest -T F -D -f hwconn \n"); + + for (i=0; i< 5; i++) + { + if (i==0) + { + sprintf (dev_name, "%s", "40"); /* just number */ + } + else if (i==1) + { + sprintf (dev_name, "%s", "x"); /* just letter */ + } + else if (i==2) + { + sprintf (dev_name, "%s", "xsdf"); /* multiple letter */ + } + else if (i==3) + { + sprintf (dev_name, "%s", "x34"); /* letter_number */ + } + else if (i==4) + { + sprintf (dev_name, "%s", "34df"); /* number_letter */ + } + + evd_handle = DAT_HANDLE_NULL; + rc=dat_ia_open ((const DAT_NAME_PTR)dev_name, 10, &evd_handle, &nic_handle); + if (DAT_GET_TYPE (rc) != DAT_PROVIDER_NOT_FOUND) + { + const char *major_msg, *minor_msg; + + DT_Tdep_PT_Printf (phead, " \ + fff not get expected result when open NIC with device name: %s\n", dev_name); + dat_strerror (rc, &major_msg, &minor_msg); + DT_Tdep_PT_Printf (phead, "ERROR: %s (%s)\n", major_msg, minor_msg); + + + if (rc==DAT_SUCCESS) + { + rc = dat_ia_close (nic_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + return 0; + } + } + return 1; +} + +/*--------------------------------------------------------*/ +int DT_hwconn_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DAT_IA_HANDLE nic_handle; + DAT_RETURN rc; + int res=1; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: Try to close nic with Nic handle is null (NIC not open)\n"); + nic_handle=0; + rc=dat_ia_close (nic_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_dat (phead, DAT_GET_TYPE (rc) ==DAT_INVALID_HANDLE); + +cleanup: + return res; +} + +/*--------------------------------------------------------*/ +int DT_hwconn_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + FFT_Connection_t conn; + DAT_RETURN rc; + int res; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "Description: Test if we can close NIC when the created \n"); + DT_Tdep_PT_Printf (phead, "endpoint has not been destroyed.\n"); + DT_Tdep_PT_Printf (phead, "The problem for this case is that once the hca is closed, \n"); + DT_Tdep_PT_Printf (phead, "there is no way to destroy the endpoint's resources\n"); + DT_Tdep_PT_Printf (phead, "thus the test leaks a small amount of memory\n"); + + res=1; + + DT_fft_init_client (params_ptr, cmd, &conn); + + /* try to close nic when vi have not destroyed */ + if (conn.ia_handle) + { + rc= dat_ia_close (conn.ia_handle, DAT_CLOSE_ABRUPT_FLAG); + if (rc !=DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Warning: dat_ia_close fails %s, reboot for cleanup\n", + DT_RetToString (rc)); + return 0; + } + } else + { + res = 0; + } + /* if nic is closed, it is impossible to destory vi and ptag */ + //DT_fft_destroy_conn_struct(&conn); + return res; + +} + + +/*-------------------------------------------------------------*/ +void DT_hwconn_test (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + int i; + int res; + DT_Tdep_Print_Head *phead; + + FFT_Testfunc_t cases_func[] = + { + { DT_hwconn_case0 }, + { DT_hwconn_case1 }, + { DT_hwconn_case2 }, + { DT_hwconn_case3 }, + }; + + phead = params_ptr->phead; + for (i=0; isize; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: Hardware connection case: %d\n", i); + res = cases_func[i].fun (params_ptr, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_mem.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_mem.c new file mode 100644 index 00000000..ae34e391 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_mem.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define CQENTRYCOUNT 100 +#define BUFFSIZE 1024 + +/*--------------------------------------------------------*/ +int DT_mem_generic (Params_t *params_ptr, FFT_Cmd_t *cmd, int flag) +{ + DAT_RETURN rc, expect; + FFT_Connection_t conn; + DAT_REGION_DESCRIPTION region; + DAT_VLEN reg_size; + DAT_LMR_HANDLE lmr_handle; + DAT_LMR_CONTEXT lmr_context; + DAT_VADDR reg_addr; + unsigned char *alloc_ptr; + int res; + DAT_VLEN buffer_size; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + rc = 0; + expect = 0; + res = 1; + lmr_handle = 0; + lmr_context = 0; + reg_addr = 0; + alloc_ptr = 0; + ia_handle = 0; + pz_handle = 0; + + DT_fft_init_client (params_ptr, cmd, &conn); + DT_assert (phead, NULL != conn.ia_handle); + + if (flag == 2) + { + buffer_size = 0; + alloc_ptr = 0; + } + else + { + buffer_size = BUFFSIZE * sizeof (unsigned char); + alloc_ptr = (unsigned char *)DT_Mdep_Malloc ((size_t)buffer_size); + DT_assert (phead, alloc_ptr); + } + + + memset (®ion, 0, sizeof (region)); + region.for_va = alloc_ptr; + + ia_handle = conn.ia_handle; + + if (flag != 3) + { + pz_handle = conn.pz_handle; + } + + if (flag != 4) + { + DT_Tdep_PT_Printf (phead, "Registering memory\n"); + rc = DT_Tdep_lmr_create (ia_handle, + DAT_MEM_TYPE_VIRTUAL, + region, + buffer_size, + conn.pz_handle, + DAT_MEM_PRIV_ALL_FLAG, + &lmr_handle, + &lmr_context, + NULL, /* FIXME */ + ®_size, + ®_addr); + if (flag == 2) + { + expect = DAT_LENGTH_ERROR; + } + else + { + expect = DAT_SUCCESS; + } + DT_assert_dat (phead, DAT_GET_TYPE (rc) == expect); + } + if (flag == 1) + { + if (lmr_handle) + { + rc = dat_lmr_free (lmr_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + lmr_handle = 0; + + rc = DT_Tdep_lmr_create (conn.ia_handle, + DAT_MEM_TYPE_VIRTUAL, + region, + buffer_size, + conn.pz_handle, + DAT_MEM_PRIV_ALL_FLAG, + &lmr_handle, + &lmr_context, + NULL, /* FIXME */ + ®_size, + ®_addr); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + +cleanup: + if (lmr_handle) + { + rc = dat_lmr_free (lmr_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (alloc_ptr) + { + DT_Mdep_Free (alloc_ptr); + } + rc = DT_fft_destroy_conn_struct (params_ptr, &conn); + DT_assert_clean (phead, rc == DAT_SUCCESS); + + return res; + +} +int DT_mem_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: Test if we can register typical size of memory\n"); + DT_Tdep_PT_Printf (phead, "\ + then deregister it.\n"); + return DT_mem_generic (params_ptr, cmd, 0); +} + +/*--------------------------------------------------------*/ +int DT_mem_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: Test if we can register typical size of memory\n"); + DT_Tdep_PT_Printf (phead, "\ + deregister, then register it again.\n"); + return DT_mem_generic (params_ptr,cmd, 1); +} + +/*--------------------------------------------------------*/ +int DT_mem_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: Try to register memory with memory size 0\n"); + return DT_mem_generic (params_ptr, cmd, 2); +} + +/*--------------------------------------------------------*/ +int DT_mem_case3 ( Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: Try to register memory with null pz\n"); + return DT_mem_generic (params_ptr, cmd, 3); +} + +/*--------------------------------------------------------*/ +int DT_mem_case4 ( Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "\ + Description: Try to deregister memory with null lmr_handle\n"); + return DT_mem_generic (params_ptr, cmd, 4); +} + +/*-------------------------------------------------------------*/ +void DT_mem_test (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + int i; + int res; + DT_Tdep_Print_Head *phead; + + FFT_Testfunc_t cases_func[] = + { + { DT_mem_case0 }, + { DT_mem_case1 }, + { DT_mem_case2 }, + { DT_mem_case3 }, + { DT_mem_case4 }, + }; + + phead = params_ptr->phead; + for (i=0; isize; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: Memory register/deregister case: %d\n", i); + res = cases_func[i].fun (params_ptr, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: not support or next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_pz.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_pz.c new file mode 100644 index 00000000..6f5d2cdf --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_pz.c @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define CQENTRYCOUNT 100 +#define BUFFSIZE 1024 + +/*--------------------------------------------------------*/ +int DT_pz_case0 ( Params_t *params_ptr, + FFT_Cmd_t *cmd) +{ + char* dev_name; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_EVD_HANDLE evd_handle; + DAT_RETURN rc; + int res; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: Test if we can normally create pz and destroy it.\n"); + + res=1; + ia_handle=0; + pz_handle =0; + evd_handle = DAT_HANDLE_NULL; + dev_name= cmd->device_name; + + rc = DT_ia_open (dev_name, &ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + rc = dat_pz_create (ia_handle, &pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + +cleanup: + if (pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + if (ia_handle) + { + rc = dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + return res; +} + +/*--------------------------------------------------------*/ +int DT_pz_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + char* dev_name; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + DAT_EP_HANDLE ep_handle; + DAT_EVD_HANDLE conn_evd, send_evd, recv_evd, cr_evd; + DAT_RETURN rc; + int res; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: try to destroy pz with vi still associated with it\n"); + + res=1; + ia_handle=0; + pz_handle =0; + ep_handle=0; + conn_evd = 0; + send_evd = 0; + recv_evd = 0; + cr_evd = 0; + dev_name= cmd->device_name; + + rc = DT_ia_open (dev_name, &ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = dat_pz_create (ia_handle, &pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = DT_ep_create (params_ptr, + ia_handle, + pz_handle, + &cr_evd, + &conn_evd, + &send_evd, + &recv_evd, + &ep_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + if (pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_INVALID_STATE); + } + +cleanup: + /* corrrect order */ + if (ep_handle) + { + rc=dat_ep_free (ep_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn_evd) + { + rc = DT_Tdep_evd_free (conn_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (send_evd) + { + rc = DT_Tdep_evd_free (send_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (recv_evd) + { + rc = DT_Tdep_evd_free (recv_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (pz_handle) + { + rc=dat_pz_free (pz_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (ia_handle) + { + rc=dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + return res; +} + +/*--------------------------------------------------------*/ +int DT_pz_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + char* dev_name; + DAT_IA_HANDLE ia_handle; + DAT_PZ_HANDLE pz_handle; + Bpool *bpool; + DAT_RETURN rc; + int res; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + DT_Tdep_PT_Printf (phead, "\ + Description: try to destroy pz with registered memory still\n"); + DT_Tdep_PT_Printf (phead, "\ + associated with it\n"); + + res=1; + ia_handle=0; + pz_handle =0; + bpool=0; + dev_name= cmd->device_name; + + rc = DT_ia_open (dev_name, &ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = dat_pz_create (ia_handle, &pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* allocate and register bpool */ + bpool = DT_BpoolAlloc (0, phead, ia_handle, pz_handle, NULL, + NULL, BUFFSIZE, 1, DAT_OPTIMAL_ALIGNMENT, + false, false); + DT_assert (phead, bpool != 0); + + if (pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_dat (phead, DAT_GET_TYPE (rc) == DAT_INVALID_STATE); + } + +cleanup: + + /* deregister and free bpool */ + if (DT_Bpool_Destroy (0, phead, bpool)==false) + { + DT_Tdep_PT_Printf (phead, "Warning: Destroy bpool fails, reboot for cleanup\n"); + return 0; + } + if (pz_handle) + { + rc=dat_pz_free (pz_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (ia_handle) + { + rc=dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + return res; +} + +/*-------------------------------------------------------------*/ +void DT_pz_test (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + int i; + int res; + DT_Tdep_Print_Head *phead; + + FFT_Testfunc_t cases_func[] = + { + { DT_pz_case0 }, + { DT_pz_case1 }, + { DT_pz_case2 }, + }; + + phead = params_ptr->phead; + for (i=0; isize; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "\ + Function feature: Protection Zone management case: %d\n", i); + res = cases_func[i].fun (params_ptr, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res ==-1) + { + DT_Tdep_PT_Printf (phead, "Result: use other test tool\n"); + } + else if (res ==-2) + { + DT_Tdep_PT_Printf (phead, "Result: not support or next stage to develop\n"); + } + + DT_Tdep_PT_Printf (phead, "\ + *********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_queryinfo.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_queryinfo.c new file mode 100644 index 00000000..04dc257e --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_queryinfo.c @@ -0,0 +1,664 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define CQENTRYCOUNT 100 +#define BUFFSIZE 1024 +#define DEFAULT_QUEUE_LEN 10 + +#if defined(WIN32) +static DAT_OS_WAIT_PROXY_AGENT NULLPROXY = +{ + (DAT_PVOID) NULL, (DAT_AGENT_FUNC) NULL +}; +#endif + +int DT_queryinfo_basic (Params_t *params_ptr, + FFT_Cmd_t *cmd, + FFT_query_enum object_to_query, + DAT_RETURN result_wanted) +{ + char *dev_name; + DAT_IA_HANDLE ia_handle; + DAT_IA_ATTR ia_attributes; + DAT_PROVIDER_ATTR provider_attributes; + DAT_EVD_HANDLE evd_handle; + DAT_EVD_HANDLE conn_evd_handle; + DAT_EVD_HANDLE cr_evd_handle; + DAT_EVD_HANDLE send_evd_handle; + DAT_EVD_HANDLE recv_evd_handle; + DAT_EP_HANDLE ep_handle; + DAT_EP_PARAM ep_param; + DAT_CNO_HANDLE cno_handle; +#ifndef __KDAPLTEST__ + DAT_CNO_PARAM cno_param; +#endif + DAT_EVD_PARAM evd_param; + DAT_PSP_HANDLE psp_handle; + DAT_PSP_PARAM psp_param; + DAT_RSP_HANDLE rsp_handle; + DAT_RSP_PARAM rsp_param; + DAT_PZ_HANDLE pz_handle; + DAT_PZ_PARAM pz_param; + DAT_LMR_HANDLE lmr_handle; + DAT_LMR_PARAM lmr_param; + DAT_LMR_CONTEXT lmr_context; + DAT_RMR_HANDLE rmr_handle; + DAT_RMR_PARAM rmr_param; + DAT_REGION_DESCRIPTION region; + DAT_VLEN reg_size; + DAT_VADDR reg_addr; + DAT_VLEN buffer_size; + unsigned char *alloc_ptr; + DT_Tdep_Print_Head *phead; + + DAT_RETURN rc; + int res = 1; + buffer_size = BUFFSIZE * sizeof (unsigned char); + phead = params_ptr->phead; + reg_addr = 0; + alloc_ptr = 0; + + ia_handle = NULL; + pz_handle = NULL; + ep_handle = NULL; + lmr_handle = NULL; + rmr_handle = NULL; + pz_handle = NULL; + psp_handle = NULL; + rsp_handle = NULL; + cno_handle = NULL; + evd_handle = DAT_HANDLE_NULL; + conn_evd_handle = DAT_HANDLE_NULL; + cr_evd_handle = DAT_HANDLE_NULL; + recv_evd_handle = DAT_HANDLE_NULL; + send_evd_handle = DAT_HANDLE_NULL; + dev_name = cmd->device_name; + + /* All functions require an ia_handle to be created */ + rc = dat_ia_open ((const DAT_NAME_PTR)dev_name, + DEFAULT_QUEUE_LEN, + &evd_handle, + &ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* These functions require a pz_handle to be created */ + if ( (object_to_query == QUERY_EVD) || + (object_to_query == QUERY_RMR) || + (object_to_query == QUERY_LMR) || + (object_to_query == QUERY_EP) || + (object_to_query == QUERY_RSP) || + (object_to_query == QUERY_PZ) ) + { + rc = dat_pz_create (ia_handle, + &pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + + /* These functions require a ep_handle to be created */ + if ( (object_to_query == QUERY_EP) || + (object_to_query == QUERY_RSP) ) + { + rc = DT_Tdep_evd_create (ia_handle, + DEFAULT_QUEUE_LEN, + cno_handle, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + &send_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = DT_Tdep_evd_create (ia_handle, + DEFAULT_QUEUE_LEN, + cno_handle, + DAT_EVD_DTO_FLAG, + &recv_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = DT_Tdep_evd_create (ia_handle, + DEFAULT_QUEUE_LEN, + cno_handle, + DAT_EVD_CONNECTION_FLAG, + &conn_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + rc = dat_ep_create (ia_handle, + pz_handle, + recv_evd_handle, + send_evd_handle, + conn_evd_handle, + NULL, + &ep_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + + /* These functions require a CR EVD to be created. */ + if ( (object_to_query == QUERY_PSP) || + (object_to_query == QUERY_RSP) ) + { + rc = DT_Tdep_evd_create (ia_handle, + DEFAULT_QUEUE_LEN, + cno_handle, + DAT_EVD_CR_FLAG, + &cr_evd_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + } + + /* Test dat_ia_query function */ + if (object_to_query == QUERY_IA) + { + if (result_wanted == DAT_SUCCESS) + { + rc = dat_ia_query (ia_handle, + &evd_handle, + DAT_IA_ALL, + &ia_attributes, + DAT_PROVIDER_FIELD_ALL, + &provider_attributes); + } + else if (result_wanted == DAT_INVALID_PARAMETER) + { + /* + * The only way to get an invalid parameter is to + * NULL out ia_attr and for the DAT_IA_ATTR_MASK to + * have values + */ + rc = dat_ia_query (ia_handle, + &evd_handle, + DAT_IA_ALL, + NULL, + DAT_PROVIDER_FIELD_ALL, + &provider_attributes); + } + else if (result_wanted == DAT_INVALID_HANDLE) + { + rc = dat_ia_query (evd_handle, + &evd_handle, + DAT_IA_ALL, + &ia_attributes, + DAT_PROVIDER_FIELD_ALL, + &provider_attributes); + } + } + + /* Test dat_cno_query function */ + else if (object_to_query == QUERY_CNO) + { + +#ifndef __KDAPLTEST__ +#if defined(WIN32) + rc = dat_cno_create (ia_handle, + NULLPROXY, + &cno_handle); +#else + rc = dat_cno_create (ia_handle, + DAT_OS_WAIT_PROXY_AGENT_NULL, + &cno_handle); +#endif + + DT_assert_dat (phead, rc == DAT_SUCCESS); + + if (result_wanted == DAT_SUCCESS) + { + rc = dat_cno_query (cno_handle, + DAT_CNO_FIELD_ALL, + &cno_param); + } + else if (result_wanted == DAT_INVALID_PARAMETER) + { + rc = dat_cno_query (cno_handle, + DAT_CNO_FIELD_ALL, + NULL); + } + else if (result_wanted == DAT_INVALID_HANDLE) + { + rc = dat_cno_query (ia_handle, + DAT_CNO_FIELD_ALL, + &cno_param); + } +#endif + } + /* Test dat_evd_query function */ + else if (object_to_query == QUERY_EVD) + { + if (result_wanted == DAT_SUCCESS) + { + rc = dat_evd_query (evd_handle, + DAT_EVD_FIELD_ALL, + &evd_param); + } + else if (result_wanted == DAT_INVALID_PARAMETER) + { + rc = dat_evd_query (evd_handle, + DAT_EVD_FIELD_ALL, + NULL); + } + else if (result_wanted == DAT_INVALID_HANDLE) + { + rc = dat_evd_query (ia_handle, + DAT_EVD_FIELD_ALL, + &evd_param); + } + } + + /* Test dat_psp_query function */ + else if (object_to_query == QUERY_PSP) + { + rc = dat_psp_create (ia_handle, + SERVER_PORT_NUMBER, + cr_evd_handle, + DAT_PSP_PROVIDER_FLAG, + &psp_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + if (result_wanted == DAT_SUCCESS) + { + rc = dat_psp_query (psp_handle, + DAT_PSP_FIELD_ALL, + &psp_param); + } + else if (result_wanted == DAT_INVALID_PARAMETER) + { + rc = dat_psp_query (psp_handle, + DAT_PSP_FIELD_ALL, + NULL); + } + else if (result_wanted == DAT_INVALID_HANDLE) + { + rc = dat_psp_query (evd_handle, + DAT_PSP_FIELD_ALL, + &psp_param); + } + } + + /* Test dat_rsp_query function */ + else if (object_to_query == QUERY_RSP) + { + rc = dat_rsp_create (ia_handle, + SERVER_PORT_NUMBER, + ep_handle, + cr_evd_handle, + &rsp_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + rc = dat_rsp_query (rsp_handle, + DAT_RSP_FIELD_ALL, + &rsp_param); + } + + /* Test dat_cr_query function */ + else if (object_to_query == QUERY_CR) + { + /* This query is tested in the conmgt test */ + res = -1; + } + + /* Test dat_ep_query function */ + else if (object_to_query == QUERY_EP) + { + rc = dat_ep_query (ep_handle, + DAT_EP_FIELD_ALL, + &ep_param); + } + + /* Test dat_pz_query function */ + else if (object_to_query == QUERY_PZ) + { + rc = dat_pz_query (pz_handle, + DAT_PZ_FIELD_ALL, + &pz_param); + } + + /* Test dat_lmr_query function */ + else if (object_to_query == QUERY_LMR) + { + alloc_ptr = (unsigned char *)DT_Mdep_Malloc ((size_t)buffer_size); + DT_assert (phead, alloc_ptr); + memset (®ion, 0, sizeof (region)); + region.for_va = alloc_ptr; + rc = DT_Tdep_lmr_create (ia_handle, + DAT_MEM_TYPE_VIRTUAL, + region, + buffer_size, + pz_handle, + DAT_MEM_PRIV_ALL_FLAG, + &lmr_handle, + &lmr_context, + NULL, /* FIXME */ + ®_size, + ®_addr); + DT_assert_dat (phead, rc == DAT_SUCCESS); + rc = dat_lmr_query (lmr_handle, + DAT_LMR_FIELD_ALL, + &lmr_param); + } + + /* Test dat_rmr_query function */ + else if (object_to_query == QUERY_RMR) + { + rc = dat_rmr_create (pz_handle, + &rmr_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + /* We don't bind the RMR to anything, so don't ask for the + * LMR_TRIPLET flag + */ + rc = dat_rmr_query (rmr_handle, + DAT_RMR_FIELD_ALL - DAT_RMR_FIELD_LMR_TRIPLET, + &rmr_param); + } + + DT_assert_dat (phead, DAT_GET_TYPE (rc) == result_wanted); + +cleanup: + if (rsp_handle) + { + rc = dat_rsp_free (rsp_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (ep_handle) + { + rc = dat_ep_free (ep_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (send_evd_handle) + { + rc = DT_Tdep_evd_free (send_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (recv_evd_handle) + { + rc = DT_Tdep_evd_free (recv_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (conn_evd_handle) + { + rc = DT_Tdep_evd_free (conn_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (lmr_handle) + { + rc = dat_lmr_free (lmr_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (rmr_handle) + { + rc = dat_rmr_free (rmr_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + +#ifndef __KDAPLTEST__ + if (cno_handle) + { + rc = dat_cno_free (cno_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } +#endif + if (psp_handle) + { + rc = dat_psp_free (psp_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (cr_evd_handle) + { + rc = DT_Tdep_evd_free (cr_evd_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (pz_handle) + { + rc = dat_pz_free (pz_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + if (ia_handle) + { + rc = dat_ia_close (ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + + return res; +} + +int DT_queryinfo_case0 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify IA Querying information is successful using\nDAT_IA_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_IA, DAT_SUCCESS); +} + +int DT_queryinfo_case1 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify CNO Querying information is successful using\nDAT_CNO_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_CNO, DAT_SUCCESS); +} + +int DT_queryinfo_case2 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify EVD Querying information is successful using\nDAT_EVD_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_EVD, DAT_SUCCESS); +} + +int DT_queryinfo_case3 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify PSP Querying information is successful using\nDAT_PSP_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_PSP, DAT_SUCCESS); +} + +int DT_queryinfo_case4 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify RSP Querying information is successful using\nDAT_RSP_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_RSP, DAT_SUCCESS); +} + +int DT_queryinfo_case5 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify CR Querying information is successful using\nDAT_CR_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_CR, DAT_SUCCESS); +} + +int DT_queryinfo_case6 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify EP Querying information is successful using\nDAT_EP_QUERY.\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_EP, DAT_SUCCESS); +} + +int DT_queryinfo_case7 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify PZ Querying information is successful using\n"); + DT_Tdep_PT_Printf (phead, "DAT_PZ_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_PZ, DAT_SUCCESS); +} + +int DT_queryinfo_case8 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify LMR Querying information is successful using\n"); + DT_Tdep_PT_Printf (phead, "DAT_LMR_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_LMR, DAT_SUCCESS); +} + +int DT_queryinfo_case9 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify RMR Querying information is successful using\n"); + DT_Tdep_PT_Printf (phead, "DAT_RMR_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_RMR, DAT_SUCCESS); +} + +int DT_queryinfo_case10 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify IA Querying fails with DAT_INVALID_PARAMETER when\n"); + DT_Tdep_PT_Printf (phead, "passing a bad parameter to DAT_IA_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_IA, DAT_INVALID_PARAMETER); +} + +int DT_queryinfo_case11 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify IA Querying fails with DAT_INVALID_HANDLE when\n"); + DT_Tdep_PT_Printf (phead, "passing an invalid handle to DAT_IA_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_IA, DAT_INVALID_HANDLE); +} + +int DT_queryinfo_case12 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify CNO Querying fails with DAT_INVALID_PARAMETER when\n"); + DT_Tdep_PT_Printf (phead, "passing a bad parameter to DAT_CNO_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_CNO, DAT_INVALID_PARAMETER); +} + +int DT_queryinfo_case13 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify CNO Querying fails with DAT_INVALID_HANDLE when\n"); + DT_Tdep_PT_Printf (phead, "passing an invalid handle to DAT_CNO_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_CNO, DAT_INVALID_HANDLE); +} + +int DT_queryinfo_case14 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify EVD Querying fails with DAT_INVALID_PARAMETER when\n"); + DT_Tdep_PT_Printf (phead, "passing a bad parameter to DAT_EVD_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_EVD, DAT_INVALID_PARAMETER); +} + +int DT_queryinfo_case15 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify EVD Querying fails with DAT_INVALID_HANDLE when\n"); + DT_Tdep_PT_Printf (phead, "passing an invalid handle to DAT_EVD_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_EVD, DAT_INVALID_HANDLE); +} + +int DT_queryinfo_case16 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify PSP Querying fails with DAT_INVALID_PARAMETER when\n"); + DT_Tdep_PT_Printf (phead, "passing a bad parameter to DAT_PSP_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_PSP, DAT_INVALID_PARAMETER); +} + +int DT_queryinfo_case17 (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + DT_Tdep_PT_Printf (phead, "Description: Verify PSP Querying fails with DAT_INVALID_HANDLE when\n"); + DT_Tdep_PT_Printf (phead, "passing an invalid handle to DAT_PSP_QUERY\n"); + return DT_queryinfo_basic (params_ptr, cmd, QUERY_PSP, DAT_INVALID_HANDLE); +} + +/*-------------------------------------------------------------*/ +void DT_queryinfo_test (Params_t *params_ptr, FFT_Cmd_t *cmd) +{ + int i; + int res; + DT_Tdep_Print_Head *phead; + FFT_Testfunc_t cases_func[] = + { + { DT_queryinfo_case0 }, + { DT_queryinfo_case1 }, + { DT_queryinfo_case2 }, + { DT_queryinfo_case3 }, + { DT_queryinfo_case4 }, + { DT_queryinfo_case5 }, + { DT_queryinfo_case6 }, + { DT_queryinfo_case7 }, + { DT_queryinfo_case8 }, + { DT_queryinfo_case9 }, + { DT_queryinfo_case10 }, + { DT_queryinfo_case11 }, +#ifndef __KDAPLTEST__ + { DT_queryinfo_case12 }, + { DT_queryinfo_case13 }, +#endif + { DT_queryinfo_case14 }, + { DT_queryinfo_case15 }, + { DT_queryinfo_case16 }, + { DT_queryinfo_case17 }, + }; + + phead = params_ptr->phead; + for (i=0; i < cmd->size; i++) + { + if (cmd->cases_flag[i]) + { + DT_Tdep_PT_Printf (phead, "*********************************************************************\n"); + DT_Tdep_PT_Printf (phead, "Function feature: Queryinfo case: %d\n", i + ); + res = cases_func[i].fun (params_ptr, cmd); + if (res==1) + { + DT_Tdep_PT_Printf (phead, "Result: PASS\n"); + } + else if (res ==0) + { + DT_Tdep_PT_Printf (phead, "Result: FAIL\n"); + } + else if (res == -1) + { + DT_Tdep_PT_Printf (phead, "Result: UNSUPP\n"); + } + + DT_Tdep_PT_Printf (phead, "*********************************************************************\n"); + } + } + return; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_test.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_test.c new file mode 100644 index 00000000..6ad462e1 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_test.c @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_cs_FFT (Params_t *params_ptr, FFT_Cmd_t * cmd) +{ + DT_Tdep_Print_Head *phead; + + phead = params_ptr->phead; + + switch (cmd->fft_type) + { + case HWCONN: + { + DT_hwconn_test (params_ptr, cmd); + break; + } + case ENDPOINT: + { + DT_endpoint_test (params_ptr, cmd); + break; + } + case PTAGMGT: + { + DT_pz_test (params_ptr, cmd); + break; + } + case MEMMGT: + { + DT_mem_test (params_ptr, cmd); + break; + } + case CONNMGT: + { + DT_connmgt_test (params_ptr, cmd); + break; + } + case QUERYINFO: + { + DT_queryinfo_test (params_ptr, cmd); + break; + } +#if 0 // not yet implemented + case CONNMGT_CLIENT: + case NS: + case ERRHAND: + case UNSUPP: + case STRESS: + case STRESS_CLIENT: + case CQMGT: + { + DT_Tdep_PT_Printf (phead, "Not Yet Implemented\n"); + break; + } +#endif + default: + { + DT_Tdep_PT_Printf (phead, "don't know this test\n"); + break; + } + } +} + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_util.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_util.c new file mode 100644 index 00000000..57d49228 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_fft_util.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define DEFAULT_QUEUE_LEN 10 + + +/* function that is called when an assertion fails, printing out the line + * that failed vi DT_Tdep_PT_Printf + */ +void DT_assert_fail (DT_Tdep_Print_Head *phead, char *exp, char *file, char *baseFile, int line) +{ + if (!strcmp (file, baseFile)) + { + DT_Tdep_PT_Printf (phead, + "%s failed in file %s, line %d\n", + exp, + file, + line); + } + else + { + DT_Tdep_PT_Printf (phead, + "%s failed in file %s (included from %s), line %d\n", + exp, + file, + baseFile, + line); + } +} + +/* helper function to open an IA */ +int DT_ia_open (DAT_NAME_PTR dev_name, DAT_IA_HANDLE *ia_handle) +{ + DAT_EVD_HANDLE evd_handle; + evd_handle = DAT_HANDLE_NULL; + return dat_ia_open (dev_name, DEFAULT_QUEUE_LEN, &evd_handle, ia_handle); +} + +/* helper function to create an endpoint and its associated EVDs */ +int DT_ep_create (Params_t *params_ptr, + DAT_IA_HANDLE ia_handle, + DAT_PZ_HANDLE pz_handle, + DAT_EVD_HANDLE *cr_evd, + DAT_EVD_HANDLE *conn_evd, + DAT_EVD_HANDLE *send_evd, + DAT_EVD_HANDLE *recv_evd, + DAT_EP_HANDLE *ep_handle) +{ + DAT_RETURN status; + DT_Tdep_Print_Head *phead; + *conn_evd= 0; + *send_evd= 0; + *recv_evd= 0; + *cr_evd= 0; + phead = params_ptr->phead; + + status = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, DAT_HANDLE_NULL, + DAT_EVD_CR_FLAG, cr_evd); + if (status != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "dat_evd_create failed %s\n", DT_RetToString (status)); + return status; + } + + status = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, DAT_HANDLE_NULL, + DAT_EVD_CONNECTION_FLAG, conn_evd); + if (status != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "dat_evd_create failed %s\n", DT_RetToString (status)); + return status; + } + + status = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, DAT_HANDLE_NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + send_evd); + if (status != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "dat_evd_create failed %s\n", DT_RetToString (status)); + return status; + } + + status = DT_Tdep_evd_create (ia_handle, DEFAULT_QUEUE_LEN, DAT_HANDLE_NULL, + DAT_EVD_DTO_FLAG, recv_evd); + if (status != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "dat_evd_create failed %s\n", DT_RetToString (status)); + return status; + } + + status = dat_ep_create (ia_handle, pz_handle, *recv_evd, + *send_evd, *conn_evd, NULL, ep_handle); + if (status != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "dat_ep_create failed %s\n", DT_RetToString (status)); + } + return status; +} + +/* function that initializes the connection struct */ +void DT_fft_init_conn_struct (FFT_Connection_t *conn) +{ + conn->ia_handle = 0; + conn->pz_handle = 0; + conn->psp_handle = 0; + conn->ep_handle = 0; + conn->cr_evd = 0; + conn->send_evd = 0; + conn->conn_evd = 0; + conn->recv_evd = 0; + conn->cr_handle = 0; + conn->remote_netaddr = 0; + conn->bpool = 0; + conn->pt_ptr = 0; + conn->connected = false; +} + +/* helper function that simplifies many dat calls for the initiialization of a + * dat "client" + */ +void DT_fft_init_client (Params_t *params_ptr, FFT_Cmd_t *cmd, FFT_Connection_t *conn) +{ + int res; + DAT_RETURN rc=0; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + /* initialize the struct's members */ + DT_fft_init_conn_struct (conn); + + /* open the IA */ + rc = DT_ia_open (cmd->device_name, &conn->ia_handle); + if (rc != DAT_SUCCESS) + { + /* make sure the handle has an invalid value */ + conn->ia_handle = NULL; + } + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* create a PZ */ + rc = dat_pz_create (conn->ia_handle, &conn->pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* create an EP and its EVDs */ + rc =DT_ep_create (params_ptr, + conn->ia_handle, + conn->pz_handle, + &conn->cr_evd, + &conn->conn_evd, + &conn->send_evd, + &conn->recv_evd, + &conn->ep_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* if a server name is given, allocate memory for a net address and set it + * up appropriately + */ + if (cmd->server_name && strlen (cmd->server_name)) + { + conn->remote_netaddr = ¶ms_ptr->server_netaddr; + } +cleanup: + return; +} + +/* helper function to break down a client or server created with one of the + * init helper functions + */ +int DT_fft_destroy_conn_struct (Params_t *params_ptr, + FFT_Connection_t *conn) +{ + DAT_RETURN rc = DAT_SUCCESS; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + if (conn->ep_handle) + { + if (conn->connected) + { + rc = dat_ep_disconnect (conn->ep_handle, DAT_CLOSE_DEFAULT); + DT_assert_clean (phead, rc == DAT_SUCCESS); + + if (!DT_disco_event_wait ( phead, conn->cr_evd, NULL )) + + { + DT_Tdep_PT_Printf (phead, "DT_fft_destroy_conn_struct: bad disconnect event\n"); + } + } + rc = dat_ep_free (conn->ep_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->bpool) + { + DT_Bpool_Destroy (0, phead, conn->bpool); + } + if (conn->psp_handle) + { + rc = dat_psp_free (conn->psp_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->cr_evd) + { + rc = DT_Tdep_evd_free (conn->cr_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->conn_evd) + { + rc = DT_Tdep_evd_free (conn->conn_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->send_evd) + { + rc = DT_Tdep_evd_free (conn->send_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->recv_evd) + { + rc = DT_Tdep_evd_free (conn->recv_evd); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->pt_ptr) + { + DT_Free_Per_Test_Data (conn->pt_ptr); + } + if (conn->pz_handle) + { + rc = dat_pz_free (conn->pz_handle); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + if (conn->ia_handle) + { + rc = dat_ia_close (conn->ia_handle, DAT_CLOSE_ABRUPT_FLAG); + DT_assert_clean (phead, rc == DAT_SUCCESS); + } + return rc; +} + +/* helper function to init a dat "server" */ +void DT_fft_init_server (Params_t *params_ptr, FFT_Cmd_t *cmd, FFT_Connection_t *conn) +{ + int res; + DAT_RETURN rc=0; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + /* init the connection struct's members */ + DT_fft_init_conn_struct (conn); + + /* open the IA */ + rc = DT_ia_open (cmd->device_name, &conn->ia_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* create a PZ */ + rc = dat_pz_create (conn->ia_handle, &conn->pz_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* create an EP and its EVDs */ + rc =DT_ep_create (params_ptr, + conn->ia_handle, + conn->pz_handle, + &conn->cr_evd, + &conn->conn_evd, + &conn->send_evd, + &conn->recv_evd, + &conn->ep_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* create a PSP */ + rc = dat_psp_create (conn->ia_handle, SERVER_PORT_NUMBER, conn->cr_evd, + DAT_PSP_CONSUMER_FLAG, &conn->psp_handle); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* allocate memory for buffers */ + conn->bpool = DT_BpoolAlloc (0, phead, conn->ia_handle, conn->pz_handle, NULL, NULL, + 8192, 2, DAT_OPTIMAL_ALIGNMENT, false, false); + DT_assert (phead, conn->bpool); +cleanup: + return; +} + +/* helper function that allows a server to listen for a connection */ +void DT_fft_listen (Params_t *params_ptr, FFT_Connection_t *conn) +{ + int res; + DAT_RETURN rc=0; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + /* wait on a CR event via the CR EVD */ + DT_assert_dat (phead, DT_cr_event_wait (phead, conn->cr_evd, &conn->cr_stat) && + DT_cr_check (phead, + &conn->cr_stat, + conn->psp_handle, + SERVER_PORT_NUMBER, + &conn->cr_handle, + "DT_fft_listen")); + + /* accept the connection */ + rc =dat_cr_accept (conn->cr_handle, conn->ep_handle, 0, (DAT_PVOID)0); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* wait on a conn event via the conn EVD */ + DT_assert (phead, + DT_conn_event_wait (phead, + conn->ep_handle, + conn->conn_evd, + &conn->event_num) == true); + conn->connected = true; +cleanup: + return; +} + +/* helper function that allows a client to connect to a server */ +int DT_fft_connect (Params_t *params_ptr, FFT_Connection_t *conn) +{ + int wait_count; + int res; + DAT_RETURN rc=0; + DT_Tdep_Print_Head *phead; + phead = params_ptr->phead; + + /* try 10 times to connect */ + for (wait_count = 0; wait_count < 10; wait_count++) + { + DT_Tdep_PT_Printf (phead, "Connection to server, attempt #%d\n", wait_count+1); + + /* attempt to connect, timeout = 10 secs */ + rc = dat_ep_connect (conn->ep_handle, conn->remote_netaddr, + SERVER_PORT_NUMBER, 10*1000000, 0, (DAT_PVOID)0, + DAT_QOS_BEST_EFFORT, DAT_CONNECT_DEFAULT_FLAG); + DT_assert_dat (phead, rc == DAT_SUCCESS); + + /* wait on conn event */ + DT_assert (phead, + DT_conn_event_wait (phead, + conn->ep_handle, + conn->conn_evd, + &conn->event_num) == true); + + /* make sure we weren't rejected by the peer */ + if (conn->event_num == DAT_CONNECTION_EVENT_PEER_REJECTED) + { + DT_Mdep_Sleep (1000); + DT_Tdep_PT_Printf (phead, "Connection rejected by peer; retrying\n"); + } + } +cleanup: + if (conn->event_num == DAT_CONNECTION_EVENT_ESTABLISHED) + { + conn->connected = true; + } + /* returns true if connected, false otherwise */ + return (conn->connected); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_limit.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_limit.c new file mode 100644 index 00000000..862b927d --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_limit.c @@ -0,0 +1,1554 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* + * Increase the size of an array of handles + */ + +static bool +more_handles (DT_Tdep_Print_Head *phead, + DAT_HANDLE **old_ptrptr, /* pointer to current pointer */ + unsigned int *old_count, /* number pointed to */ + unsigned int size) /* size of one datum */ +{ + unsigned int count = *old_count; + DAT_HANDLE *old_handles = *old_ptrptr; + DAT_HANDLE *handle_tmp = DT_Mdep_Malloc (count * 2 * size); + + if (!handle_tmp) + { + DT_Tdep_PT_Printf (phead, "Out of memory for more DAT_HANDLEs\n"); + return (false); + } + + memcpy (handle_tmp, old_handles, count*size); + DT_Mdep_Free (old_handles); + *old_ptrptr = handle_tmp; + *old_count = count * 2; + return (true); +} + + +/* + * Limit test workhorse. + * + * This test creates the sequence of DAT objects needed to move + * data back and forth, attempting to find the limits supported + * for the DAT object indicated by 'depth'. For example, if + * depth == LIM_LMR, the test will create a set of {IA,PZ,CNO,EVD,EP} + * before trying to exhaust LMR creation using the {IA,PZ,CNO,EVD,EP} set. + * + * The 'cmd->width' parameter can be used to control how may of these + * parallel DAT object sets we create before stopping to beat upon + * the constructor for the object indicated by 'depth', providing for + * increased (or at least different) stress on the DAPL. + */ +static bool +limit_test ( DT_Tdep_Print_Head *phead, + Limit_Cmd_t *cmd, + Limit_Index depth) +{ + DAT_EVD_HANDLE conn_handle; + typedef struct obj_set + { + DAT_IA_HANDLE ia_handle; + DAT_EVD_HANDLE ia_async_handle; + DAT_PZ_HANDLE pz_handle; + DAT_CNO_HANDLE cno_handle; + DAT_EVD_HANDLE evd_handle; + DAT_EP_HANDLE ep_handle; + DAT_LMR_HANDLE lmr_handle; + char * lmr_buffer; + DAT_LMR_CONTEXT lmr_context; + DAT_RMR_HANDLE rmr_handle; + DAT_RMR_CONTEXT rmr_context; + } Obj_Set; + + Obj_Set *hdl_sets = (Obj_Set *) NULL; + bool retval = false; + char *module = "LimitTest"; + +#if defined (WIN32) + /* + * The Windows compiler will not deal with complex definitions + * in macros, so create a variable here. + */ +#if defined (DAT_OS_WAIT_PROXY_AGENT_NULL) +#undef DAT_OS_WAIT_PROXY_AGENT_NULL +#endif + DAT_OS_WAIT_PROXY_AGENT DAT_OS_WAIT_PROXY_AGENT_NULL = {NULL, NULL}; +#endif + + DAT_RETURN ret; + +#ifdef DFLT_QLEN +#undef DFLT_QLEN +#endif + +# define DFLT_QLEN 10 /* a small event queue size */ +# define START_COUNT 1024 /* initial # handles */ +# define DFLT_BUFFSZ 4096 /* default size for buffer */ +# define CONN_QUAL0 0xAffab1e + + /* Allocate 'width' Obj_Sets */ + if (depth && ! (hdl_sets = DT_Mdep_Malloc (sizeof (Obj_Set) * cmd->width))) + { + DT_Tdep_PT_Printf (phead, "%s: No memory for handle array!\n", module); + goto clean_up_now; + } + + /* ----------- + * IA handling + */ + if (depth > LIM_IA) + { + /* + * The abuse is not for us this time, just prep Obj_Set. + */ + unsigned int w; + + DT_Tdep_PT_Debug (1, (phead, + "%s: dat_ia_open X %d\n", + module, + cmd->width)); + for (w = 0; w < cmd->width; w++) + { + /* Specify that we want to get back an async EVD. */ + hdl_sets[w].ia_async_handle = DAT_HANDLE_NULL; + ret = dat_ia_open (cmd->device_name, + DFLT_QLEN, + &hdl_sets[w].ia_async_handle, + &hdl_sets[w].ia_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_open (%s) #%d fails: %s\n", + module, cmd->device_name, + w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + hdl_sets[w].ia_async_handle = DAT_HANDLE_NULL; + hdl_sets[w].ia_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + } + } + else if (depth == LIM_IA) + { + /* + * See how many IAs we can create + */ + typedef struct _ia + { + DAT_IA_HANDLE ia_handle; + DAT_EVD_HANDLE ia_async_handle; + } + OneOpen; + unsigned int count = START_COUNT; + OneOpen *hdlptr = (OneOpen *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + + /* IA Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead, "%s: Exhausting dat_ia_open\n", module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: IAs opened: %d\n", module, w); + retval = true; + break; + } + /* Specify that we want to get back an async EVD. */ + hdlptr[w].ia_async_handle = DAT_HANDLE_NULL; + ret = dat_ia_open (cmd->device_name, + DFLT_QLEN, + &hdlptr[w].ia_async_handle, + &hdlptr[w].ia_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_open (%s) #%d fails: %s\n", + module, cmd->device_name, + w+1, DT_RetToString (ret)); + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: IAs opened: %d\n", module, w); + retval = true; + + /* IA Cleanup loop */ + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = dat_ia_close (hdlptr[tmp].ia_handle, + DAT_CLOSE_GRACEFUL_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_close (graceful) fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + ret = dat_ia_close (hdlptr[tmp].ia_handle, + DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_close (abrupt) fails: %s\n", + module, DT_RetToString (ret)); + } + } + } + DT_Mdep_Free (hdlptr); + } + } /* End IA handling */ + + /* ----------- + * PZ handling + */ + if (depth > LIM_PZ) + { + /* + * The abuse is not for us this time, just prep Obj_Set. + */ + unsigned int w; + + DT_Tdep_PT_Debug (1,(phead, + "%s: dat_pz_create X %d\n", + module, + cmd->width)); + for (w = 0; w < cmd->width; w++) + { + ret = dat_pz_create (hdl_sets[w].ia_handle, + &hdl_sets[w].pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_pz_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + hdl_sets[w].pz_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + } + } + else if (depth == LIM_PZ) + { + /* + * See how many PZs we can create + */ + unsigned int count = START_COUNT; + DAT_PZ_HANDLE *hdlptr = (DAT_PZ_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + + /* PZ Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead, + "%s: Exhausting dat_pz_create\n", + module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: PZs created: %d\n", module, w); + retval = true; + break; + } + ret = dat_pz_create (hdl_sets[w % cmd->width].ia_handle, + &hdlptr[w]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_pz_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: PZs created: %d\n", module, w); + retval = true; + + /* PZ Cleanup loop */ + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = dat_pz_free (hdlptr[tmp]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_pz_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + DT_Mdep_Free (hdlptr); + } + } /* End PZ handling */ + +#ifndef __KDAPLTEST__ + /* ----------- + * CNO handling + */ + + if (depth > LIM_CNO) + { + /* + * The abuse is not for us this time, just prep Obj_Set. + */ + unsigned int w; + + DT_Tdep_PT_Debug (1,(phead, + "%s: dat_cno_create X %d\n", + module, + cmd->width)); + for (w = 0; w < cmd->width; w++) + { + ret = dat_cno_create (hdl_sets[w].ia_handle, + DAT_OS_WAIT_PROXY_AGENT_NULL, + &hdl_sets[w].cno_handle); + if (DAT_GET_TYPE (ret) == DAT_NOT_IMPLEMENTED) + { + DT_Tdep_PT_Printf (phead, "%s: dat_cno_create unimplemented\n", module); + hdl_sets[w].cno_handle = DAT_HANDLE_NULL; + /* ignore this error */ + break; + } + else if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_cno_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + hdl_sets[w].cno_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + } + } + else if (depth == LIM_CNO) + { + /* + * See how many CNOs we can create + */ + unsigned int count = START_COUNT; + DAT_CNO_HANDLE *hdlptr = (DAT_CNO_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + + /* CNO Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead, + "%s: Exhausting dat_cno_create\n", + module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: CNOs created: %d\n", module, w); + retval = true; + break; + } + ret = dat_cno_create (hdl_sets[w % cmd->width].ia_handle, + DAT_OS_WAIT_PROXY_AGENT_NULL, + &hdlptr[w]); + if (DAT_GET_TYPE (ret) == DAT_NOT_IMPLEMENTED) + { + DT_Tdep_PT_Printf (phead, "%s: dat_cno_create unimplemented\n", + module); + retval = true; + break; + } + else if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_cno_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: CNOs created: %d\n", module, w); + retval = true; + + /* CNO Cleanup loop */ + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = dat_cno_free (hdlptr[tmp]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_cno_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + DT_Mdep_Free (hdlptr); + } + } /* End CNO handling */ +#endif /* __KDAPLTEST__ */ + + /* ----------- + * EVD handling + */ + if (depth > LIM_EVD) + { + /* + * The abuse is not for us this time, just prep Obj_Set. + */ + unsigned int w=0; + DAT_EVD_FLAGS flags = ( DAT_EVD_DTO_FLAG + /* | DAT_EVD_SOFTWARE_FLAG */ + | DAT_EVD_CR_FLAG + | DAT_EVD_RMR_BIND_FLAG ); /* not ASYNC */ + + DT_Tdep_PT_Debug (1,(phead, + "%s: dat_evd_create X %d\n", + module, + cmd->width)); + /* + * First create a connection EVD to be used for EP creation + */ + + ret = DT_Tdep_evd_create (hdl_sets[0].ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_CONNECTION_FLAG, + &conn_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: conn dat_evd_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + conn_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + for (w = 0; w < cmd->width; w++) + { + ret = DT_Tdep_evd_create (hdl_sets[w].ia_handle, + DFLT_QLEN, + hdl_sets[w].cno_handle, + flags, + &hdl_sets[w].evd_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_evd_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + hdl_sets[w].evd_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + } + } + else if (depth == LIM_EVD) + { + /* + * See how many EVDs we can create + */ + unsigned int count = START_COUNT; + DAT_EVD_HANDLE *hdlptr = (DAT_EVD_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + DAT_EVD_FLAGS flags = ( DAT_EVD_DTO_FLAG + | DAT_EVD_RMR_BIND_FLAG + | DAT_EVD_CR_FLAG); + + /* EVD Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead, + "%s: Exhausting dat_evd_create\n", + module)); + /* + * First create a connection EVD to be used for EP creation + */ + ret = DT_Tdep_evd_create (hdl_sets[0].ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_CONNECTION_FLAG, + &conn_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: conn dat_evd_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + conn_handle = DAT_HANDLE_NULL; + } + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: EVDs created: %d\n", module, w); + retval = true; + break; + } + ret = DT_Tdep_evd_create (hdl_sets[w % cmd->width].ia_handle, + DFLT_QLEN, + hdl_sets[w % cmd->width].cno_handle, + flags, + &hdlptr[w]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_evd_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: EVDs created: %d\n", module, w); + retval = true; + + /* EVD Cleanup loop */ + if (conn_handle != DAT_HANDLE_NULL) + { + ret = DT_Tdep_evd_free (conn_handle); + conn_handle = DAT_HANDLE_NULL; + } + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = DT_Tdep_evd_free (hdlptr[tmp]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_evd_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + DT_Mdep_Free (hdlptr); + } + } /* End EVD handling */ + + /* ----------- + * EP handling + */ + if (depth > LIM_EP) + { + /* + * The abuse is not for us this time, just prep Obj_Set. + */ + unsigned int w; + + DT_Tdep_PT_Debug (1,(phead, + "%s: dat_ep_create X %d\n", + module, + cmd->width)); + for (w = 0; w < cmd->width; w++) + { + ret = dat_ep_create (hdl_sets[w].ia_handle, + hdl_sets[w].pz_handle, + hdl_sets[w].evd_handle, /* recv */ + hdl_sets[w].evd_handle, /* request */ + conn_handle, /* connect */ + (DAT_EP_ATTR *) NULL, + &hdl_sets[w].ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + hdl_sets[w].ep_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + } + } + else if (depth == LIM_EP) + { + /* + * See how many EPs we can create + */ + unsigned int count = START_COUNT; + DAT_EP_HANDLE *hdlptr = (DAT_EP_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + + /* EP Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead,"%s: Exhausting dat_ep_create\n", module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: EPs created: %d\n", module, w); + retval = true; + break; + } + ret = dat_ep_create (hdl_sets[w % cmd->width].ia_handle, + hdl_sets[w % cmd->width].pz_handle, + hdl_sets[w % cmd->width].evd_handle, + hdl_sets[w % cmd->width].evd_handle, + conn_handle, /* connect */ + (DAT_EP_ATTR *) NULL, + &hdlptr[w]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: EPs created: %d\n", module, w); + retval = true; + + /* EP Cleanup loop */ + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = dat_ep_free (hdlptr[tmp]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + DT_Mdep_Free (hdlptr); + } + } /* End EP handling */ + + /* ----------- + * RSP handling + * + * if (depth > LIM_RSP) { + * Since RSPs are not part of the Obj_Set, + * there's nothing to do. + * } else ... + */ + if (depth == LIM_RSP) + { + /* + * See how many RSPs we can create + */ + unsigned int count = START_COUNT; + DAT_RSP_HANDLE *hdlptr = (DAT_RSP_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + DAT_EP_HANDLE *epptr = (DAT_EP_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*epptr)); + + /* RSP Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead,"%s: Exhausting dat_rsp_create\n", module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count) + { + unsigned int count1 = count; + unsigned int count2 = count; + + if (!more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count1, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: RSPs created: %d\n", module, w); + retval = true; + break; + } + if (!more_handles (phead, (DAT_HANDLE **) &epptr, + &count2, + sizeof (*epptr))) + { + DT_Tdep_PT_Printf (phead, "%s: RSPs created: %d\n", module, w); + retval = true; + break; + } + + if (count1 != count2) + { + DT_Tdep_PT_Printf (phead, "%s: Mismatch in allocation of handle arrays at point %d\n", + module, w); + retval = true; + break; + } + + count = count1; + } + + /* + * Each RSP needs a unique EP, so create one first + */ + ret = dat_ep_create (hdl_sets[w % cmd->width].ia_handle, + hdl_sets[w % cmd->width].pz_handle, + hdl_sets[w % cmd->width].evd_handle, + hdl_sets[w % cmd->width].evd_handle, + conn_handle, + (DAT_EP_ATTR *) NULL, + &epptr[w]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_create #%d fails: %s testing RSPs\n", + module, w+1, DT_RetToString (ret)); + retval = true; + break; + } + + ret = dat_rsp_create (hdl_sets[w % cmd->width].ia_handle, + CONN_QUAL0 + w, + epptr[w], + hdl_sets[w % cmd->width].evd_handle, + &hdlptr[w]); + if (DAT_GET_TYPE (ret) == DAT_NOT_IMPLEMENTED) + { + DT_Tdep_PT_Printf (phead, "%s: dat_rsp_create unimplemented\n", + module); + /* ignore this error */ + retval = true; + break; + } + else if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_rsp_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* Cleanup the EP; no-one else will. */ + ret = dat_ep_free (epptr[w]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_free (internal cleanup @ #%d) fails: %s\n", + module, w+1, DT_RetToString (ret)); + } + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: RSPs created: %d\n", module, w); + retval = true; + + /* RSP Cleanup loop */ + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = dat_rsp_free (hdlptr[tmp]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_rsp_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + /* Free EPs */ + ret = dat_ep_free (epptr[tmp]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_free fails: %s for RSPs\n", + module, DT_RetToString (ret)); + retval = false; + } + } + DT_Mdep_Free (hdlptr); + } + } /* End RSP handling */ + + /* ----------- + * PSP handling + * + * if (depth > LIM_PSP) { + * Since PSPs are not part of the Obj_Set, + * there's nothing to do. + * } else ... + */ + if (depth == LIM_PSP) + { + /* + * See how many PSPs we can create + */ + unsigned int count = START_COUNT; + DAT_PSP_HANDLE *hdlptr = (DAT_PSP_HANDLE *) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + + /* PSP Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead,"%s: Exhausting dat_psp_create\n", module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: PSPs created: %d\n", module, w); + retval = true; + break; + } + ret = dat_psp_create (hdl_sets[w % cmd->width].ia_handle, + CONN_QUAL0 + w, + hdl_sets[w % cmd->width].evd_handle, + DAT_PSP_CONSUMER_FLAG, + &hdlptr[w]); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_psp_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + retval = true; + hdlptr[w] = DAT_HANDLE_NULL; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: PSPs created: %d\n", module, w); + retval = true; + + /* PSP Cleanup loop */ + for (tmp = 0; tmp < w; tmp++) + { + DT_Mdep_Schedule(); + ret = dat_psp_free (hdlptr[tmp]); + if (DAT_GET_TYPE (ret) == DAT_NOT_IMPLEMENTED) + { + DT_Tdep_PT_Printf (phead, "%s: dat_psp_free unimplemented\n" + "\tNB: Expect EVD+IA cleanup errors!\n", + module); + break; + } + else if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_psp_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + DT_Mdep_Free (hdlptr); + } + } /* End PSP handling */ + + /* ----------- + * LMR handling + */ + if (depth > LIM_LMR) + { + /* + * The abuse is not for us this time, just prep Obj_Set. + */ + unsigned int w; + + DT_Tdep_PT_Debug (1,(phead,"%s: dat_lmr_create X %d\n", module, cmd->width)); + for (w = 0; w < cmd->width; w++) + { + DAT_REGION_DESCRIPTION region; + DAT_VLEN reg_size; + DAT_VADDR reg_addr; + + hdl_sets[w].lmr_buffer = DT_Mdep_Malloc (DFLT_BUFFSZ); + if (!hdl_sets[w].lmr_buffer) + { + DT_Tdep_PT_Printf (phead, "%s: no memory for LMR buffers\n", module); + goto clean_up_now; + } + memset (®ion, 0, sizeof (region)); + region.for_va = hdl_sets[w].lmr_buffer; + + ret = DT_Tdep_lmr_create (hdl_sets[w].ia_handle, + DAT_MEM_TYPE_VIRTUAL, + region, + DFLT_BUFFSZ, + hdl_sets[w].pz_handle, + DAT_MEM_PRIV_ALL_FLAG, + &hdl_sets[w].lmr_handle, + &hdl_sets[w].lmr_context, + NULL, /* FIXME */ + ®_size, ®_addr); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_lmr_create #%d fails: %s\n", + module, w+1, DT_RetToString (ret)); + /* handle contents undefined on failure */ + hdl_sets[w].lmr_handle = DAT_HANDLE_NULL; + goto clean_up_now; + } + if ((uintptr_t)reg_addr > (uintptr_t)hdl_sets[w].lmr_buffer + || (reg_size < DFLT_BUFFSZ + ((uintptr_t)reg_addr + - (uintptr_t)hdl_sets[w].lmr_buffer))) + { + DT_Tdep_PT_Printf (phead, "%s: dat_lmr_create bogus outputs " + "in: 0x%p, %x out 0x%llx, %llx\n", + module, + hdl_sets[w].lmr_buffer, DFLT_BUFFSZ, + reg_addr, reg_size); + goto clean_up_now; + } + } + } + else if (depth == LIM_LMR) + { + /* + * See how many LMRs we can create + */ + unsigned int count = START_COUNT; + Bpool **hdlptr = (Bpool **) + DT_Mdep_Malloc (count * sizeof (*hdlptr)); + + /* LMR Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int tmp; + + DT_Tdep_PT_Debug (1,(phead,"%s: Exhausting dat_lmr_create\n", module)); + for (w = 0; w < cmd->maximum; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: no memory for LMR handles\n", + module); + DT_Tdep_PT_Printf (phead, "%s: LMRs created: %d\n", module, w); + retval = true; + break; + } + /* + * Let BpoolAlloc do the hard work; this means that + * we're testing unique memory registrations rather + * than repeatedly binding the same buffer set. + */ + hdlptr[w] = DT_BpoolAlloc ((Per_Test_Data_t *)0, + phead, + hdl_sets[w % cmd->width].ia_handle, + hdl_sets[w % cmd->width].pz_handle, + hdl_sets[w % cmd->width].ep_handle, + hdl_sets[w % cmd->width].evd_handle, + DFLT_BUFFSZ, + 1, + DAT_OPTIMAL_ALIGNMENT, + false, + false); + if (!hdlptr[w]) + { + DT_Tdep_PT_Printf (phead, "%s: LMRs created: %d\n", module, w); + retval = true; + break; + } + } + + DT_Tdep_PT_Printf (phead, "%s: LMRs created: %d\n", module, w); + retval = true; + + /* LMR Cleanup loop */ + for (tmp = 0; tmp <= w; tmp++) + { + DT_Mdep_Schedule(); + if (hdlptr[tmp]) + { + /* ignore rval - DT_Bpool_Destroy will complain */ + (void) DT_Bpool_Destroy ((Per_Test_Data_t *)0, phead, hdlptr[tmp]); + } + } + DT_Mdep_Free (hdlptr); + } + } /* End LMR handling */ + + /* ----------- + * Posted receive buffer handling + */ + if (depth == LIM_RPOST) + { + /* + * See how many receive buffers we can post (to each EP). + * We are posting the same buffer 'cnt' times, deliberately, + * but that should be OK. + */ + unsigned int count = START_COUNT; + DAT_LMR_TRIPLET *hdlptr = (DAT_LMR_TRIPLET *) + DT_Mdep_Malloc (count * cmd->width * sizeof (*hdlptr)); + + /* Recv-Post Exhaustion test loop */ + if (hdlptr) + { + unsigned int w = 0; + unsigned int i = 0; + unsigned int done = 0; + + DT_Tdep_PT_Debug (1,(phead,"%s: Exhausting posting of recv buffers\n", module)); + for (w = 0; w < cmd->maximum && !done; w++) + { + DT_Mdep_Schedule(); + if (w == count + && !more_handles (phead, (DAT_HANDLE **) &hdlptr, + &count, + cmd->width * sizeof (*hdlptr))) + { + DT_Tdep_PT_Printf (phead, "%s: no memory for IOVs \n", + module); + DT_Tdep_PT_Printf (phead, "%s: recv buffers posted per EP: %d\n" + "\t\t (total posted: %d)\n", + module, + w, + w * cmd->width); + done = retval = true; + break; + } + for (i = 0; i < cmd->width; i++) + { + DAT_LMR_TRIPLET *iovp = &hdlptr[w * cmd->width + i]; + DAT_DTO_COOKIE cookie; + + iovp->virtual_address = (DAT_VADDR) (uintptr_t) + hdl_sets[i].lmr_buffer; + iovp->segment_length = DFLT_BUFFSZ; + iovp->lmr_context = hdl_sets[i].lmr_context; + cookie.as_64 = (DAT_UINT64)0UL; + cookie.as_ptr = (DAT_PVOID) hdl_sets[i].lmr_buffer; + + DT_Tdep_PT_Printf (phead, "%s: dat_ep_post_recv #%d\n", module, + w * cmd->width + i + 1); + ret = dat_ep_post_recv (hdl_sets[i].ep_handle, + 1, + iovp, + cookie, + DAT_COMPLETION_DEFAULT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_post_recv fails: %s\n", + module, DT_RetToString (ret)); + DT_Tdep_PT_Printf (phead, "%s: recv buffers posted per EP: %d\n" + "\t\t (total posted: %d)\n", + module, + w, + w * cmd->width + i); + done = retval = true; + break; + } + } /* end for each EP wide */ + } /* end forever (!done) loop */ + + retval = true; + DT_Tdep_PT_Printf (phead, "%s: recv buffers posted per EP: %d\n" + "\t\t (total posted: %d)\n", + module, + w, + w * cmd->width); + + /* Rpost Cleanup loop */ + for (i = 0; i < cmd->width; i++) + { + DAT_EVENT event; + + /* + * Disconnecting an unconnected EP should complete + * outstanding recv DTOs in error, and otherwise + * be a no-op. + */ + ret = dat_ep_reset (hdl_sets[i].ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_disconnect (abrupt) fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + else + { + /* + * Remove all DTOs. The disconnect above should have + * flushed all posted operations, so this is just a + * clean up. + */ + do + { + ret = DT_Tdep_evd_dequeue ( hdl_sets[i].evd_handle, + &event); + } while (ret == DAT_SUCCESS); + } + + } + DT_Mdep_Free (hdlptr); + } + } /* end depth == LIM_RPOST */ + + /* ----------- + * Test maximum size of LMR allowed + */ + if (depth == LIM_SIZE_LMR) + { + DAT_COUNT last_size = 0; + DAT_COUNT test_size = DFLT_BUFFSZ; + Bpool *test_bpool; + for (;;) + { + DT_Mdep_Schedule(); + test_bpool = DT_BpoolAlloc ((Per_Test_Data_t *)0, + phead, + hdl_sets[0].ia_handle, + hdl_sets[0].pz_handle, + hdl_sets[0].ep_handle, + hdl_sets[0].evd_handle, + test_size, + 1, + DAT_OPTIMAL_ALIGNMENT, + false, + false); + + if (!test_bpool) + { + DT_Tdep_PT_Printf (phead, "%s: Largest LMR was 0x%x bytes\n" + "\t (failed attempting 0x%x bytes)\n", + module, last_size, test_size); + retval = true; + break; + } + else if (!DT_Bpool_Destroy ((Per_Test_Data_t *)0, phead, test_bpool)) + { + DT_Tdep_PT_Printf (phead, "%s: Largest LMR was 0x%x bytes\n", + module, test_size); + retval = true; + break; + } + + last_size = test_size; + test_size <<= 1; + if (test_size < last_size) + { + /* could conceivably wrap on 32-bit architectures */ + DT_Tdep_PT_Printf (phead, "%s: LMR of 0x%x bytes OK - %s\n", + module, last_size, "stopping now."); + retval = true; + break; + } + } /* end forever loop */ + } /* end depth == LIM_SIZE_LMR */ + + DT_Tdep_PT_Debug (1,(phead,"%s: Limit Testing Completed - %s\n", + module, + retval ? "Successfully" : "with errors")); + + /* ---------------------------------------------------------- + * Clean up and go home + */ +clean_up_now: + + DT_Tdep_PT_Debug (1,(phead,"%s: Cleaning up ...\n", module)); + if (depth > LIM_LMR) + { + unsigned int w; + + for (w = 0; w < cmd->width; w++) + { + if (hdl_sets[w].lmr_handle) + { + ret = dat_lmr_free (hdl_sets[w].lmr_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_lmr_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + if ((void *) hdl_sets[w].lmr_buffer) + { + DT_Mdep_Free ((void *) hdl_sets[w].lmr_buffer); + } + } + } /* end LIM_LMR cleanup */ + + /* + * if (depth == LIM_PSP) { + * Since PSPs are not part of the Obj_Set, + * there's no cleanup to do. + * } + * + * if (depth == LIM_RSP) { + * Since RSPs are not part of the Obj_Set, + * there'no cleanup nothing to do. + * } + */ + + if (depth > LIM_EP) + { + unsigned int w; + + for (w = 0; w < cmd->width; w++) + { + if (hdl_sets[w].ep_handle) + { + ret = dat_ep_free (hdl_sets[w].ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + } + } /* end LIM_EP cleanup */ + + if (depth > LIM_EVD) + { + unsigned int w; + + if (conn_handle != DAT_HANDLE_NULL) + { + ret = DT_Tdep_evd_free (conn_handle); + conn_handle = DAT_HANDLE_NULL; + } + for (w = 0; w < cmd->width; w++) + { + if (hdl_sets[w].evd_handle) + { + ret = DT_Tdep_evd_free (hdl_sets[w].evd_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_evd_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + } + } /* end LIM_EVD cleanup */ + +#ifndef __KDAPLTEST__ + if (depth > LIM_CNO) + { + unsigned int w; + + for (w = 0; w < cmd->width; w++) + { + if (hdl_sets[w].cno_handle) + { + ret = dat_cno_free (hdl_sets[w].cno_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_cno_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + } + } /* end LIM_CNO cleanup */ +#endif + + if (depth > LIM_PZ) + { + unsigned int w; + + for (w = 0; w < cmd->width; w++) + { + if (hdl_sets[w].pz_handle) + { + ret = dat_pz_free (hdl_sets[w].pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_pz_free fails: %s\n", + module, DT_RetToString (ret)); + retval = false; + } + } + } + } /* end LIM_PZ cleanup */ + + if (depth > LIM_IA) + { + unsigned int w; + + for (w = 0; w < cmd->width; w++) + { + if (hdl_sets[w].ia_handle) + { + /* dat_ia_close cleans up async evd handle, too */ + ret = dat_ia_close (hdl_sets[w].ia_handle, + DAT_CLOSE_GRACEFUL_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_close (graceful) error: %s\n", + module, DT_RetToString (ret)); + /* + * Since we take some pains to clean up properly, + * this really is an error. But if we get here, + * we may as well try the largest hammer we have. + */ + retval = false; + ret = dat_ia_close (hdl_sets[w].ia_handle, + DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_close (abrupt) error: %s\n", + module, DT_RetToString (ret)); + } + } + } + } + } /* end LIM_IA cleanup */ + + if (depth && hdl_sets) + { + DT_Mdep_Free (hdl_sets); + } + + DT_Tdep_PT_Debug (1,(phead,"%s: testing and cleanup complete.\n", module)); + + return ( retval ); +} + + +/********************************************************************* + * Framework to run through all of the limit tests + */ +void +DT_cs_Limit (Params_t *params, Limit_Cmd_t * cmd) +{ + DT_Tdep_Print_Head *phead; + char *star = + "**********************************************************************" ; + + phead = params->phead; + + if (cmd->Test_List[ LIM_IA ]) + { + char list[] = + { + "Limitation Test limit_ia\n" + "Description: Test max num of opens for the same physical IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_IA)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_PZ ]) + { + char list[] = + { + "Limitation Test limit_pz\n" + "Description: Test max num of PZs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_PZ)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + +#ifndef __KDAPLTEST__ + if (cmd->Test_List[ LIM_CNO ]) + { + char list[] = + { + "Limitation Test limit_cno\n" + "Description: Test max num of CNOs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_CNO)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } +#endif + + if (cmd->Test_List[ LIM_EVD ]) + { + char list[] = + { + "Limitation Test limit_evd\n" + "Description: Test max num of EVDs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_EVD)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_EP ]) + { + char list[] = + { + "Limitation Test limit_ep\n" + "Description: Test max num of EPs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_EP)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_RSP ]) + { + char list[] = + { + "Limitation Test limit_rsp\n" + "Description: Test max num of RSPs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_RSP)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_PSP ]) + { + char list[] = + { + "Limitation Test limit_psp\n" + "Description: Test max num of PSPs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_PSP)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_LMR ]) + { + char list[] = + { + "Limitation Test limit_lmr\n" + "Description: Test max num of LMRs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_LMR)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_RPOST ]) + { + char list[] = + { + "Limitation Test limit_rpost\n" + "Description: Test max num of receive buffers posted to an EP" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_RPOST)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + if (cmd->Test_List[ LIM_SIZE_LMR ]) + { + char list[] = + { + "Limitation Test limit_size_lmr\n" + "Description: Test max size of LMRs that are supported by an IA" + } + ; + + DT_Tdep_PT_Printf (phead, "%s\n", star); + DT_Tdep_PT_Printf (phead, "%s\n", list); + if (!limit_test (phead, cmd, LIM_SIZE_LMR)) + { + goto error; + } + DT_Tdep_PT_Printf (phead, "%s\n", star); + } + + /* More tests TBS ... */ + + return; + +error: + DT_Tdep_PT_Printf (phead, "error occurs, can not continue with limit test\n"); + DT_Tdep_PT_Printf (phead, "%s\n", star); + return; +} + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_memlist.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_memlist.c new file mode 100644 index 00000000..7fd878fe --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_memlist.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_MemListInit (Per_Test_Data_t * pt_ptr) +{ + DT_Mdep_LockInit (&pt_ptr->MemListLock); + pt_ptr->MemListHead = 0; +} + +void * +DT_MemListAlloc (Per_Test_Data_t * pt_ptr, + char *file, + mem_type_e t, + int size) +{ + void *buffptr; + MemListEntry_t *entry_ptr; + buffptr = 0; + entry_ptr = 0; + + buffptr = DT_Mdep_Malloc (size); + if (buffptr == 0) + { + return 0; + } + if (pt_ptr == 0) /* not use mem_list */ + { + return buffptr; + } + entry_ptr = (MemListEntry_t *) DT_Mdep_Malloc (sizeof (MemListEntry_t)); + if (entry_ptr == 0) + { + DT_Mdep_Free (buffptr); + return 0; + } + strcpy (entry_ptr->filename, file); + entry_ptr->MemType = t; + entry_ptr->mem_ptr = buffptr; + + DT_Mdep_Lock (&pt_ptr->MemListLock); + entry_ptr->next = pt_ptr->MemListHead; + pt_ptr->MemListHead = entry_ptr; + DT_Mdep_Unlock (&pt_ptr->MemListLock); + + return buffptr; +} + +void +DT_MemListFree (Per_Test_Data_t * pt_ptr, void *ptr) +{ + MemListEntry_t *pre, *cur; + if (pt_ptr == 0) /* not use mem_list */ + { + DT_Mdep_Free (ptr); + return; + } + DT_Mdep_Lock (&pt_ptr->MemListLock); + pre = 0; + cur = pt_ptr->MemListHead; + while (cur) + { + if (cur->mem_ptr == ptr) + { + if (!pre) /* first entry */ + { + pt_ptr->MemListHead = cur->next; + cur->next = 0; + } + else + { + pre->next = cur->next; + cur->next = 0; + } + DT_Mdep_Free (ptr); + DT_Mdep_Free (cur); + goto unlock_and_return; + } + pre = cur; + cur = cur->next; + } +unlock_and_return: + DT_Mdep_Unlock (&pt_ptr->MemListLock); +} + +void +DT_PrintMemList (Per_Test_Data_t * pt_ptr) +{ + char *type[10] = + { + "BPOOL", "BUFF", "PERTESTDATA", "NIC", "NETADDRESS", + "TRANSACTIONTEST", "THREAD", "EPCONTEXT" + }; + DT_Tdep_Print_Head *phead; + MemListEntry_t *cur; + + phead = pt_ptr->Params.phead; + DT_Mdep_Lock (&pt_ptr->MemListLock); + cur = pt_ptr->MemListHead; + if (cur != 0) + { + DT_Tdep_PT_Printf (phead, "the allocated memory that have not been returned are:\n"); + } + while (cur) + { + DT_Tdep_PT_Printf (phead, "file: dapl_%s, \tMemType:%s\n", + cur->filename, type[cur->MemType]); + cur = cur->next; + } + DT_Mdep_Unlock (&pt_ptr->MemListLock); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_client.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_client.c new file mode 100644 index 00000000..5c0cb3b8 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_client.c @@ -0,0 +1,520 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define MAX_CONN_RETRY 8 + +/****************************************************************************/ +void +DT_Performance_Test_Client ( + Params_t *params_ptr, + Per_Test_Data_t *pt_ptr, + DAT_IA_HANDLE *ia_handle, + DAT_IA_ADDRESS_PTR remote_ia_addr) +{ + Performance_Test_t *test_ptr = NULL; + int connected = 1; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + DT_Tdep_PT_Debug (1,(phead,"Client: Starting performance test\n")); + + if ( !DT_Performance_Test_Create (pt_ptr, + ia_handle, + remote_ia_addr, + false, + pt_ptr->Server_Info.is_little_endian, + &test_ptr) ) + { + DT_Tdep_PT_Debug (1,(phead,"Client: Resource Creation Failed\n")); + connected = 0; + } + else if ( !DT_Performance_Test_Client_Connect (phead, test_ptr) ) + { + DT_Tdep_PT_Debug (1,(phead,"Client: Connection Failed\n")); + connected = 0; + } + + if ( connected ) + { + if ( !DT_Performance_Test_Client_Exchange (params_ptr,phead, test_ptr) ) + { + DT_Tdep_PT_Debug (1,(phead,"Client: Test Failed\n")); + } + } + /* If we never connected, then the test will hang here + * because in the destroy of the test it waits for a + * disconnect event which will never arrive, simply + * because there was never a connection. + */ + + DT_Performance_Test_Destroy (pt_ptr, test_ptr, false); + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + DT_Tdep_PT_Debug (1,(phead,"Client: Finished performance test\n")); +} + + +/****************************************************************************/ +bool +DT_Performance_Test_Client_Connect ( + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr) +{ + DAT_RETURN ret; + DAT_EVENT_NUMBER event_num; + unsigned int retry_cnt = 0; + + /* + * Client - connect + */ + DT_Tdep_PT_Debug (1,(phead,"Client[" F64x "]: Connect on port 0x" F64x "\n", + test_ptr->base_port, test_ptr->ep_context.port)); + +retry: + ret = dat_ep_connect (test_ptr->ep_context.ep_handle, + test_ptr->remote_ia_addr, + test_ptr->ep_context.port, + DAT_TIMEOUT_INFINITE, + 0, + (DAT_PVOID) 0, /* no private data */ + test_ptr->cmd->qos, + DAT_CONNECT_DEFAULT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ep_connect error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + return false; + } + + /* wait for DAT_CONNECTION_EVENT_ESTABLISHED */ + if (!DT_conn_event_wait (phead, + test_ptr->ep_context.ep_handle, + test_ptr->conn_evd_hdl, + &event_num)) + { + if ( event_num == DAT_CONNECTION_EVENT_PEER_REJECTED ) + { + DT_Mdep_Sleep (1000); + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: retrying connection...\n", + test_ptr->base_port); + retry_cnt++; + if (retry_cnt < MAX_CONN_RETRY) + { + goto retry; + } + } + /* error message printed by DT_cr_event_wait */ + return false; + } + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + DT_Tdep_PT_Debug (1,(phead,"Client[" F64x "]: Got Connection\n", test_ptr->base_port)); + + return true; +} + + +/****************************************************************************/ +static bool +DT_Performance_Test_Client_Phase1 ( + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr, + Performance_Stats_t *stats ) +{ + DT_Mdep_TimeStamp pre_ts; + DT_Mdep_TimeStamp post_ts; + DT_CpuStat pre_cpu_stat; + DT_CpuStat post_cpu_stat; + unsigned int post_cnt; + unsigned int reap_cnt; + + /* + * measure bandwidth, OPS, and CPU utilization + */ + + if ( !DT_Mdep_GetCpuStat (&pre_cpu_stat) ) + { + return false; + } + + pre_ts = DT_Mdep_GetTimeStamp (); + + /* + * Fill the pipe + */ + + for ( post_cnt = 0; post_cnt < (unsigned int)test_ptr->ep_context.pipeline_len; post_cnt++ ) + { + if ( !DT_performance_post_rdma_op (&test_ptr->ep_context, + test_ptr->reqt_evd_hdl, + stats) ) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Post %i failed\n", + test_ptr->base_port, + post_cnt)); + return false; + } + } + + /* + * Reap completions and repost + */ + + for ( reap_cnt = 0; reap_cnt < test_ptr->cmd->num_iterations; ) + { + unsigned int cur_reap_cnt; + unsigned int cur_post_cnt; + unsigned int cur_post_i; + + cur_reap_cnt = DT_performance_reap (phead, + test_ptr->reqt_evd_hdl, + test_ptr->cmd->mode, + stats); + + if ( 0 == cur_reap_cnt ) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Poll %i failed\n", + test_ptr->base_port, + reap_cnt)); + return false; + } + + /* repost */ + cur_post_cnt = DT_min (test_ptr->cmd->num_iterations - post_cnt, + cur_reap_cnt); + + for ( cur_post_i = 0; cur_post_i < cur_post_cnt; cur_post_i++) + { + if ( !DT_performance_post_rdma_op (&test_ptr->ep_context, + test_ptr->reqt_evd_hdl, + stats) ) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Post %i failed\n", + test_ptr->base_port, + post_cnt)); + return false; + } + } + + reap_cnt += cur_reap_cnt; + post_cnt += cur_post_cnt; + } + + /* end time and update stats */ + post_ts = DT_Mdep_GetTimeStamp (); + stats->time_ts = post_ts - pre_ts; + stats->num_ops = test_ptr->cmd->num_iterations; + + if ( !DT_Mdep_GetCpuStat (&post_cpu_stat) ) + { + return false; + } + + /* calculate CPU utilization */ + { + unsigned long int system; + unsigned long int user; + unsigned long int idle; + unsigned long int total; + + system = post_cpu_stat.system - pre_cpu_stat.system; + user = post_cpu_stat.user - pre_cpu_stat.user; + idle = post_cpu_stat.idle - pre_cpu_stat.idle; + + total = system + user + idle; + + if ( 0 == total ) + { + stats->cpu_utilization = 0.0; + } + else + { + stats->cpu_utilization = (double)1.0 - ((double) idle / (double) total ); + stats->cpu_utilization *= 100.0; + } + } + + return true; +} + + +/****************************************************************************/ +static bool +DT_Performance_Test_Client_Phase2 ( + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr, + Performance_Stats_t *stats ) +{ + DAT_LMR_TRIPLET *iov; + DAT_RMR_TRIPLET rmr_triplet; + DAT_DTO_COOKIE cookie; + DAT_EVENT event; + DAT_RETURN ret; + Performance_Ep_Context_t *ep_context; + Performance_Test_Op_t *op; + DT_Mdep_TimeStamp pre_ts; + DT_Mdep_TimeStamp post_ts; + unsigned long int bytes; + unsigned int i; + + /* + * measure latency + */ + + ep_context = &test_ptr->ep_context; + op = &ep_context->op; + iov = DT_Bpool_GetIOV (op->bp, 0); + + bytes = op->seg_size * op->num_segs; + + /* Prep the inputs */ + for (i = 0; i < op->num_segs; i++) + { + iov[i].virtual_address = (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (op->bp, i); + iov[i].segment_length = op->seg_size; + iov[i].lmr_context = DT_Bpool_GetLMR (op->bp, i); + } + + rmr_triplet.virtual_address = op->Rdma_Address; + rmr_triplet.segment_length = op->seg_size * op->num_segs; + rmr_triplet.rmr_context = op->Rdma_Context; + + cookie.as_ptr = NULL; + + for ( i = 0; i < test_ptr->cmd->num_iterations; i++ ) + { + if ( RDMA_WRITE == op->transfer_type ) + { + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = dat_ep_post_rdma_write (ep_context->ep_handle, + op->num_segs, + iov, + cookie, + &rmr_triplet, + DAT_COMPLETION_DEFAULT_FLAG); + } + else + { + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = dat_ep_post_rdma_read (ep_context->ep_handle, + op->num_segs, + iov, + cookie, + &rmr_triplet, + DAT_COMPLETION_DEFAULT_FLAG); + } + + if ( DAT_SUCCESS != ret ) + { + return false; + } + + for (;;) + { + DT_Mdep_Schedule(); + ret = DT_Tdep_evd_dequeue ( test_ptr->reqt_evd_hdl, + &event); + + post_ts = DT_Mdep_GetTimeStamp (); + + if (DAT_GET_TYPE (ret) == DAT_QUEUE_EMPTY) + { + continue; + } + else if ( DAT_SUCCESS != ret ) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_dequeue failed: %s\n", + DT_RetToString (ret)); + return false; + } + else if (event.event_number == DAT_DTO_COMPLETION_EVENT) + { + DT_performance_stats_record_latency (stats, post_ts - pre_ts); + break; + } + else /* error */ + { + DT_Tdep_PT_Printf ( + phead, + "Warning: dapl_performance_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + + return false; + } + } + } + + return true; +} + + +/****************************************************************************/ +bool +DT_Performance_Test_Client_Exchange ( + Params_t *params_ptr, + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr) +{ + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_DTO_COOKIE dto_cookie; + Performance_Stats_t stats; + RemoteMemoryInfo *rmi; + + test_ptr->ep_context.op.bp = + DT_BpoolAlloc (test_ptr->pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context.ep_handle, + test_ptr->reqt_evd_hdl, + test_ptr->ep_context.op.seg_size, + test_ptr->ep_context.op.num_segs, + DAT_OPTIMAL_ALIGNMENT, + false, + false); + + if ( !test_ptr->ep_context.op.bp ) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for buffers (RDMA/RD)\n", + test_ptr->base_port); + return false; + } + + /* + * Recv the other side's info + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Waiting for Sync Msg\n", + test_ptr->base_port)); + + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context.bp, + DT_PERF_SYNC_RECV_BUFFER_ID); + if ( !DT_dto_event_wait (phead, test_ptr->recv_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + test_ptr->ep_context.ep_handle, + DT_PERF_SYNC_BUFF_SIZE, + dto_cookie, + "Received Sync_Msg") ) + { + return false; + } + + /* + * Extract what we need + */ + DT_Tdep_PT_Debug (1, (phead,"Test[" F64x "]: Sync Msg Received\n", test_ptr->base_port)); + rmi = (RemoteMemoryInfo *) DT_Bpool_GetBuffer (test_ptr->ep_context.bp, + DT_PERF_SYNC_RECV_BUFFER_ID); + + /* + * If the client and server are of different endiannesses, + * we must correct the endianness of the handle and address + * we pass to the other side. The other side cannot (and + * better not) interpret these values. + */ + if (DT_local_is_little_endian != test_ptr->is_remote_little_endian) + { + rmi->rmr_context = DT_EndianMemHandle (rmi->rmr_context); + rmi->mem_address.as_64 =DT_EndianMemAddress (rmi->mem_address.as_64); + } + + test_ptr->ep_context.op.Rdma_Context = rmi->rmr_context; + test_ptr->ep_context.op.Rdma_Address = rmi->mem_address.as_64; + + DT_Tdep_PT_Debug (3, (phead, + "Got RemoteMemInfo [ va=" F64x ", ctx=%x ]\n", + test_ptr->ep_context.op.Rdma_Address, + test_ptr->ep_context.op.Rdma_Context)); + + /* + * Get to work ... + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Begin...\n", test_ptr->base_port)); + + DT_performance_stats_init (&stats); + + if ( !DT_Performance_Test_Client_Phase1 (phead, test_ptr, &stats) ) + { + return false; + } + + if ( !DT_Performance_Test_Client_Phase2 (phead, test_ptr, &stats) ) + { + return false; + } + + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Sending Sync Msg\n", test_ptr->base_port)); + + if (!DT_post_send_buffer (phead, + test_ptr->ep_context.ep_handle, + test_ptr->ep_context.bp, + DT_PERF_SYNC_SEND_BUFFER_ID, + DT_PERF_SYNC_BUFF_SIZE)) + { + /* error message printed by DT_post_send_buffer */ + return false; + } + + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context.bp, + DT_PERF_SYNC_SEND_BUFFER_ID); + if (!DT_dto_event_wait (phead, test_ptr->reqt_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + test_ptr->ep_context.ep_handle, + DT_PERF_SYNC_BUFF_SIZE, + dto_cookie, + "Client_Sync_Send")) + { + return false; + } + DT_performance_stats_print (params_ptr,phead, &stats, test_ptr->cmd, test_ptr); + + return true; +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_server.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_server.c new file mode 100644 index 00000000..4c173af5 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_server.c @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/****************************************************************************/ +void +DT_Performance_Test_Server ( + void *var) +{ + Per_Test_Data_t *pt_ptr = var; + Performance_Test_t *test_ptr = NULL; + DT_Tdep_Print_Head *phead; + int success = 1; + + phead = pt_ptr->Params.phead; + DT_Tdep_PT_Debug (1,(phead,"Server: Starting performance test\n")); + + if ( !DT_Performance_Test_Create (pt_ptr, + pt_ptr->ps_ptr->ia_handle, + (DAT_IA_ADDRESS_PTR) 0, + true, + pt_ptr->Client_Info.is_little_endian, + &test_ptr) ) + { + DT_Tdep_PT_Printf (phead, "Server: Resource Creation Failed\n"); + success = 0; + } + if ( 1 == success ) + { + if (! DT_Performance_Test_Server_Connect (phead, test_ptr) ) + { + success = 0; + DT_Tdep_PT_Printf (phead, "Server: Connection Failed\n"); + } + } + + if ( 1 == success ) + { + if ( ! DT_Performance_Test_Server_Exchange (phead, test_ptr) ) + { + success = 0; + DT_Tdep_PT_Printf (phead, "Server: Test Failed\n"); + } + } + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + + + DT_Performance_Test_Destroy (pt_ptr, test_ptr, true); + + DT_Tdep_PT_Printf (phead, "Server: Finished performance test. Detaching.\n"); + + DT_Mdep_Thread_Detach (DT_Mdep_Thread_SELF ()); /* AMM */ + DT_Thread_Destroy (pt_ptr->thread, pt_ptr); /* destroy Master thread */ + + DT_Mdep_Lock (&pt_ptr->ps_ptr->num_clients_lock); + pt_ptr->ps_ptr->num_clients--; + DT_Mdep_Unlock (&pt_ptr->ps_ptr->num_clients_lock); + + DT_PrintMemList (pt_ptr); /* check if we return all space allocated */ + DT_Mdep_LockDestroy (&pt_ptr->Thread_counter_lock); + DT_Mdep_LockDestroy (&pt_ptr->MemListLock); + DT_Free_Per_Test_Data (pt_ptr); + + DT_Mdep_Unlock (&g_PerfTestLock); + DT_Tdep_PT_Printf (phead, "Server: Finished performance test. Exiting.\n"); + + DT_Mdep_Thread_EXIT (NULL); +} + + +/****************************************************************************/ +bool +DT_Performance_Test_Server_Connect ( + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr) +{ + DAT_RETURN ret; + bool status; + DAT_RSP_HANDLE rsp_handle; + DAT_PSP_HANDLE psp_handle; + + DAT_CR_ARRIVAL_EVENT_DATA cr_stat; + DAT_CR_HANDLE cr_handle; + DAT_EVENT_NUMBER event_num; + + rsp_handle = DAT_HANDLE_NULL; + psp_handle = DAT_HANDLE_NULL; +#if 0 /* FIXME */ + if (test_ptr->cmd->use_rsp) + { + /* + * Server - create a single-use RSP and + * await a connection for this EP + */ + ret = dat_rsp_create (test_ptr->ia_handle, + test_ptr->ep_context.port, + test_ptr->ep_context.ep_handle, + test_ptr->creq_evd_hdl, + &rsp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_rsp_create error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + status = false; + goto psp_free; + } + + DT_Tdep_PT_Debug (1,(phead,"Server[" F64x "]: Listen on RSP port 0x" F64x "\n", + test_ptr->base_port, test_ptr->ep_context.port)); + + /* wait for the connection request */ + if (!DT_cr_event_wait (test_ptr->conn_evd_hdl, &cr_stat) || + !DT_cr_check ( &cr_stat, + DAT_HANDLE_NULL, + test_ptr->ep_context.port, + &cr_handle, + "Server") ) + { + status = false; + goto psp_free; + } + + /* what, me query? just try to accept the connection */ + ret = dat_cr_accept (cr_handle, + test_ptr->ep_context.ep_handle, + 0, (DAT_PVOID)0 /* no private data */ ); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_cr_accept error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* cr_handle consumed on failure */ + status = false; + goto psp_free; + } + + /* wait for DAT_CONNECTION_EVENT_ESTABLISHED */ + if (!DT_conn_event_wait ( test_ptr->ep_context.ep_handle, + test_ptr->conn_evd_hdl, + &event_num)) + { + /* error message printed by DT_conn_event_wait */ + status = false; + goto psp_free; + } + + } + else +#endif /* FIXME */ + { + /* + * Server - use a short-lived PSP instead of an RSP + */ + status = true; + + ret = dat_psp_create (test_ptr->ia_handle, + test_ptr->ep_context.port, + test_ptr->creq_evd_hdl, + DAT_PSP_CONSUMER_FLAG, + &psp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_psp_create error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + status = false; + psp_handle = DAT_HANDLE_NULL; + return (status); + } + + } + + /* + * Here's where we tell the main server process that + * this thread is ready to wait for a connection request + * from the remote end. + */ + DT_Mdep_wait_object_wakeup (&test_ptr->pt_ptr->synch_wait_object); + + DT_Tdep_PT_Debug (1,(phead,"Server[" F64x "]: Listen on PSP port 0x" F64x "\n", + test_ptr->base_port, test_ptr->ep_context.port)); + + /* wait for a connection request */ + if (!DT_cr_event_wait (phead, test_ptr->creq_evd_hdl, &cr_stat) || + !DT_cr_check ( phead, + &cr_stat, + psp_handle, + test_ptr->ep_context.port, + &cr_handle, + "Server") ) + { + status = false; + goto psp_free; + } + + /* what, me query? just try to accept the connection */ + ret = dat_cr_accept (cr_handle, + test_ptr->ep_context.ep_handle, + 0, + (DAT_PVOID)0 /* no private data */ ); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_cr_accept error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* cr_handle consumed on failure */ + status = false; + goto psp_free; + } + + /* wait for DAT_CONNECTION_EVENT_ESTABLISHED */ + if (!DT_conn_event_wait (phead, + test_ptr->ep_context.ep_handle, + test_ptr->conn_evd_hdl, + &event_num ) ) + { + /* error message printed by DT_cr_event_wait */ + status = false; + goto psp_free; + } + + DT_Tdep_PT_Debug (1, (phead,"Server[" F64x "]: Accept on port 0x" F64x "\n", + test_ptr->base_port, test_ptr->ep_context.port)); +psp_free: + if ( DAT_HANDLE_NULL != psp_handle ) + { + /* throw away single-use PSP */ + ret = dat_psp_free (psp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_psp_free error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + status = false; + } + } + if ( DAT_HANDLE_NULL != rsp_handle ) + { + /* throw away single-use PSP */ + ret = dat_rsp_free (rsp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_rsp_free error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + status = false; + } + } /* end short-lived PSP */ + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + return status; +} + + + +/****************************************************************************/ +bool +DT_Performance_Test_Server_Exchange ( + DT_Tdep_Print_Head *phead, + Performance_Test_t *test_ptr) +{ + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + RemoteMemoryInfo *rmi; + DAT_DTO_COOKIE dto_cookie; + + test_ptr->ep_context.op.bp = + DT_BpoolAlloc (test_ptr->pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context.ep_handle, + test_ptr->reqt_evd_hdl, + test_ptr->ep_context.op.seg_size, + test_ptr->ep_context.op.num_segs, + DAT_OPTIMAL_ALIGNMENT, + true, + true); + + if ( !test_ptr->ep_context.op.bp ) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for buffers (RDMA/RD)\n", + test_ptr->base_port); + return false; + } + + test_ptr->ep_context.op.Rdma_Context = + DT_Bpool_GetRMR (test_ptr->ep_context.op.bp, 0); + test_ptr->ep_context.op.Rdma_Address = (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (test_ptr->ep_context.op.bp, 0); + + /* + * Prep send buffer with memory information + */ + rmi = (RemoteMemoryInfo *) DT_Bpool_GetBuffer (test_ptr->ep_context.bp, + DT_PERF_SYNC_SEND_BUFFER_ID); + + rmi->rmr_context = test_ptr->ep_context.op.Rdma_Context; + rmi->mem_address.as_64 =test_ptr->ep_context.op.Rdma_Address; + + if ( rmi->mem_address.as_ptr ) + { + DT_Tdep_PT_Debug (3, (phead, + "RemoteMemInfo va=" F64x ", ctx=%x\n", + rmi->mem_address.as_64, + rmi->rmr_context)); + } + + /* + * If the client and server are of different endiannesses, + * we must correct the endianness of the handle and address + * we pass to the other side. The other side cannot (and + * better not) interpret these values. + */ + if (DT_local_is_little_endian != test_ptr->is_remote_little_endian) + { + rmi->rmr_context = DT_EndianMemHandle (rmi->rmr_context); + rmi->mem_address.as_64 =DT_EndianMemAddress (rmi->mem_address.as_64); + } + + /* + * Send our memory info + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Sending Sync Msg\n", test_ptr->base_port)); + + /* post the send buffer */ + if (!DT_post_send_buffer (phead, + test_ptr->ep_context.ep_handle, + test_ptr->ep_context.bp, + DT_PERF_SYNC_SEND_BUFFER_ID, + DT_PERF_SYNC_BUFF_SIZE)) + { + /* error message printed by DT_post_send_buffer */ + return false; + } + + /* reap the send and verify it */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context.bp, + DT_PERF_SYNC_SEND_BUFFER_ID); + if ( !DT_dto_event_wait (phead, test_ptr->reqt_evd_hdl, &dto_stat) || + !DT_dto_check (phead, + &dto_stat, + test_ptr->ep_context.ep_handle, + DT_PERF_SYNC_BUFF_SIZE, + dto_cookie, + "Send Sync_Msg") ) + { + return false; + } + + /* + * Recv the other side's info + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Waiting for Sync Msg\n", + test_ptr->base_port)); + + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context.bp, + DT_PERF_SYNC_RECV_BUFFER_ID); + if ( !DT_dto_event_wait (phead, test_ptr->recv_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + test_ptr->ep_context.ep_handle, + DT_PERF_SYNC_BUFF_SIZE, + dto_cookie, + "Received Sync_Msg") ) + { + return false; + } + + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Received Sync Msg\n", + test_ptr->base_port)); + + return true; +} + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_stats.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_stats.c new file mode 100644 index 00000000..7a834515 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_stats.c @@ -0,0 +1,399 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_performance_stats_init ( + Performance_Stats_t *stats) +{ + stats->num_ops = 0; + stats->bytes = 0; + stats->post_ctxt_switch_num = 0; + stats->reap_ctxt_switch_num = 0; + + stats->cpu_utilization = 0.0; + stats->time_ts = 0; + + stats->posts_sans_ctxt.num = 0; + stats->posts_sans_ctxt.total_ts = 0; + stats->posts_sans_ctxt.max_ts = 0; + stats->posts_sans_ctxt.min_ts = ~0; + + stats->posts_with_ctxt.num = 0; + stats->posts_with_ctxt.total_ts = 0; + stats->posts_with_ctxt.max_ts = 0; + stats->posts_with_ctxt.min_ts = ~0; + + stats->reaps_sans_ctxt.num = 0; + stats->reaps_sans_ctxt.total_ts = 0; + stats->reaps_sans_ctxt.max_ts = 0; + stats->reaps_sans_ctxt.min_ts = ~0; + + stats->reaps_with_ctxt.num = 0; + stats->reaps_with_ctxt.total_ts = 0; + stats->reaps_with_ctxt.max_ts = 0; + stats->reaps_with_ctxt.min_ts = ~0; + + stats->latency.num = 0; + stats->latency.total_ts = 0; + stats->latency.max_ts = 0; + stats->latency.min_ts = ~0; +} + + +void +DT_performance_stats_record_post ( + Performance_Stats_t *stats, + unsigned long ctxt_switch_num, + DT_Mdep_TimeStamp ts) +{ + if ( ctxt_switch_num ) + { + stats->posts_with_ctxt.num++; + stats->posts_with_ctxt.total_ts += ts; + stats->posts_with_ctxt.max_ts = + DT_max (stats->posts_with_ctxt.max_ts, ts); + stats->posts_with_ctxt.min_ts = + DT_min (stats->posts_with_ctxt.min_ts, ts); + + stats->post_ctxt_switch_num += ctxt_switch_num; + } + else + { + stats->posts_sans_ctxt.num++; + stats->posts_sans_ctxt.total_ts += ts; + stats->posts_sans_ctxt.max_ts = + DT_max (stats->posts_sans_ctxt.max_ts, ts); + stats->posts_sans_ctxt.min_ts = + DT_min (stats->posts_sans_ctxt.min_ts, ts); + } +} + + +void +DT_performance_stats_record_reap ( + Performance_Stats_t *stats, + unsigned long ctxt_switch_num, + DT_Mdep_TimeStamp ts) +{ + if ( ctxt_switch_num ) + { + stats->reaps_with_ctxt.num++; + stats->reaps_with_ctxt.total_ts += ts; + stats->reaps_with_ctxt.max_ts = + DT_max (stats->reaps_with_ctxt.max_ts, ts); + stats->reaps_with_ctxt.min_ts = + DT_min (stats->reaps_with_ctxt.min_ts, ts); + + stats->reap_ctxt_switch_num += ctxt_switch_num; + } + else + { + stats->reaps_sans_ctxt.num++; + stats->reaps_sans_ctxt.total_ts += ts; + stats->reaps_sans_ctxt.max_ts = + DT_max (stats->reaps_sans_ctxt.max_ts, ts); + stats->reaps_sans_ctxt.min_ts = + DT_min (stats->reaps_sans_ctxt.min_ts, ts); + } +} + + +void +DT_performance_stats_record_latency ( + Performance_Stats_t *stats, + DT_Mdep_TimeStamp ts) +{ + stats->latency.num++; + stats->latency.total_ts += ts; + stats->latency.max_ts = + DT_max (stats->latency.max_ts, ts); + stats->latency.min_ts = + DT_min (stats->latency.min_ts, ts); +} + +void +DT_performance_stats_data_combine ( + Performance_Stats_Data_t *dest, + Performance_Stats_Data_t *src_a, + Performance_Stats_Data_t *src_b) +{ + dest->num = src_a->num + src_b->num; + dest->total_ts = src_a->total_ts + src_b->total_ts; + dest->max_ts = DT_max (src_a->max_ts, src_b->max_ts); + dest->min_ts = DT_min (src_a->min_ts, src_b->min_ts); +} + + +void +DT_performance_stats_combine ( + Performance_Stats_t *dest, + Performance_Stats_t *src_a, + Performance_Stats_t *src_b) +{ + dest->num_ops = + src_a->num_ops + src_b->num_ops; + + dest->bytes = + src_a->bytes + src_b->bytes; + + dest->post_ctxt_switch_num = + src_a->post_ctxt_switch_num + src_b->post_ctxt_switch_num; + + dest->reap_ctxt_switch_num = + src_b->reap_ctxt_switch_num + src_b->reap_ctxt_switch_num; + + dest->cpu_utilization = DT_max (src_a->cpu_utilization, + src_b->cpu_utilization); + dest->time_ts = DT_max (src_a->time_ts, src_b->time_ts); + + DT_performance_stats_data_combine (&dest->posts_sans_ctxt, + &src_a->posts_sans_ctxt, + &src_b->posts_sans_ctxt); + + DT_performance_stats_data_combine (&dest->posts_with_ctxt, + &src_a->posts_with_ctxt, + &src_b->posts_with_ctxt); + + DT_performance_stats_data_combine (&dest->reaps_sans_ctxt, + &src_a->reaps_sans_ctxt, + &src_b->reaps_sans_ctxt); + + DT_performance_stats_data_combine (&dest->reaps_with_ctxt, + &src_a->reaps_with_ctxt, + &src_b->reaps_with_ctxt); + + DT_performance_stats_data_combine (&dest->latency, + &src_a->latency, + &src_b->latency); +} + + +double +DT_performance_stats_data_print ( + DT_Tdep_Print_Head *phead, + Performance_Stats_Data_t *data, + double cpu_mhz) +{ + double average; + + average = (int64_t)data->total_ts / (data->num * cpu_mhz); + + DT_Tdep_PT_Printf (phead, + " Arithmetic mean : %d.%d us\n" + " maximum : %d.%d us\n" + " minimum : %d.%d us\n", + DT_whole(average),DT_hundredths(average), + DT_whole((int64_t)data->max_ts / cpu_mhz),DT_hundredths((int64_t)data->max_ts / cpu_mhz), + DT_whole((int64_t)data->min_ts / cpu_mhz),DT_hundredths((int64_t)data->min_ts / cpu_mhz)); + return average; +} + + +void +DT_performance_stats_print ( + Params_t *params_ptr, + DT_Tdep_Print_Head *phead, + Performance_Stats_t *stats, + Performance_Cmd_t *cmd, + Performance_Test_t *test) +{ + double cpu_mhz; + double time_s; + double mbytes; + double ops_per_sec; + double bandwidth; + double latency; + double time_per_post; + double time_per_reap; + + cpu_mhz = params_ptr->cpu_mhz; + latency = 0; +#if defined(WIN32) + /* + * The Microsoft compiler is unable to do a 64 bit conversion when + * working with double. time_ts is a 64 bit value, so we + * potentially lose precision, so limit it to the Windows + * platform. Trying to do the operation below without casting + * a 64 bit value to an unsigned int results in the error when + * using Visual C 6.0: + * + * Compiler Error C2520: conversion from unsigned __int64 to + * double not implemented, use signed __int64. + * + * Note that signed __int64 doesn't work either! + */ + time_s = (double) ((unsigned int)stats->time_ts / (1000000.0 * cpu_mhz)); +#else + time_s = (double) (stats->time_ts / (1000000.0 * cpu_mhz)); +#endif + mbytes = (double) (1.0 * stats->bytes) / 1024 / 1024; + + if ( 0.0 == time_s ) + { + DT_Tdep_PT_Printf (phead, "Error determining time\n"); + return; + } + else if ( 0 == stats->num_ops ) + { + DT_Tdep_PT_Printf (phead, "Error determining number of operations\n"); + return; + } + else if ( 0.0 == cpu_mhz ) + { + DT_Tdep_PT_Printf (phead, "Error determining CPU speed\n"); + return; + } + + ops_per_sec = stats->num_ops / time_s; + bandwidth = mbytes / time_s; + DT_Tdep_PT_Printf (phead, "------------------------- Statistics -------------------------\n"); + DT_Tdep_PT_Printf (phead, + " Mode : %s\n" + " Operation Type : %s\n" + " Number of Operations : %u\n" + " Segment Size : %u\n" + " Number of Segments : %u bytes\n" + " Pipeline Length : %u\n\n", + DT_PerformanceModeToString (cmd->mode), + DT_TransferTypeToString (cmd->op.transfer_type), + cmd->num_iterations, + cmd->op.seg_size, + cmd->op.num_segs, + test->ep_context.pipeline_len); + + DT_Tdep_PT_Printf (phead, + " Total Time : %d.%d sec\n" + " Total Data Exchanged : %d.%d MB\n" + " CPU Utilization : %d.%d\n" + " Operation Throughput : %d.%d ops/sec\n" + " Bandwidth : %d.%d MB/sec\n", + DT_whole(time_s),DT_hundredths(time_s), + DT_whole(mbytes),DT_hundredths(mbytes), + DT_whole(stats->cpu_utilization),DT_hundredths(stats->cpu_utilization), + DT_whole(ops_per_sec),DT_hundredths(ops_per_sec), + DT_whole(bandwidth),DT_hundredths(bandwidth)); + + DT_Tdep_PT_Printf (phead, "\nLatency\n"); + + if ( stats->latency.num ) + { + latency = DT_performance_stats_data_print (phead, + &stats->latency, + cpu_mhz); + } + DT_Tdep_PT_Printf (phead, "\n" + "Time Per Post\n" + " %u posts without context switches\n", + stats->posts_sans_ctxt.num); + + if ( stats->posts_sans_ctxt.num ) + { + DT_performance_stats_data_print (phead, + &stats->posts_sans_ctxt, + cpu_mhz); + } + + DT_Tdep_PT_Printf (phead, "\n" + " %u posts with context switches\n", + stats->posts_with_ctxt.num); + + if ( stats->posts_with_ctxt.num ) + { + DT_Tdep_PT_Printf (phead, " %u number of context switches\n", + stats->post_ctxt_switch_num); + DT_performance_stats_data_print (phead, + &stats->posts_with_ctxt, + cpu_mhz); + } + + DT_Tdep_PT_Printf (phead, "\n" + "Time Per Reap\n" + " %u reaps without context switches\n", + stats->reaps_sans_ctxt.num); + + if ( stats->reaps_sans_ctxt.num ) + { + DT_performance_stats_data_print (phead, + &stats->reaps_sans_ctxt, + cpu_mhz); + } + + + DT_Tdep_PT_Printf (phead, "\n" + " %u reaps with context switches\n", + stats->reaps_with_ctxt.num); + + if ( stats->reaps_with_ctxt.num ) + { + DT_Tdep_PT_Printf (phead, "\n" + " %u number of context switches\n", + stats->reap_ctxt_switch_num); + + DT_performance_stats_data_print (phead, + &stats->reaps_with_ctxt, + cpu_mhz); + } + + time_per_post = + (int64_t) (stats->posts_sans_ctxt.total_ts + stats->posts_with_ctxt.total_ts) / + (cpu_mhz * (stats->posts_sans_ctxt.num + stats->posts_with_ctxt.num)); + + time_per_reap = + (int64_t) (stats->reaps_sans_ctxt.total_ts + stats->reaps_with_ctxt.total_ts) / + (cpu_mhz * (stats->reaps_sans_ctxt.num + stats->reaps_with_ctxt.num)); + + DT_Tdep_PT_Printf (phead, + "\nNOTE: 1 MB = 1024 KB = 1048576 B \n"); + DT_Tdep_PT_Printf (phead, + "---------------------------------------------------------------------\n"); + DT_Tdep_PT_Printf (phead, + "raw: %s, %u, %u, %u, %u, %d.%d, %d.%d, %d.%d, %d.%d, %d.%d, %d.%d \n", + DT_TransferTypeToString (cmd->op.transfer_type), + cmd->num_iterations, + cmd->op.seg_size, + cmd->op.num_segs, + test->ep_context.pipeline_len, + DT_whole (stats->cpu_utilization), + DT_hundredths (stats->cpu_utilization), + DT_whole (ops_per_sec), + DT_hundredths (ops_per_sec), + DT_whole (bandwidth), + DT_hundredths (bandwidth), + DT_whole (latency), + DT_hundredths (latency), + DT_whole (time_per_post), + DT_hundredths (time_per_post), + DT_whole (time_per_reap), + DT_hundredths (time_per_reap)); + DT_Tdep_PT_Printf (phead, + "---------------------------------------------------------------------\n"); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_util.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_util.c new file mode 100644 index 00000000..05eca701 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_performance_util.c @@ -0,0 +1,678 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define DT_Mdep_GetContextSwitchNum() 0 /* FIXME */ + +/****************************************************************************/ +bool +DT_Performance_Test_Create ( + Per_Test_Data_t *pt_ptr, + DAT_IA_HANDLE *ia_handle, + DAT_IA_ADDRESS_PTR remote_ia_addr, + DAT_BOOLEAN is_server, + DAT_BOOLEAN is_remote_little_endian, + Performance_Test_t **perf_test) +{ + Performance_Test_t *test_ptr; + DAT_COUNT pipeline_len; + DAT_RETURN ret; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + test_ptr = DT_MemListAlloc (pt_ptr, + "transaction_test_t", + TRANSACTIONTEST, + sizeof (Performance_Test_t)); + if ( NULL == test_ptr ) + { + return false; + } + + *perf_test = test_ptr; + + test_ptr->pt_ptr = pt_ptr; + test_ptr->remote_ia_addr = remote_ia_addr; + test_ptr->is_remote_little_endian = is_remote_little_endian; + test_ptr->base_port = (DAT_CONN_QUAL) pt_ptr->Server_Info.first_port_number; + test_ptr->ia_handle = ia_handle; + test_ptr->cmd = &pt_ptr->Params.u.Performance_Cmd; + + ret = dat_ia_query (test_ptr->ia_handle, + NULL, + DAT_IA_ALL, + &test_ptr->ia_attr, + 0, + NULL); + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ia_query error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + return false; + } + + pipeline_len = DT_min ( + DT_min (test_ptr->cmd->num_iterations, + test_ptr->cmd->pipeline_len), + DT_min (test_ptr->ia_attr.max_dto_per_ep, + test_ptr->ia_attr.max_evd_qlen)); + + if ( RDMA_READ == test_ptr->cmd->op.transfer_type ) + { + pipeline_len = DT_min (pipeline_len, + test_ptr->ia_attr.max_rdma_read_per_ep); + } + + test_ptr->reqt_evd_length = pipeline_len; + test_ptr->recv_evd_length = DT_PERF_DFLT_EVD_LENGTH; + test_ptr->conn_evd_length = DT_PERF_DFLT_EVD_LENGTH; + test_ptr->creq_evd_length = DT_PERF_DFLT_EVD_LENGTH; + + /* create a protection zone */ + ret = dat_pz_create (test_ptr->ia_handle, &test_ptr->pz_handle); + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_pz_create error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->pz_handle = DAT_HANDLE_NULL; + return false; + } + + /* create 4 EVDs - recv, request+RMR, conn-request, connect */ + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->recv_evd_length, + test_ptr->cno_handle, + DAT_EVD_DTO_FLAG, + &test_ptr->recv_evd_hdl); /* recv */ + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (recv) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->recv_evd_hdl = DAT_HANDLE_NULL; + return false; + } + + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->reqt_evd_length, + test_ptr->cno_handle, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + &test_ptr->reqt_evd_hdl); /* request + rmr bind */ + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (request) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->reqt_evd_hdl = DAT_HANDLE_NULL; + return false; + } + + + if ( is_server ) + { + /* Client-side doesn't need CR events */ + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->creq_evd_length, + DAT_HANDLE_NULL, + DAT_EVD_CR_FLAG, + &test_ptr->creq_evd_hdl); /* cr */ + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (cr) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->creq_evd_hdl = DAT_HANDLE_NULL; + return false; + } + } + + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->conn_evd_length, + DAT_HANDLE_NULL, + DAT_EVD_CONNECTION_FLAG, + &test_ptr->conn_evd_hdl); /* conn */ + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (conn) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->conn_evd_hdl = DAT_HANDLE_NULL; + return false; + } + + /* + * Set up the EP context: + * create the EP + * allocate buffers for remote memory info and sync message + * post the receive buffers + * connect + * set up buffers and remote memory info + * send across our info + * recv the other side's info and extract what we need + */ + test_ptr->ep_context.ep_attr = test_ptr->pt_ptr->ep_attr; + test_ptr->ep_context.ep_attr.max_request_dtos = pipeline_len; + + /* Create EP */ + ret = dat_ep_create (test_ptr->ia_handle, /* IA */ + test_ptr->pz_handle, /* PZ */ + test_ptr->recv_evd_hdl, /* recv */ + test_ptr->reqt_evd_hdl, /* request */ + test_ptr->conn_evd_hdl, /* connect */ + &test_ptr->ep_context.ep_attr, /* EP attrs */ + &test_ptr->ep_context.ep_handle); + if ( DAT_SUCCESS != ret) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ep_create error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->ep_context.ep_handle = DAT_HANDLE_NULL; + return false; + } + + /* + * Allocate a buffer pool so we can exchange the + * remote memory info and initialize. + */ + test_ptr->ep_context.bp = DT_BpoolAlloc (test_ptr->pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context.ep_handle, + DAT_HANDLE_NULL, /* rmr */ + DT_PERF_SYNC_BUFF_SIZE, + 2, /* 2 RMIs */ + DAT_OPTIMAL_ALIGNMENT, + false, + false); + if ( !test_ptr->ep_context.bp ) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for remote memory buffers\n", + test_ptr->base_port); + return false; + } + + DT_Tdep_PT_Debug (3, (phead, + "0: SYNC_SEND %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (test_ptr->ep_context.bp, + DT_PERF_SYNC_SEND_BUFFER_ID))); + DT_Tdep_PT_Debug (3, (phead, + "1: SYNC_RECV %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (test_ptr->ep_context.bp, + DT_PERF_SYNC_RECV_BUFFER_ID))); + + /* + * Post recv and sync buffers + */ + if ( !DT_post_recv_buffer (phead, + test_ptr->ep_context.ep_handle, + test_ptr->ep_context.bp, + DT_PERF_SYNC_RECV_BUFFER_ID, + DT_PERF_SYNC_BUFF_SIZE) ) + { + /* error message printed by DT_post_recv_buffer */ + return false; + } + + /* + * Fill in the test_ptr with relevant command info + */ + test_ptr->ep_context.op.transfer_type + = test_ptr->cmd->op.transfer_type; + test_ptr->ep_context.op.num_segs + = test_ptr->cmd->op.num_segs; + test_ptr->ep_context.op.seg_size + = test_ptr->cmd->op.seg_size; + + /* + * Exchange remote memory info: If we're going to participate + * in an RDMA, we need to allocate memory buffers and advertise + * them to the other side. + */ + test_ptr->ep_context.op.Rdma_Context = (DAT_RMR_CONTEXT) 0; + test_ptr->ep_context.op.Rdma_Address = 0; + test_ptr->ep_context.port = test_ptr->base_port; + test_ptr->ep_context.pipeline_len = pipeline_len; + + return true; +} + + +/****************************************************************************/ +void +DT_Performance_Test_Destroy ( + Per_Test_Data_t *pt_ptr, + Performance_Test_t *test_ptr, + DAT_BOOLEAN is_server) +{ + DAT_RETURN ret; + DAT_EP_HANDLE ep_handle; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + ep_handle = DAT_HANDLE_NULL; + + /* Free the per-op buffers */ + if (test_ptr->ep_context.op.bp) + { + if (!DT_Bpool_Destroy (test_ptr->pt_ptr, + phead, + test_ptr->ep_context.op.bp)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Warning: Bpool destroy fails\n", + test_ptr->base_port); + /* carry on trying, regardless */ + } + } + + /* Free the remote memory info exchange buffers */ + if (test_ptr->ep_context.bp) + { + if (!DT_Bpool_Destroy (test_ptr->pt_ptr, + phead, + test_ptr->ep_context.bp)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Warning: Bpool destroy fails\n", + test_ptr->base_port); + /* carry on trying, regardless */ + } + } + + /* + * Disconnect -- we may have left recv buffers posted, if we + * bailed out mid-setup, or ran to completion + * normally, so we use abrupt closure. + */ + if (test_ptr->ep_context.ep_handle) + { + ret = dat_ep_disconnect (test_ptr->ep_context.ep_handle, + DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Warning: dat_ep_disconnect error %s\n", + test_ptr->base_port, + DT_RetToString (ret)); + /* carry on trying, regardless */ + } + else if (!DT_disco_event_wait ( phead, test_ptr->conn_evd_hdl, + &ep_handle)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: bad disconnect event\n", + test_ptr->base_port); + } + } + + if ( DAT_HANDLE_NULL != ep_handle) + { + /* Destroy the EP */ + ret = dat_ep_free (ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ep_free error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* carry on trying, regardless */ + } + } + + /* clean up the EVDs */ + if (test_ptr->conn_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->conn_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (conn) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + if (is_server) + { + if (test_ptr->creq_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->creq_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (creq) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + } + if (test_ptr->reqt_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->reqt_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (reqt) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + if (test_ptr->recv_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->recv_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (recv) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + + /* clean up the PZ */ + if (test_ptr->pz_handle) + { + ret = dat_pz_free (test_ptr->pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_pz_free error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + + DT_MemListFree (test_ptr->pt_ptr, test_ptr); + DT_Tdep_PT_Debug (1, (phead,"Test[" F64x "]: cleanup is done\n", test_ptr->base_port)); + +} + + +/****************************************************************************/ +bool +DT_performance_post_rdma_op ( + Performance_Ep_Context_t *ep_context, + DAT_EVD_HANDLE reqt_evd_hdl, + Performance_Stats_t *stats) +{ + unsigned int j; + unsigned long int bytes; + unsigned long pre_ctxt_num; + unsigned long post_ctxt_num; + DT_Mdep_TimeStamp pre_ts; + DT_Mdep_TimeStamp post_ts; + DAT_DTO_COOKIE cookie; + DAT_RETURN ret; + Performance_Test_Op_t *op = &ep_context->op; + DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV (op->bp, 0); + DAT_RMR_TRIPLET rmr_triplet; + + bytes = op->seg_size * op->num_segs; + + /* Prep the inputs */ + for (j = 0; j < op->num_segs; j++) + { + iov[j].virtual_address = (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (op->bp, j); + iov[j].segment_length = op->seg_size; + iov[j].lmr_context = DT_Bpool_GetLMR (op->bp, j); + } + + rmr_triplet.virtual_address = op->Rdma_Address; + rmr_triplet.segment_length = op->seg_size * op->num_segs; + rmr_triplet.rmr_context = op->Rdma_Context; + + cookie.as_ptr = NULL; + + if ( RDMA_WRITE == op->transfer_type ) + { + pre_ctxt_num = DT_Mdep_GetContextSwitchNum (); + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = dat_ep_post_rdma_write (ep_context->ep_handle, + op->num_segs, + iov, + cookie, + &rmr_triplet, + DAT_COMPLETION_DEFAULT_FLAG); + + post_ts = DT_Mdep_GetTimeStamp (); + post_ctxt_num = DT_Mdep_GetContextSwitchNum (); + + stats->bytes += bytes; + } + else + { + pre_ctxt_num = DT_Mdep_GetContextSwitchNum (); + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = dat_ep_post_rdma_read (ep_context->ep_handle, + op->num_segs, + iov, + cookie, + &rmr_triplet, + DAT_COMPLETION_DEFAULT_FLAG); + + post_ts = DT_Mdep_GetTimeStamp (); + post_ctxt_num = DT_Mdep_GetContextSwitchNum (); + + stats->bytes += bytes; + } + + if ( DAT_SUCCESS != ret ) + { + return false; + } + + DT_performance_stats_record_post (stats, + post_ctxt_num - pre_ctxt_num, + post_ts - pre_ts); + + return true; +} + +/****************************************************************************/ +unsigned int +DT_performance_reap ( + DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + Performance_Mode_Type mode, + Performance_Stats_t *stats) +{ + if ( BLOCKING_MODE == mode ) + { + return DT_performance_wait (phead, evd_handle, stats); + } + else + { + return DT_performance_poll (phead, evd_handle, stats); + } +} + +/****************************************************************************/ +unsigned int +DT_performance_wait ( + DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + Performance_Stats_t *stats) +{ + DAT_COUNT i; + DAT_COUNT queue_size; + DAT_RETURN ret; + DAT_EVENT event; + unsigned long pre_ctxt_num; + unsigned long post_ctxt_num; + DT_Mdep_TimeStamp pre_ts; + DT_Mdep_TimeStamp post_ts; + + queue_size = 0; + + pre_ctxt_num = DT_Mdep_GetContextSwitchNum (); + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = DT_Tdep_evd_wait ( evd_handle, + DAT_TIMEOUT_INFINITE, + &event); + + post_ts = DT_Mdep_GetTimeStamp (); + post_ctxt_num = DT_Mdep_GetContextSwitchNum (); + + if ( DAT_SUCCESS != ret ) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_dequeue failed: %s\n", + DT_RetToString (ret)); + return 0; + } + else if (event.event_number == DAT_DTO_COMPLETION_EVENT) + { + DT_performance_stats_record_reap (stats, + post_ctxt_num - pre_ctxt_num, + post_ts - pre_ts); + } + else + { + /* This should not happen. There has been an error if it does. */ + + DT_Tdep_PT_Printf (phead, "Warning: dapl_performance_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + + return 0; + } + + for ( i = 0; i < queue_size; i++ ) + { + pre_ctxt_num = DT_Mdep_GetContextSwitchNum (); + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = DT_Tdep_evd_dequeue ( evd_handle, + &event); + + post_ts = DT_Mdep_GetTimeStamp (); + post_ctxt_num = DT_Mdep_GetContextSwitchNum (); + + if (DAT_GET_TYPE (ret) == DAT_QUEUE_EMPTY) + { + continue; + } + else if ( DAT_SUCCESS != ret ) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_dequeue failed: %s\n", + DT_RetToString (ret)); + return 0; + } + else if (event.event_number == DAT_DTO_COMPLETION_EVENT) + { + DT_performance_stats_record_reap (stats, + post_ctxt_num - pre_ctxt_num, + post_ts - pre_ts); + } + else + { + /* This should not happen. There has been an error if it does. */ + + DT_Tdep_PT_Printf (phead, "Warning: dapl_performance_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + + return 0; + } + } + + return ++queue_size; +} + +/****************************************************************************/ +unsigned int +DT_performance_poll ( + DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + Performance_Stats_t *stats) +{ + DAT_RETURN ret; + DAT_EVENT event; + unsigned long pre_ctxt_num; + unsigned long post_ctxt_num; + DT_Mdep_TimeStamp pre_ts; + DT_Mdep_TimeStamp post_ts; + + for (;;) + { + pre_ctxt_num = DT_Mdep_GetContextSwitchNum (); + pre_ts = DT_Mdep_GetTimeStamp (); + + ret = DT_Tdep_evd_dequeue ( evd_handle, + &event); + + post_ts = DT_Mdep_GetTimeStamp (); + post_ctxt_num = DT_Mdep_GetContextSwitchNum (); + + if (DAT_GET_TYPE (ret) == DAT_QUEUE_EMPTY) + { + continue; + } + else if ( DAT_SUCCESS != ret ) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_dequeue failed: %s\n", + DT_RetToString (ret)); + return 0; + } + else if (event.event_number == DAT_DTO_COMPLETION_EVENT) + { + DT_performance_stats_record_reap (stats, + post_ctxt_num - pre_ctxt_num, + post_ts - pre_ts); + return 1; + } + else + { + /* This should not happen. There has been an error if it does. */ + + DT_Tdep_PT_Printf (phead, "Warning: dapl_performance_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + + return 0; + } + } + + /*never reached */ + return 0; +} + +void +DT_Performance_Cmd_PT_Print ( DT_Tdep_Print_Head *phead, + Performance_Cmd_t *cmd) +{ + DT_Tdep_PT_Printf (phead, "-------------------------------------\n"); + DT_Tdep_PT_Printf (phead, "PerfCmd.server_name : %s\n", + cmd->server_name); + DT_Tdep_PT_Printf (phead, "PerfCmd.dapl_name : %s\n", + cmd->dapl_name); + DT_Tdep_PT_Printf (phead, "PerfCmd.mode : %s\n", + (cmd->mode == BLOCKING_MODE) ? "BLOCKING" : "POLLING"); + DT_Tdep_PT_Printf (phead, "PerfCmd.num_iterations : %d\n", + cmd->num_iterations); + DT_Tdep_PT_Printf (phead, "PerfCmd.pipeline_len : %d\n", + cmd->pipeline_len); + DT_Tdep_PT_Printf (phead, "PerfCmd.op.transfer_type : %s\n", + cmd->op.transfer_type == RDMA_READ ? "RDMA_READ" : + cmd->op.transfer_type == RDMA_WRITE ? "RDMA_WRITE" : + "SEND_RECV"); + DT_Tdep_PT_Printf (phead, "PerfCmd.op.num_segs : %d\n", + cmd->op.num_segs); + DT_Tdep_PT_Printf (phead, "PerfCmd.op.seg_size : %d\n", + cmd->op.seg_size); +} + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_quit_util.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_quit_util.c new file mode 100644 index 00000000..57c14859 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_quit_util.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_Quit_Cmd_PT_Print (DT_Tdep_Print_Head *phead, Quit_Cmd_t * cmd) +{ + /* DT_Mdep_printf ("Quit_Cmd.server_name: %s\n", cmd->server_name); + DT_Mdep_printf ("Quit_Cmd.device_name: %s\n", cmd->device_name); + */ + DT_Tdep_PT_Printf (phead,"Quit_Cmd.server_name: %s\n", cmd->server_name); + DT_Tdep_PT_Printf (phead,"Quit_Cmd.device_name: %s\n", cmd->device_name); + +} + + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_server.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_server.c new file mode 100644 index 00000000..dd579a4c --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_server.c @@ -0,0 +1,885 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#ifdef DFLT_QLEN +#undef DFLT_QLEN +#endif + +#define DFLT_QLEN 8 /* default event queue length */ + +int send_control_data (DT_Tdep_Print_Head *phead, + unsigned char *buffp, + Per_Server_Data_t *ps_ptr, + Per_Test_Data_t *pt_ptr); + +void +DT_cs_Server (Params_t * params_ptr) +{ + Server_Cmd_t *Server_Cmd = ¶ms_ptr->u.Server_Cmd; + Client_Info_t *Client_Info = NULL; + Transaction_Cmd_t *Transaction_Cmd= NULL; + Quit_Cmd_t *Quit_Cmd = NULL; + Performance_Cmd_t *Performance_Cmd= NULL; + Per_Server_Data_t *ps_ptr = NULL; + Per_Test_Data_t *pt_ptr = NULL; + Started_server_t *temp_list = NULL; + Started_server_t *pre_list = NULL; + unsigned char *buffp = NULL; + char *module = "DT_cs_Server"; + + DAT_DTO_COOKIE dto_cookie; + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_RETURN ret; + DT_Tdep_Print_Head *phead; + + phead = params_ptr->phead; + + /* Check if device from command line already in use */ + temp_list = DT_started_server_list; + while (temp_list) + { + if (strcmp (temp_list->devicename, Server_Cmd->dapl_name) == 0) + { + DT_Tdep_PT_Printf (phead, + "NOTICE: server already started for this NIC: %s\n", + Server_Cmd->dapl_name); + return; + } + temp_list = temp_list->next; + } + + /* Alloc memory for server list */ + temp_list = (Started_server_t *) DT_Mdep_Malloc (sizeof (Started_server_t)); + if (temp_list == 0) + { + DT_Tdep_PT_Printf (phead, "no memory for server_list\n"); + return; + } + strcpy (temp_list->devicename, Server_Cmd->dapl_name); + temp_list->next = DT_started_server_list; + DT_started_server_list = temp_list; + + if (Server_Cmd->debug) + { + /* Echo our inputs if debugging */ + DT_Server_Cmd_PT_Print (phead, Server_Cmd); + } + + /* Allocate memory for Per_Server_Data */ + ps_ptr = (Per_Server_Data_t *) DT_Mdep_Malloc (sizeof (Per_Server_Data_t)); + if (ps_ptr == 0) + { + DT_Tdep_PT_Printf (phead, "no memory for ps_data\n"); + goto server_exit; + } + DT_Mdep_LockInit (&ps_ptr->num_clients_lock); + ps_ptr->NextPortNumber = SERVER_PORT_NUMBER + 1; + ps_ptr->num_clients = 0; + + /* Open the IA */ + ret = dat_ia_open (Server_Cmd->dapl_name, + DFLT_QLEN, + &ps_ptr->async_evd_hdl, + &ps_ptr->ia_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: Could not open %s (%s)\n", + module, + Server_Cmd->dapl_name, + DT_RetToString (ret)); + ps_ptr->ia_handle = DAT_HANDLE_NULL; + goto server_exit; + } + DT_Tdep_PT_Debug (1,(phead,"%s: IA %s opened\n", module, Server_Cmd->dapl_name)); + + /* Create a PZ */ + ret = dat_pz_create (ps_ptr->ia_handle, &ps_ptr->pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_pz_create error: %s\n", + module, + DT_RetToString (ret)); + ps_ptr->pz_handle = DAT_HANDLE_NULL; + goto server_exit; + } + DT_Tdep_PT_Debug (1,(phead,"%s: PZ created\n", module)); + + /* Create 4 events - recv, request, connection-request, connect */ + ret = DT_Tdep_evd_create (ps_ptr->ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_DTO_FLAG, + &ps_ptr->recv_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (recv) failed %s\n", + module, DT_RetToString (ret)); + ps_ptr->recv_evd_hdl = DAT_HANDLE_NULL; + goto server_exit; + } + ret = DT_Tdep_evd_create (ps_ptr->ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + &ps_ptr->reqt_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (send) failed %s\n", + module, DT_RetToString (ret)); + ps_ptr->reqt_evd_hdl = DAT_HANDLE_NULL; + goto server_exit; + } + ret = DT_Tdep_evd_create (ps_ptr->ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_CR_FLAG, + &ps_ptr->creq_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (cr) failed %s\n", + module, DT_RetToString (ret)); + ps_ptr->creq_evd_hdl = DAT_HANDLE_NULL; + goto server_exit; + } + ret = DT_Tdep_evd_create (ps_ptr->ia_handle, + DFLT_QLEN, + NULL, + DAT_EVD_CONNECTION_FLAG, + &ps_ptr->conn_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_create (conn) failed %s\n", + module, DT_RetToString (ret)); + ps_ptr->conn_evd_hdl = DAT_HANDLE_NULL; + goto server_exit; + } + + /* Create the EP */ + ret = dat_ep_create (ps_ptr->ia_handle, /* IA */ + ps_ptr->pz_handle, /* PZ */ + ps_ptr->recv_evd_hdl, /* recv */ + ps_ptr->reqt_evd_hdl, /* request */ + ps_ptr->conn_evd_hdl, /* connect */ + (DAT_EP_ATTR *) NULL, + &ps_ptr->ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ep_create error: %s\n", + module, + DT_RetToString (ret)); + ps_ptr->ep_handle = DAT_HANDLE_NULL; + goto server_exit; + } + DT_Tdep_PT_Debug (1,(phead,"%s: EP created\n", module)); + + /* Create PSP */ + ret = dat_psp_create (ps_ptr->ia_handle, + SERVER_PORT_NUMBER, + ps_ptr->creq_evd_hdl, + DAT_PSP_CONSUMER_FLAG, + &ps_ptr->psp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_psp_create error: %s\n", + module, + DT_RetToString (ret)); + ps_ptr->psp_handle = DAT_HANDLE_NULL; + goto server_exit; + } + DT_Tdep_PT_Debug (1,(phead,"%s: PSP created\n", module)); + + /* + * Create two buffers, large enough to hold ClientInfo and the largest + * command we'll use. + */ + ps_ptr->bpool = DT_BpoolAlloc (NULL, + phead, + ps_ptr->ia_handle, + ps_ptr->pz_handle, + ps_ptr->ep_handle, + DAT_HANDLE_NULL, /* no RMR */ + DT_RoundSize (sizeof (Transaction_Cmd_t), 8192), + 3, /* num_buffers */ + DAT_OPTIMAL_ALIGNMENT, + false, + false); + if (ps_ptr->bpool == 0) + { + DT_Tdep_PT_Printf (phead, + "%s: no memory for command buffer pool.\n", + module); + goto server_exit; + } + + DT_Tdep_PT_Debug (3, (phead, + "Recv 0 %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (ps_ptr->bpool, 0))); + DT_Tdep_PT_Debug (3, (phead, + "Recv 1 %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (ps_ptr->bpool, 1))); + DT_Tdep_PT_Debug (3, (phead, + "SrvInfo 2 %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer (ps_ptr->bpool, 2))); + + /* Initialize the performance test lock in case an incoming test + * is a performance test, so we can allow only one at a time. + * Otherwise, multiple performance tests cause a race condition + * between the server creating a new thread trying to allocate a + * PSP with the same ID as another thread that is either running + * a test on that same ID or hasn't yet destroyed it. Only one + * PSP with a particular ID can exist at a time. It's a + * de-facto shared resource that must be protected. + */ + /************************************************************************ + * Loop accepting connections and acting on them + */ + for (;/* EVER */;) + { + DAT_CR_HANDLE cr_handle; + DAT_CR_ARRIVAL_EVENT_DATA cr_stat; + DAT_EVENT_NUMBER event_num; + + /* Set up the Per_Test_Data */ + pt_ptr = DT_Alloc_Per_Test_Data (phead); + if (!pt_ptr) + { + DT_Tdep_PT_Printf (phead, + "%s: no memory for Per_Test_Data\n", + module); + goto server_exit; + } + DT_MemListInit (pt_ptr); + DT_Thread_Init (pt_ptr); + pt_ptr->local_is_server = true; + pt_ptr->ps_ptr = ps_ptr; + memcpy ( (void *) (uintptr_t) &pt_ptr->Params, + (const void *) params_ptr, + sizeof (Params_t)); + + /* Server_Info, Client_Info, Params set up below */ + + /* Gather whatever info we want about defaults */ + if (!DT_query (pt_ptr, ps_ptr->ia_handle, ps_ptr->ep_handle)) + { + goto server_exit; + } + + /* Post recv buffers for ClientInfo and Transaction_Cmd_t */ + DT_Tdep_PT_Debug (1,(phead,"%s: Posting 2 recvs\n", module)); + if (!DT_post_recv_buffer (phead, + ps_ptr->ep_handle, + ps_ptr->bpool, + 0, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 0))) + { + DT_Tdep_PT_Printf (phead, + "%s: cannot post ClientInfo recv buffer\n", + module); + goto server_exit; + } + if (!DT_post_recv_buffer (phead, + ps_ptr->ep_handle, + ps_ptr->bpool, + 1, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 1))) + { + DT_Tdep_PT_Printf (phead, + "%s: cannot post Transaction_Cmd_t recv buffer\n", + module); + goto server_exit; + } + + /* message to help automated test scripts know when to start the client */ + DT_Tdep_PT_Printf (phead, + "Dapltest: Service Point Ready - %s\n", + Server_Cmd->dapl_name); + + DT_Mdep_flush (); + + DT_Tdep_PT_Debug (1,(phead,"%s: Waiting for Connection Request\n", module)); + if (!DT_cr_event_wait (phead, ps_ptr->creq_evd_hdl, &cr_stat) || + !DT_cr_check (phead, + &cr_stat, + ps_ptr->psp_handle, + SERVER_PORT_NUMBER, + &cr_handle, + module)) + { + + DT_Tdep_PT_Printf (phead, + "CR Check failed, file %s line %d\n", __FILE__, + __LINE__); + goto server_exit; + } + + DT_Tdep_PT_Debug (1,(phead,"%s: Accepting Connection Request\n", module)); + ret = dat_cr_accept (cr_handle, ps_ptr->ep_handle, 0, (DAT_PVOID)0); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_cr_accept error: %s\n", + module, + DT_RetToString (ret)); + goto server_exit; + } + + DT_Tdep_PT_Debug (1,(phead,"%s: Awaiting connection ...\n", module)); + if (!DT_conn_event_wait (phead, + ps_ptr->ep_handle, + ps_ptr->conn_evd_hdl, + &event_num)) + { + DT_Tdep_PT_Printf (phead, + "%s: error awaiting conn-established event\n", + module); + goto server_exit; + } + + if (DT_dapltest_debug) + { + DT_Tdep_PT_Debug (1,(phead, "%s: Connected!\n", module)); + get_ep_connection_state (phead, ps_ptr->ep_handle); + } + + /* Wait for Client_Info */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( ps_ptr->bpool, 0); + DT_Tdep_PT_Debug (1,(phead,"%s: Waiting for Client_Info\n", module)); + if (!DT_dto_event_wait (phead, ps_ptr->recv_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + ps_ptr->ep_handle, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 0), + dto_cookie, + "Client_Info_Recv")) + { + goto server_exit; + } + DT_Tdep_PT_Debug (1,(phead,"%s: Got Client_Info\n", module)); + + /* Extract the Client_Info */ + Client_Info = (Client_Info_t*) DT_Bpool_GetBuffer (ps_ptr->bpool, 0); + DT_Client_Info_Endian (Client_Info); + memcpy ( (void *) (uintptr_t)&pt_ptr->Client_Info, + (const void *)Client_Info, + sizeof (Client_Info_t)); + + /* Wait for client's command info */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( ps_ptr->bpool, 1); + DT_Tdep_PT_Debug (1,(phead,"%s: Waiting for Client_Cmd_Info\n", module)); + if (!DT_dto_event_wait (phead, ps_ptr->recv_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + ps_ptr->ep_handle, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 1), + dto_cookie, + "Client_Cmd_Recv")) + { + goto server_exit; + } + + /* Extract the client's command info */ + switch (Client_Info->test_type) + { + case TRANSACTION_TEST: + { + Transaction_Cmd = (Transaction_Cmd_t *) + DT_Bpool_GetBuffer (ps_ptr->bpool, 1); + DT_Transaction_Cmd_Endian (Transaction_Cmd, false); + memcpy ( (void *) (uintptr_t)&pt_ptr->Params.u.Transaction_Cmd, + (const void *)Transaction_Cmd, + sizeof (Transaction_Cmd_t)); + break; + } + + case QUIT_TEST: + { + Quit_Cmd = (Quit_Cmd_t*)DT_Bpool_GetBuffer (ps_ptr->bpool, 1); + DT_Quit_Cmd_Endian (Quit_Cmd, false); + memcpy ( (void *) (uintptr_t)&pt_ptr->Params.u.Quit_Cmd, + (const void *)Quit_Cmd, + sizeof (Quit_Cmd_t)); + break; + } + + case PERFORMANCE_TEST: + { + Performance_Cmd = (Performance_Cmd_t *) + DT_Bpool_GetBuffer (ps_ptr->bpool, 1); + DT_Performance_Cmd_Endian (Performance_Cmd); + memcpy ( (void *) (uintptr_t)&pt_ptr->Params.u.Performance_Cmd, + (const void *)Performance_Cmd, + sizeof (Performance_Cmd_t)); + break; + } + + default: + { + DT_Tdep_PT_Printf (phead, "Unknown TestType received\n"); + goto server_exit; + break; + } + } + + /* Setup Server Info */ + DT_Tdep_PT_Debug (1,(phead,"%s: Send Server_Info\n", module)); + pt_ptr->Server_Info.dapltest_version = DAPLTEST_VERSION; + pt_ptr->Server_Info.is_little_endian = DT_local_is_little_endian; + pt_ptr->Server_Info.first_port_number = ps_ptr->NextPortNumber; + ps_ptr->NextPortNumber += pt_ptr->Client_Info.total_threads; + + /* This had to be done here because the pt_ptr is being fed to + * the thread as its context, and if it isn't properly + * initialized before the thread spawns then the thread may + * incorrectly set up its PSP and the server will be listening + * on the WRONG PORT! + */ + + switch (Client_Info->test_type) + { + case TRANSACTION_TEST: + { + /* create a thread to handle this pt_ptr; */ + DT_Tdep_PT_Debug (1,(phead,"%s: Creating Transaction Test Thread\n", module)); + pt_ptr->thread = DT_Thread_Create (pt_ptr, + DT_Transaction_Test_Server, + pt_ptr, + DT_MDEP_DEFAULT_STACK_SIZE); + if (pt_ptr->thread == 0) + { + DT_Tdep_PT_Printf (phead, "no memory to create thread\n"); + goto server_exit; + } + break; + } + + case QUIT_TEST: + { + DT_Tdep_PT_Debug (1,(phead,"Client Requests Server to Quit\n")); + (void) send_control_data (phead, buffp, ps_ptr, pt_ptr); + goto server_exit; + break; + } + + case LIMIT_TEST: + { + DT_Tdep_PT_Debug (1,(phead,"Limit Test is Client-side Only!\n")); + (void) send_control_data (phead, buffp, ps_ptr, pt_ptr); + goto server_exit; + break; + } + + case PERFORMANCE_TEST: + { + /* create a thread to handle this pt_ptr; */ + DT_Tdep_PT_Debug (1,(phead,"%s: Creating Performance Test Thread\n", module)); + pt_ptr->thread = DT_Thread_Create (pt_ptr, + DT_Performance_Test_Server, + pt_ptr, + DT_MDEP_DEFAULT_STACK_SIZE); + if (pt_ptr->thread == 0) + { + DT_Tdep_PT_Printf (phead, "no memory to create thread\n"); + goto server_exit; + } + /* take the performance test lock to serialize */ + DT_Mdep_Lock ( &g_PerfTestLock ); + + break; + } + + case FFT_TEST: + + default: + { + DT_Tdep_PT_Printf (phead, "Unknown TestType received\n"); + (void) send_control_data (phead, buffp, ps_ptr, pt_ptr); + goto server_exit; + break; + } + } + + /* Start the new test thread */ + DT_Tdep_PT_Debug (1,(phead,"%s: Starting Test Thread\n", module)); + if (DT_Thread_Start (pt_ptr->thread) == false) + { + DT_Tdep_PT_Debug (1,(phead,"failed to start test thread\n")); + goto server_exit; + } + + buffp = DT_Bpool_GetBuffer (ps_ptr->bpool, 2); /* 3rd buffer */ + memcpy ( (void *)buffp, + (const void *)&pt_ptr->Server_Info, + sizeof (Server_Info_t)); + DT_Server_Info_Endian ((Server_Info_t *) buffp); + + /* Perform obligatory version check */ + if (pt_ptr->Client_Info.dapltest_version != DAPLTEST_VERSION) + { + DT_Tdep_PT_Printf (phead, + "%s: %s: Server %d, Client %d\n", + module, + "DAPLTEST VERSION MISMATCH", + DAPLTEST_VERSION, + pt_ptr->Client_Info.dapltest_version); + goto server_exit; + } + DT_Tdep_PT_Debug (1,(phead,"%s: Version OK!\n", module)); + + DT_Mdep_wait_object_wait ( + &pt_ptr->synch_wait_object, + DAT_TIMEOUT_INFINITE); + + /* Send the Server_Info */ + DT_Tdep_PT_Debug (1,(phead,"%s: Send Server_Info\n", module)); + + if (!DT_post_send_buffer ( phead, ps_ptr->ep_handle, + ps_ptr->bpool, + 2, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 2))) + { + DT_Tdep_PT_Printf (phead, "%s: cannot send Server_Info\n", module); + goto server_exit; + } + /* reap the send and verify it */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( ps_ptr->bpool, 2); + if (!DT_dto_event_wait (phead, ps_ptr->reqt_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + ps_ptr->ep_handle, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 2), + dto_cookie, + "Server_Info_Send")) + { + goto server_exit; + } + + + /* Count this new client and get ready for the next */ + DT_Mdep_Lock (&ps_ptr->num_clients_lock); + ps_ptr->num_clients++; + DT_Mdep_Unlock (&ps_ptr->num_clients_lock); + + /* we passed the pt_ptr to the thread and must now 'forget' it */ + pt_ptr = 0; + +#ifdef CM_BUSTED + DT_Tdep_PT_Debug (1,(phead,"%s: Server exiting because provider does not support\n" + " multiple connections to the same service point\n", + module)); + /* Until connections are healthier we run just one test */ + break; +#else + ret = dat_ep_disconnect (ps_ptr->ep_handle, DAT_CLOSE_GRACEFUL_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_disconnect fails: %s\n", + module, DT_RetToString (ret)); + goto server_exit; + } + if (!DT_disco_event_wait ( phead, ps_ptr->conn_evd_hdl, NULL)) + { + DT_Tdep_PT_Printf (phead, "%s: bad disconnect event\n", module); + goto server_exit; + } + + /* reset the EP to get back into the game */ + dat_ep_reset (ps_ptr->ep_handle); + DT_Tdep_PT_Debug (1,(phead,"%s: Waiting for another client...\n", module)); +#endif + } /* end loop accepting connections */ + + /************************************************************************ + * Finished (or had an error) so clean up and go home + */ +server_exit: + + /* Wait until all of our clients are gone */ + DT_Tdep_PT_Debug (1,(phead,"%s: Waiting for clients to all go away...\n", module)); + while (ps_ptr && ps_ptr->num_clients > 0) + { + DT_Mdep_Sleep (100); + } + + /* Clean up the Per_Test_Data (if any) */ + DT_Tdep_PT_Debug (1,(phead,"%s: Cleaning up ...\n", module)); + if (pt_ptr) + { + DT_Mdep_LockDestroy (&pt_ptr->Thread_counter_lock); + DT_Mdep_LockDestroy (&pt_ptr->MemListLock); + DT_Free_Per_Test_Data (pt_ptr); + } + + /* Clean up the Per_Server_Data */ + if (ps_ptr) + { + /* + * disconnect the most recent EP + * + * we also get here on error, hence abrupt closure to + * flush any lingering buffers posted. + */ + if (ps_ptr->ep_handle) + { + ret = dat_ep_disconnect (ps_ptr->ep_handle, + DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_disconnect fails: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + else if (!DT_disco_event_wait ( phead, + ps_ptr->conn_evd_hdl, + NULL)) + { + DT_Tdep_PT_Printf (phead, + "%s: bad disconnect event\n", + module); + } + } + + /* Destroy the Bpool */ + if (ps_ptr->bpool) + { + if (!DT_Bpool_Destroy (NULL, phead, ps_ptr->bpool)) + { + DT_Tdep_PT_Printf (phead, + "%s: error destroying buffer pool\n", module); + /* keep trying */ + } + } + + /* Free the PSP */ + if (ps_ptr->psp_handle) + { + ret = dat_psp_free (ps_ptr->psp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_psp_free error: %s\n", + module, + DT_RetToString (ret)); + /* keep trying */ + } + } + + /* Free the EP */ + if (ps_ptr->ep_handle) + { + ret = dat_ep_free (ps_ptr->ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_free error: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + } + + /* Free the 4 EVDs */ + if (ps_ptr->conn_evd_hdl) + { + ret = DT_Tdep_evd_free (ps_ptr->conn_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (conn) error: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + } + if (ps_ptr->creq_evd_hdl) + { + ret = DT_Tdep_evd_free (ps_ptr->creq_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (creq) error: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + } + if (ps_ptr->reqt_evd_hdl) + { + ret = DT_Tdep_evd_free (ps_ptr->reqt_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (reqt) error: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + } + if (ps_ptr->recv_evd_hdl) + { + ret = DT_Tdep_evd_free (ps_ptr->recv_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_evd_free (recv) error: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + } + + /* Free the PZ */ + if (ps_ptr->pz_handle) + { + ret = dat_pz_free (ps_ptr->pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_pz_free error: %s\n", + module, DT_RetToString (ret)); + /* keep trying */ + } + } + + /* Close the IA */ + if (ps_ptr->ia_handle) + { + /* dat_ia_close cleans up async evd handle, too */ + ret = dat_ia_close (ps_ptr->ia_handle, DAT_CLOSE_GRACEFUL_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ia_close (graceful) error: %s\n", + module, DT_RetToString (ret)); + ret = dat_ia_close (ps_ptr->ia_handle, DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "%s: dat_ia_close (abrupt) error: %s\n", + module, DT_RetToString (ret)); + } + /* keep trying */ + } + else + { + DT_Tdep_PT_Debug (1,(phead,"%s: IA %s closed\n", + module, + Server_Cmd->dapl_name)); + } + } + + /* Destroy the ps_ptr */ + DT_Mdep_LockDestroy (&ps_ptr->num_clients_lock); + DT_Mdep_Free (ps_ptr); + } /* end if ps_ptr */ + + /* Clean up the server list */ + pre_list = 0; + temp_list = DT_started_server_list; + while (temp_list) + { + if (strcmp (temp_list->devicename, Server_Cmd->dapl_name) == 0) + { + if (pre_list == 0) /* first one */ + { + DT_started_server_list = temp_list->next; + } + else + { + pre_list->next = temp_list->next; + } + DT_Mdep_Free (temp_list); + break; + } + pre_list = temp_list; + temp_list = temp_list->next; + } + + DT_Tdep_PT_Printf (phead, + "%s (%s): Exiting.\n", module, Server_Cmd->dapl_name); +} + + +int +send_control_data ( + DT_Tdep_Print_Head *phead, + unsigned char *buffp, + Per_Server_Data_t *ps_ptr, + Per_Test_Data_t *pt_ptr) +{ + char *module = "send_control_data"; + DAT_DTO_COOKIE dto_cookie; + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + + buffp = DT_Bpool_GetBuffer (ps_ptr->bpool, 2); /* 3rd buffer */ + memcpy ( (void *)buffp, + (const void *)&pt_ptr->Server_Info, + sizeof (Server_Info_t)); + DT_Server_Info_Endian ((Server_Info_t *) buffp); + + if (!DT_post_send_buffer ( phead, ps_ptr->ep_handle, + ps_ptr->bpool, + 2, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 2))) + { + DT_Tdep_PT_Printf (phead, "%s: cannot send Server_Info\n", module); + return 1; + } + /* reap the send and verify it */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) (uintptr_t) DT_Bpool_GetBuffer ( ps_ptr->bpool, 2); + if (!DT_dto_event_wait (phead, ps_ptr->reqt_evd_hdl, &dto_stat) || + !DT_dto_check (phead, + &dto_stat, + ps_ptr->ep_handle, + DT_Bpool_GetBuffSize (ps_ptr->bpool, 2), + dto_cookie, + "Server_Info_Send")) + { + return 1; + } + + return 0; +} + +void +DT_Server_Cmd_PT_Print (DT_Tdep_Print_Head *phead, Server_Cmd_t * Server_Cmd) +{ + DT_Tdep_PT_Printf (phead, "Server_Cmd.debug: %d\n", Server_Cmd->debug); + DT_Tdep_PT_Printf (phead, "Server_Cmd.dapl_name: %s\n", Server_Cmd->dapl_name); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_server_info.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_server_info.c new file mode 100644 index 00000000..41bf462e --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_server_info.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +Started_server_t *DT_started_server_list = 0; + +void +DT_Server_Info_Endian (Server_Info_t * server_info) +{ + server_info->dapltest_version = DT_Endian32 (server_info->dapltest_version); + server_info->is_little_endian = DT_Endian32 (server_info->is_little_endian); + server_info->first_port_number= DT_Endian32 (server_info->first_port_number); +} + +void +DT_Server_Info_Print (DT_Tdep_Print_Head *phead, Server_Info_t * server_info) +{ + DT_Tdep_PT_Printf (phead, "-------------------------------------\n"); + DT_Tdep_PT_Printf (phead, "Server_Info.dapltest_version : %d\n", + server_info->dapltest_version); + DT_Tdep_PT_Printf (phead, "Server_Info.is_little_endian : %d\n", + server_info->is_little_endian); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_test_data.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_test_data.c new file mode 100644 index 00000000..e8a6ffae --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_test_data.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +DT_Mdep_LockType g_PerfTestLock; +/* + * check memory leaking int alloc_count; DT_Mdep_LockType + * Alloc_Count_Lock; + */ + +Per_Test_Data_t * +DT_Alloc_Per_Test_Data (DT_Tdep_Print_Head *phead) +{ + Per_Test_Data_t *pt_ptr; + pt_ptr = 0; + + pt_ptr = DT_Mdep_Malloc (sizeof (Per_Test_Data_t)); + if (!pt_ptr) + { + DT_Tdep_PT_Printf (phead, "No Memory to create per_test_data!\n"); + } + + return (pt_ptr); +} + +void +DT_Free_Per_Test_Data (Per_Test_Data_t * pt_ptr) +{ + DT_Mdep_Free (pt_ptr); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_test_util.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_test_util.c new file mode 100644 index 00000000..e91dc4af --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_test_util.c @@ -0,0 +1,720 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* ----------------------------------------------------------- + * Gather info about default attributes + */ +DAT_BOOLEAN +DT_query ( Per_Test_Data_t *pt_ptr, + DAT_IA_HANDLE ia_handle, + DAT_EP_HANDLE ep_handle) +{ + char *module = "DT_query"; + DAT_EVD_HANDLE async_evd_hdl; /* not used */ + DAT_EP_PARAM ep_params; + DAT_RETURN ret; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + /* Query the IA */ + ret = dat_ia_query (ia_handle, + &async_evd_hdl, + DAT_IA_ALL, + &pt_ptr->ia_attr, + DAT_PROVIDER_FIELD_ALL, + &pt_ptr->provider_attr); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ia_query error: %s\n", + module, + DT_RetToString (ret)); + return ( false ); + } + + /* Query the EP */ + ret = dat_ep_query ( ep_handle, + DAT_EP_FIELD_ALL, + &ep_params); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "%s: dat_ep_query error: %s\n", + module, + DT_RetToString (ret)); + return ( false ); + } + pt_ptr->ep_attr = ep_params.ep_attr; + + /* + * If debugging, print out some interesting attributes + */ + if (DT_dapltest_debug) + { + DAT_SOCK_ADDR6 *ip6_addr; + struct sockaddr_in *ip_addr; + + DT_Tdep_PT_Printf (phead, "***** DAPL Characteristics *****\n"); + DT_Tdep_PT_Printf (phead, "Provider: %s Version %d.%d DAPL %d.%d\n", + pt_ptr->provider_attr.provider_name, + pt_ptr->provider_attr.provider_version_major, + pt_ptr->provider_attr.provider_version_minor, + pt_ptr->provider_attr.dapl_version_major, + pt_ptr->provider_attr.dapl_version_minor ); + DT_Tdep_PT_Printf (phead, "Adapter: %s by %s Version %d.%d\n", + pt_ptr->ia_attr.adapter_name, + pt_ptr->ia_attr.vendor_name, + pt_ptr->ia_attr.hardware_version_major, + pt_ptr->ia_attr.hardware_version_minor ); + DT_Tdep_PT_Printf (phead, "Supporting:\n"); + DT_Tdep_PT_Printf (phead, "\t%d EPs with %d DTOs and %d RDMA/RDs each\n", + pt_ptr->ia_attr.max_eps, + pt_ptr->ia_attr.max_dto_per_ep, + pt_ptr->ia_attr.max_rdma_read_per_ep ); + DT_Tdep_PT_Printf (phead, "\t%d EVDs of up to %d entries " + " (default S/R size is %d/%d)\n", + pt_ptr->ia_attr.max_evds, + pt_ptr->ia_attr.max_evd_qlen, + pt_ptr->ep_attr.max_request_dtos, + pt_ptr->ep_attr.max_recv_dtos ); + DT_Tdep_PT_Printf (phead, "\tIOVs of up to %d elements\n", + pt_ptr->ia_attr.max_iov_segments_per_dto ); + DT_Tdep_PT_Printf (phead, "\t%d LMRs (and %d RMRs) of up to 0x" F64x " bytes\n", + pt_ptr->ia_attr.max_lmrs, + pt_ptr->ia_attr.max_rmrs, + pt_ptr->ia_attr.max_lmr_block_size ); + DT_Tdep_PT_Printf (phead, "\tMaximum MTU 0x" F64x " bytes, RDMA 0x" F64x " bytes\n", + pt_ptr->ia_attr.max_mtu_size, + pt_ptr->ia_attr.max_rdma_size ); + DT_Tdep_PT_Printf (phead, "\tMaximum Private data size %d bytes\n", + pt_ptr->provider_attr.max_private_data_size ); + + ip6_addr = (DAT_SOCK_ADDR6 *)pt_ptr->ia_attr.ia_address_ptr; + if (ip6_addr->sin6_family == AF_INET6 ) + { + DT_Tdep_PT_Printf (phead, "\tLocal IP address %x:%x:%x:%x:%x:%x:%x:%x:\n", + ip6_addr->sin6_addr.s6_addr[0], + ip6_addr->sin6_addr.s6_addr[1], + ip6_addr->sin6_addr.s6_addr[2], + ip6_addr->sin6_addr.s6_addr[3], + ip6_addr->sin6_addr.s6_addr[4], + ip6_addr->sin6_addr.s6_addr[5], + ip6_addr->sin6_addr.s6_addr[6], + ip6_addr->sin6_addr.s6_addr[7]); + DT_Tdep_PT_Printf (phead, "%x:%x:%x:%x:%x:%x:%x:%x\n", + ip6_addr->sin6_addr.s6_addr[8], + ip6_addr->sin6_addr.s6_addr[9], + ip6_addr->sin6_addr.s6_addr[10], + ip6_addr->sin6_addr.s6_addr[11], + ip6_addr->sin6_addr.s6_addr[12], + ip6_addr->sin6_addr.s6_addr[13], + ip6_addr->sin6_addr.s6_addr[14], + ip6_addr->sin6_addr.s6_addr[15]); + } + else if (ip6_addr->sin6_family == AF_INET ) + + { + int rval; + + ip_addr = (struct sockaddr_in *)pt_ptr->ia_attr.ia_address_ptr; + rval = (int) ip_addr->sin_addr.s_addr; + + DT_Tdep_PT_Printf (phead, "\tLocal IP address %d.%d.%d.%d\n", + (rval >> 0) & 0xff, + (rval >> 8) & 0xff, + (rval >> 16) & 0xff, + (rval >> 24) & 0xff); + } + + DT_Tdep_PT_Printf (phead, "***** ***** ***** ***** ***** *****\n"); + } + + return ( true ); +} + + +/* ----------------------------------------------------------- + * Post a recv buffer + */ +DAT_BOOLEAN +DT_post_recv_buffer (DT_Tdep_Print_Head *phead, + DAT_EP_HANDLE ep_handle, + Bpool * bp, + int index, + int size) +{ + unsigned char *buff = DT_Bpool_GetBuffer (bp, index); + DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV (bp, index); + DAT_LMR_CONTEXT lmr_c = DT_Bpool_GetLMR (bp, index); + DAT_DTO_COOKIE cookie; + DAT_RETURN ret; + + /* + * Prep the inputs + */ + iov->virtual_address = (DAT_VADDR) (uintptr_t) buff; + iov->segment_length = size; + iov->lmr_context = lmr_c; + cookie.as_64 = (DAT_UINT64)0UL; + cookie.as_ptr = (DAT_PVOID) buff; + + DT_Tdep_PT_Debug (3, (phead, "Post-Recv #%d [%p, %x]\n", index, buff, size)); + + /* Post the recv buffer */ + ret = dat_ep_post_recv (ep_handle, + 1, + iov, + cookie, + DAT_COMPLETION_DEFAULT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dat_ep_post_recv failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + return false; + } + return true; +} + + +/* ----------------------------------------------------------- + * Post a send buffer + */ +DAT_BOOLEAN +DT_post_send_buffer (DT_Tdep_Print_Head *phead, + DAT_EP_HANDLE ep_handle, + Bpool * bp, + int index, + int size) +{ + unsigned char *buff = DT_Bpool_GetBuffer (bp, index); + DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV (bp, index); + DAT_LMR_CONTEXT lmr_c = DT_Bpool_GetLMR (bp, index); + DAT_DTO_COOKIE cookie; + DAT_RETURN ret; + + /* + * Prep the inputs + */ + iov->virtual_address = (DAT_VADDR) (uintptr_t) buff; + iov->segment_length = size; + iov->lmr_context = lmr_c; + cookie.as_64 = (DAT_UINT64)0UL; + cookie.as_ptr = (DAT_PVOID) buff; + + DT_Tdep_PT_Debug (3, (phead, "Post-Send #%d [%p, %x]\n", index, buff, size)); + + /* Post the recv buffer */ + ret = dat_ep_post_send (ep_handle, + 1, + iov, + cookie, + DAT_COMPLETION_DEFAULT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dat_ep_post_send failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + return false; + } + return true; +} + + +/* ----------------------------------------------------------- + * Wait for a CR event, returning false on error. + */ +bool +DT_cr_event_wait ( DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + DAT_CR_ARRIVAL_EVENT_DATA *cr_stat_p) +{ + int err_cnt; + + err_cnt = 0; + + for (;;) + { + DAT_RETURN ret; + DAT_EVENT event; + + ret = DT_Tdep_evd_wait (evd_handle, DAT_TIMEOUT_INFINITE, &event); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_wait (CR) failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + /* + * If we get an error due to the client breaking the + * connection early or some transients, just ignore it + * and keep going. If we get a bunch of errors, bail + * out. + */ + /* if ( err_cnt++ < 10 ) */ + /* { */ + /* continue; */ + /* } */ + + break; + } + + if (event.event_number == DAT_CONNECTION_REQUEST_EVENT) + { + /* + * Pass back what we know, if requested. + */ + if (cr_stat_p) + { + *cr_stat_p = event.event_data.cr_arrival_event_data; + } + return (true); + } + + DT_Tdep_PT_Printf (phead, "Warning: cr_event_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + } + + return (false); +} + + +/* ----------------------------------------------------------- + * Wait for a connection event, returning false on error. + */ +bool +DT_conn_event_wait (DT_Tdep_Print_Head *phead, + DAT_EP_HANDLE ep_handle, + DAT_EVD_HANDLE evd_handle, + DAT_EVENT_NUMBER *event_number) + +{ + for (;;) + { + DAT_RETURN ret; + DAT_EVENT event; + + ret = DT_Tdep_evd_wait (evd_handle, DAT_TIMEOUT_INFINITE, &event); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_wait (CONN) failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + break; + } + *event_number = event.event_number; + if (event.event_number == DAT_CONNECTION_EVENT_PEER_REJECTED + || event.event_number == DAT_CONNECTION_EVENT_NON_PEER_REJECTED + || event.event_number == DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR + || event.event_number == DAT_CONNECTION_EVENT_DISCONNECTED + || event.event_number == DAT_CONNECTION_EVENT_BROKEN + || event.event_number == DAT_CONNECTION_EVENT_UNREACHABLE + || event.event_number == DAT_CONNECTION_EVENT_TIMED_OUT) + { + DT_Tdep_PT_Printf (phead, "Warning: conn_event_wait %s\n", + DT_EventToSTr (event.event_number)); + break; + } + if (event.event_number == DAT_CONNECTION_EVENT_ESTABLISHED) + { + /* + * Could return DAT_CONNECTION_EVENT_DATA and verify: + * event.event_data.connect_event_data.ep_handle + * event.event_data.connect_event_data.private_data_size + * event.event_data.connect_event_data.private_data + */ + return (true); + } + + DT_Tdep_PT_Printf (phead, "Warning: conn_event_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + } + + return (false); +} + + +/* ----------------------------------------------------------- + * Wait for a disconnection event, returning false on error. + */ +bool +DT_disco_event_wait ( DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + DAT_EP_HANDLE *ep_handle ) +{ + for (;;) + { + DAT_RETURN ret; + DAT_EVENT event; + + ret = DT_Tdep_evd_wait (evd_handle, DAT_TIMEOUT_INFINITE, &event); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_wait (DISCONN) failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + break; + } + if (event.event_number == DAT_CONNECTION_EVENT_PEER_REJECTED + || event.event_number == DAT_CONNECTION_EVENT_NON_PEER_REJECTED + || event.event_number == DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR + || event.event_number == DAT_CONNECTION_EVENT_BROKEN + || event.event_number == DAT_CONNECTION_EVENT_UNREACHABLE + || event.event_number == DAT_CONNECTION_EVENT_TIMED_OUT) + { + DT_Tdep_PT_Printf (phead, "Warning: disconn_event_wait %s\n", + DT_EventToSTr (event.event_number)); + break; + } + + + if (event.event_number == DAT_CONNECTION_EVENT_DISCONNECTED) + { + if ( ep_handle != NULL ) + { + *ep_handle = event.event_data.connect_event_data.ep_handle; + } + return (true); + } + + DT_Tdep_PT_Printf (phead, "Warning: conn_event_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + } + + return (false); +} + + +/* ----------------------------------------------------------- + * Reap a DTO event using a wait or polling, returning false on error. + */ +bool +DT_dto_event_reap (DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + bool poll, + DAT_DTO_COMPLETION_EVENT_DATA *dto_statusp) +{ + if (poll) + { + return DT_dto_event_poll (phead, evd_handle, dto_statusp); + } + else + { + return DT_dto_event_wait (phead, evd_handle, dto_statusp); + } +} + + +/* ----------------------------------------------------------- + * Poll for a DTO event, returning false on error. + */ +bool +DT_dto_event_poll (DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + DAT_DTO_COMPLETION_EVENT_DATA *dto_statusp) +{ + for (;;) + { + DAT_RETURN ret; + DAT_EVENT event; + + ret = DT_Tdep_evd_dequeue ( evd_handle, + &event); + + if (DAT_GET_TYPE (ret) == DAT_QUEUE_EMPTY) + { + continue; + } + + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_wait (DTO) failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + break; + } + + if (event.event_number == DAT_DTO_COMPLETION_EVENT) + { + /* + * Pass back all the useful bits if requested: + * ep_handle, user_cookie.as_ptr + * status, transfered_length + */ + if (dto_statusp) + { + *dto_statusp = event.event_data.dto_completion_event_data; + } + + return (true); + } + + DT_Tdep_PT_Printf (phead, "Warning: dto_event_poll swallowing %s event\n", + DT_EventToSTr (event.event_number)); + } + + return (false); +} + + +/* ----------------------------------------------------------- + * Wait for a DTO event, returning false on error. + */ +bool +DT_dto_event_wait (DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + DAT_DTO_COMPLETION_EVENT_DATA *dto_statusp) +{ + for (;;) + { + DAT_RETURN ret; + DAT_EVENT event; + + ret = DT_Tdep_evd_wait (evd_handle, DAT_TIMEOUT_INFINITE, &event); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_wait (DTO) failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + break; + } + + if (event.event_number == DAT_DTO_COMPLETION_EVENT) + { + /* + * Pass back all the useful bits if requested: + * ep_handle, user_cookie.as_ptr + * status, transfered_length + */ + if (dto_statusp) + { + *dto_statusp = event.event_data.dto_completion_event_data; + } + return (true); + } + + DT_Tdep_PT_Printf (phead, "Warning: dto_event_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + } + + return (false); +} + + +/* ----------------------------------------------------------- + * Wait for a RMR event, returning false on error. + */ +bool +DT_rmr_event_wait (DT_Tdep_Print_Head *phead, + DAT_EVD_HANDLE evd_handle, + DAT_RMR_BIND_COMPLETION_EVENT_DATA *rmr_statusp) +{ + for (;;) + { + DAT_RETURN ret; + DAT_EVENT event; + + ret = DT_Tdep_evd_wait (evd_handle, DAT_TIMEOUT_INFINITE, &event); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dapl_event_wait (RMR) failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + break; + } + + if (event.event_number == DAT_RMR_BIND_COMPLETION_EVENT) + { + /* + * Pass back all the useful bits if requested: + * rmr_handle, user_cookie, status + */ + if (rmr_statusp) + { + *rmr_statusp = event.event_data.rmr_completion_event_data; + } + return (true); + } + + DT_Tdep_PT_Printf (phead, "Warning: rmr_event_wait swallowing %s event\n", + DT_EventToSTr (event.event_number)); + } + + return (false); +} + + +/* ----------------------------------------------------------- + * Check a DTO and print some debug info if anything is amiss. + */ +bool +DT_dto_check ( DT_Tdep_Print_Head *phead, + DAT_DTO_COMPLETION_EVENT_DATA *dto_p, + DAT_EP_HANDLE ep_expected, + DAT_COUNT len_expected, + DAT_DTO_COOKIE cookie_expected, + char *message ) +{ + if ( ( (ep_expected != NULL) && (dto_p->ep_handle != ep_expected) ) + || dto_p->transfered_length != len_expected + || dto_p->user_cookie.as_64 != cookie_expected.as_64 + || dto_p->status != DAT_DTO_SUCCESS ) + { + DT_Tdep_PT_Printf (phead, "Test Error: %s-reaping DTO problem, status = %s\n", + message, + (dto_p->status == DAT_DTO_SUCCESS ? "OK" + : (dto_p->status == DAT_DTO_FAILURE ? "FAILURE" + : "LengthError"))); + DT_Test_Error (); + if ( (ep_expected != NULL) && (dto_p->ep_handle != ep_expected) ) + { + DT_Tdep_PT_Printf (phead, "\tEndPoint mismatch (got %p wanted %p)\n", + dto_p->ep_handle, + ep_expected); + } + if (dto_p->transfered_length != len_expected) + { + DT_Tdep_PT_Printf ( phead, + "\tLength mismatch (xfer 0x" F64x " wanted 0x%x)\n", + dto_p->transfered_length, + len_expected); + } + if (dto_p->user_cookie.as_64 != cookie_expected.as_64) + { + DT_Tdep_PT_Printf (phead, "\tCookie mismatch (got " F64x " wanted " F64x ")\n", + dto_p->user_cookie.as_64, + cookie_expected.as_64); + } + return ( false ); + } + + return ( true ); +} + + +/* ----------------------------------------------------------- + * Check an RMR Bind and print some debug info if anything is amiss. + */ +bool +DT_rmr_check ( DT_Tdep_Print_Head *phead, + DAT_RMR_BIND_COMPLETION_EVENT_DATA *rmr_p, + DAT_RMR_HANDLE rmr_expected, + DAT_PVOID cookie_expected, + char *message) +{ + if ( rmr_p->rmr_handle != rmr_expected + || rmr_p->user_cookie.as_ptr != cookie_expected + || rmr_p->status != DAT_RMR_BIND_SUCCESS ) + { + + DT_Tdep_PT_Printf (phead, "Test Error: %s RMR bind problem, status = %s\n", + message, + (rmr_p->status == DAT_RMR_BIND_SUCCESS + ? "OK" : "FAILURE")); + DT_Test_Error (); + if (rmr_p->rmr_handle != rmr_expected) + { + DT_Tdep_PT_Printf (phead, "\tRMR handle mismatch (got 0x%p wanted 0x%p)\n", + rmr_p->rmr_handle, + rmr_expected); + } + if (rmr_p->user_cookie.as_ptr != cookie_expected) + { + DT_Tdep_PT_Printf (phead, "\tCookie mismatch (got %p wanted %p)\n", + rmr_p->user_cookie.as_ptr, + cookie_expected); + } + return ( false ); + } + + return ( true ); +} + + +/* ----------------------------------------------------------- + * Check a CR and print some debug info if anything is amiss. + */ +bool +DT_cr_check ( DT_Tdep_Print_Head *phead, + DAT_CR_ARRIVAL_EVENT_DATA *cr_stat_p, + DAT_PSP_HANDLE psp_handle_expected, + DAT_CONN_QUAL port_expected, + DAT_CR_HANDLE *cr_handlep, + char *message) +{ + DAT_RETURN ret; + + if (cr_handlep) + { + *cr_handlep = (DAT_CR_HANDLE) 0; + } + + if (cr_stat_p->conn_qual != port_expected || + (psp_handle_expected && + cr_stat_p->sp_handle.psp_handle != psp_handle_expected)) + { + + DT_Tdep_PT_Printf (phead, "Test Error: %s CR data problem\n", message); + DT_Test_Error (); + if (cr_stat_p->conn_qual != port_expected) + { + DT_Tdep_PT_Printf (phead, "\tCR conn_qual mismatch " + " (got 0x" F64x " wanted 0x" F64x ")\n", + cr_stat_p->conn_qual, port_expected); + } + if (psp_handle_expected && + cr_stat_p->sp_handle.psp_handle != psp_handle_expected) + { + DT_Tdep_PT_Printf (phead, "\tPSP mismatch (got 0x%p wanted 0x%p)\n", + cr_stat_p->sp_handle.psp_handle, + psp_handle_expected); + } + if (!cr_stat_p->cr_handle) + { + DT_Tdep_PT_Printf (phead, "\tGot NULL cr_handle\n"); + } + else + { + ret = dat_cr_reject (cr_stat_p->cr_handle, 0, NULL); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "\tdat_cr_reject error: %s\n", + DT_RetToString (ret)); + } + } + return ( false ); + } + + if (cr_handlep) + { + *cr_handlep = cr_stat_p->cr_handle; + } + return ( true ); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_thread.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_thread.c new file mode 100644 index 00000000..c7f2d6f0 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_thread.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" +/* + * Class Thread + * + * Threads subsystem initialization + */ +void +DT_Thread_Init (Per_Test_Data_t * pt_ptr) +{ + DT_Mdep_LockInit (&pt_ptr->Thread_counter_lock); + pt_ptr->Thread_counter = 0; + + /* + * Initialize the synchronization event in the pt_ptr so it's ready + * to be signalled when the time comes. The countdown counter + * lets me coordinate with all the test threads so that the server + * thread doesn't get notified that the test endpoints are ready + * until they actually are. Only transaction tests use this * + * functionality; if the performance test gets changed to use + * multiple threads on the server side then that code semantic + * will need to be added for final test endpoint setup + * notification or there will continue to be a race condition + * between the main server thread and the server test threads. + */ + DT_Mdep_wait_object_init (&pt_ptr->synch_wait_object); + pt_ptr->Countdown_Counter = 0; + +} + +/* + * Threads subsystem destroying + */ +void +DT_Thread_End (Per_Test_Data_t * pt_ptr) +{ + DT_Mdep_LockDestroy (&pt_ptr->Thread_counter_lock); + + /* + * destroy the wait object created by init. + */ + DT_Mdep_wait_object_destroy ( + &pt_ptr->synch_wait_object ); + +} + +/* + * Thread constructor + * + * NOTE: This routine does NOT create a thread as the name implies. The thread + * is created in DT_Thread_Start (which is counter intuitive) + */ +Thread * +DT_Thread_Create (Per_Test_Data_t * pt_ptr, + void (*fn) (void *), + void *param, + unsigned int stacksize) +{ + Thread *thread_ptr; + thread_ptr = (Thread *) DT_MemListAlloc (pt_ptr, "thread.c", THREAD, sizeof (Thread)); + if (thread_ptr == NULL) + { + return NULL; + } + thread_ptr->param = param; + thread_ptr->function = fn; + thread_ptr->thread_handle = 0; + thread_ptr->stacksize = stacksize; + + DT_Mdep_Lock (&pt_ptr->Thread_counter_lock); + pt_ptr->Thread_counter++; + DT_Mdep_Unlock (&pt_ptr->Thread_counter_lock); + + DT_Mdep_Thread_Init_Attributes (thread_ptr); + + return thread_ptr; +} + +/* + * Thread destructor + */ +void +DT_Thread_Destroy (Thread * thread_ptr, Per_Test_Data_t * pt_ptr) +{ + if (thread_ptr) + { + DT_Mdep_Lock (&pt_ptr->Thread_counter_lock); + pt_ptr->Thread_counter--; + DT_Mdep_Unlock (&pt_ptr->Thread_counter_lock); + + DT_Mdep_Thread_Destroy_Attributes (thread_ptr); + DT_MemListFree (pt_ptr, thread_ptr); + } +} + +/* + * Start thread execution NOTE: This routine DOES create a thread in addition + * to starting it whereas DT_Thread_Create just sets up some data structures. + * (this is counter-intuitive) + */ +bool +DT_Thread_Start (Thread * thread_ptr) +{ + return DT_Mdep_Thread_Start (thread_ptr); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_stats.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_stats.c new file mode 100644 index 00000000..0001ee77 --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_stats.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_init_transaction_stats (Transaction_Stats_t * transaction_stats, unsigned int num) +{ + DT_Mdep_LockInit (&transaction_stats->lock); + + transaction_stats->wait_count = num; + transaction_stats->num_ops = 0; + transaction_stats->time_ms = 0; + transaction_stats->bytes_send = 0; + transaction_stats->bytes_recv = 0; + transaction_stats->bytes_rdma_read = 0; + transaction_stats->bytes_rdma_write = 0; +} + +void +DT_transaction_stats_set_ready (DT_Tdep_Print_Head *phead, + Transaction_Stats_t * transaction_stats) +{ + DT_Mdep_Lock (&transaction_stats->lock); + transaction_stats->wait_count--; + + DT_Tdep_PT_Debug (1,(phead,"Received Sync Message from server (%d left)\n", + transaction_stats->wait_count)); + DT_Mdep_Unlock (&transaction_stats->lock); +} + +bool +DT_transaction_stats_wait_for_all (DT_Tdep_Print_Head *phead, + Transaction_Stats_t * transaction_stats) +{ + unsigned int loop_count; + loop_count = 100 * 10; /* 100 * 10ms * 10 = 10 seconds */ + while (transaction_stats->wait_count != 0 && loop_count != 0) + { + DT_Mdep_Sleep (10); + loop_count--; + } + if (loop_count == 0) + { + DT_Tdep_PT_Printf (phead, "FAIL: %d Server test connections did not report ready.\n", + transaction_stats->wait_count); + return false; + } + return true; +} + + +/* + * + */ +void +DT_update_transaction_stats (Transaction_Stats_t * transaction_stats, + unsigned int num_ops, + unsigned int time_ms, + unsigned int bytes_send, + unsigned int bytes_recv, + unsigned int bytes_rdma_read, + unsigned int bytes_rdma_write) +{ + DT_Mdep_Lock (&transaction_stats->lock); + + /* look for the longest time... */ + if (time_ms > transaction_stats->time_ms) + { + transaction_stats->time_ms = time_ms; + } + + transaction_stats->num_ops += num_ops; + transaction_stats->bytes_send += bytes_send; + transaction_stats->bytes_recv += bytes_recv; + transaction_stats->bytes_rdma_read += bytes_rdma_read; + transaction_stats->bytes_rdma_write += bytes_rdma_write; + DT_Mdep_Unlock (&transaction_stats->lock); +} + +/* + * + */ +void +DT_print_transaction_stats (DT_Tdep_Print_Head *phead, + Transaction_Stats_t * transaction_stats, + unsigned int num_threads, + unsigned int num_EPs) +{ + double time_s; + double mbytes_send; + double mbytes_recv; + double mbytes_rdma_read; + double mbytes_rdma_write; + int total_ops; + DT_Mdep_Lock (&transaction_stats->lock); + time_s = (double) (transaction_stats->time_ms) / 1000; + if (time_s == 0.0) + { + DT_Tdep_PT_Printf (phead, + "----- Test completed successfully, but cannot calculate stats as not\n" + "----- enough time has lapsed.\n" + "----- Try running the test with more iterations.\n"); + goto unlock_and_return; + } + mbytes_send = (double) transaction_stats->bytes_send / 1000 / 1000; + mbytes_recv = (double) transaction_stats->bytes_recv / 1000 / 1000; + mbytes_rdma_read = (double) transaction_stats->bytes_rdma_read / 1000 / 1000; + mbytes_rdma_write = (double) transaction_stats->bytes_rdma_write / 1000 / 1000; + total_ops = transaction_stats->num_ops; + + if ( 0 == total_ops ) + { + DT_Tdep_PT_Printf (phead, "----- Test completed successfully, but no operations!\n"); + goto unlock_and_return; + } + + DT_Tdep_PT_Printf (phead, "----- Stats ---- : %u threads, %u EPs\n", + num_threads, num_EPs); + DT_Tdep_PT_Printf (phead, "Total WQE : %7d.%02d WQE/Sec\n", + whole (total_ops / time_s), + hundredths (total_ops / time_s)); + DT_Tdep_PT_Printf (phead, "Total Time : %7d.%02d sec\n", + whole (time_s), + hundredths (time_s)); + DT_Tdep_PT_Printf (phead, "Total Send : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_send), + hundredths (mbytes_send), + whole (mbytes_send / time_s), + hundredths (mbytes_send / time_s)); + DT_Tdep_PT_Printf (phead, "Total Recv : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_recv), + hundredths (mbytes_recv), + whole (mbytes_recv / time_s), + hundredths (mbytes_recv / time_s)); + DT_Tdep_PT_Printf (phead, "Total RDMA Read : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_rdma_read), + hundredths (mbytes_rdma_read), + whole (mbytes_rdma_read / time_s), + hundredths (mbytes_rdma_read / time_s)); + DT_Tdep_PT_Printf (phead, "Total RDMA Write : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_rdma_write), + hundredths (mbytes_rdma_write), + whole (mbytes_rdma_write / time_s), + hundredths (mbytes_rdma_write / time_s)); + +unlock_and_return: + DT_Mdep_Unlock (&transaction_stats->lock); +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_test.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_test.c new file mode 100644 index 00000000..72811abc --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_test.c @@ -0,0 +1,2010 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +#define RMI_SEND_BUFFER_ID 0 +#define RMI_RECV_BUFFER_ID 1 +#define SYNC_SEND_BUFFER_ID 2 +#define SYNC_RECV_BUFFER_ID 3 + +/* + * The sync buffers are sent to say "Go!" to the other side. + * This is a handy place to test whether a zero-sized send into + * a zero-sized buffer actually works. If the client side hangs + * in 'Wait for Sync Message' when this is zero, it's a DAPL bug. + */ +#define SYNC_BUFF_SIZE 64 + +#define DFLT_QLEN 8 /* default event queue length */ +#define DFLT_TMO 10 /* default timeout (seconds) */ +#define MAX_CONN_RETRY 8 + +/****************************************************************************/ +void +DT_Transaction_Test_Client (Per_Test_Data_t * pt_ptr, + DAT_IA_HANDLE ia_handle, + DAT_IA_ADDRESS_PTR remote_ia_addr) +{ + Transaction_Cmd_t *cmd = &pt_ptr->Params.u.Transaction_Cmd; + unsigned int i; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + DT_init_transaction_stats (&pt_ptr->Client_Stats, + cmd->num_threads * cmd->eps_per_thread); + + /* Now go set up the client test threads */ + for (i = 0; i < cmd->num_threads; i++) + { + unsigned int port_num = pt_ptr->Server_Info.first_port_number + + i * cmd->eps_per_thread; + + DT_Tdep_PT_Debug (1,(phead,"Client: Starting Client side of test\n")); + if (!DT_Transaction_Create_Test (pt_ptr, + ia_handle, + false, + port_num, + pt_ptr->Server_Info.is_little_endian, + remote_ia_addr)) + { + DT_Tdep_PT_Printf (phead, "Client: Cannot Create Test!\n"); + break; + } + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + } + + /* Wait until end of all threads */ + while (pt_ptr->Thread_counter > 0) + { + DT_Mdep_Sleep (100); + } + + DT_print_transaction_stats (phead, + &pt_ptr->Client_Stats, + cmd->num_threads, + cmd->eps_per_thread); +} + + +/****************************************************************************/ +void +DT_Transaction_Test_Server (void *params) +{ + Per_Test_Data_t *pt_ptr = (Per_Test_Data_t *) params; + Transaction_Cmd_t *cmd = &pt_ptr->Params.u.Transaction_Cmd; + unsigned int i; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + pt_ptr->Countdown_Counter = cmd->num_threads; + + for (i = 0; i < cmd->num_threads; i++) + { + unsigned int port_num = pt_ptr->Server_Info.first_port_number + + i * cmd->eps_per_thread; + + if (!DT_Transaction_Create_Test (pt_ptr, + pt_ptr->ps_ptr->ia_handle, + true, + port_num, + pt_ptr->Client_Info.is_little_endian, + (DAT_IA_ADDRESS_PTR) 0)) + { + DT_Tdep_PT_Printf (phead, "Server: Cannot Create Test!\n"); + break; + } + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + } + + /* Wait until end of all sub-threads */ + while (pt_ptr->Thread_counter > 1) + { + DT_Mdep_Sleep (100); + } + DT_Thread_Destroy (pt_ptr->thread, pt_ptr); /* destroy Master thread */ + + DT_Mdep_Lock (&pt_ptr->ps_ptr->num_clients_lock); + pt_ptr->ps_ptr->num_clients--; + DT_Mdep_Unlock (&pt_ptr->ps_ptr->num_clients_lock); + + /* NB: Server has no pt_ptr->remote_netaddr */ + DT_PrintMemList (pt_ptr); /* check if we return all space allocated */ + DT_Mdep_LockDestroy (&pt_ptr->Thread_counter_lock); + DT_Mdep_LockDestroy (&pt_ptr->MemListLock); + DT_Free_Per_Test_Data (pt_ptr); + DT_Tdep_PT_Printf (phead, "Server: Transaction Test Finished for this client\n"); + /* + * check memory leaking DT_Tdep_PT_Printf(phead, "Server: App allocated Memory Left: + * %d\n", alloc_count); + */ +} + + +/****************************************************************************/ +/* + * DT_Transaction_Create_Test() + * + * Initialize what we can in the test structure. Then fork a thread to do the + * work. + */ + +bool +DT_Transaction_Create_Test (Per_Test_Data_t * pt_ptr, + DAT_IA_HANDLE *ia_handle, + DAT_BOOLEAN is_server, + unsigned int port_num, + DAT_BOOLEAN remote_is_little_endian, + DAT_IA_ADDRESS_PTR remote_ia_addr) +{ + Transaction_Test_t *test_ptr; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + test_ptr = (Transaction_Test_t *) DT_MemListAlloc (pt_ptr, + "transaction_test_t", + TRANSACTIONTEST, + sizeof (Transaction_Test_t)); + if (!test_ptr) + { + DT_Tdep_PT_Printf (phead, "No Memory to create transaction test structure!\n"); + return false; + } + + /* Unused fields zeroed by allocator */ + test_ptr->remote_is_little_endian = remote_is_little_endian; + test_ptr->is_server = is_server; + test_ptr->pt_ptr = pt_ptr; + test_ptr->ia_handle = ia_handle; + test_ptr->base_port = (DAT_CONN_QUAL) port_num; + test_ptr->cmd = &pt_ptr->Params.u.Transaction_Cmd; + test_ptr->time_out = DFLT_TMO * 1000; /* DFLT_TMO seconds */ + + /* FIXME more analysis needs to go into determining the minimum */ + /* possible value for DFLT_QLEN. This evd_length value will be */ + /* used for all EVDs. There are a number of dependencies imposed */ + /* by this design (ex. min(cr_evd_len) != min(recv_evd_len) ). */ + /* In the future it may be best to use individual values. */ + test_ptr->evd_length = DT_max ( + DFLT_QLEN, + test_ptr->cmd->eps_per_thread * test_ptr->cmd->num_ops); + + test_ptr->remote_ia_addr = remote_ia_addr; + + test_ptr->thread = DT_Thread_Create (pt_ptr, + DT_Transaction_Main, + test_ptr, + DT_MDEP_DEFAULT_STACK_SIZE); + if (test_ptr->thread == 0) + { + DT_Tdep_PT_Printf (phead, "No memory!\n"); + DT_MemListFree (test_ptr->pt_ptr, test_ptr); + return false; + } + DT_Thread_Start (test_ptr->thread); + + return true; +} + + +/****************************************************************************/ +/* + * Main Transaction Test Execution Routine + * + * Both client and server threads start here, with IA already open. + * Each test thread establishes a connection with its counterpart. + * They swap remote memory information (if necessary), then set up + * buffers and local data structures. When ready, the two sides + * synchronize, then testing begins. + */ +void +DT_Transaction_Main (void *param) +{ + Transaction_Test_t *test_ptr = (Transaction_Test_t *) param; + DAT_RETURN ret; + DAT_UINT32 i, j; + bool success = false; + Per_Test_Data_t *pt_ptr; + Thread *thread; + DAT_DTO_COOKIE dto_cookie; + char *private_data_str; + DAT_EVENT_NUMBER event_num; + DT_Tdep_Print_Head *phead; + + pt_ptr = test_ptr->pt_ptr; + thread = test_ptr->thread; + phead = pt_ptr->Params.phead; +#ifdef CM_BUSTED + private_data_str = ""; +#else + private_data_str = "DAPL and RDMA rule! Test 4321."; +#endif + + /* create a protection zone */ + ret = dat_pz_create (test_ptr->ia_handle, &test_ptr->pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_pz_create error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->pz_handle = DAT_HANDLE_NULL; + goto test_failure; + } + + /* create 4 EVDs - recv, request+RMR, conn-request, connect */ + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->evd_length, + NULL, + DAT_EVD_DTO_FLAG, + &test_ptr->recv_evd_hdl); /* recv */ + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (recv) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->recv_evd_hdl = DAT_HANDLE_NULL; + goto test_failure; + } + + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->evd_length, + NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, + &test_ptr->reqt_evd_hdl); /* request + rmr bind */ + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (request) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->reqt_evd_hdl = DAT_HANDLE_NULL; + goto test_failure; + } + + if (pt_ptr->local_is_server) + { + /* Client-side doesn't need CR events */ + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->evd_length, + NULL, + DAT_EVD_CR_FLAG, + &test_ptr->creq_evd_hdl); /* cr */ + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (cr) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->creq_evd_hdl = DAT_HANDLE_NULL; + goto test_failure; + } + } + + ret = DT_Tdep_evd_create (test_ptr->ia_handle, + test_ptr->evd_length, + NULL, + DAT_EVD_CONNECTION_FLAG, + &test_ptr->conn_evd_hdl); /* conn */ + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_create (conn) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + test_ptr->conn_evd_hdl = DAT_HANDLE_NULL; + goto test_failure; + } + + /* Allocate per-EP data */ + test_ptr->ep_context = (Ep_Context_t *) + DT_MemListAlloc (pt_ptr, + "transaction_test", + EPCONTEXT, + test_ptr->cmd->eps_per_thread + * sizeof (Ep_Context_t)); + if (!test_ptr->ep_context) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for EP context\n", + test_ptr->base_port); + goto test_failure; + } + + /* + * Set up the per-EP contexts: + * create the EP + * allocate buffers for remote memory info exchange + * post the receive buffers + * connect + * set up buffers and remote memory info + * send across our info + * recv the other side's info and extract what we need + */ + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_EP_ATTR ep_attr; + DAT_UINT32 buff_size = MAX_OPS * sizeof (RemoteMemoryInfo); + + /* + * Adjust default EP attributes to fit the requested test. + * This is simplistic; in that we don't count ops of each + * type and direction, checking EP limits. We just try to + * be sure the EP's WQs are large enough. The "+2" is for + * the RemoteMemInfo and Sync receive buffers. + */ + ep_attr = pt_ptr->ep_attr; + if (ep_attr.max_recv_dtos < test_ptr->cmd->num_ops + 2) + { + ep_attr.max_recv_dtos = test_ptr->cmd->num_ops + 2; + } + if (ep_attr.max_request_dtos < test_ptr->cmd->num_ops + 2) + { + ep_attr.max_request_dtos = test_ptr->cmd->num_ops + 2; + } + + /* Create EP */ + ret = dat_ep_create (test_ptr->ia_handle, /* IA */ + test_ptr->pz_handle, /* PZ */ + test_ptr->recv_evd_hdl, /* recv */ + test_ptr->reqt_evd_hdl, /* request */ + test_ptr->conn_evd_hdl, /* connect */ + &ep_attr, /* EP attrs */ + &test_ptr->ep_context[i].ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ep_create #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + test_ptr->ep_context[i].ep_handle = DAT_HANDLE_NULL; + goto test_failure; + } + + /* + * Allocate a buffer pool so we can exchange the + * remote memory info and initialize. + */ + test_ptr->ep_context[i].bp = DT_BpoolAlloc (pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context[i].ep_handle, + DAT_HANDLE_NULL, /* rmr */ + buff_size, + 4, + DAT_OPTIMAL_ALIGNMENT, + false, + false); + if (!test_ptr->ep_context[i].bp) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for remote memory buffers\n", + test_ptr->base_port); + goto test_failure; + } + + DT_Tdep_PT_Debug (3, (phead, + "0: RMI_SEND %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, 0))); + DT_Tdep_PT_Debug (3, (phead, + "1: RMI_RECV %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, 1))); + DT_Tdep_PT_Debug (3, (phead, + "2: SYNC_SEND %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, 2))); + DT_Tdep_PT_Debug (3, (phead, + "3: SYNC_RECV %p\n", + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, 3))); + + /* + * Post recv and sync buffers + */ + if (!DT_post_recv_buffer ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + RMI_RECV_BUFFER_ID, + buff_size)) + { + /* error message printed by DT_post_recv_buffer */ + goto test_failure; + } + if (!DT_post_recv_buffer ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + SYNC_RECV_BUFFER_ID, + SYNC_BUFF_SIZE)) + { + /* error message printed by DT_post_recv_buffer */ + goto test_failure; + } + + /* + * Establish the connection + */ + test_ptr->ep_context[i].ia_port = test_ptr->base_port + i; + + if (pt_ptr->local_is_server) + { + if (test_ptr->cmd->use_rsp) + { + /* + * Server - create a single-use RSP and + * await a connection for this EP + */ + + ret = dat_rsp_create (test_ptr->ia_handle, + test_ptr->ep_context[i].ia_port, + test_ptr->ep_context[i].ep_handle, + test_ptr->creq_evd_hdl, + &test_ptr->ep_context[i].rsp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_rsp_create #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + goto test_failure; + } + } + else + { + ret = dat_psp_create (test_ptr->ia_handle, + test_ptr->ep_context[i].ia_port, + test_ptr->creq_evd_hdl, + DAT_PSP_CONSUMER_FLAG, + &test_ptr->ep_context[i].psp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_psp_create #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + goto test_failure; + } + + DT_Tdep_PT_Debug (1,(phead,"Server[" F64x "]: Listen #%d on PSP port 0x" F64x "\n", + test_ptr->base_port, i, + test_ptr->ep_context[i].ia_port)); + } + } + } + + /* Here's where we tell the server process that this thread is + * ready to wait for connection requests from the remote end. + * Modify the synch wait semantics at your own risk - if these + * signals and waits aren't here, there will be chronic + * connection rejection timing problems. + */ + if (pt_ptr->local_is_server) + { + DT_Mdep_Lock (&pt_ptr->Thread_counter_lock); + pt_ptr->Countdown_Counter--; + /* Deliberate pre-decrement. Post decrement won't + * work here, so don't do it. + */ + if (pt_ptr->Countdown_Counter <= 0 ) + { + DT_Mdep_wait_object_wakeup (&pt_ptr->synch_wait_object); + } + + DT_Mdep_Unlock (&pt_ptr->Thread_counter_lock); + } + + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_UINT32 buff_size = MAX_OPS * sizeof (RemoteMemoryInfo); + RemoteMemoryInfo *RemoteMemInfo; + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_CR_ARRIVAL_EVENT_DATA cr_stat; + DAT_CR_HANDLE cr_handle; + + /* + * Establish the connection + */ + + if (pt_ptr->local_is_server) + { + DAT_CR_PARAM cr_param; + + if (test_ptr->cmd->use_rsp) + { + + /* wait for the connection request */ + if (!DT_cr_event_wait (phead, + test_ptr->creq_evd_hdl, + &cr_stat) || + !DT_cr_check ( phead, + &cr_stat, + test_ptr->ep_context[i].rsp_handle, + test_ptr->ep_context[i].ia_port, + &cr_handle, + "Server") ) + { + goto test_failure; + } + + ret = dat_cr_query (cr_handle, + DAT_CR_FIELD_ALL, + &cr_param); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_cr_query #%d error:(%x) %s\n", + test_ptr->base_port, i, ret, DT_RetToString (ret)); + } + else + { + if ( strncmp ((char *)cr_param.private_data, + private_data_str, + strlen(private_data_str) ) != 0 ) + { + DT_Tdep_PT_Printf (phead, "--Private Data mismatch!\n"); + } + else + { + DT_Tdep_PT_Debug (1, (phead,"--Private Data: %d: <%s>\n", + cr_param.private_data_size, + (char *)cr_param.private_data) ); + } + } + + /* what, me query? just try to accept the connection */ + ret = dat_cr_accept (cr_handle, + 0, /* NULL for RSP */ + 0, (DAT_PVOID)0 /* no private data */ ); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_cr_accept #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + /* cr_handle consumed on failure */ + goto test_failure; + } + + /* wait for DAT_CONNECTION_EVENT_ESTABLISHED */ + if (!DT_conn_event_wait ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->conn_evd_hdl, + &event_num)) + { + /* error message printed by DT_conn_event_wait */ + goto test_failure; + } + /* throw away single-use PSP */ + ret = dat_rsp_free (test_ptr->ep_context[i].rsp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_rsp_free #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + goto test_failure; + } + + } + else + { + /* + * Server - use a short-lived PSP instead of an RSP + */ + /* wait for a connection request */ + if (!DT_cr_event_wait (phead, + test_ptr->creq_evd_hdl, + &cr_stat) ) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_psp_create #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + goto test_failure; + } + + if ( !DT_cr_check ( phead, + &cr_stat, + test_ptr->ep_context[i].psp_handle, + test_ptr->ep_context[i].ia_port, + &cr_handle, + "Server") ) + { + goto test_failure; + } + + ret = dat_cr_query (cr_handle, + DAT_CR_FIELD_ALL, + &cr_param); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_cr_query #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + } + else + { + if ( strncmp ((char *)cr_param.private_data, + private_data_str, + strlen(private_data_str) ) != 0 ) + { + DT_Tdep_PT_Printf (phead, "--Private Data mismatch!\n"); + } + else + { + DT_Tdep_PT_Debug (1, (phead,"--Private Data: %d: <%s>\n", + cr_param.private_data_size, + (char *)cr_param.private_data) ); + } + } + + + /* what, me query? just try to accept the connection */ + ret = dat_cr_accept (cr_handle, + test_ptr->ep_context[i].ep_handle, + 0, (DAT_PVOID)0 /* no private data */ ); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_cr_accept #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + /* cr_handle consumed on failure */ + (void) dat_psp_free (test_ptr->ep_context[i].psp_handle); + goto test_failure; + } + + /* wait for DAT_CONNECTION_EVENT_ESTABLISHED */ + if (!DT_conn_event_wait ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->conn_evd_hdl, + &event_num)) + { + /* error message printed by DT_cr_event_wait */ + (void) dat_psp_free (&test_ptr->ep_context[i].psp_handle); + goto test_failure; + } + + /* throw away single-use PSP */ + ret = dat_psp_free (test_ptr->ep_context[i].psp_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_psp_free #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + goto test_failure; + } + } /* end short-lived PSP */ + + DT_Tdep_PT_Debug (1,(phead,"Server[" F64x "]: Accept #%d on port 0x" F64x "\n", + test_ptr->base_port, i, + test_ptr->ep_context[i].ia_port)); + } + else + { + /* + * Client - connect + */ + unsigned int retry_cnt = 0; + DAT_UINT32 buff_size = MAX_OPS * sizeof (RemoteMemoryInfo); + + DT_Tdep_PT_Debug (1,(phead,"Client[" F64x "]: Connect #%d on port 0x" F64x "\n", + test_ptr->base_port, i, + test_ptr->ep_context[i].ia_port)); + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + +retry: + ret = dat_ep_connect (test_ptr->ep_context[i].ep_handle, + test_ptr->remote_ia_addr, + test_ptr->ep_context[i].ia_port, + DAT_TIMEOUT_INFINITE, + strlen (private_data_str), private_data_str, + pt_ptr->Params.ReliabilityLevel, + DAT_CONNECT_DEFAULT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ep_connect #%d error: %s (0x%x)\n", + test_ptr->base_port, i, DT_RetToString (ret), ret); + goto test_failure; + } + + /* wait for DAT_CONNECTION_EVENT_ESTABLISHED */ + if (!DT_conn_event_wait ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->conn_evd_hdl, + &event_num)) + { + /* error message printed by DT_cr_event_wait */ + if ( event_num == DAT_CONNECTION_EVENT_PEER_REJECTED ) + { + DT_Mdep_Sleep (1000); + /* + * See if any buffers were flushed as a result of + * the REJECT; clean them up and repost if so + */ + { + DAT_EVENT event; + DAT_COUNT drained = 0; + + dat_ep_reset (test_ptr->ep_context[i].ep_handle); + do + { + ret = DT_Tdep_evd_dequeue ( test_ptr->recv_evd_hdl, + &event); + drained++; + } while (DAT_GET_TYPE(ret) != DAT_QUEUE_EMPTY); + + if (drained > 1) + { + /* + * Post recv and sync buffers + */ + if (!DT_post_recv_buffer ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + RMI_RECV_BUFFER_ID, + buff_size)) + { + /* error message printed by DT_post_recv_buffer */ + goto test_failure; + } + if (!DT_post_recv_buffer ( phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + SYNC_RECV_BUFFER_ID, + SYNC_BUFF_SIZE)) + { + /* error message printed by DT_post_recv_buffer */ + goto test_failure; + } + } + } + DT_Tdep_PT_Printf (phead, "Client[" F64x "]: retrying connection...\n", + test_ptr->base_port); + retry_cnt++; + if (retry_cnt < MAX_CONN_RETRY) + { + goto retry; + } + } + /* error message printed by DT_cr_event_wait */ + goto test_failure; + } + + DT_Tdep_PT_Debug (1,(phead,"Client[" F64x "]: Got Connection #%d\n", + test_ptr->base_port, i)); + } + +#ifdef CM_BUSTED + /***** XXX Chill out a bit to give the kludged CM a chance ... + *****/DT_Mdep_Sleep (5000); +#endif + + /* + * Fill in the test_ptr with relevant command info + */ + for (j = 0; j < test_ptr->cmd->num_ops; j++) + { + test_ptr->ep_context[i].op[j].server_initiated + = test_ptr->cmd->op[j].server_initiated; + test_ptr->ep_context[i].op[j].transfer_type + = test_ptr->cmd->op[j].transfer_type; + test_ptr->ep_context[i].op[j].num_segs + = test_ptr->cmd->op[j].num_segs; + test_ptr->ep_context[i].op[j].seg_size + = test_ptr->cmd->op[j].seg_size; + test_ptr->ep_context[i].op[j].reap_send_on_recv + = test_ptr->cmd->op[j].reap_send_on_recv; + } + + /* + * Exchange remote memory info: If we're going to participate + * in an RDMA, we need to allocate memory buffers and advertise + * them to the other side. + */ + for (j = 0; j < test_ptr->cmd->num_ops; j++) + { + DAT_BOOLEAN us; + + us = (pt_ptr->local_is_server && + test_ptr->ep_context[i].op[j].server_initiated) || + (!pt_ptr->local_is_server && + !test_ptr->ep_context[i].op[j].server_initiated); + + test_ptr->ep_context[i].op[j].Rdma_Context = (DAT_RMR_CONTEXT) 0; + test_ptr->ep_context[i].op[j].Rdma_Address = 0; + + switch (test_ptr->ep_context[i].op[j].transfer_type) + { + case RDMA_READ: + { + test_ptr->ep_context[i].op[j].bp = + DT_BpoolAlloc (pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context[i].ep_handle, + test_ptr->reqt_evd_hdl, + test_ptr->ep_context[i].op[j].seg_size, + test_ptr->ep_context[i].op[j].num_segs, + DAT_OPTIMAL_ALIGNMENT, + false, + !us ? true : false); + if (!test_ptr->ep_context[i].op[j].bp) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for buffers (RDMA/RD)\n", + test_ptr->base_port); + goto test_failure; + } + if (!us) + { + test_ptr->ep_context[i].op[j].Rdma_Context = + DT_Bpool_GetRMR (test_ptr->ep_context[i].op[j].bp, 0); + test_ptr->ep_context[i].op[j].Rdma_Address = + (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (test_ptr->ep_context[i].op[j].bp, + 0); + DT_Tdep_PT_Debug (3, (phead, + "not-us: RDMA/RD [ va=" F64x ", ctxt=%x ]\n", + test_ptr->ep_context[i].op[j].Rdma_Address, + test_ptr->ep_context[i].op[j].Rdma_Context)); + } + break; + } + + case RDMA_WRITE: + { + test_ptr->ep_context[i].op[j].bp = + DT_BpoolAlloc (pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context[i].ep_handle, + test_ptr->reqt_evd_hdl, + test_ptr->ep_context[i].op[j].seg_size, + test_ptr->ep_context[i].op[j].num_segs, + DAT_OPTIMAL_ALIGNMENT, + !us ? true : false, + false); + if (!test_ptr->ep_context[i].op[j].bp) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for buffers (RDMA/WR)\n", + test_ptr->base_port); + goto test_failure; + } + if (!us) + { + test_ptr->ep_context[i].op[j].Rdma_Context = + DT_Bpool_GetRMR (test_ptr->ep_context[i].op[j].bp, 0); + test_ptr->ep_context[i].op[j].Rdma_Address = + (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (test_ptr->ep_context[i].op[j].bp, + 0); + DT_Tdep_PT_Debug (3, (phead, + "not-us: RDMA/WR [ va=" F64x ", ctxt=%x ]\n", + test_ptr->ep_context[i].op[j].Rdma_Address, + test_ptr->ep_context[i].op[j].Rdma_Context)); + } + break; + } + + case SEND_RECV: + { + test_ptr->ep_context[i].op[j].bp = + DT_BpoolAlloc (pt_ptr, + phead, + test_ptr->ia_handle, + test_ptr->pz_handle, + test_ptr->ep_context[i].ep_handle, + DAT_HANDLE_NULL, /* rmr */ + test_ptr->ep_context[i].op[j].seg_size, + test_ptr->ep_context[i].op[j].num_segs, + DAT_OPTIMAL_ALIGNMENT, + false, + false); + if (!test_ptr->ep_context[i].op[j].bp) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: no memory for buffers (S/R)\n", + test_ptr->base_port); + goto test_failure; + } + + DT_Tdep_PT_Debug (3, (phead, + "%d: S/R [ va=%p ]\n", j, (DAT_PVOID) + DT_Bpool_GetBuffer ( test_ptr->ep_context[i].op[j].bp, 0))); + break; + } + } + } /* end foreach op */ + + /* + * Prep send buffer with memory information + */ + RemoteMemInfo = (RemoteMemoryInfo *) + DT_Bpool_GetBuffer (test_ptr->ep_context[i].bp, + RMI_SEND_BUFFER_ID); + + for (j = 0; j < test_ptr->cmd->num_ops; j++) + { + RemoteMemInfo[j].rmr_context = + test_ptr->ep_context[i].op[j].Rdma_Context; + RemoteMemInfo[j].mem_address.as_64 = + test_ptr->ep_context[i].op[j].Rdma_Address; + if (RemoteMemInfo[j].mem_address.as_64) + { + DT_Tdep_PT_Debug (3, (phead, + "RemoteMemInfo[%d] va=" F64x ", ctx=%x\n", + j, + RemoteMemInfo[j].mem_address.as_64, + RemoteMemInfo[j].rmr_context)); + } + /* + * If the client and server are of different endiannesses, + * we must correct the endianness of the handle and address + * we pass to the other side. The other side cannot (and + * better not) interpret these values. + */ + if (DT_local_is_little_endian != test_ptr->remote_is_little_endian) + { + RemoteMemInfo[j].rmr_context = + DT_EndianMemHandle (RemoteMemInfo[j].rmr_context); + RemoteMemInfo[j].mem_address.as_64 = + DT_EndianMemAddress (RemoteMemInfo[j].mem_address.as_64); + } + } /* end foreach op */ + + /* + * Send our memory info. The client performs the first send to comply + * with the iWARP MPA protocol's "Connection Startup Rules". + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Sending %s Memory Info\n", + test_ptr->base_port, + test_ptr->is_server ? "Server" : "Client")); + + if (!test_ptr->is_server ) { + + /* post the send buffer */ + if (!DT_post_send_buffer (phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + RMI_SEND_BUFFER_ID, + buff_size)) + { + /* error message printed by DT_post_send_buffer */ + goto test_failure; + } + /* reap the send and verify it */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, + RMI_SEND_BUFFER_ID); + if (!DT_dto_event_wait (phead, test_ptr->reqt_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + test_ptr->ep_context[i].ep_handle, + buff_size, + dto_cookie, + test_ptr->is_server ? "Client_Mem_Info_Send" + : "Server_Mem_Info_Send")) + { + goto test_failure; + } + } + + /* + * Recv the other side's info + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Waiting for %s Memory Info\n", + test_ptr->base_port, + test_ptr->is_server ? "Client" : "Server")); + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, + RMI_RECV_BUFFER_ID); + if (!DT_dto_event_wait (phead, test_ptr->recv_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + test_ptr->ep_context[i].ep_handle, + buff_size, + dto_cookie, + test_ptr->is_server ? "Client_Mem_Info_Recv" + : "Server_Mem_Info_Recv")) + { + goto test_failure; + } + + if (test_ptr->is_server ) { + /* post the send buffer */ + if (!DT_post_send_buffer (phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + RMI_SEND_BUFFER_ID, + buff_size)) + { + /* error message printed by DT_post_send_buffer */ + goto test_failure; + } + /* reap the send and verify it */ + dto_cookie.as_64 = LZERO; + dto_cookie.as_ptr = + (DAT_PVOID) DT_Bpool_GetBuffer ( + test_ptr->ep_context[i].bp, + RMI_SEND_BUFFER_ID); + if (!DT_dto_event_wait (phead, test_ptr->reqt_evd_hdl, &dto_stat) || + !DT_dto_check ( phead, + &dto_stat, + test_ptr->ep_context[i].ep_handle, + buff_size, + dto_cookie, + test_ptr->is_server ? "Client_Mem_Info_Send" + : "Server_Mem_Info_Send")) + { + goto test_failure; + } + } + + /* + * Extract what we need + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Memory Info received \n", + test_ptr->base_port)); + RemoteMemInfo = (RemoteMemoryInfo *) + DT_Bpool_GetBuffer (test_ptr->ep_context[i].bp, + RMI_RECV_BUFFER_ID); + for (j = 0; j < test_ptr->cmd->num_ops; j++) + { + DAT_BOOLEAN us; + + us = (pt_ptr->local_is_server && + test_ptr->ep_context[i].op[j].server_initiated) || + (!pt_ptr->local_is_server && + !test_ptr->ep_context[i].op[j].server_initiated); + if (us && + (test_ptr->ep_context[i].op[j].transfer_type == RDMA_READ || + test_ptr->ep_context[i].op[j].transfer_type == RDMA_WRITE)) + { + test_ptr->ep_context[i].op[j].Rdma_Context = + RemoteMemInfo[j].rmr_context; + test_ptr->ep_context[i].op[j].Rdma_Address = + RemoteMemInfo[j].mem_address.as_64; + DT_Tdep_PT_Debug (3, (phead, + "Got RemoteMemInfo [ va=" F64x ", ctx=%x ]\n", + test_ptr->ep_context[i].op[j].Rdma_Address, + test_ptr->ep_context[i].op[j].Rdma_Context)); + } + } + } /* end foreach EP context */ + + /* + * Dump out the state of the world if we're debugging + */ + if (test_ptr->cmd->debug) + { + DT_Print_Transaction_Test (phead, test_ptr); + } + + /* + * Finally! Run the test. + */ + success = DT_Transaction_Run (phead, test_ptr); + + /* + * Now clean up and go home + */ +test_failure: + if (test_ptr->ep_context) + { + + /* Foreach EP */ + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_EP_HANDLE ep_handle; + + ep_handle = DAT_HANDLE_NULL; + + /* Free the per-op buffers */ + for (j = 0; j < test_ptr->cmd->num_ops; j++) + { + if (test_ptr->ep_context[i].op[j].bp) + { + if (!DT_Bpool_Destroy (pt_ptr, + phead, + test_ptr->ep_context[i].op[j].bp)) + { + DT_Tdep_PT_Printf (phead, "test[" F64x "]: Warning: Bpool destroy fails\n", + test_ptr->base_port); + /* carry on trying, regardless */ + } + } + } + + /* Free the remote memory info exchange buffers */ + if (test_ptr->ep_context[i].bp) + { + if (!DT_Bpool_Destroy (pt_ptr, + phead, + test_ptr->ep_context[i].bp)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Warning: Bpool destroy fails\n", + test_ptr->base_port); + /* carry on trying, regardless */ + } + } + + /* + * Disconnect -- we may have left recv buffers posted, if we + * bailed out mid-setup, or ran to completion + * normally, so we use abrupt closure. + */ + if (test_ptr->ep_context[i].ep_handle) + { + ret = dat_ep_disconnect (test_ptr->ep_context[i].ep_handle, + DAT_CLOSE_ABRUPT_FLAG); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Warning: dat_ep_disconnect (%s) " + "#%d error %s\n", + test_ptr->base_port, + success ? "graceful" : "abrupt", + i, DT_RetToString (ret)); + /* carry on trying, regardless */ + } + } + + /* + * Wait on each of the outstanding EP handles. Some of them + * may be disconnected by the remote side, we are racing + * here. + */ + + if ( success ) /* Ensure DT_Transaction_Run did not return error otherwise may get stuck waiting for disconnect event*/ + { + if (!DT_disco_event_wait ( phead, + test_ptr->conn_evd_hdl, + &ep_handle)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: bad disconnect event\n", + test_ptr->base_port); + } + else + { + /* + * We have successfully obtained a completed EP. We are + * racing with the remote node on disconnects, so we + * don't know which EP this is. Run the list and + * remove it so we don't disconnect a disconnected EP + */ + for (j = 0; j < test_ptr->cmd->eps_per_thread; j++) + { + if ( test_ptr->ep_context[j].ep_handle == ep_handle ) + { + test_ptr->ep_context[j].ep_handle = NULL; + } + } + } + } + else /* !success - QP may be in error state */ + { + ep_handle = test_ptr->ep_context[i].ep_handle; + } + + /* + * Free the handle returned by the disconnect event. + * With multiple EPs, it may not be the EP we just + * disconnected as we are racing with the remote side + * disconnects. + */ + if ( DAT_HANDLE_NULL != ep_handle) + { + DAT_EVENT event; + /* + * Drain off outstanding DTOs that may have been + * generated by racing disconnects + */ + do + { + ret = DT_Tdep_evd_dequeue ( test_ptr->recv_evd_hdl, + &event); + } while (ret == DAT_SUCCESS); + /* Destroy the EP */ + ret = dat_ep_free (ep_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_ep_free #%d error: %s\n", + test_ptr->base_port, i, DT_RetToString (ret)); + /* carry on trying, regardless */ + } + } + } /* end foreach per-EP context */ + + DT_MemListFree (pt_ptr, test_ptr->ep_context); + } + + /* clean up the EVDs */ + if (test_ptr->conn_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->conn_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (conn) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + if (pt_ptr->local_is_server) + { + if (test_ptr->creq_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->creq_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (creq) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + } + if (test_ptr->reqt_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->reqt_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (reqt) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + if (test_ptr->recv_evd_hdl) + { + ret = DT_Tdep_evd_free (test_ptr->recv_evd_hdl); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_evd_free (recv) error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + + /* clean up the PZ */ + if (test_ptr->pz_handle) + { + ret = dat_pz_free (test_ptr->pz_handle); + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: dat_pz_free error: %s\n", + test_ptr->base_port, DT_RetToString (ret)); + /* fall through, keep trying */ + } + } + + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: cleanup is done\n", test_ptr->base_port)); + DT_MemListFree ( pt_ptr, test_ptr ); + DT_Thread_Destroy ( thread, pt_ptr ); + DT_Mdep_Thread_Detach ( DT_Mdep_Thread_SELF () ); /* AMM */ + DT_Mdep_Thread_EXIT (NULL); /* AMM */ +} + + +/* ----------------------------------------------------------------------- + * The actual performance test + */ +bool +DT_Transaction_Run (DT_Tdep_Print_Head *phead, Transaction_Test_t * test_ptr) +{ + unsigned int op; + unsigned int iteration; + int bytes; + bool ours; + bool success = false; + bool repost_recv; + unsigned int i; + + /* pre-post all receive buffers */ + for (op = 0; op < test_ptr->cmd->num_ops; op++) + { + /* if it is a SEND/RECV, we must post receive buffers */ + if (test_ptr->ep_context[0].op[op].transfer_type == SEND_RECV) + { + ours = (test_ptr->is_server == + test_ptr->ep_context[0].op[op].server_initiated); + if (!ours) + { + if (!DT_handle_post_recv_buf (phead, + test_ptr->ep_context, + test_ptr->cmd->eps_per_thread, + op)) + { + goto bail; + } + } + } + } + + /* initialize data if we are validating it */ + if (test_ptr->cmd->validate) + { + DT_Transaction_Validation_Fill (phead, test_ptr, 0); + } + + /* + * Now that we've posted our receive buffers... + * synchronize with the other side. + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Synchronize with the other side\n", + test_ptr->base_port)); + + + /* + * Each server thread sends a sync message to the corresponding + * client thread. All clients wait until all server threads + * have sent their sync messages. Then all clients send + * sync message. + * + * Since all of the events are directed to the same EVD, + * we do not use DT_dto_check(.) to verify the attributes + * of the sync message event. DT_dto_check(.) requires the + * comsumer to pass the expected EP, but we do not know + * what to expect. DAPL does not guarantee the order of + * completions across EPs. Therfore we only know that + * test_ptr->cmd->eps_per_thread number of completion events + * will be generated but not the order in which they will + * complete. + */ + + if (test_ptr->is_server) + { + /* + * Server + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Send Sync to Client\n", + test_ptr->base_port)); + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + if (!DT_post_send_buffer (phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + SYNC_SEND_BUFFER_ID, + SYNC_BUFF_SIZE)) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Server sync send error\n", + test_ptr->base_port)); + goto bail; + } + } + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + + if ( !DT_dto_event_wait (phead, + test_ptr->reqt_evd_hdl, + &dto_stat) ) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Server sync send error\n", + test_ptr->base_port)); + + goto bail; + } + } + + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Wait for Sync Message\n", + test_ptr->base_port)); + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + + if ( !DT_dto_event_wait (phead, + test_ptr->recv_evd_hdl, + &dto_stat) ) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Server sync recv error\n", + test_ptr->base_port)); + goto bail; + } + } + } + else + { + /* + * Client + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Wait for Sync Message\n", + test_ptr->base_port)); + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + + if ( !DT_dto_event_wait (phead, + test_ptr->recv_evd_hdl, + &dto_stat) ) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Client sync recv error\n", + test_ptr->base_port)); + goto bail; + } + DT_transaction_stats_set_ready (phead, + &test_ptr->pt_ptr->Client_Stats); + } + + /* check if it is time for client to send sync */ + if (!DT_transaction_stats_wait_for_all (phead, + &test_ptr->pt_ptr->Client_Stats)) + { + goto bail; + } + + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Send Sync Msg\n", test_ptr->base_port)); + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + if (!DT_post_send_buffer (phead, + test_ptr->ep_context[i].ep_handle, + test_ptr->ep_context[i].bp, + SYNC_SEND_BUFFER_ID, + SYNC_BUFF_SIZE)) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Client sync send error\n", + test_ptr->base_port)); + goto bail; + } + } + for (i = 0; i < test_ptr->cmd->eps_per_thread; i++) + { + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + + if ( !DT_dto_event_wait (phead, + test_ptr->reqt_evd_hdl, + &dto_stat) ) + { + goto bail; + } + } + } + + /* + * Get to work ... + */ + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: Begin...\n", test_ptr->base_port)); + test_ptr->stats.start_time = DT_Mdep_GetTime (); + + for (iteration = 0; + iteration < test_ptr->cmd->num_iterations; + iteration++) + { + + DT_Tdep_PT_Debug (1,(phead,"iteration: %d\n", iteration)); + + /* repost unless this is the last iteration */ + repost_recv = (iteration + 1 != test_ptr->cmd->num_iterations); + + for (op = 0; op < test_ptr->cmd->num_ops; op++) + { + ours = (test_ptr->is_server == + test_ptr->ep_context[0].op[op].server_initiated); + bytes = (test_ptr->ep_context[0].op[op].seg_size * + test_ptr->ep_context[0].op[op].num_segs * + test_ptr->cmd->eps_per_thread); + + switch (test_ptr->ep_context[0].op[op].transfer_type) + { + case RDMA_READ: + { + test_ptr->stats.stat_bytes_rdma_read += bytes; + if (ours) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: RdmaRead [%d]\n", + test_ptr->base_port, + op)); + if (!DT_handle_rdma_op (phead, + test_ptr->ep_context, + test_ptr->reqt_evd_hdl, + test_ptr->cmd->eps_per_thread, + RDMA_READ, + op, + test_ptr->cmd->poll)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: RdmaRead error[%d]\n", + test_ptr->base_port, op); + goto bail; + } + } + break; + } + + case RDMA_WRITE: + { + test_ptr->stats.stat_bytes_rdma_write += bytes; + if (ours) + { + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: RdmaWrite [%d]\n", + test_ptr->base_port, + op)); + if (!DT_handle_rdma_op (phead, + test_ptr->ep_context, + test_ptr->reqt_evd_hdl, + test_ptr->cmd->eps_per_thread, + RDMA_WRITE, + op, + test_ptr->cmd->poll)) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: RdmaWrite error[%d]\n", + test_ptr->base_port, op); + goto bail; + } + } + break; + } + + case SEND_RECV: + { + if (ours) + { + test_ptr->stats.stat_bytes_send += bytes; + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: postsend [%d] \n", + test_ptr->base_port, op)); + /* send data */ + if (!DT_handle_send_op (phead, + test_ptr->ep_context, + test_ptr->reqt_evd_hdl, + test_ptr->cmd->eps_per_thread, + op, + test_ptr->cmd->poll)) + { + goto bail; + } + } + else + { + test_ptr->stats.stat_bytes_recv += bytes; + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: RecvWait and Re-Post [%d] \n", + test_ptr->base_port, op)); + + if (!DT_handle_recv_op (phead, + test_ptr->ep_context, + test_ptr->recv_evd_hdl, + test_ptr->reqt_evd_hdl, + test_ptr->cmd->eps_per_thread, + op, + test_ptr->cmd->poll, + repost_recv)) + { + goto bail; + } + } + + /* now before going on, is it time to validate? */ + if (test_ptr->cmd->validate) + { + if (!test_ptr->pt_ptr->local_is_server) /* CLIENT */ + { + /* the client validates on the third to last op */ + if (op == test_ptr->cmd->num_ops - 3) + { + if (!DT_Transaction_Validation_Check (phead, + test_ptr, + iteration)) + { + goto bail; + } + DT_Transaction_Validation_Fill (phead, + test_ptr, + iteration + 1); + } + } + else /* SERVER */ + { + /* the server validates on the second to last op */ + if (op == test_ptr->cmd->num_ops - 2) + { + if (!DT_Transaction_Validation_Check (phead, + test_ptr, + iteration)) + { + goto bail; + } + DT_Transaction_Validation_Fill (phead, + test_ptr, + iteration + 1); + } + } + } /* end validate */ + break; + } + } /* end switch for transfer type */ + } /* end loop for each op */ + } /* end loop for iteration */ + + /* end time and print stats */ + test_ptr->stats.end_time = DT_Mdep_GetTime (); + if (!test_ptr->pt_ptr->local_is_server) + { + DT_update_transaction_stats (&test_ptr->pt_ptr->Client_Stats, + test_ptr->cmd->eps_per_thread * test_ptr->cmd->num_ops * + test_ptr->cmd->num_iterations, + test_ptr->stats.end_time - test_ptr->stats.start_time, + test_ptr->stats.stat_bytes_send, + test_ptr->stats.stat_bytes_recv, + test_ptr->stats.stat_bytes_rdma_read, + test_ptr->stats.stat_bytes_rdma_write); + } + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: End Successfully\n", test_ptr->base_port)); + success = true; + +bail: + return ( success ); +} + + +/*------------------------------------------------------------------------------ */ +void +DT_Transaction_Validation_Fill ( DT_Tdep_Print_Head *phead, + Transaction_Test_t * test_ptr, + unsigned int iteration) +{ + bool ours; + unsigned int op; + unsigned int i; + unsigned int j; + unsigned int ind; + unsigned char *buff; + + if (iteration >= test_ptr->cmd->num_iterations) + { + return; + } + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: FILL Buffers Iteration %d\n", + test_ptr->base_port, iteration)); + + /* + * fill all but the last three ops, which + * were added to create barriers for data validation + */ + for (ind = 0; ind < test_ptr->cmd->eps_per_thread; ind++) + { + for (op = 0; op < test_ptr->cmd->num_ops - 3; op++) + { + ours = (test_ptr->is_server == + test_ptr->ep_context[ind].op[op].server_initiated); + + switch (test_ptr->ep_context[ind].op[op].transfer_type) + + { + case RDMA_READ: + { + if (!ours) + { + for (i = 0; + i < test_ptr->ep_context[ind].op[op].num_segs; + i++) + { + + buff = DT_Bpool_GetBuffer ( + test_ptr->ep_context[ind].op[op].bp, i); + for (j = 0; + j < test_ptr->ep_context[ind].op[op].seg_size; + j++) + { + /* Avoid using all zero bits the 1st time */ + buff[j] = (iteration + 1) & 0xFF; + } + } + } + break; + } + + case RDMA_WRITE: + { + if (ours) + { + for (i = 0; + i < test_ptr->ep_context[ind].op[op].num_segs; + i++) + { + + buff = DT_Bpool_GetBuffer ( + test_ptr->ep_context[ind].op[op].bp, i); + for (j = 0; + j < test_ptr->ep_context[ind].op[op].seg_size; + j++) + { + /* Avoid using all zero bits the 1st time */ + buff[j] = (iteration + 1) & 0xFF; + } + } + } + break; + } + + case SEND_RECV: + { + if (ours) + { + for (i = 0; + i < test_ptr->ep_context[ind].op[op].num_segs; + i++) + { + + buff = DT_Bpool_GetBuffer ( + test_ptr->ep_context[ind].op[op].bp, + i); + /***** + DT_Tdep_PT_Printf(phead, + "\tFill: wq=%d op=%d seg=%d ptr=[%p, %d]\n", + ind, op, i, buff, j); + *****/ + for (j = 0; + j < test_ptr->ep_context[ind].op[op].seg_size; + j++) + { + /* Avoid using all zero bits the 1st time */ + buff[j] = (iteration + 1) & 0xFF; + } + } + } + break; + } + } /* end switch transfer_type */ + } /* end for each op */ + } /* end for each ep per thread */ +} + + +/*------------------------------------------------------------------------------ */ +bool +DT_Transaction_Validation_Check (DT_Tdep_Print_Head *phead, + Transaction_Test_t * test_ptr, + int iteration) +{ + bool ours; + bool success = true; + unsigned int op; + unsigned int i; + unsigned int j; + unsigned int ind; + unsigned char *buff; + unsigned char expect; + unsigned char got; + + DT_Tdep_PT_Debug (1,(phead,"Test[" F64x "]: VALIDATE Buffers Iteration %d\n", + test_ptr->base_port, + iteration)); + + /* + * fill all but the last three ops, which + * were added to create barriers for data validation + */ + for (ind = 0; ind < test_ptr->cmd->eps_per_thread; ind++) + { + for (op = 0; op < test_ptr->cmd->num_ops - 3; op++) + { + ours = (test_ptr->is_server == + test_ptr->ep_context[ind].op[op].server_initiated); + + switch (test_ptr->ep_context[ind].op[op].transfer_type) + { + case RDMA_READ: + { + if (ours) + { + for (i = 0; + i < test_ptr->ep_context[ind].op[op].num_segs; i++) + { + + buff = DT_Bpool_GetBuffer ( + test_ptr->ep_context[ind].op[op].bp, i); + + for (j = 0; + j < test_ptr->ep_context[ind].op[op].seg_size; + j++) + { + + expect = (iteration + 1) & 0xFF; + got = buff[j]; + if (expect != got) + { + DT_Tdep_PT_Printf (phead, + "Test[" F64x "]: Validation Error :: %d\n", + test_ptr->base_port, + op); + DT_Tdep_PT_Printf (phead, + "Test[" F64x "]: Expected %x Got %x\n", + test_ptr->base_port, + expect, + got); + DT_Tdep_PT_Debug (3, + (phead, + "\twq=%d op=%d seg=%d byte=%d ptr=%p\n", + ind, op, i, j, buff)); + success = false; + break; + } + } + } + } + break; + } + + case RDMA_WRITE: + { + if (!ours) + { + for (i = 0; + i < test_ptr->ep_context[ind].op[op].num_segs; + i++) + { + + buff = DT_Bpool_GetBuffer ( + test_ptr->ep_context[ind].op[op].bp, i); + for (j = 0; + j < test_ptr->ep_context[ind].op[op].seg_size; + j++) + { + + expect = (iteration + 1) & 0xFF; + got = buff[j]; + if (expect != got) + { + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Validation Error :: %d\n", + test_ptr->base_port, + op); + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Expected %x Got %x\n", + test_ptr->base_port, + expect, + got); + DT_Tdep_PT_Debug (3, + (phead, + "\twq=%d op=%d seg=%d byte=%d ptr=%p\n", + ind, op, i, j, buff)); + success = false; + break; + } + } + } + } + break; + } + + case SEND_RECV: + { + if (!ours) + { + for (i = 0; + i < test_ptr->ep_context[ind].op[op].num_segs; + i++) + { + + buff = DT_Bpool_GetBuffer ( + test_ptr->ep_context[ind].op[op].bp, i); + DT_Tdep_PT_Debug (3, ( + phead, + "\tCheck:wq=%d op=%d seg=%d ptr=[%p, %d]\n", + ind, op, i, buff, + test_ptr->ep_context[ind].op[op].seg_size)); + + for (j = 0; + j < test_ptr->ep_context[ind].op[op].seg_size; + j++) + { + + expect = (iteration + 1) & 0xFF; + got = buff[j]; + if (expect != got) + { + DT_Tdep_PT_Printf (phead, + "Test[" F64x "]: Validation Error :: %d\n", + test_ptr->base_port, + op); + DT_Tdep_PT_Printf (phead, "Test[" F64x "]: Expected %x Got %x\n", + test_ptr->base_port, + expect, + got); + DT_Tdep_PT_Debug (3, + (phead, + "\twq=%d op=%d seg=%d byte=%d ptr=%p\n", + ind, op, i, j, buff)); + success = false; + break; + } + } + } + } + break; + } + } /* end switch transfer_type */ + } /* end for each op */ + } /* end for each ep per thread */ + + return ( success ); +} + + +/*------------------------------------------------------------------------------ */ +void +DT_Print_Transaction_Test (DT_Tdep_Print_Head *phead, Transaction_Test_t * test_ptr) +{ + DT_Tdep_PT_Printf (phead, "-------------------------------------\n"); + DT_Tdep_PT_Printf (phead, "TransTest.is_server : %d\n", + test_ptr->is_server); + DT_Tdep_PT_Printf (phead, "TransTest.remote_little_endian : %d\n", + test_ptr->remote_is_little_endian); + DT_Tdep_PT_Printf (phead, "TransTest.base_port : " F64x "\n", + test_ptr->base_port); + DT_Tdep_PT_Printf (phead, "TransTest.pz_handle : %p\n", + test_ptr->pz_handle); + /* statistics */ + DT_Tdep_PT_Printf (phead, "TransTest.bytes_send : %d\n", + test_ptr->stats.stat_bytes_send); + DT_Tdep_PT_Printf (phead, "TransTest.bytes_recv : %d\n", + test_ptr->stats.stat_bytes_recv); + DT_Tdep_PT_Printf (phead, "TransTest.bytes_rdma_read : %d\n", + test_ptr->stats.stat_bytes_rdma_read); + DT_Tdep_PT_Printf (phead, "TransTest.bytes_rdma_write : %d\n", + test_ptr->stats.stat_bytes_rdma_write); +} + + +/*------------------------------------------------------------------------------ */ +void +DT_Print_Transaction_Stats (DT_Tdep_Print_Head *phead, Transaction_Test_t * test_ptr) +{ + double time; + double mbytes_send; + double mbytes_recv; + double mbytes_rdma_read; + double mbytes_rdma_write; + int total_ops; + time = (double) (test_ptr->stats.end_time - test_ptr->stats.start_time) / 1000; + mbytes_send = (double) test_ptr->stats.stat_bytes_send / 1024 / 1024; + mbytes_recv = (double) test_ptr->stats.stat_bytes_recv / 1024 / 1024; + mbytes_rdma_read = (double) test_ptr->stats.stat_bytes_rdma_read / 1024 / 1024; + mbytes_rdma_write = (double) test_ptr->stats.stat_bytes_rdma_write / 1024 / 1024; + total_ops = test_ptr->cmd->num_ops * test_ptr->cmd->num_iterations; + + DT_Tdep_PT_Printf (phead, "Test[: " F64x "] ---- Stats ----\n", test_ptr->base_port); + DT_Tdep_PT_Printf (phead, "Iterations : %u\n", test_ptr->cmd->num_iterations); + DT_Tdep_PT_Printf (phead, "Ops : %7d.%02d Ops/Sec\n", + whole (total_ops / time), + hundredths (total_ops / time)); + DT_Tdep_PT_Printf (phead, "Time : %7d.%02d sec\n", + whole (time), + hundredths (time)); + DT_Tdep_PT_Printf (phead, "Sent : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_send), + hundredths (mbytes_send), + whole (mbytes_send / time), + hundredths (mbytes_send / time)); + DT_Tdep_PT_Printf (phead, "Recv : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_recv), + hundredths (mbytes_recv), + whole (mbytes_recv / time), + hundredths (mbytes_recv / time)); + DT_Tdep_PT_Printf (phead, "RDMA Read : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_rdma_read), + hundredths (mbytes_rdma_read), + whole (mbytes_rdma_read / time), + hundredths (mbytes_rdma_read / time)); + DT_Tdep_PT_Printf (phead, "RDMA Write : %7d.%02d MB - %7d.%02d MB/Sec\n", + whole (mbytes_rdma_write), + hundredths (mbytes_rdma_write), + whole (mbytes_rdma_write / time), + hundredths (mbytes_rdma_write / time)); +} + diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_util.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_util.c new file mode 100644 index 00000000..4b886faa --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_transaction_util.c @@ -0,0 +1,800 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* ----------------------------------------------------------- + * Post a recv buffer on each of this thread's EPs. + */ +bool +DT_handle_post_recv_buf (DT_Tdep_Print_Head *phead, + Ep_Context_t * ep_context, + unsigned int num_eps, + int op_indx) +{ + unsigned int i, j; + + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op = &ep_context[i].op[op_indx]; + DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV (op->bp, 0); + DAT_DTO_COOKIE cookie; + DAT_RETURN ret; + + /* Prep the inputs */ + for (j = 0; j < op->num_segs; j++) + { + iov[j].virtual_address = (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (op->bp, j); + iov[j].segment_length = op->seg_size; + iov[j].lmr_context = DT_Bpool_GetLMR (op->bp, j); + } + cookie.as_64 = + ((((DAT_UINT64) i) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) & 0xffffffffUL)); + + /* Post the recv */ + ret = dat_ep_post_recv ( ep_context[i].ep_handle, + op->num_segs, + iov, + cookie, + DAT_COMPLETION_DEFAULT_FLAG); + + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "Test Error: dat_ep_post_recv failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + return false; + } + } + + return true; +} + + +/* ----------------------------------------------------------- + * Post a send buffer on each of this thread's EPs. + */ +bool +DT_handle_send_op (DT_Tdep_Print_Head *phead, + Ep_Context_t * ep_context, + DAT_EVD_HANDLE reqt_evd_hdl, + unsigned int num_eps, + int op_indx, + bool poll) +{ + unsigned int i, j; + unsigned char *completion_reaped; + + completion_reaped = DT_Mdep_Malloc (num_eps * sizeof (unsigned char)); + + if (!completion_reaped) + { + return false; + } + + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op = &ep_context[i].op[op_indx]; + DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV (op->bp, 0); + DAT_DTO_COOKIE cookie; + DAT_RETURN ret; + + /* Prep the inputs */ + for (j = 0; j < op->num_segs; j++) + { + iov[j].virtual_address = (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (op->bp, j); + iov[j].segment_length = op->seg_size; + iov[j].lmr_context = DT_Bpool_GetLMR (op->bp, j); + } + cookie.as_64 = + ((((DAT_UINT64) i) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) & 0xffffffffUL)); + + /* Post the send */ + ret = dat_ep_post_send ( ep_context[i].ep_handle, + op->num_segs, + iov, + cookie, + DAT_COMPLETION_DEFAULT_FLAG); + + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, + "Test Error: dat_ep_post_send failed: %s\n", + DT_RetToString (ret)); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return false; + } + } + + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op = &ep_context[i].op[op_indx]; + + if (op->reap_send_on_recv && !op->server_initiated) + { + /* we will reap the send on the recv (Client SR) */ + DT_Mdep_Free (completion_reaped); + return true; + } + } + + bzero ((void *) completion_reaped, sizeof (unsigned char) * num_eps); + + /* reap the send completion */ + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op; + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_DTO_COOKIE dto_cookie; + unsigned int epnum; + + if (!DT_dto_event_reap (phead, reqt_evd_hdl, poll, &dto_stat)) + { + DT_Mdep_Free (completion_reaped); + return false; + } + + epnum = dto_stat.user_cookie.as_64 >> 32; + if (epnum > num_eps) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Send: Invalid endpoint completion reaped.\n" + "\tEndpoint: 0x%p, Cookie: 0x" F64x ", Length: " F64u "\n", + dto_stat.ep_handle, dto_stat.user_cookie.as_64, + dto_stat.transfered_length); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return false; + } + + op = &ep_context[epnum].op[op_indx]; + + dto_cookie.as_64 = + ((((DAT_UINT64) epnum) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) & 0xffffffffUL)); + + if (!DT_dto_check (phead, + &dto_stat, + ep_context[epnum].ep_handle, + op->num_segs * op->seg_size, + dto_cookie, + "Send")) + { + DT_Mdep_Free (completion_reaped); + return false; + } + + if (completion_reaped[epnum]) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Send: Secondary completion seen for endpoint 0x%p (%d)\n", + ep_context[epnum].ep_handle, epnum); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return ( false ); + } + completion_reaped[epnum] = 1; + } + + for (i = 0; i < num_eps; i++) + { + if (completion_reaped[i] == 0) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Send: No completion seen for endpoint 0x%p (#%d)\n", + ep_context[i].ep_handle, i); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return ( false ); + } + } + + DT_Mdep_Free (completion_reaped); + return true; +} + + +/* ----------------------------------------------------------- + * Reap a recv op on each of this thread's EPs, + * then if requested reap the corresponding send ops, + * and re-post all of the recv buffers. + */ +bool +DT_handle_recv_op (DT_Tdep_Print_Head *phead, + Ep_Context_t * ep_context, + DAT_EVD_HANDLE recv_evd_hdl, + DAT_EVD_HANDLE reqt_evd_hdl, + unsigned int num_eps, + int op_indx, + bool poll, + bool repost_recv) +{ + unsigned int i; + unsigned char *recv_completion_reaped; + unsigned char *send_completion_reaped; + + recv_completion_reaped = DT_Mdep_Malloc (num_eps); + if (recv_completion_reaped == NULL) + { + return false; + } + + send_completion_reaped = DT_Mdep_Malloc (num_eps); + if (send_completion_reaped == NULL) + { + DT_Mdep_Free (recv_completion_reaped); + return false; + } + + /* Foreach EP, reap */ + bzero ((void *) recv_completion_reaped, sizeof (unsigned char) * num_eps); + bzero ((void *) send_completion_reaped, sizeof (unsigned char) * num_eps); + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op; + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_DTO_COOKIE dto_cookie; + unsigned int epnum; + + /* First reap the recv DTO event */ + if (!DT_dto_event_reap (phead, recv_evd_hdl, poll, &dto_stat)) + { + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + epnum = dto_stat.user_cookie.as_64 >> 32; + if (epnum > num_eps) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Receive: Invalid endpoint completion reaped.\n" + "\tEndpoint: 0x%p, Cookie: 0x" F64x ", Length: " F64u "\n", + dto_stat.ep_handle, dto_stat.user_cookie.as_64, + dto_stat.transfered_length); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + op = &ep_context[epnum].op[op_indx]; + dto_cookie.as_64 = + ((((DAT_UINT64) epnum) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) & 0xffffffffUL)); + + if (!DT_dto_check (phead, + &dto_stat, + ep_context[epnum].ep_handle, + op->num_segs * op->seg_size, + dto_cookie, + "Recv")) + { + DT_Tdep_PT_Printf (phead, + "Test Error: recv DTO problem\n"); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + if (recv_completion_reaped[epnum]) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Receive: Secondary completion seen for endpoint 0x%p (%d)\n", + ep_context[epnum].ep_handle, epnum); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return ( false ); + } + recv_completion_reaped[epnum] = 1; + + /* + * Check the current op to see whether we are supposed + * to reap the previous send op now. + */ + if (op->reap_send_on_recv && op->server_initiated) + { + if (op_indx <= 0) + /* shouldn't happen, but let's be certain */ + { + DT_Tdep_PT_Printf (phead, + "Internal Error: reap_send_on_recv" + " but current op == #%d\n", op_indx); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + if (!DT_dto_event_reap (phead, reqt_evd_hdl, poll, &dto_stat)) + { + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + epnum = dto_stat.user_cookie.as_64 >> 32; + if (epnum > num_eps) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Send (ror): Invalid endpoint completion reaped.\n" + "\tEndpoint: 0x%p, Cookie: 0x" F64x ", Length: "F64u "\n", + dto_stat.ep_handle, dto_stat.user_cookie.as_64, + dto_stat.transfered_length); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + /* + * We're reaping the last transaction, a + * send completion that we skipped when it was sent. + */ + op = &ep_context[epnum].op[op_indx - 1]; + + dto_cookie.as_64 = + ((((DAT_UINT64) epnum) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) + & 0xffffffffUL)); + + /* + * If we have multiple EPs we can't guarantee the order of + * completions, so disable ep_handle check + */ + if (!DT_dto_check (phead, + &dto_stat, + num_eps == 1?ep_context[i].ep_handle: NULL, + op->num_segs * op->seg_size, + dto_cookie, + "Send-reaped-on-recv")) + { + DT_Tdep_PT_Printf (phead, + "Test Error: send DTO problem\n"); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + + if (send_completion_reaped[epnum]) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Send (ror): Secondary completion seen for endpoint 0x%p (%d)\n", + ep_context[epnum].ep_handle, epnum); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return ( false ); + } + send_completion_reaped[epnum] = 1; + } + } + + for (i = 0; i < num_eps; i++) + { + if (recv_completion_reaped[i] == 0) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Receive: No completion seen for endpoint 0x%p (#%d)\n", + ep_context[i].ep_handle, i); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return ( false ); + } + } + + if (ep_context[0].op[op_indx].reap_send_on_recv + && ep_context[0].op[op_indx].server_initiated) + { + for (i = 0; i < num_eps; i++) + { + if (send_completion_reaped[i] == 0) + { + DT_Tdep_PT_Printf (phead, + "Test Error: Send (ror): No completion seen for endpoint 0x%p (#%d)\n", + ep_context[i].ep_handle, i); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return ( false ); + } + } + } + + if (repost_recv) + { + /* repost the receive buffer */ + if (!DT_handle_post_recv_buf (phead, ep_context, num_eps, op_indx)) + { + DT_Tdep_PT_Printf (phead, + "Test Error: recv re-post problem\n"); + DT_Test_Error (); + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return false; + } + } + + DT_Mdep_Free (recv_completion_reaped); + DT_Mdep_Free (send_completion_reaped); + return true; +} + + +/* ----------------------------------------------------------- + * Initiate an RDMA op (synchronous) on each of this thread's EPs. + */ +bool +DT_handle_rdma_op (DT_Tdep_Print_Head *phead, + Ep_Context_t * ep_context, + DAT_EVD_HANDLE reqt_evd_hdl, + unsigned int num_eps, + DT_Transfer_Type opcode, + int op_indx, + bool poll) +{ + unsigned int i, j; + DAT_RETURN ret; + unsigned char *completion_reaped; + + completion_reaped = DT_Mdep_Malloc (num_eps * sizeof (unsigned char)); + + if (!completion_reaped) + { + return false; + } + + /* Initiate the operation */ + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op = &ep_context[i].op[op_indx]; + DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV (op->bp, 0); + DAT_DTO_COOKIE cookie; + DAT_RMR_TRIPLET rmr_triplet; + + /* Prep the inputs */ + for (j = 0; j < op->num_segs; j++) + { + iov[j].virtual_address = (DAT_VADDR) (uintptr_t) + DT_Bpool_GetBuffer (op->bp, j); + iov[j].segment_length = op->seg_size; + iov[j].lmr_context = DT_Bpool_GetLMR (op->bp, j); + } + cookie.as_64 = + ((((DAT_UINT64) i) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) & 0xffffffffUL)); + + rmr_triplet.virtual_address = (DAT_VADDR) (uintptr_t) op->Rdma_Address; + rmr_triplet.segment_length = op->seg_size * op->num_segs; + rmr_triplet.rmr_context = op->Rdma_Context; + + DT_Tdep_PT_Debug (3, (phead, + "Call dat_ep_post_rdma_%s [" F64x ", sz=" F64x ", ctxt=%x]\n", + (opcode == RDMA_WRITE ? "write" : "read"), + rmr_triplet.virtual_address, + rmr_triplet.segment_length, + rmr_triplet.rmr_context )); + + /* Post the operation */ + if (opcode == RDMA_WRITE) + { + + ret = dat_ep_post_rdma_write (ep_context[i].ep_handle, + op->num_segs, + iov, + cookie, + &rmr_triplet, + DAT_COMPLETION_DEFAULT_FLAG); + + } + else /* opcode == RDMA_READ */ + { + + ret = dat_ep_post_rdma_read ( ep_context[i].ep_handle, + op->num_segs, + iov, + cookie, + &rmr_triplet, + DAT_COMPLETION_DEFAULT_FLAG); + + } + if (ret != DAT_SUCCESS) + { + DT_Tdep_PT_Printf (phead, "Test Error: dat_ep_post_rdma_%s failed: %s\n", + (opcode == RDMA_WRITE ? "write" : "read"), + DT_RetToString (ret)); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return ( false ); + } + else + { + DT_Tdep_PT_Debug (3, (phead, + "Done dat_ep_post_rdma_%s %s\n", + (opcode == RDMA_WRITE ? "write" : "read"), + " () Waiting ...")); + } + } + + bzero ((void *) completion_reaped, sizeof (unsigned char) * num_eps); + /* Wait for it to happen */ + for (i = 0; i < num_eps; i++) + { + Transaction_Test_Op_t *op; + DAT_DTO_COMPLETION_EVENT_DATA dto_stat; + DAT_DTO_COOKIE dto_cookie; + unsigned int epnum; + + if (!DT_dto_event_reap (phead, reqt_evd_hdl, poll, &dto_stat)) + { + DT_Mdep_Free (completion_reaped); + return ( false ); + } + + epnum = dto_stat.user_cookie.as_64 >> 32; + if (epnum > num_eps) + { + DT_Tdep_PT_Printf (phead, "Test Error: %s: Invalid endpoint completion reaped.\n" + "\tEndpoint: 0x%p, Cookie: 0x" F64x ", Length: " F64u "\n", + opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", + dto_stat.ep_handle, dto_stat.user_cookie.as_64, + dto_stat.transfered_length); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return false; + } + op = &ep_context[epnum].op[op_indx]; + + dto_cookie.as_64 = + ((((DAT_UINT64) epnum) << 32) + | (((uintptr_t) DT_Bpool_GetBuffer (op->bp, 0)) & 0xffffffffUL)); + + if (!DT_dto_check (phead, + &dto_stat, + ep_context[epnum].ep_handle, + op->num_segs * op->seg_size, + dto_cookie, + (opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))) + { + DT_Mdep_Free (completion_reaped); + return ( false ); + } + + if (completion_reaped[epnum]) + { + DT_Tdep_PT_Printf (phead, "Test Error: %s: Secondary completion seen for endpoint 0x%p (%d)\n", + opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", + ep_context[epnum].ep_handle, epnum); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return ( false ); + } + completion_reaped[epnum] = 1; + + DT_Tdep_PT_Debug (3, (phead, + "dat_ep_post_rdma_%s OK\n", + (opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))); + } + + for (i = 0; i < num_eps; i++) + { + if (completion_reaped[i] == 0) + { + DT_Tdep_PT_Printf (phead, "Test Error: %s: No completion seen for endpoint 0x%p (#%d)\n", + opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", + ep_context[i].ep_handle, i); + DT_Test_Error (); + DT_Mdep_Free (completion_reaped); + return ( false ); + } + } + + DT_Mdep_Free (completion_reaped); + + return ( true ); +} + + +/* ----------------------------------------------------------- + * Verify whether we (the client side) can support + * the requested 'T' test. + */ +bool +DT_check_params (Per_Test_Data_t *pt_ptr, + char *module) +{ + Transaction_Cmd_t * cmd = &pt_ptr->Params.u.Transaction_Cmd; + unsigned long num_recvs = 0U; + unsigned long num_sends = 0U; + unsigned long num_rdma_rd = 0U; + unsigned long num_rdma_wr = 0U; + unsigned long max_size = 0U; + unsigned long max_segs = 0U; + bool rval = true; + unsigned int i; + DT_Tdep_Print_Head *phead; + + phead = pt_ptr->Params.phead; + + /* Count up what's requested (including -V appended sync points) */ + for (i = 0; i < cmd->num_ops; i++) + { + unsigned int xfer_size; + + xfer_size = cmd->op[i].num_segs * cmd->op[i].seg_size; + if (xfer_size > max_size) + { + max_size = xfer_size; + } + if (cmd->op[i].num_segs > max_segs) + { + max_segs = cmd->op[i].num_segs; + } + + switch (cmd->op[i].transfer_type) + { + case SEND_RECV: + { + if (cmd->op[i].server_initiated) + { + num_recvs++; + } + else + { + num_sends++; + } + break; + } + + case RDMA_READ: + { + num_rdma_rd++; + break; + } + + case RDMA_WRITE: + { + num_rdma_wr++; + break; + } + } + } + + /* + * Now check the IA and EP attributes, and check for some of the + * more obvious resource problems. This is hardly exhaustive, + * and some things will inevitably fall through to run-time. + * + * We don't compare + * num_rdma_rd > pt_ptr->ia_attr.max_rdma_read_per_ep + * num_rdma_wr > pt_ptr->ia_attr.max_dto_per_ep + * because each thread has its own EPs, and transfers are issued + * synchronously (across a thread's EPs, and ignoring -f, which allows + * a per-EP pipeline depth of at most 2 and applies only to SR ops), + * so dapltest actually attempts almost no pipelining on a single EP. + * But we do check that pre-posted recv buffers will all fit. + */ + if (num_recvs > pt_ptr->ia_attr.max_dto_per_ep || + num_sends > pt_ptr->ia_attr.max_dto_per_ep) + { + DT_Tdep_PT_Printf ( + phead, + "%s: S/R: cannot supply %ld SR ops (maximum: %d)\n", + module, + num_recvs > num_sends ? num_recvs : num_sends, + pt_ptr->ia_attr.max_dto_per_ep); + rval = false; + } + if (max_size > pt_ptr->ia_attr.max_lmr_block_size) + { + DT_Tdep_PT_Printf ( + phead, + "%s: buffer too large: 0x%lx (maximum: " F64x " bytes)\n", + module, + max_size, + pt_ptr->ia_attr.max_lmr_block_size); + rval = false; + } + if (max_segs > pt_ptr->ep_attr.max_recv_iov || + max_segs > pt_ptr->ep_attr.max_request_iov) + { + /* + * In an ideal world, we'd just ask for more segments + * when creating the EPs for the test, rather than + * checking against default EP attributes. + */ + DT_Tdep_PT_Printf ( + phead, + "%s: cannot use %ld segments (maxima: S %d, R %d)\n", + module, + max_segs, + pt_ptr->ep_attr.max_request_iov, + pt_ptr->ep_attr.max_recv_iov ); + rval = false; + } + + return ( rval ); +} + +/* Empty function in which to set breakpoints. */ +void +DT_Test_Error (void) +{ + ; +} + +void +DT_Transaction_Cmd_PT_Print (DT_Tdep_Print_Head *phead, Transaction_Cmd_t * cmd) +{ + unsigned int i; + DT_Tdep_PT_Printf (phead, "-------------------------------------\n"); + DT_Tdep_PT_Printf (phead, "TransCmd.server_name : %s\n", + cmd->server_name); + DT_Tdep_PT_Printf (phead, "TransCmd.num_iterations : %d\n", + cmd->num_iterations); + DT_Tdep_PT_Printf (phead, "TransCmd.num_threads : %d\n", + cmd->num_threads); + DT_Tdep_PT_Printf (phead, "TransCmd.eps_per_thread : %d\n", + cmd->eps_per_thread); + DT_Tdep_PT_Printf (phead, "TransCmd.validate : %d\n", + cmd->validate); + DT_Tdep_PT_Printf (phead, "TransCmd.dapl_name : %s\n", + cmd->dapl_name); + DT_Tdep_PT_Printf (phead, "TransCmd.num_ops : %d\n", + cmd->num_ops); + + for (i = 0; i < cmd->num_ops; i++) + { + DT_Tdep_PT_Printf (phead, "TransCmd.op[%d].transfer_type : %s %s\n", + i, + cmd->op[i].transfer_type == 0 ? "RDMA_READ" : + cmd->op[i].transfer_type == 1 ? "RDMA_WRITE" : + "SEND_RECV", + cmd->op[i].server_initiated ? " (server)" : " (client)" ); + DT_Tdep_PT_Printf (phead, "TransCmd.op[%d].seg_size : %d\n", + i, + cmd->op[i].seg_size); + DT_Tdep_PT_Printf (phead, "TransCmd.op[%d].num_segs : %d\n", + i, + cmd->op[i].num_segs); + DT_Tdep_PT_Printf (phead, "TransCmd.op[%d].reap_send_on_recv : %d\n", + i, + cmd->op[i].reap_send_on_recv); + } +} diff --git a/trunk/ulp/dapl2/test/dapltest/test/dapl_util.c b/trunk/ulp/dapl2/test/dapltest/test/dapl_util.c new file mode 100644 index 00000000..c61b787a --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/test/dapl_util.c @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +/* + * Map DAT_RETURN values to readable strings, + * but don't assume the values are zero-based or contiguous. + */ +const char * +DT_RetToString (DAT_RETURN ret_value) +{ + const char *major_msg, *minor_msg; + int sz; + char *errmsg; + + dat_strerror (ret_value, &major_msg, &minor_msg); + + sz = strlen (major_msg) + strlen (minor_msg) + 2; +/* + * FIXME: The callers of this function are not freeing + * the errmsg string. Hence there is a memory leak + * (this function is likely only used on error paths, + * so the consequences may not be that dire). + */ + errmsg = DT_Mdep_Malloc (sz); + strcpy (errmsg, major_msg); + strcat (errmsg, " "); + strcat (errmsg, minor_msg); + + return errmsg; +} + +/* + * Map DAT_RETURN values to readable strings, + * but don't assume the values are zero-based or contiguous. + */ +const char * +DT_TransferTypeToString (DT_Transfer_Type type) +{ + static char *DT_Type[] = + { + "RR", + "RW", + "SR" + }; + + if ( (0 <= type) && (type <= 2) ) + { + return DT_Type[type]; + } + else + { + return "Error: Unkown Transfer Type"; + } +} + + +/* + * Map DAT_ASYNC_ERROR_CODE values to readable strings + */ +const char * +DT_AsyncErr2Str (DAT_EVENT_NUMBER error_code) +{ + unsigned int i; + static struct + { + const char *name; + DAT_RETURN value; + } dat_errors[] = + { +# define DATxx(x) { # x, x } + DATxx (DAT_DTO_COMPLETION_EVENT), + DATxx (DAT_RMR_BIND_COMPLETION_EVENT), + DATxx (DAT_CONNECTION_REQUEST_EVENT), + DATxx (DAT_CONNECTION_EVENT_ESTABLISHED), + DATxx (DAT_CONNECTION_EVENT_PEER_REJECTED), + DATxx (DAT_CONNECTION_EVENT_NON_PEER_REJECTED), + DATxx (DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR), + DATxx (DAT_CONNECTION_EVENT_DISCONNECTED), + DATxx (DAT_CONNECTION_EVENT_BROKEN), + DATxx (DAT_CONNECTION_EVENT_TIMED_OUT), + DATxx (DAT_ASYNC_ERROR_EVD_OVERFLOW), + DATxx (DAT_ASYNC_ERROR_IA_CATASTROPHIC), + DATxx (DAT_ASYNC_ERROR_EP_BROKEN), + DATxx (DAT_ASYNC_ERROR_TIMED_OUT), + DATxx (DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR), + DATxx (DAT_SOFTWARE_EVENT) +# undef DATxx + }; +# define NUM_ERRORS (sizeof (dat_errors)/sizeof (dat_errors[0])) + + for (i = 0; i < NUM_ERRORS; i++) + { + if (dat_errors[i].value == error_code) + { + return ( dat_errors[i].name ); + } + } + + return ( "Invalid_DAT_EVENT_NUMBER" ); +} + +/* + * Map DAT_EVENT_CODE values to readable strings + */ +const char * +DT_EventToSTr (DAT_EVENT_NUMBER event_code) +{ + unsigned int i; + static struct + { + const char *name; + DAT_RETURN value; + } + dat_events[] = + { +# define DATxx(x) { # x, x } + DATxx (DAT_DTO_COMPLETION_EVENT), + DATxx (DAT_RMR_BIND_COMPLETION_EVENT), + DATxx (DAT_CONNECTION_REQUEST_EVENT), + DATxx (DAT_CONNECTION_EVENT_ESTABLISHED), + DATxx (DAT_CONNECTION_EVENT_PEER_REJECTED), + DATxx (DAT_CONNECTION_EVENT_NON_PEER_REJECTED), + DATxx (DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR), + DATxx (DAT_CONNECTION_EVENT_DISCONNECTED), + DATxx (DAT_CONNECTION_EVENT_BROKEN), + DATxx (DAT_CONNECTION_EVENT_TIMED_OUT), + DATxx (DAT_CONNECTION_EVENT_UNREACHABLE), + DATxx (DAT_ASYNC_ERROR_EVD_OVERFLOW), + DATxx (DAT_ASYNC_ERROR_IA_CATASTROPHIC), + DATxx (DAT_ASYNC_ERROR_EP_BROKEN), + DATxx (DAT_ASYNC_ERROR_TIMED_OUT), + DATxx (DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR), + DATxx (DAT_SOFTWARE_EVENT) +# undef DATxx + }; +# define NUM_EVENTS (sizeof (dat_events)/sizeof (dat_events[0])) + + for (i = 0; i < NUM_EVENTS; i++) + { + if (dat_events[i].value == event_code) + { + return ( dat_events[i].name ); + } + } + + return ( "Invalid_DAT_EVENT_NUMBER" ); +} + + +/* + * Map DAT_EP_STATE_CODE values to readable strings + */ +const char * +DT_State2Str (DAT_EP_STATE state_code) +{ + unsigned int i; + static struct + { + const char *name; + DAT_RETURN value; + } + dat_state[] = + { +# define DATxx(x) { # x, x } + DATxx (DAT_EP_STATE_UNCONNECTED), + DATxx (DAT_EP_STATE_RESERVED), + DATxx (DAT_EP_STATE_PASSIVE_CONNECTION_PENDING), + DATxx (DAT_EP_STATE_ACTIVE_CONNECTION_PENDING), + DATxx (DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING), + DATxx (DAT_EP_STATE_CONNECTED), + DATxx (DAT_EP_STATE_DISCONNECT_PENDING), + DATxx (DAT_EP_STATE_ERROR) +# undef DATxx + }; +# define NUM_STATES (sizeof (dat_state)/sizeof (dat_state[0])) + + for (i = 0; i < NUM_STATES; i++) + { + if (dat_state[i].value == state_code) + { + return ( dat_state[i].name ); + } + } + + return ( "Invalid_DAT_STATE_NUMBER" ); +} + + +/* + * A couple of round-up routines (for pointers and counters) + * which both assume a power-of-two 'align' factor, + * and do the correct thing if align == 0. + */ +unsigned char * +DT_AlignPtr (void * val, DAT_COUNT align) +{ + if (align) + { +#if defined(_WIN64) + return ( (unsigned char *) + (((uint64_t)val+((uint64_t)align) - 1) & ~ (((uint64_t)align) - 1))); +#else + return ( (unsigned char *) + (((unsigned long)val + ((unsigned long)align) - 1) & ~ (((unsigned long)align) - 1))); +#endif + } + return (val); +} + +DAT_COUNT +DT_RoundSize (DAT_COUNT val, DAT_COUNT align) +{ + if (align) + { + return ( ((val + align - 1) & ~ (align - 1)) ); + } + return ( val ); +} diff --git a/trunk/ulp/dapl2/test/dapltest/udapl/udapl_tdep.c b/trunk/ulp/dapl2/test/dapltest/udapl/udapl_tdep.c new file mode 100644 index 00000000..073ed20b --- /dev/null +++ b/trunk/ulp/dapl2/test/dapltest/udapl/udapl_tdep.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * in the file LICENSE.txt in the root directory. The license is also + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is in the file + * LICENSE2.txt in the root directory. The license is also available from + * the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is in the file LICENSE3.txt in the root directory. The + * license is also available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +#include "dapl_proto.h" + +void +DT_Tdep_Init (void) +{ + DT_Mdep_LockInit (&g_PerfTestLock); /* For kDAPL, this is done in kdapl_module.c */ +} + +void +DT_Tdep_End (void) +{ + DT_Mdep_LockDestroy (&g_PerfTestLock); /* For kDAPL, this is done in kdapl_module.c */ +} + +void +DT_Tdep_Execute_Test (Params_t *params_ptr) +{ + DT_Execute_Test (params_ptr); +} + +DAT_RETURN +DT_Tdep_lmr_create (DAT_IA_HANDLE ia_handle, + DAT_MEM_TYPE mem_type, + DAT_REGION_DESCRIPTION region, + DAT_VLEN len, + DAT_PZ_HANDLE pz_handle, + DAT_MEM_PRIV_FLAGS priv_flag, + DAT_LMR_HANDLE *lmr_handle_ptr, + DAT_LMR_CONTEXT *lmr_context_ptr, + DAT_RMR_CONTEXT *rmr_context_ptr, + DAT_VLEN *reg_size_ptr, + DAT_VADDR *reg_addr_ptr) +{ + return dat_lmr_create (ia_handle, + mem_type, + region, + len, + pz_handle, + priv_flag, + DAT_VA_TYPE_VA, + lmr_handle_ptr, + lmr_context_ptr, + rmr_context_ptr, + reg_size_ptr, + reg_addr_ptr); +} + +DAT_RETURN +DT_Tdep_evd_create (DAT_IA_HANDLE ia_handle, + DAT_COUNT evd_min_qlen, + DAT_CNO_HANDLE cno_handle, + DAT_EVD_FLAGS evd_flags, + DAT_EVD_HANDLE *evd_handle_ptr) +{ + return dat_evd_create (ia_handle, + evd_min_qlen, + DAT_HANDLE_NULL, + evd_flags, + evd_handle_ptr); +} + +DAT_RETURN +DT_Tdep_evd_free (DAT_EVD_HANDLE evd_handle) +{ + return dat_evd_free (evd_handle); +} + +DAT_RETURN +DT_Tdep_evd_wait (DAT_EVD_HANDLE evd_handle, + DAT_TIMEOUT timeout, + DAT_EVENT *event) +{ + DAT_COUNT count; + + return dat_evd_wait (evd_handle, + timeout, + 1, + event, + &count); +} + + +void +DT_Tdep_PT_Printf (DT_Tdep_Print_Head *phead, const char * fmt, ...) +{ + char buffer[256]; + va_list args; + + va_start (args, fmt); + vsprintf (buffer, fmt, args); + va_end (args); + + printf (buffer); +} + +DAT_RETURN +DT_Tdep_evd_dequeue (DAT_EVD_HANDLE evd_handle, + DAT_EVENT *event) +{ + return dat_evd_dequeue (evd_handle, event); +} + diff --git a/trunk/ulp/dapl2/test/dirs b/trunk/ulp/dapl2/test/dirs new file mode 100644 index 00000000..5c041791 --- /dev/null +++ b/trunk/ulp/dapl2/test/dirs @@ -0,0 +1 @@ +DIRS=dapltest dtest diff --git a/trunk/ulp/dapl2/test/dtest/GETOPT.C b/trunk/ulp/dapl2/test/dtest/GETOPT.C new file mode 100644 index 00000000..9de14fb3 --- /dev/null +++ b/trunk/ulp/dapl2/test/dtest/GETOPT.C @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------ +** +** Copyright (c) 1996, Intel Corp. All rights reserved. +** +** THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF +** Intel Corp. +** The copyright notice above does not evidence any +** actual or intended publication of such source code. +** +** +** Module: getopt.c +** +** Source: PAL, Intel Corp. Mike Ripley +** +** Abstract: +** This is source for the standard getopt() library. +** Original source was obtained from the public domain, +** and modified to better suit DOS and Win32 environments. +** +** Environment: +** Microsoft Win32 +** Microsoft Visual C++ 4.0 + + +--------------------------------------------------------------------- + + LIMITATION OF LIABILITY AND DISCLAIMER + OF + WARRANTY FOR SOFTWARE + + + + + +THE SOFTWARE THAT YOU ARE RECEIVING IS LICENSED FREE OF CHARGE. THE SOFTWARE +IS PROVIDED "AS IS." INTEL MAKES NO WARRANTY OF ANY KIND REGARDING THE +SOFTWARE, TO THE EXTENT PERMITTED BY APPLICABLE STATE LAW. INTEL WILL NOT +PROVIDE ANY INSTALLATION OR TECHNICAL SUPPORT, OR ANY OTHER KIND OF +ASSISTANCE TO YOU REGARDING YOUR USE OF THE SOFTWARE. IF THE SOFTWARE PROVES +DEFECTIVE ANYWAY, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. INTEL WILL NOT PROVIDE ANY UPDATES, ENHANCEMENTS OR EXTENSIONS +TO THE SOFTWARE. + +ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +WARRANTIES MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE EXCLUDED, +AND DO NOT APPLY TO THE SOFTWARE YOU ARE RECEIVING. + +IN ADDITION TO ANY OTHER DISCLAIMER SET FORTH HEREIN, INTEL WILL NOT HAVE ANY +LIABILITY TO YOU FOR: + + a) any defects in the software you are receiving; or + + b) any inability by you to operate the software you are receiving to any + performance specification; or + + c) any claim by you or any third party with respect to, or arising out + of, the use of the software. + + +INTEL DOES NOT MAKE ANY WARRANTIES OF ANY KIND THAT THE SOFTWARE DOES NOT OR +WILL NOT INFRINGE ANY COPYRIGHT, PATENT, TRADE SECRET OR ANY OTHER +INTELLECTUAL PROPERTY RIGHT OF ANY THIRD PARTY IN ANY COUNTRY. + +IN NO EVENT, SHALL INTEL BE LIABLE TO YOU OR ANY THIRD PARTY FOR ANY INDIRECT, +SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, INCLUDING BUT NOT +LIMITED TO LOSS OF PROFITS, LOSS OF USE OF DATA, OR INTERRUPTION OF BUSINESS, +EVEN IF ADVISED OF THE POSSIBILITIES OF SUCH DAMAGES. + + +-------------------------------------------------------------------------*/ + + + +/* Comment header from original: + * + * Here's something you've all been waiting for: the AT&T public domain + * source for getopt(3). It is the code which was given out at the 1985 + * UNIFORUM conference in Dallas. I obtained it by electronic mail + * directly from AT&T. The people there assure me that it is indeed + * in the public domain. + * + * There is no manual page. That is because the one they gave out at + * UNIFORUM was slightly different from the current System V Release 2 + * manual page. The difference apparently involved a note about the + * famous rules 5 and 6, recommending using white space between an option + * and its first argument, and not grouping options that have arguments. + * Getopt itself is currently lenient about both of these things. White + * space is allowed, but not mandatory, and the last option in a group can + * have an argument. That particular version of the man page evidently + * has no official existence, and my source at AT&T did not send a copy. + * The current SVR2 man page reflects the actual behavor of this getopt. + * However, I am not about to post a copy of anything licensed by AT&T. + */ + +#include +#include + +int opterr = 1; +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char **argv, char *opts) +{ + static int sp = 1; + register int c; + register char *cp; + + if(sp == 1) + if(optind >= argc || + argv[optind][0] != '-' || argv[optind][1] == '\0') + return(EOF); + else if(strcmp(argv[optind], "--") == 0) { + optind++; + return(EOF); + } + optopt = c = argv[optind][sp]; + if(c == ':' || (cp=strchr(opts, c)) == NULL) { + if (opterr) + fprintf(stderr,"%s: illegal option -- %c\n",argv[0],c); + if(argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return('?'); + } + if(*++cp == ':') { + if(argv[optind][sp+1] != '\0') + optarg = &argv[optind++][sp+1]; + else if(++optind >= argc) { + if (opterr) + fprintf(stderr, + "%s: option requires argument -- %c\n", + argv[0], c); + sp = 1; + return('?'); + } else + optarg = argv[optind++]; + sp = 1; + } else { + if(argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return(c); +} diff --git a/trunk/ulp/dapl2/test/dtest/GETOPT.H b/trunk/ulp/dapl2/test/dtest/GETOPT.H new file mode 100644 index 00000000..b159cbb9 --- /dev/null +++ b/trunk/ulp/dapl2/test/dtest/GETOPT.H @@ -0,0 +1,78 @@ +/*------------------------------------------------------------------------ +** +** Copyright (c) 1994, Intel Corp. All rights reserved. +** +** THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF Intel Corp. +** The copyright notice above does not evidence any +** actual or intended publication of such source code. +** +** +** Module: getopt.h +** +** Source: PAL, Intel Corp. Mike Ripley +** +** Abstract: +** This an include file for the standard getopt() library. +** +** Environment: +** Microsoft Win32 +** Microsoft Visual C++ 4.0 +** +--------------------------------------------------------------------- + + LIMITATION OF LIABILITY AND DISCLAIMER + OF + WARRANTY FOR SOFTWARE + + + + + +THE SOFTWARE THAT YOU ARE RECEIVING IS LICENSED FREE OF CHARGE. THE SOFTWARE +IS PROVIDED "AS IS." INTEL MAKES NO WARRANTY OF ANY KIND REGARDING THE +SOFTWARE, TO THE EXTENT PERMITTED BY APPLICABLE STATE LAW. INTEL WILL NOT +PROVIDE ANY INSTALLATION OR TECHNICAL SUPPORT, OR ANY OTHER KIND OF +ASSISTANCE TO YOU REGARDING YOUR USE OF THE SOFTWARE. IF THE SOFTWARE PROVES +DEFECTIVE ANYWAY, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. INTEL WILL NOT PROVIDE ANY UPDATES, ENHANCEMENTS OR EXTENSIONS +TO THE SOFTWARE. + +ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +WARRANTIES MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE EXCLUDED, +AND DO NOT APPLY TO THE SOFTWARE YOU ARE RECEIVING. + +IN ADDITION TO ANY OTHER DISCLAIMER SET FORTH HEREIN, INTEL WILL NOT HAVE ANY +LIABILITY TO YOU FOR: + + a) any defects in the software you are receiving; or + + b) any inability by you to operate the software you are receiving to any + performance specification; or + + c) any claim by you or any third party with respect to, or arising out + of, the use of the software. + + +INTEL DOES NOT MAKE ANY WARRANTIES OF ANY KIND THAT THE SOFTWARE DOES NOT OR +WILL NOT INFRINGE ANY COPYRIGHT, PATENT, TRADE SECRET OR ANY OTHER +INTELLECTUAL PROPERTY RIGHT OF ANY THIRD PARTY IN ANY COUNTRY. + +IN NO EVENT, SHALL INTEL BE LIABLE TO YOU OR ANY THIRD PARTY FOR ANY INDIRECT, +SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, INCLUDING BUT NOT +LIMITED TO LOSS OF PROFITS, LOSS OF USE OF DATA, OR INTERRUPTION OF BUSINESS, +EVEN IF ADVISED OF THE POSSIBILITIES OF SUCH DAMAGES. + + +-------------------------------------------------------------------------*/ + +#ifndef _GETOPT_H_ +#define _GETOPT_H_ + +extern int getopt(int argc, char **argv, char *opts); + +extern int optind; +extern int optopt; +extern char *optarg; +extern int opterr; + +#endif // _GETOPT_H_ diff --git a/trunk/ulp/dapl2/test/dtest/README b/trunk/ulp/dapl2/test/dtest/README new file mode 100644 index 00000000..30c7c100 --- /dev/null +++ b/trunk/ulp/dapl2/test/dtest/README @@ -0,0 +1,22 @@ +Linux +----- + +simple dapl test just for initial openIB uDAPL testing... + + dtest/dtest.c + dtest/makefile + dtest/dat.conf + +to build (default uDAPL name == IB1, ib device == mthca0, port == 1) + edit makefile and change path (DAT_LIB) to appropriate libdat.so + edit dat.conf and change path to appropriate libdapl.so + cp dat.conf to /etc/dat.conf + +to run: + server: dtest + client: dtest -h hostname + +for verbose uDAPL and uDAT debug: + + export DAPL_DBG_TYPE=0xffff + export DAT_DBG_TYPE=0xffff diff --git a/trunk/ulp/dapl2/test/dtest/SOURCES b/trunk/ulp/dapl2/test/dtest/SOURCES new file mode 100644 index 00000000..126623c3 --- /dev/null +++ b/trunk/ulp/dapl2/test/dtest/SOURCES @@ -0,0 +1,28 @@ +TARGETNAME=dtest2 +TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=PROGRAM +UMTYPE=console +USE_CRTDLL=1 + +SOURCES=dtest.rc \ + dtest.c \ + getopt.c + +INCLUDES=.;..\..\dat\include;$(SDK_INC_PATH); + +RCOPTIONS=/I..\..\..\..\inc; + +# Set defines particular to the driver. +#USER_C_FLAGS=$(USER_C_FLAGS) /DDAT_EXTENSIONS + +!if $(FREEBUILD) +DATLIB=dat2.lib +!else +DATLIB=dat2d.lib +!endif + +TARGETLIBS=$(TARGETPATH)\*\$(DATLIB) $(SDK_LIB_PATH)\ws2_32.lib + +# XXX do this ASAP - MSC_WARNING_LEVEL= /W3 +MSC_WARNING_LEVEL= /W1 + diff --git a/trunk/ulp/dapl2/test/dtest/dtest.c b/trunk/ulp/dapl2/test/dtest/dtest.c new file mode 100644 index 00000000..5e45f29a --- /dev/null +++ b/trunk/ulp/dapl2/test/dtest/dtest.c @@ -0,0 +1,1904 @@ +/* + * Copyright (c) 2007 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: $ + */ +#include +#include +#include + +#if defined(_WIN32) || defined(_WIN64) + +#include +#include +#include +#include +#include +#include + +#define getpid _getpid +#define PRIx64 "%llx" + +#else + +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +/* Debug: 1 == connect & close only, otherwise full-meal deal */ +#define CONNECT_ONLY 0 + +#ifdef DAPL_PROVIDER +#undef DAPL_PROVIDER +#endif +#define DAPL_PROVIDER "ibnic0" + +#define MAX_POLLING_CNT 50000 +#define MAX_RDMA_RD 4 +#define MAX_PROCS 1000 + +/* Header files needed for DAT/uDAPL */ +#include "dat/udat.h" + +/* definitions */ +#define SERVER_CONN_QUAL 45248 +#define DTO_TIMEOUT (1000*1000*5) +#define DTO_FLUSH_TIMEOUT (1000*1000*2) +#define CONN_TIMEOUT (1000*1000*10) +#define SERVER_TIMEOUT (1000*1000*20) +#define RDMA_BUFFER_SIZE (64) + +/* Global DAT vars */ +static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL; +static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL; +static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL; +static DAT_PSP_HANDLE h_psp = DAT_HANDLE_NULL; +static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL; + +static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL; +static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL; +static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL; +static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL; +static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL; +static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL; + +/* RDMA buffers */ +static DAT_LMR_HANDLE h_lmr_send = DAT_HANDLE_NULL; +static DAT_LMR_HANDLE h_lmr_recv = DAT_HANDLE_NULL; +static DAT_LMR_CONTEXT lmr_context_send; +static DAT_LMR_CONTEXT lmr_context_recv; +static DAT_RMR_CONTEXT rmr_context_send; +static DAT_RMR_CONTEXT rmr_context_recv; +static DAT_VLEN registered_size_send; +static DAT_VLEN registered_size_recv; +static DAT_VADDR registered_addr_send; +static DAT_VADDR registered_addr_recv; + +/* Initial msg receive buf, RMR exchange, and Rdma-write notification */ +#define MSG_BUF_COUNT 3 +#define MSG_IOV_COUNT 4 +static DAT_RMR_TRIPLET rmr_recv_msg[MSG_BUF_COUNT]; +static DAT_LMR_HANDLE h_lmr_recv_msg = DAT_HANDLE_NULL; +static DAT_LMR_CONTEXT lmr_context_recv_msg; +static DAT_RMR_CONTEXT rmr_context_recv_msg; +static DAT_VLEN registered_size_recv_msg; +static DAT_VADDR registered_addr_recv_msg; + +/* message send buffer */ +static DAT_RMR_TRIPLET rmr_send_msg; +static DAT_LMR_HANDLE h_lmr_send_msg = DAT_HANDLE_NULL; +static DAT_LMR_CONTEXT lmr_context_send_msg; +static DAT_RMR_CONTEXT rmr_context_send_msg; +static DAT_VLEN registered_size_send_msg; +static DAT_VADDR registered_addr_send_msg; +static DAT_EP_ATTR ep_attr; +char hostname[256] = {0}; +char provider[64] = DAPL_PROVIDER; + +/* rdma pointers */ +char *rbuf = NULL; +char *sbuf = NULL; +int status; + +/* timers */ +double start,stop,total_us,total_sec; + +struct dt_time { + double total; + double open; + double reg; + double unreg; + double pzc; + double pzf; + double evdc; + double evdf; + double cnoc; + double cnof; + double epc; + double epf; + double rdma_wr; + double rdma_rd[MAX_RDMA_RD]; + double rdma_rd_total; + double rtt; + double close; +}; + +struct dt_time time; + +/* defaults */ +static int parent=1; +static int connected=0; +static int burst=10; +static int server=1; +static int verbose=0; +static int polling=0; +static int poll_count=0; +static int rdma_wr_poll_count=0; +static int rdma_rd_poll_count[MAX_RDMA_RD]={0}; +static int pin_memory=0; +static int delay=0; +static int buf_len=RDMA_BUFFER_SIZE; +static int use_cno=0; +static int post_recv_count=MSG_BUF_COUNT; +static int recv_msg_index=0; +static int burst_msg_posted=0; +static int burst_msg_index=0; + +static int child[MAX_PROCS+1]; + +/* forward prototypes */ +const char * DT_RetToString (DAT_RETURN ret_value); +const char * DT_EventToSTr (DAT_EVENT_NUMBER event_code); +void print_usage(void); +double get_time(void); +void init_data(void); + +DAT_RETURN send_msg( void *data, + DAT_COUNT size, + DAT_LMR_CONTEXT context, + DAT_DTO_COOKIE cookie, + DAT_COMPLETION_FLAGS flags ); + +DAT_RETURN connect_ep( char *hostname, int conn_id ); +void disconnect_ep( void ); +DAT_RETURN register_rdma_memory( void ); +DAT_RETURN unregister_rdma_memory( void ); +DAT_RETURN create_events( void ); +DAT_RETURN destroy_events(void); +DAT_RETURN do_rdma_write_with_msg( void ); +DAT_RETURN do_rdma_read_with_msg( void ); +DAT_RETURN do_ping_pong_msg( void ); + +#define LOGPRINTF1(_format, _aa) if (verbose) printf(_format, _aa) +#define LOGPRINTF2(_format, _aa,_bb) if (verbose) printf(_format, _aa,_bb) +#define LOGPRINTF3(_format, _aa,_bb,_cc) \ + if (verbose) printf(_format, _aa,_bb,_cc) +#define LOGPRINTF4(_format, _aa,_bb,_cc,_dd) \ + if (verbose) printf(_format, _aa,_bb,_cc,_dd) +#define LOGPRINTF5(_format, _aa,_bb,_cc,_dd,_ee) \ + if (verbose) printf(_format, _aa,_bb,_cc,_dd,_ee) + + +main(int argc, char **argv) +{ + int i,c; + DAT_RETURN ret; + + /* parse arguments */ + while ((c = getopt(argc, argv, "scvpb:d:B:h:P:")) != -1) + { + switch(c) + { + case 's': + server = 1; + fflush(stdout); + break; + case 'c': + use_cno = 1; + printf("%d Creating CNO for DTO EVD's\n",getpid()); + fflush(stdout); + break; + case 'v': + verbose = 1; + printf("%d Verbose\n",getpid()); + fflush(stdout); + break; + case 'p': + polling = 1; + printf("%d Polling\n",getpid()); + fflush(stdout); + break; + case 'B': + burst = atoi(optarg); + break; + case 'd': + delay = atoi(optarg); + break; + case 'b': + buf_len = atoi(optarg); + break; + case 'h': + server = 0; + strcpy (hostname, optarg); + break; + case 'P': + strcpy (provider, optarg); + break; + default: + print_usage(); + exit(-12); + } + } + +#if defined(_WIN32) || defined(_WIN64) + { + WSADATA wsaData; + + i = WSAStartup ( MAKEWORD(2,2), &wsaData ); + if ( i != 0 ) { + printf("%s WSAStartup(2.2) failed? (0x%x)\n",argv[0],i); + fflush(stdout); + exit(1); + } + } +#endif + + if (!server) { + printf("%d Running as client - %s\n",getpid(),provider); fflush(stdout); + } else { + printf("%d Running as server - %s\n",getpid(),provider); fflush(stdout); + } + + /* allocate send and receive buffers */ + if (((rbuf = malloc(buf_len*burst)) == NULL) || + ((sbuf = malloc(buf_len*burst)) == NULL)) { + perror("malloc"); + exit(1); + } + memset( &time, 0, sizeof(struct dt_time) ); + LOGPRINTF4("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n", + getpid(), rbuf, sbuf, buf_len); + + /* dat_ia_open, dat_pz_create */ + h_async_evd = DAT_HANDLE_NULL; + start = get_time(); + ret = dat_ia_open( provider, 8, &h_async_evd, &h_ia ); + stop = get_time(); + time.open += ((stop - start)*1.0e6); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d: Error Adaptor open: %s\n", + getpid(),DT_RetToString(ret)); + exit(1); + } else + LOGPRINTF1("%d Opened Interface Adaptor\n",getpid()); + + /* Create Protection Zone */ + start = get_time(); + LOGPRINTF1("%d Create Protection Zone\n",getpid()); + ret = dat_pz_create(h_ia, &h_pz); + stop = get_time(); + time.pzc += ((stop - start)*1.0e6); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error creating Protection Zone: %s\n", + getpid(),DT_RetToString(ret)); + exit(1); + } else + LOGPRINTF1("%d Created Protection Zone\n",getpid()); + + /* Register memory */ + LOGPRINTF1("%d Register RDMA memory\n", getpid()); + ret = register_rdma_memory(); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error registering RDMA memory: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else + LOGPRINTF1("%d Register RDMA memory done\n", getpid()); + + LOGPRINTF1("%d Create events\n", getpid()); + ret = create_events(); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error creating events: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else { + LOGPRINTF1("%d Create events done\n", getpid()); + } + + /* create EP */ + memset( &ep_attr, 0, sizeof(ep_attr) ); + ep_attr.service_type = DAT_SERVICE_TYPE_RC; + ep_attr.max_rdma_size = 0x10000; + ep_attr.qos = 0; + ep_attr.recv_completion_flags = 0; + ep_attr.max_recv_dtos = MSG_BUF_COUNT + (burst*3); + ep_attr.max_request_dtos = MSG_BUF_COUNT + (burst*3) + MAX_RDMA_RD; + ep_attr.max_recv_iov = MSG_IOV_COUNT; + ep_attr.max_request_iov = MSG_IOV_COUNT; + ep_attr.max_rdma_read_in = MAX_RDMA_RD; + ep_attr.max_rdma_read_out = MAX_RDMA_RD; + ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG; + ep_attr.ep_transport_specific_count = 0; + ep_attr.ep_transport_specific = NULL; + ep_attr.ep_provider_specific_count = 0; + ep_attr.ep_provider_specific = NULL; + + start = get_time(); + ret = dat_ep_create( h_ia, h_pz, h_dto_rcv_evd, + h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep ); + stop = get_time(); + time.epc += ((stop - start)*1.0e6); + time.total += time.epc; + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_ep_create: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else + LOGPRINTF2("%d EP created %p \n", getpid(), h_ep); + + /* + * register message buffers, establish connection, and + * exchange DMA RMR information info via messages + */ + ret = connect_ep( hostname, SERVER_CONN_QUAL ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error connect_ep: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else + LOGPRINTF1("%d connect_ep complete\n", getpid()); + +#if CONNECT_ONLY +#if defined(_WIN32) || defined(_WIN64) + Sleep(1*1000); +#else + sleep(1); +#endif + goto cleanup; +#endif + + /*********** RDMA write data *************/ + ret = do_rdma_write_with_msg(); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else + LOGPRINTF1("%d do_rdma_write_with_msg complete\n", getpid()); + + /*********** RDMA read data *************/ + ret = do_rdma_read_with_msg(); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else + LOGPRINTF1("%d do_rdma_read_with_msg complete\n", getpid()); + + /*********** PING PING messages ************/ + ret = do_ping_pong_msg(); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error do_ping_pong_msg: %s\n", + getpid(),DT_RetToString(ret)); + goto cleanup; + } else + LOGPRINTF1("%d do_ping_pong_msg complete\n", getpid()); + +cleanup: + /* disconnect and free EP resources */ + if ( h_ep != DAT_HANDLE_NULL ) { + /* unregister message buffers and tear down connection */ + LOGPRINTF2("%d Disconnect and Free EP %p \n",getpid(),h_ep); + disconnect_ep(); + } + + /* free EP */ + LOGPRINTF2("%d Free EP %p \n",getpid(),h_ep); + start = get_time(); + ret = dat_ep_free( h_ep ); + stop = get_time(); + time.epf += ((stop - start)*1.0e6); + time.total += time.epf; + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error freeing EP: %s\n", + getpid(), DT_RetToString(ret)); + } else { + LOGPRINTF1("%d Freed EP\n",getpid()); + h_ep = DAT_HANDLE_NULL; + } + + /* free EVDs */ + LOGPRINTF1("%d destroy events\n", getpid()); + ret = destroy_events(); + if(ret != DAT_SUCCESS) + fprintf(stderr, "%d Error destroy_events: %s\n", + getpid(),DT_RetToString(ret)); + else + LOGPRINTF1("%d destroy events done\n", getpid()); + + + ret = unregister_rdma_memory(); + LOGPRINTF1("%d unregister_rdma_memory \n", getpid()); + if(ret != DAT_SUCCESS) + fprintf(stderr, "%d Error unregister_rdma_memory: %s\n", + getpid(),DT_RetToString(ret)); + else + LOGPRINTF1("%d unregister_rdma_memory done\n", getpid()); + + /* Free protection domain */ + LOGPRINTF1("%d Freeing pz\n",getpid()); + start = get_time(); + ret = dat_pz_free( h_pz ); + stop = get_time(); + time.pzf += ((stop - start)*1.0e6); + if (ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error freeing PZ: %s\n", + getpid(), DT_RetToString(ret)); + } else { + LOGPRINTF1("%d Freed pz\n",getpid()); + h_pz = NULL; + } + + /* close the device */ + LOGPRINTF1("%d Closing Interface Adaptor\n",getpid()); + start = get_time(); + ret = dat_ia_close( h_ia, DAT_CLOSE_ABRUPT_FLAG ); + //ret = dat_ia_close( h_ia, DAT_CLOSE_GRACEFUL_FLAG ); + stop = get_time(); + time.close += ((stop - start)*1.0e6); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d: Error Adaptor close: %s\n", + getpid(),DT_RetToString(ret)); + exit(1); + } else + LOGPRINTF1("%d Closed Interface Adaptor\n",getpid()); + + printf( "\n%d: DAPL Test Complete.\n\n",getpid()); + printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf" + " usec, pc=%d\n", + getpid(), time.rtt, burst, time.rtt/burst, poll_count ); + printf( "%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf" + " usec, pc=%d\n", + getpid(), time.rdma_wr, burst, + time.rdma_wr/burst, rdma_wr_poll_count ); + for(i=0;itv_sec = now.wMinute * 60; + t->tv_sec += now.wSecond; + t->tv_usec = now.wMilliseconds; +} + +#endif + +double get_time(void) +{ + struct timeval tp; + + gettimeofday(&tp, NULL); + return ((double) tp.tv_sec + (double) tp.tv_usec * 1e-6); +} + +void init_data(void) +{ + memset(rbuf, 'a', buf_len); + memset(sbuf, 'b', buf_len); +} + + +DAT_RETURN +send_msg( void *data, + DAT_COUNT size, + DAT_LMR_CONTEXT context, + DAT_DTO_COOKIE cookie, + DAT_COMPLETION_FLAGS flags ) +{ + DAT_LMR_TRIPLET iov; + DAT_EVENT event; + DAT_COUNT nmore; + DAT_RETURN ret; + + iov.lmr_context = context; + iov.virtual_address = (DAT_VADDR)data; + iov.segment_length = size; + + LOGPRINTF1("%d calling post_send\n", getpid()); + cookie.as_64 = 0xaaaa; + ret = dat_ep_post_send( h_ep, + 1, + &iov, + cookie, + flags ); + + if (ret != DAT_SUCCESS) { + fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n", + getpid(),DT_RetToString(ret)); + return ret; + } + + if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) { + if ( polling ) { + printf("%d Polling post send completion...\n",getpid()); + while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY) ; + } + else { + LOGPRINTF1("%d waiting for post_send completion event\n", getpid()); + if (use_cno) { + DAT_EVD_HANDLE evd = DAT_HANDLE_NULL; + ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd ); + LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd); + if ( evd != h_dto_req_evd ) { + fprintf(stderr, + "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n", + getpid()); + return( DAT_ABORT ); + } + } + /* use wait to dequeue */ + ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore ); + if (ret != DAT_SUCCESS) { + fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n", + getpid(),DT_RetToString(ret)); + return ret; + } + } + + /* validate event number, len, cookie, and status */ + if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) { + fprintf(stderr, "%d: ERROR: DTO event number %s\n", + getpid(),DT_EventToSTr(event.event_number)); + return( DAT_ABORT ); + } + + if ((event.event_data.dto_completion_event_data.transfered_length != size ) || + (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0xaaaa )) { + fprintf(stderr, "%d: ERROR: DTO len %d or cookie %llx\n", + getpid(), + event.event_data.dto_completion_event_data.transfered_length, + event.event_data.dto_completion_event_data.user_cookie.as_64 ); + return( DAT_ABORT ); + + } + if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) { + fprintf(stderr, "%d: ERROR: DTO event status %s\n", + getpid(),DT_RetToString(ret)); + return( DAT_ABORT ); + } + } + + return DAT_SUCCESS; +} + + +DAT_RETURN +connect_ep( char *hostname, int conn_id ) +{ + DAT_SOCK_ADDR remote_addr; + DAT_RETURN ret; + DAT_REGION_DESCRIPTION region; + DAT_EVENT event; + DAT_COUNT nmore; + DAT_LMR_TRIPLET l_iov; + DAT_RMR_TRIPLET r_iov; + DAT_DTO_COOKIE cookie; + int i; + + /* Register send message buffer */ + LOGPRINTF3("%d Registering send Message Buffer %p, len %d\n", + getpid(), &rmr_send_msg, sizeof(DAT_RMR_TRIPLET) ); + region.for_va = &rmr_send_msg; + ret = dat_lmr_create( h_ia, + DAT_MEM_TYPE_VIRTUAL, + region, + sizeof(DAT_RMR_TRIPLET), + h_pz, + DAT_MEM_PRIV_LOCAL_WRITE_FLAG, + DAT_VA_TYPE_VA, + &h_lmr_send_msg, + &lmr_context_send_msg, + &rmr_context_send_msg, + ®istered_size_send_msg, + ®istered_addr_send_msg ); + + if (ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error registering send msg buffer: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF2("%d Registered send Message Buffer %p \n", + getpid(),region.for_va ); + + /* Register Receive buffers */ + LOGPRINTF2("%d Registering Receive Message Buffer %p\n", + getpid(), rmr_recv_msg ); + region.for_va = rmr_recv_msg; + ret = dat_lmr_create( h_ia, + DAT_MEM_TYPE_VIRTUAL, + region, + sizeof(DAT_RMR_TRIPLET)*MSG_BUF_COUNT, + h_pz, + DAT_MEM_PRIV_LOCAL_WRITE_FLAG, + DAT_VA_TYPE_VA, + &h_lmr_recv_msg, + &lmr_context_recv_msg, + &rmr_context_recv_msg, + ®istered_size_recv_msg, + ®istered_addr_recv_msg ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error registering recv msg buffer: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF2("%d Registered Receive Message Buffer %p\n", + getpid(),region.for_va); + + for ( i = 0; i < MSG_BUF_COUNT; i++ ) { + cookie.as_64 = i; + l_iov.lmr_context = lmr_context_recv_msg; + l_iov.virtual_address = (DAT_VADDR)&rmr_recv_msg[ i ]; + l_iov.segment_length = sizeof(DAT_RMR_TRIPLET); + + LOGPRINTF2("%d Posting Receive Message Buffer %p\n", + getpid(), &rmr_recv_msg[ i ]); + ret = dat_ep_post_recv( h_ep, + 1, + &l_iov, + cookie, + DAT_COMPLETION_DEFAULT_FLAG ); + + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error registering recv msg buffer: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF2("%d Registered Receive Message Buffer %p\n", + getpid(),region.for_va); + + } + + /* setup receive rdma buffer to initial string to be overwritten */ + strcpy( (char*)rbuf, "blah, blah, blah\n" ); + + if ( server ) { /* SERVER */ + + /* create the service point for server listen */ + LOGPRINTF1("%d Creating service point for listen\n",getpid()); + ret = dat_psp_create( h_ia, + conn_id, + h_cr_evd, + DAT_PSP_CONSUMER_FLAG, + &h_psp ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_psp_create: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF1("%d dat_psp_created for server listen\n", getpid()); + + printf("%d Server waiting for connect request..\n", getpid()); + ret = dat_evd_wait( h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_evd_wait: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF1("%d dat_evd_wait for cr_evd completed\n", getpid()); + + if ( event.event_number != DAT_CONNECTION_REQUEST_EVENT ) { + fprintf(stderr, "%d Error unexpected cr event : %s\n", + getpid(),DT_EventToSTr(event.event_number)); + return( DAT_ABORT ); + } + if ( (event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) || + (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != h_psp) ) { + fprintf(stderr, "%d Error wrong cr event data : %s\n", + getpid(),DT_EventToSTr(event.event_number)); + return( DAT_ABORT ); + } + + /* use to test rdma_cma timeout logic */ +#if defined(_WIN32) || defined(_WIN64) + if (delay) Sleep(delay*1000); +#else + if (delay) sleep(delay); +#endif + /* accept connect request from client */ + h_cr = event.event_data.cr_arrival_event_data.cr_handle; + LOGPRINTF1("%d Accepting connect request from client\n",getpid()); + ret = dat_cr_accept( h_cr, h_ep, 0, (DAT_PVOID)0 ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_cr_accept: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF1("%d dat_cr_accept completed\n", getpid()); + } + else { /* CLIENT */ + struct addrinfo *target; + int rval; + +#if defined(_WIN32) || defined(_WIN64) + if ((rval=getaddrinfo (hostname, "ftp", NULL, &target)) != 0) { + printf("\n remote name resolution failed! %s\n",gai_strerror(rval)); + exit ( 1 ); + } + rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr; +#else + if (getaddrinfo (hostname, NULL, NULL, &target) != 0) { + perror("\n remote name resolution failed!"); + exit ( 1 ); + } + rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.sin_addr; +#endif + printf ("%d Server Name: %s \n", getpid(), hostname); + printf ("%d Server Net Address: %d.%d.%d.%d\n", getpid(), + (rval >> 0) & 0xff, + (rval >> 8) & 0xff, + (rval >> 16) & 0xff, + (rval >> 24) & 0xff); + + remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr); + + LOGPRINTF1("%d Connecting to server\n",getpid()); + ret = dat_ep_connect( h_ep, + &remote_addr, + conn_id, + CONN_TIMEOUT, + 0, + (DAT_PVOID)0, + 0, + DAT_CONNECT_DEFAULT_FLAG ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_ep_connect: %s\n", + getpid(), DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF1("%d dat_ep_connect completed\n", getpid()); + } + + printf("%d Waiting for connect response\n",getpid()); + + ret = dat_evd_wait( h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_evd_wait: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF1("%d dat_evd_wait for h_conn_evd completed\n", getpid()); + + if ( event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED ) { + fprintf(stderr, "%d Error unexpected conn event : %s\n", + getpid(),DT_EventToSTr(event.event_number)); + return( DAT_ABORT ); + } + printf("\n%d CONNECTED!\n\n",getpid()); + connected = 1; + +#if CONNECT_ONLY + return 0; +#endif + + /* + * Setup our remote memory and tell the other side about it + */ + rmr_send_msg.rmr_context = rmr_context_recv; + rmr_send_msg.virtual_address = (DAT_VADDR)rbuf; + rmr_send_msg.segment_length = RDMA_BUFFER_SIZE; + + printf("%d Send RMR to remote: snd_msg: r_key_ctx=%x,va=%llx,len=0x%x\n", + getpid(), rmr_send_msg.rmr_context, + rmr_send_msg.virtual_address, rmr_send_msg.segment_length ); + + ret = send_msg( &rmr_send_msg, + sizeof( DAT_RMR_TRIPLET ), + lmr_context_send_msg, + cookie, + DAT_COMPLETION_SUPPRESS_FLAG ); + + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error send_msg: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else + LOGPRINTF1("%d send_msg completed\n", getpid()); + + /* + * Wait for remote RMR information for RDMA + */ + if ( polling ) { + printf("%d Polling for remote to send RMR data\n",getpid()); + while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY ); + } + else { + printf("%d Waiting for remote to send RMR data\n",getpid()); + if (use_cno) + { + DAT_EVD_HANDLE evd = DAT_HANDLE_NULL; + ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd ); + LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd); + if ( evd != h_dto_rcv_evd ) { + fprintf(stderr, + "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n", + getpid()); + return( DAT_ABORT ); + } + } + /* use wait to dequeue */ + ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error waiting on h_dto_rcv_evd: %s\n", + getpid(),DT_RetToString(ret)); + return(ret); + } + else { + LOGPRINTF1("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid()); + } + } + + printf("%d remote RMR data arrived!\n",getpid()); + + if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) { + fprintf(stderr, "%d Error unexpected DTO event : %s\n", + getpid(),DT_EventToSTr(event.event_number)); + return( DAT_ABORT ); + } + if ((event.event_data.dto_completion_event_data.transfered_length != + sizeof( DAT_RMR_TRIPLET )) || + (event.event_data.dto_completion_event_data.user_cookie.as_64 != + recv_msg_index) ) { + fprintf(stderr,"ERR recv event: len=%d cookie= %llx expected %d/%d\n", + (int)event.event_data.dto_completion_event_data.transfered_length, + (int)event.event_data.dto_completion_event_data.user_cookie.as_64, + sizeof(DAT_RMR_TRIPLET), recv_msg_index ); + return( DAT_ABORT ); + } + + r_iov = rmr_recv_msg[ recv_msg_index ]; + + printf("%d Received RMR from remote: r_iov: r_key_ctx=%x,va=%llx,len=0x%x\n", + getpid(), r_iov.rmr_context, + r_iov.virtual_address, r_iov.segment_length ); + + recv_msg_index++; + + return ( DAT_SUCCESS ); +} + + +void +disconnect_ep() +{ + DAT_RETURN ret; + DAT_EVENT event; + DAT_COUNT nmore; + + if (connected) { + + /* + * Only the client needs to call disconnect. The server _should_ be able + * to just wait on the EVD associated with connection events for a + * disconnect request and then exit. + */ + if ( !server ) { + LOGPRINTF1("%d dat_ep_disconnect\n", getpid()); + ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_ep_disconnect: %s\n", + getpid(),DT_RetToString(ret)); + } + else { + LOGPRINTF1("%d dat_ep_disconnect completed\n", getpid()); + } + } + else { + LOGPRINTF1("%d Server waiting for disconnect...\n", getpid()); + } + + ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_evd_wait: %s\n", + getpid(),DT_RetToString(ret)); + } + else { + LOGPRINTF1("%d dat_evd_wait for h_conn_evd completed\n", getpid()); + } + } + + /* destroy service point */ + if (( server ) && ( h_psp != DAT_HANDLE_NULL )) { + ret = dat_psp_free( h_psp ); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error dat_psp_free: %s\n", + getpid(),DT_RetToString(ret)); + } + else { + LOGPRINTF1("%d dat_psp_free completed\n", getpid()); + } + } + + /* Unregister Send message Buffer */ + if ( h_lmr_send_msg != DAT_HANDLE_NULL ) { + LOGPRINTF2("%d Unregister send message h_lmr %p \n",getpid(),h_lmr_send_msg); + ret = dat_lmr_free(h_lmr_send_msg); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error deregistering send msg mr: %s\n", + getpid(), DT_RetToString(ret)); + } else { + LOGPRINTF1("%d Unregistered send message Buffer\n",getpid()); + h_lmr_send_msg = NULL; + } + } + + /* Unregister recv message Buffer */ + if ( h_lmr_recv_msg != DAT_HANDLE_NULL ) { + LOGPRINTF2("%d Unregister recv message h_lmr %p \n",getpid(),h_lmr_recv_msg); + ret = dat_lmr_free(h_lmr_recv_msg); + if(ret != DAT_SUCCESS) { + fprintf(stderr, "%d Error deregistering recv msg mr: %s\n", + getpid(), DT_RetToString(ret)); + } else { + LOGPRINTF1("%d Unregistered recv message Buffer\n",getpid()); + h_lmr_recv_msg = NULL; + } + } + return; +} + + +DAT_RETURN +do_rdma_write_with_msg( ) +{ + DAT_EVENT event; + DAT_COUNT nmore; + DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT]; + DAT_RMR_TRIPLET r_iov; + DAT_DTO_COOKIE cookie; + DAT_RETURN ret; + int i; + + printf("\n %d RDMA WRITE DATA with SEND MSG\n\n",getpid()); + + cookie.as_64 = 0x5555; + + if ( recv_msg_index >= MSG_BUF_COUNT ) + return( DAT_ABORT ); + + /* get RMR information from previously received message */ + r_iov = rmr_recv_msg[ recv_msg_index-1 ]; + + if ( server ) + strcpy( (char*)sbuf, "server RDMA write data..." ); + else + strcpy( (char*)sbuf, "client RDMA write data..." ); + + for (i=0;i= MSG_BUF_COUNT ) + return( DAT_ABORT ); + + /* get RMR information from previously received message */ + r_iov = rmr_recv_msg[ recv_msg_index-1 ]; + + /* setup rdma read buffer to initial string to be overwritten */ + strcpy( (char*)sbuf, "blah, blah, blah\n" ); + + if ( server ) + strcpy( (char*)rbuf, "server RDMA read data..." ); + else + strcpy( (char*)rbuf, "client RDMA read data..." ); + + l_iov.lmr_context = lmr_context_send; + l_iov.virtual_address = (DAT_VADDR)sbuf; + l_iov.segment_length = buf_len; + + for (i=0;i + +#define VER_FILETYPE VFT_APP +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#if DBG +#define VER_FILEDESCRIPTION_STR "Simple DAPL/DAT svr/cli test Application (Debug)" +#else +#define VER_FILEDESCRIPTION_STR "Simple DAPL/DAT svr/cli test Application" +#endif + +#define VER_INTERNALNAME_STR "dapl2test.exe" +#define VER_ORIGINALFILENAME_STR "dapl2test.exe" + +#include diff --git a/trunk/ulp/dapl2/test/dtest/makefile b/trunk/ulp/dapl2/test/dtest/makefile new file mode 100644 index 00000000..a0c06273 --- /dev/null +++ b/trunk/ulp/dapl2/test/dtest/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\inc\openib.def -- 2.41.0