UMTYPE=console\r
USE_CRTDLL=1\r
\r
-SOURCES=main.c\r
+SOURCES=cmtest_main.c\r
\r
INCLUDES=..\..\..\inc;..\..\..\inc\user;\r
\r
--- /dev/null
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.\r
+ * Copyright (c) 1996-2002 Intel Corporation. All rights reserved. \r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id$\r
+ */\r
+\r
+\r
+/*\r
+ * Abstract:\r
+ * Command line interface for cmtest.\r
+ *\r
+ * Environment:\r
+ * User Mode\r
+ */\r
+\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <stdarg.h>\r
+#include <ctype.h>\r
+#include <complib/cl_atomic.h>\r
+#include <complib/cl_debug.h>\r
+#include <complib/cl_event.h>\r
+#include <complib/cl_math.h>\r
+#include <complib/cl_mutex.h>\r
+#include <complib/cl_qlist.h>\r
+#include <complib/cl_thread.h>\r
+#include <complib/cl_timer.h>\r
+#include <iba/ib_types.h>\r
+#include <iba/ib_al.h>\r
+\r
+\r
+/* Globals */\r
+#define CMT_DBG_VERBOSE 1\r
+\r
+#define CMT_BASE_SVC_ID 0xFFEE\r
+#define CMT_ACCESS_CTRL (IB_AC_LOCAL_WRITE + IB_AC_RDMA_READ + IB_AC_RDMA_WRITE)\r
+#define BAD_PKEY_INDEX 0xFFFF\r
+\r
+\r
+typedef enum _cmtest_state\r
+{\r
+ test_idle, test_connecting, test_transfering, test_disconnecting\r
+\r
+} cmtest_state_t;\r
+\r
+\r
+\r
+typedef struct _ib_root\r
+{\r
+ ib_al_handle_t h_al;\r
+ ib_pd_handle_t h_pd;\r
+\r
+ /* Input parameters to control test. */\r
+ int32_t num_nodes;\r
+ uint32_t num_msgs;\r
+ boolean_t per_msg_buf;\r
+ cl_mutex_t mutex;\r
+\r
+ cmtest_state_t state;\r
+ atomic32_t num_connected;\r
+ uint32_t conn_index; /* current connection id */\r
+ uint32_t total_sent;\r
+ uint32_t total_recv;\r
+\r
+ uint32_t num_iter;\r
+\r
+ uint32_t msg_size;\r
+\r
+ ib_ca_handle_t h_ca;\r
+ ib_net16_t l_lid;\r
+ ib_net16_t r_lid;\r
+ ib_net64_t ca_guid;\r
+ ib_net64_t port_guid;\r
+ uint8_t port_num;\r
+ uint16_t num_pkeys;\r
+ ib_net16_t *p_pkey_table;\r
+\r
+ /* cm info */\r
+ boolean_t is_server;\r
+ ib_listen_handle_t h_listen;\r
+ ib_path_rec_t path_rec;\r
+\r
+ /* CQ info. */\r
+ boolean_t is_polling;\r
+\r
+ struct _ib_node *p_nodes;\r
+ ib_qp_create_t qp_create;\r
+ ib_qp_mod_t qp_mod_reset;\r
+ ib_qp_mod_t qp_mod_init;\r
+\r
+ /* reg mem info */\r
+ ib_mr_handle_t h_mr;\r
+ uint32_t lkey;\r
+ uint32_t rkey;\r
+ uint8_t *p_mem;\r
+ uint8_t *p_mem_recv;\r
+ uint8_t *p_mem_send;\r
+\r
+ uint64_t conn_start_time;\r
+\r
+ /*\r
+ * Connection parameters are initialized once to improve connection\r
+ * establishment rate.\r
+ */\r
+ ib_cm_req_t cm_req;\r
+ ib_cm_rep_t cm_rep;\r
+ ib_cm_rtu_t cm_rtu;\r
+ ib_cm_dreq_t cm_dreq;\r
+ ib_cm_drep_t cm_drep;\r
+\r
+ uint32_t inst_id;\r
+\r
+} ib_root_t;\r
+\r
+\r
+\r
+typedef enum _cmnode_state\r
+{\r
+ node_idle, node_conn, node_dreq_sent, node_dreq_rcvd\r
+\r
+} cmnode_state_t;\r
+\r
+\r
+\r
+typedef struct _ib_node\r
+{\r
+ uint64_t id;\r
+\r
+ ib_cq_handle_t h_send_cq;\r
+ ib_cq_handle_t h_recv_cq;\r
+ ib_qp_handle_t h_qp;\r
+ uint32_t max_inline;\r
+\r
+ cmnode_state_t state;\r
+ ib_cm_handle_t h_cm_req;\r
+ ib_cm_handle_t h_cm_dreq;\r
+\r
+ uint32_t send_cnt;\r
+ uint32_t recv_cnt;\r
+\r
+} ib_node_t;\r
+\r
+\r
+\r
+uint32_t cmt_dbg_lvl = 0x80000000;\r
+\r
+ib_root_t g_root;\r
+\r
+\r
+static char *wc_type_text[] = {\r
+ "IB_WC_SEND",\r
+ "IB_WC_RDMA_WRITE",\r
+ "IB_WC_RECV",\r
+ "IB_WC_RDMA_READ",\r
+ "IB_WC_MW_BIND",\r
+ "IB_WC_FETCH_ADD",\r
+ "IB_WC_COMPARE_SWAP",\r
+ "IB_WC_RECV_RDMA_WRITE"\r
+};\r
+\r
+static char *wc_status_text[] = {\r
+ "IB_WCS_SUCCESS",\r
+ "IB_WCS_LOCAL_LEN_ERR",\r
+ "IB_WCS_LOCAL_OP_ERR",\r
+ "IB_WCS_LOCAL_EEC_OP_ERR",\r
+ "IB_WCS_LOCAL_PROTECTION_ERR",\r
+ "IB_WCS_WR_FLUSHED_ERR",\r
+ "IB_WCS_MEM_WINDOW_BIND_ERR",\r
+ "IB_WCS_REM_ACCESS_ERR",\r
+ "IB_WCS_REM_OP_ERR",\r
+ "IB_WCS_RNR_RETRY_ERR",\r
+ "IB_WCS_TIMEOUT_RETRY_ERR",\r
+ "IB_WCS_REM_INVALID_REQ_ERR",\r
+ "IB_WCS_REM_INVALID_RD_REQ_ERR",\r
+ "IB_WCS_INVALID_EECN",\r
+ "IB_WCS_INVALID_EEC_STATE",\r
+ "IB_WCS_UNMATCHED_RESPONSE", \r
+ "IB_WCS_CANCELED" \r
+};\r
+\r
+\r
+\r
+static void\r
+__req_cb(\r
+ IN ib_cm_req_rec_t *p_cm_req_rec );\r
+\r
+static void\r
+__rep_cb(\r
+ IN ib_cm_rep_rec_t *p_cm_rep_rec );\r
+\r
+static void\r
+__rtu_cb(\r
+ IN ib_cm_rtu_rec_t *p_cm_rtu_rec );\r
+\r
+static void\r
+__rej_cb(\r
+ IN ib_cm_rej_rec_t *p_cm_rej_rec );\r
+\r
+static void\r
+__mra_cb(\r
+ IN ib_cm_mra_rec_t *p_cm_mra_rec );\r
+\r
+static void\r
+__apr_cb(\r
+ IN ib_cm_apr_rec_t *p_cm_apr_rec );\r
+\r
+static void\r
+__lap_cb(\r
+ IN ib_cm_lap_rec_t *p_cm_lap_rec );\r
+\r
+static void\r
+__dreq_cb(\r
+ IN ib_cm_dreq_rec_t *p_cm_dreq_rec );\r
+\r
+static void\r
+__drep_cb(\r
+ IN ib_cm_drep_rec_t *p_cm_drep_rec );\r
+\r
+static boolean_t\r
+__poll_cq(\r
+ IN ib_node_t *p_node,\r
+ IN ib_cq_handle_t h_cq );\r
+\r
+\r
+/**********************************************************************\r
+ **********************************************************************/\r
+static void\r
+__show_usage()\r
+{\r
+ printf( "\n------- cmtest - Usage and options ----------------------\n" );\r
+ printf( "Usage: cmtest [options]\n");\r
+ printf( "Options:\n" );\r
+ printf( "-s\n"\r
+ "--server\n"\r
+ " This option directs cmtest to act as a Server\n" );\r
+ printf( "-l <lid>\n"\r
+ "--local <lid>\n"\r
+ " This option specifies the local endpoint.\n" );\r
+ printf( "-r <lid>\n"\r
+ "--remote <lid>\n"\r
+ " This option specifies the remote endpoint.\n" );\r
+ printf( "-c <number>\n"\r
+ "--connect <number>\n"\r
+ " This option specifies the number of connections to open.\n"\r
+ " Default of 1.\n" );\r
+ printf( "-m <bytes>\n"\r
+ "--msize <bytes>\n"\r
+ " This option specifies the byte size of each message.\n"\r
+ " Default is 100 bytes.\n" );\r
+ printf( "-n <number>\n"\r
+ "--nmsgs <number>\n"\r
+ " This option specifies the number of messages to send at a time.\n" );\r
+ printf( "-p\n"\r
+ "--permsg\n"\r
+ " This option indicates if a separate buffer should be used per message.\n"\r
+ " Default is one buffer for all messages.\n" );\r
+ printf( "-i <number>\n"\r
+ "--iterate <number>\n"\r
+ " This option specifies the number of times to loop through 'nmsgs'.\n"\r
+ " Default of 1.\n" );\r
+ printf( "-v\n"\r
+ "--verbose\n"\r
+ " This option enables verbosity level to debug console.\n" );\r
+ printf( "-h\n"\r
+ "--help\n"\r
+ " Display this usage info then exit.\n\n" );\r
+}\r
+\r
+\r
+/* Windows support. */\r
+struct option\r
+{\r
+ const char *long_name;\r
+ unsigned long flag;\r
+ void *pfn_handler;\r
+ char short_name;\r
+};\r
+\r
+static char *optarg;\r
+\r
+#define strtoull strtoul\r
+\r
+\r
+char\r
+getopt_long(\r
+ int argc,\r
+ char *argv[],\r
+ const char *short_option,\r
+ const struct option *long_option,\r
+ void *unused )\r
+{\r
+ static int i = 1;\r
+ int j;\r
+ char ret = 0;\r
+\r
+ UNUSED_PARAM( unused );\r
+\r
+ if( i == argc )\r
+ return -1;\r
+\r
+ if( argv[i][0] != '-' )\r
+ return ret;\r
+\r
+ /* find the first character of the value. */\r
+ for( j = 1; isalpha( argv[i][j] ); j++ )\r
+ ;\r
+ optarg = &argv[i][j];\r
+\r
+ if( argv[i][1] == '-' )\r
+ {\r
+ /* Long option. */\r
+ for( j = 0; long_option[j].long_name; j++ )\r
+ {\r
+ if( strncmp( &argv[i][2], long_option[j].long_name,\r
+ optarg - argv[i] - 2 ) )\r
+ {\r
+ continue;\r
+ }\r
+\r
+ switch( long_option[j].flag )\r
+ {\r
+ case 1:\r
+ if( *optarg == '\0' )\r
+ return 0;\r
+ default:\r
+ break;\r
+ }\r
+ ret = long_option[j].short_name;\r
+ break;\r
+ }\r
+ }\r
+ else\r
+ {\r
+ for( j = 0; short_option[j] != '\0'; j++ )\r
+ {\r
+ if( !isalpha( short_option[j] ) )\r
+ return 0;\r
+\r
+ if( short_option[j] == argv[i][1] )\r
+ {\r
+ ret = short_option[j];\r
+ break;\r
+ }\r
+\r
+ if( short_option[j+1] == ':' )\r
+ {\r
+ if( *optarg == '\0' )\r
+ return 0;\r
+ j++;\r
+ }\r
+ }\r
+ }\r
+ i++;\r
+ return ret;\r
+}\r
+\r
+\r
+static boolean_t\r
+__parse_options(\r
+ int argc,\r
+ char* argv[] )\r
+{\r
+ uint32_t next_option;\r
+ const char* const short_option = "esl:r:c:m:n:i:pvh";\r
+\r
+ /*\r
+ In the array below, the 2nd parameter specified the number\r
+ of arguments as follows:\r
+ 0: no arguments\r
+ 1: argument\r
+ 2: optional\r
+ */\r
+ const struct option long_option[] =\r
+ {\r
+ { "event", 2, NULL, 'e'},\r
+ { "server", 2, NULL, 's'},\r
+ { "local", 1, NULL, 'l'},\r
+ { "remote", 1, NULL, 'r'},\r
+ { "connect", 1, NULL, 'c'},\r
+ { "msize", 1, NULL, 'm'},\r
+ { "nmsgs", 1, NULL, 'n'},\r
+ { "iterate", 1, NULL, 'i'},\r
+ { "permsg", 0, NULL, 'p'},\r
+ { "verbose", 0, NULL, 'v'},\r
+ { "help", 0, NULL, 'h'},\r
+ { NULL, 0, NULL, 0 } /* Required at end of array */\r
+ };\r
+\r
+ /* Set the default options. */\r
+ g_root.msg_size = 100;\r
+ g_root.num_nodes = 1;\r
+ g_root.num_msgs = 0;\r
+ g_root.num_iter = 1;\r
+ g_root.is_polling = TRUE;\r
+\r
+ /* parse cmd line arguments as input params */\r
+ do\r
+ {\r
+ next_option = getopt_long( argc, argv, short_option,\r
+ long_option, NULL );\r
+\r
+ switch( next_option )\r
+ {\r
+ case 's':\r
+ g_root.is_server = TRUE;\r
+ printf( "\tServer mode\n" );\r
+ break;\r
+\r
+ case 'd':\r
+ g_root.inst_id = strtoull( optarg, NULL, 0 );\r
+ printf( "\tinstance_id..: %d\n", g_root.inst_id );\r
+ break;\r
+\r
+ case 'c':\r
+ g_root.num_nodes = strtoull( optarg, NULL, 0 );\r
+ printf( "\tconnections..: %d\n", g_root.num_nodes );\r
+ break;\r
+\r
+ case 'l':\r
+ g_root.l_lid = cl_ntoh16( (uint16_t)strtoull( optarg, NULL, 0 ) );\r
+ printf( "\tlocal lid....: x%x\n", g_root.l_lid );\r
+ break;\r
+\r
+ case 'r':\r
+ g_root.r_lid = cl_ntoh16( (uint16_t)strtoull( optarg, NULL, 0 ) );\r
+ printf( "\tremote lid...: x%x\n", g_root.r_lid );\r
+ break;\r
+\r
+ case 'm':\r
+ g_root.msg_size = strtoull( optarg, NULL, 0 );\r
+ printf( "\tmsg size.....: %d bytes\n", g_root.msg_size );\r
+ break;\r
+\r
+ case 'n':\r
+ g_root.num_msgs = strtoull( optarg, NULL, 0 );\r
+ printf( "\tnum msgs.....: %d\n", g_root.num_msgs );\r
+ break;\r
+\r
+ case 'i':\r
+ g_root.num_iter = strtoull( optarg, NULL, 0 );\r
+ printf( "\titerate......: %d\n", g_root.num_iter );\r
+ break;\r
+\r
+ case 'p':\r
+ g_root.per_msg_buf = TRUE;\r
+ printf( "\tper message data buffer\n" );\r
+ break;\r
+\r
+ case 'v':\r
+ cmt_dbg_lvl = 0xFFFFFFFF;\r
+ printf( "\tverbose\n" );\r
+ break;\r
+\r
+ case 'e':\r
+ g_root.is_polling = FALSE;\r
+ printf( "\tevent driven completions\n" );\r
+ break;\r
+\r
+ case 'h':\r
+ __show_usage();\r
+ return FALSE;\r
+\r
+ case -1:\r
+ break;\r
+\r
+ default: /* something wrong */\r
+ __show_usage();\r
+ return FALSE;\r
+ }\r
+ } while( next_option != -1 );\r
+\r
+ return TRUE;\r
+}\r
+\r
+\r
+/**********************************************************************\r
+ **********************************************************************/\r
+static void\r
+__init_conn_info()\r
+{\r
+ /* Initialize connection request parameters. */\r
+ g_root.cm_req.svc_id = CMT_BASE_SVC_ID + g_root.inst_id;\r
+ g_root.cm_req.max_cm_retries = 5;\r
+ g_root.cm_req.p_primary_path = &g_root.path_rec;\r
+ g_root.cm_req.pfn_cm_rep_cb = __rep_cb;\r
+ g_root.cm_req.qp_type = IB_QPT_RELIABLE_CONN;\r
+ g_root.cm_req.resp_res = 3;\r
+ g_root.cm_req.init_depth = 1;\r
+ g_root.cm_req.remote_resp_timeout = 20;\r
+ g_root.cm_req.flow_ctrl = TRUE;\r
+ g_root.cm_req.local_resp_timeout = 20;\r
+ g_root.cm_req.rnr_nak_timeout = 6;\r
+ g_root.cm_req.rnr_retry_cnt = 3;\r
+ g_root.cm_req.retry_cnt = 5;\r
+ g_root.cm_req.pfn_cm_mra_cb = __mra_cb;\r
+ g_root.cm_req.pfn_cm_rej_cb = __rej_cb;\r
+\r
+ /* Initialize connection reply parameters. */\r
+ g_root.cm_rep.qp_type = IB_QPT_RELIABLE_CONN;\r
+ g_root.cm_rep.access_ctrl = CMT_ACCESS_CTRL;\r
+ g_root.cm_rep.sq_depth = 0;\r
+ g_root.cm_rep.rq_depth = 0;\r
+ g_root.cm_rep.init_depth = 1;\r
+ g_root.cm_rep.target_ack_delay = 7;\r
+ g_root.cm_rep.failover_accepted = IB_FAILOVER_ACCEPT_UNSUPPORTED;\r
+ g_root.cm_rep.flow_ctrl = TRUE;\r
+ g_root.cm_rep.rnr_nak_timeout = 7;\r
+ g_root.cm_rep.rnr_retry_cnt = 6;\r
+ g_root.cm_rep.pfn_cm_rej_cb = __rej_cb;\r
+ g_root.cm_rep.pfn_cm_mra_cb = __mra_cb;\r
+ g_root.cm_rep.pfn_cm_rtu_cb = __rtu_cb;\r
+ g_root.cm_rep.pfn_cm_lap_cb = __lap_cb;\r
+ g_root.cm_rep.pfn_cm_dreq_cb = __dreq_cb;\r
+\r
+ /* Initialize connection RTU parameters. */\r
+ g_root.cm_rtu.pfn_cm_apr_cb = __apr_cb;\r
+ g_root.cm_rtu.pfn_cm_dreq_cb = __dreq_cb;\r
+\r
+ /* Initialize disconnection request parameters. */\r
+ g_root.cm_dreq.pfn_cm_drep_cb = __drep_cb;\r
+ g_root.cm_dreq.qp_type = IB_QPT_RELIABLE_CONN;\r
+\r
+ /* Disconnection reply parameters are all zero. */\r
+}\r
+\r
+\r
+\r
+static uint16_t\r
+__get_pkey_index()\r
+{\r
+ uint16_t i;\r
+\r
+ for( i = 0; i < g_root.num_pkeys; i++ )\r
+ {\r
+ if( g_root.p_pkey_table[i] == g_root.path_rec.pkey )\r
+ return i;\r
+ }\r
+\r
+ return BAD_PKEY_INDEX;\r
+}\r
+\r
+\r
+\r
+static void\r
+__init_qp_info()\r
+{\r
+ /* Set common QP attributes for all create calls. */\r
+ g_root.qp_create.qp_type = IB_QPT_RELIABLE_CONN;\r
+ if( g_root.num_msgs )\r
+ {\r
+ g_root.qp_create.sq_depth = g_root.num_msgs;\r
+ g_root.qp_create.rq_depth = g_root.num_msgs;\r
+ }\r
+ else\r
+ {\r
+ /* Minimal queue depth of one. */\r
+ g_root.qp_create.sq_depth = 1;\r
+ g_root.qp_create.rq_depth = 1;\r
+ }\r
+\r
+ g_root.qp_create.sq_signaled = FALSE;\r
+ g_root.qp_create.sq_sge = 1;\r
+ g_root.qp_create.rq_sge = 1;\r
+\r
+ /* Set the QP attributes when modifying the QP to the reset state. */\r
+ g_root.qp_mod_reset.req_state = IB_QPS_RESET;\r
+\r
+ /* Set the QP attributes when modifying the QP to the init state. */\r
+ g_root.qp_mod_init.req_state = IB_QPS_INIT;\r
+ g_root.qp_mod_init.state.init.access_ctrl = CMT_ACCESS_CTRL;\r
+ g_root.qp_mod_init.state.init.primary_port = g_root.port_num;\r
+ g_root.qp_mod_init.state.init.pkey_index = __get_pkey_index();\r
+}\r
+\r
+\r
+\r
+static ib_api_status_t\r
+__post_recvs(\r
+ IN ib_node_t *p_node )\r
+{\r
+ ib_api_status_t status = IB_SUCCESS;\r
+ ib_recv_wr_t recv_wr;\r
+ ib_recv_wr_t *p_recv_failure;\r
+ ib_local_ds_t ds_array;\r
+ uint32_t i;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ if( !g_root.num_msgs )\r
+ {\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return IB_SUCCESS;\r
+ }\r
+\r
+ cl_memclr( &recv_wr, sizeof( ib_recv_wr_t ) );\r
+ ds_array.length = g_root.msg_size;\r
+ ds_array.lkey = g_root.lkey;\r
+ recv_wr.ds_array = &ds_array;\r
+ recv_wr.num_ds = (( g_root.msg_size <= 4 )? 0: 1 );\r
+\r
+ for( i = 0; i < g_root.num_msgs; i++ )\r
+ {\r
+ CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, (".") );\r
+\r
+ if( g_root.per_msg_buf )\r
+ {\r
+ ds_array.vaddr = (uintn_t)(g_root.p_mem_recv + (i * g_root.msg_size) +\r
+ (p_node->id * g_root.num_msgs * g_root.msg_size));\r
+ }\r
+ else\r
+ {\r
+ ds_array.vaddr = (uintn_t)g_root.p_mem;\r
+ }\r
+\r
+ recv_wr.wr_id = i;\r
+ \r
+ status = ib_post_recv( p_node->h_qp, &recv_wr, &p_recv_failure );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_post_recv failed [%s]!\n", ib_get_err_str(status) );\r
+ break;\r
+ }\r
+ }\r
+\r
+ CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, ("\n") );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return status;\r
+}\r
+\r
+\r
+\r
+static void\r
+__ca_async_event_cb(\r
+ ib_async_event_rec_t *p_err_rec )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ \r
+ CL_TRACE( CMT_DBG_VERBOSE, cmt_dbg_lvl, \r
+ ( "p_err_rec->code is %d\n", p_err_rec->code ) );\r
+ \r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__cm_listen_err_cb(\r
+ IN ib_listen_err_rec_t *p_listen_err_rec )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ if( !p_listen_err_rec )\r
+ printf( "__listen_err_cb NULL p_listen_err_rec\n" );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__cancel_listen_cb(\r
+ IN void *context )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ if( !context )\r
+ printf( "__cancel_listen_cb NULL context\n" );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+/* We need to halt the test and recover from the reject error. */\r
+static void\r
+__rej_cb(\r
+ IN ib_cm_rej_rec_t *p_cm_rej_rec )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ /*\r
+ * Note - because this callback exits the app, any output beyond the\r
+ * the first time may report junk. There have been instances where\r
+ * the callback is invoked more times than there are connection requests\r
+ * but that behavior disapeared if the call to exit below is removed.\r
+ */\r
+ printf( "Connection was rejected, status: 0x%x\n",\r
+ p_cm_rej_rec->rej_status );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ exit( 1 );\r
+}\r
+\r
+\r
+\r
+static void\r
+__req_cb(\r
+ IN ib_cm_req_rec_t *p_cm_req_rec )\r
+{\r
+ ib_api_status_t status;\r
+ ib_node_t *p_node;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ \r
+ CL_ASSERT( p_cm_req_rec );\r
+\r
+ /* Record the starting time for the server. */\r
+ if( !g_root.conn_start_time )\r
+ g_root.conn_start_time = cl_get_time_stamp( );\r
+\r
+ /*\r
+ * Do not send replies until the server is ready to establish all\r
+ * connections.\r
+ */\r
+ cl_mutex_acquire( &g_root.mutex );\r
+ p_node = &g_root.p_nodes[g_root.conn_index++];\r
+\r
+ if( g_root.state == test_connecting )\r
+ {\r
+ /* Get a node for this connection and send the reply. */\r
+ g_root.cm_rep.h_qp = p_node->h_qp;\r
+ status = ib_cm_rep( p_cm_req_rec->h_cm_req, &g_root.cm_rep );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "Call to ib_cm_rep failed\n" );\r
+ exit( 1 );\r
+ }\r
+ }\r
+ else\r
+ {\r
+ p_node->h_cm_req = p_cm_req_rec->h_cm_req;\r
+ }\r
+ cl_mutex_release( &g_root.mutex );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__rep_cb(\r
+ IN ib_cm_rep_rec_t *p_cm_rep_rec )\r
+{\r
+ ib_api_status_t status;\r
+ ib_node_t *p_node;\r
+ uint8_t pdata[IB_RTU_PDATA_SIZE];\r
+ ib_cm_mra_t mra;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_rep_rec );\r
+ \r
+ p_node = (ib_node_t*)p_cm_rep_rec->qp_context;\r
+ CL_ASSERT( p_node );\r
+\r
+ mra.p_mra_pdata = NULL;\r
+ mra.mra_length = 0;\r
+ mra.svc_timeout = 0xff;\r
+\r
+ ib_cm_mra( p_cm_rep_rec->h_cm_rep, &mra );\r
+\r
+ __post_recvs( p_node );\r
+\r
+ /* Mark that we're connected before sending the RTU. */\r
+ p_node->state = node_conn;\r
+\r
+ g_root.cm_rtu.p_rtu_pdata = pdata;\r
+ g_root.cm_rtu.rtu_length = IB_RTU_PDATA_SIZE;\r
+\r
+ status = ib_cm_rtu( p_cm_rep_rec->h_cm_rep, &g_root.cm_rtu );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "Call to ib_cm_rtu returned %s\n", ib_get_err_str( status ) );\r
+ exit( 1 );\r
+ }\r
+\r
+ cl_atomic_inc( &g_root.num_connected );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__rtu_cb(\r
+ IN ib_cm_rtu_rec_t *p_cm_rtu_rec )\r
+{\r
+ ib_node_t *p_node;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_rtu_rec );\r
+ \r
+ p_node = (ib_node_t*)p_cm_rtu_rec->qp_context;\r
+ p_node->state = node_conn;\r
+\r
+ __post_recvs( p_node );\r
+ cl_atomic_inc( &g_root.num_connected );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__mra_cb(\r
+ IN ib_cm_mra_rec_t *p_cm_mra_rec )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_mra_rec );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__apr_cb(\r
+ IN ib_cm_apr_rec_t *p_cm_apr_rec )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_apr_rec );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__lap_cb(\r
+ IN ib_cm_lap_rec_t *p_cm_lap_rec )\r
+{\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_lap_rec );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__dreq_cb(\r
+ IN ib_cm_dreq_rec_t *p_cm_dreq_rec )\r
+{\r
+ ib_node_t *p_node;\r
+ ib_api_status_t status;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_dreq_rec );\r
+ p_node = (ib_node_t*)p_cm_dreq_rec->qp_context;\r
+ CL_ASSERT( p_node );\r
+\r
+ /*\r
+ * Record that we've already received a DREQ to avoid trying to\r
+ * disconnect the QP a second time. Synchronize with the DREQ call\r
+ * using the mutex.\r
+ */\r
+ cl_mutex_acquire( &g_root.mutex );\r
+\r
+ /* If we need to send or receive more data, don't disconnect yet. */\r
+ if( g_root.state == test_disconnecting )\r
+ {\r
+ /* Send the DREP. */\r
+ status = ib_cm_drep( p_cm_dreq_rec->h_cm_dreq, &g_root.cm_drep );\r
+\r
+ /* If the DREP was successful, we're done with this connection. */\r
+ if( status == IB_SUCCESS )\r
+ {\r
+ p_node->state = node_idle;\r
+ cl_atomic_dec( &g_root.num_connected );\r
+ }\r
+ }\r
+ else\r
+ {\r
+ /* Record that we need to disconnect, but don't send the DREP yet. */\r
+ p_node->state = node_dreq_rcvd;\r
+ p_node->h_cm_dreq = p_cm_dreq_rec->h_cm_dreq;\r
+ }\r
+ cl_mutex_release( &g_root.mutex );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__drep_cb(\r
+ IN ib_cm_drep_rec_t *p_cm_drep_rec )\r
+{\r
+ ib_node_t *p_node;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ CL_ASSERT( p_cm_drep_rec );\r
+ p_node = (ib_node_t*)p_cm_drep_rec->qp_context;\r
+ CL_ASSERT( p_node );\r
+\r
+ /* We're done with this connection. */\r
+ cl_mutex_acquire( &g_root.mutex );\r
+ p_node->state = node_idle;\r
+ cl_atomic_dec( &g_root.num_connected );\r
+ cl_mutex_release( &g_root.mutex );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static void\r
+__cq_cb(\r
+ IN const ib_cq_handle_t h_cq,\r
+ IN void* cq_context )\r
+{\r
+ ib_node_t *p_node = (ib_node_t*)cq_context;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ if( !g_root.is_polling )\r
+ {\r
+ if( !__poll_cq( p_node, h_cq ) )\r
+ exit( 1 );\r
+ }\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+static ib_api_status_t\r
+__create_qp(\r
+ IN ib_node_t *p_node )\r
+{\r
+ ib_api_status_t status;\r
+ ib_qp_attr_t attr;\r
+\r
+ /* Set the per node QP attributes. */\r
+ g_root.qp_create.h_sq_cq = p_node->h_send_cq; \r
+ g_root.qp_create.h_rq_cq = p_node->h_recv_cq;\r
+\r
+ /* Allocate the QP. */\r
+ status = ib_create_qp( g_root.h_pd, &g_root.qp_create, p_node, NULL,\r
+ &p_node->h_qp );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "[%d] ib_create_qp failed [%s]!\n", __LINE__, \r
+ ib_get_err_str(status) );\r
+ return status;\r
+ }\r
+\r
+ /* Store the max inline size. */\r
+ status = ib_query_qp( p_node->h_qp, &attr );\r
+ if( status != IB_SUCCESS )\r
+ p_node->max_inline = 0;\r
+ else\r
+ p_node->max_inline = attr.sq_max_inline;\r
+\r
+ /*\r
+ * Transition the QP to the initialize state. This prevents the CM\r
+ * from having to make this QP transition and improves the connection\r
+ * establishment rate.\r
+ */\r
+ status = ib_modify_qp( p_node->h_qp, &g_root.qp_mod_reset );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_modify_qp to IB_QPS_RESET returned %s\n",\r
+ ib_get_err_str(status) );\r
+ return status;\r
+ }\r
+\r
+ status = ib_modify_qp( p_node->h_qp, &g_root.qp_mod_init );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_modify_qp to IB_QPS_INIT returned %s\n",\r
+ ib_get_err_str(status) );\r
+ return status;\r
+ }\r
+\r
+ return status;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Allocate new QPs for all nodes.\r
+ */\r
+static ib_api_status_t\r
+__create_qps()\r
+{\r
+ uint64_t start_time, total_time;\r
+ int32_t i;\r
+ ib_api_status_t status;\r
+\r
+ printf( "Creating QPs...\n" );\r
+ start_time = cl_get_time_stamp();\r
+\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ /* Allocate a new QP. */\r
+ status = __create_qp( &g_root.p_nodes[i] );\r
+ if( status != IB_SUCCESS )\r
+ break;\r
+ }\r
+\r
+ total_time = cl_get_time_stamp() - start_time;\r
+ printf( "Allocation time: %"PRId64" ms\n", total_time/1000 );\r
+\r
+ return status;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Destroy all QPs for all nodes.\r
+ */\r
+static void\r
+__destroy_qps()\r
+{\r
+ uint64_t start_time, total_time;\r
+ int32_t i;\r
+\r
+ printf( "Destroying QPs...\n" );\r
+ start_time = cl_get_time_stamp();\r
+\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ /* Destroy the QP. */\r
+ if( g_root.p_nodes[i].h_qp )\r
+ {\r
+ ib_destroy_qp( g_root.p_nodes[i].h_qp, ib_sync_destroy );\r
+ g_root.p_nodes[i].h_qp = NULL;\r
+ }\r
+ }\r
+\r
+ total_time = cl_get_time_stamp() - start_time;\r
+ printf( "Destruction time: %"PRId64" ms\n", total_time/1000 );\r
+}\r
+\r
+\r
+\r
+static boolean_t\r
+__init_node(\r
+ IN OUT ib_node_t* p_node )\r
+{\r
+ ib_api_status_t status;\r
+ ib_cq_create_t cq_create;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ /* Create the CQs. */\r
+ cl_memclr( &cq_create, sizeof(ib_cq_create_t) );\r
+ if( g_root.num_msgs )\r
+ cq_create.size = g_root.num_msgs;\r
+ else\r
+ cq_create.size = 1; /* minimal of one entry */\r
+\r
+ cq_create.pfn_comp_cb = __cq_cb;\r
+ status = ib_create_cq( g_root.h_ca, &cq_create, p_node, NULL,\r
+ &p_node->h_send_cq );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_create_cq failed for send CQ [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+ if( !g_root.is_polling )\r
+ {\r
+ status = ib_rearm_cq( p_node->h_send_cq, FALSE );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_rearm_cq failed for send CQ [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+ }\r
+\r
+ status = ib_create_cq( g_root.h_ca, &cq_create, p_node, NULL,\r
+ &p_node->h_recv_cq );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_create_cq failed for recv CQ [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+ if( !g_root.is_polling )\r
+ {\r
+ status = ib_rearm_cq( p_node->h_recv_cq, FALSE );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_rearm_cq failed for recv CQ [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+ }\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return TRUE;\r
+}\r
+\r
+static boolean_t\r
+__destroy_node(\r
+ IN OUT ib_node_t* p_node )\r
+{\r
+ ib_api_status_t status = IB_SUCCESS;\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ if (!p_node )\r
+ return (FALSE);\r
+ if ( p_node->h_send_cq )\r
+ {\r
+ status = ib_destroy_cq( p_node->h_send_cq, ib_sync_destroy );\r
+ p_node->h_send_cq = NULL;\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_destroy_cq failed for send CQ [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ }\r
+ }\r
+ if (p_node->h_recv_cq)\r
+ {\r
+ status = ib_destroy_cq( p_node->h_recv_cq, ib_sync_destroy );\r
+ p_node->h_recv_cq = NULL;\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_destroy_cq failed for recv CQ [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ }\r
+ }\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return (status == IB_SUCCESS);\r
+}\r
+\r
+\r
+\r
+static boolean_t\r
+__create_nodes()\r
+{\r
+ int32_t i;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ g_root.p_nodes[i].id = i;\r
+\r
+ CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, \r
+ ("--> create connection %d of instance %d\n", i, g_root.inst_id) );\r
+ \r
+ if( !__init_node( &g_root.p_nodes[i] ) )\r
+ return FALSE;\r
+ }\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return TRUE;\r
+}\r
+\r
+static boolean_t\r
+__destroy_nodes()\r
+{\r
+ int32_t i;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ if( !__destroy_node( &g_root.p_nodes[i] ) )\r
+ return FALSE;\r
+ }\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return TRUE;\r
+}\r
+\r
+/* query function called by ib_query() */\r
+static void\r
+__sa_query_cb(\r
+ IN ib_query_rec_t *p_query_rec )\r
+{\r
+ ib_path_rec_t *p_path;\r
+ ib_api_status_t status;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ CL_ASSERT( p_query_rec );\r
+\r
+ if( p_query_rec->status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_query failed [%d]\n", p_query_rec->status ); \r
+ return;\r
+ }\r
+\r
+ if( p_query_rec->query_type != IB_QUERY_PATH_REC_BY_LIDS )\r
+ {\r
+ printf( "Unexpected query type returned.\n" ); \r
+ return;\r
+ }\r
+\r
+ if( !p_query_rec->p_result_mad )\r
+ {\r
+ printf( "No result MAD returned from ib_query.\n" ); \r
+ return;\r
+ }\r
+\r
+ /* copy the 1st (zero'th) path record to local storage. */\r
+ p_path = ib_get_query_path_rec( p_query_rec->p_result_mad, 0 );\r
+ memcpy( (void*)&g_root.path_rec, (void*)p_path, \r
+ sizeof(ib_path_rec_t) );\r
+\r
+ CL_TRACE( CMT_DBG_VERBOSE, cmt_dbg_lvl,\r
+ ( "path{ slid:0x%x, dlid:0x%x }\n", \r
+ g_root.path_rec.slid, g_root.path_rec.dlid) );\r
+\r
+ /* release response MAD(s) back to AL pool */\r
+ if( p_query_rec->p_result_mad )\r
+ {\r
+ status = ib_put_mad( p_query_rec->p_result_mad );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_put_mad() failed [%s]\n",\r
+ ib_get_err_str(status) );\r
+ }\r
+ }\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+}\r
+\r
+\r
+\r
+static boolean_t\r
+__query_for_path()\r
+{\r
+ ib_api_status_t status;\r
+ ib_query_req_t query_rec;\r
+ ib_lid_pair_t lid_pair;\r
+\r
+ /* Query the SA for a path record. */\r
+ query_rec.query_type = IB_QUERY_PATH_REC_BY_LIDS;\r
+\r
+ lid_pair.src_lid = g_root.l_lid;\r
+ lid_pair.dest_lid = g_root.r_lid;\r
+\r
+ query_rec.p_query_input = (void*)&lid_pair;\r
+ query_rec.port_guid = g_root.port_guid;\r
+ query_rec.timeout_ms = 5 * 1000; // seconds\r
+ query_rec.retry_cnt = 2;\r
+ query_rec.flags = IB_FLAGS_SYNC;\r
+ query_rec.query_context = &g_root;\r
+ query_rec.pfn_query_cb = __sa_query_cb;\r
+\r
+ status = ib_query( g_root.h_al, &query_rec, NULL );\r
+ if( ( status != IB_SUCCESS ) || ( !g_root.path_rec.dlid ) )\r
+ {\r
+ printf( "ib_query failed.\n" );\r
+ return FALSE;\r
+ }\r
+\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+static boolean_t\r
+__create_messages()\r
+{\r
+ ib_mr_create_t mr_create;\r
+ uint32_t buf_size;\r
+ ib_api_status_t status;\r
+\r
+ /* If we're not sending messages, just return. */\r
+ if( !g_root.num_msgs || !g_root.msg_size )\r
+ return TRUE;\r
+\r
+ /* Allocate the message memory - we ignore the data, so just one buffer. */\r
+ if( g_root.per_msg_buf )\r
+ buf_size = (g_root.num_nodes * g_root.num_msgs * g_root.msg_size) << 1;\r
+ else\r
+ buf_size = g_root.msg_size;\r
+ g_root.p_mem = cl_zalloc( buf_size );\r
+ if( !g_root.p_mem )\r
+ {\r
+ printf( "Not enough memory for transfers!\n" );\r
+ return FALSE;\r
+ }\r
+ memset (g_root.p_mem, 0xae, buf_size);\r
+ g_root.p_mem_recv = g_root.p_mem;\r
+ g_root.p_mem_send = g_root.p_mem + (buf_size >> 1);\r
+\r
+ /* Register the memory with AL. */\r
+ mr_create.vaddr = g_root.p_mem;\r
+ mr_create.length = buf_size;\r
+ mr_create.access_ctrl = IB_AC_LOCAL_WRITE | IB_AC_MW_BIND;\r
+ status = ib_reg_mem( g_root.h_pd, &mr_create, &g_root.lkey, \r
+ &g_root.rkey, &g_root.h_mr );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_reg_mem failed [%s]!\n", ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * PnP callback handler. Record the port GUID of an active port.\r
+ */\r
+static ib_api_status_t\r
+__pnp_cb(\r
+ IN ib_pnp_rec_t *p_pnp_rec )\r
+{\r
+ ib_pnp_port_rec_t* p_port_rec;\r
+ uint32_t size;\r
+\r
+ p_port_rec = (ib_pnp_port_rec_t*)p_pnp_rec;\r
+\r
+ /*\r
+ * Ignore PNP events that are not related to port active, or if\r
+ * we already have an active port.\r
+ */\r
+ if( p_pnp_rec->pnp_event != IB_PNP_PORT_ACTIVE || g_root.port_guid )\r
+ return IB_SUCCESS;\r
+\r
+ /* Find the proper port for the given local LID. */\r
+ if( g_root.l_lid )\r
+ {\r
+ if( g_root.l_lid != p_port_rec->p_port_attr->lid )\r
+ return IB_SUCCESS;\r
+ }\r
+ else\r
+ {\r
+ g_root.l_lid = p_port_rec->p_port_attr->lid;\r
+ printf( "\tlocal lid....: x%x\n", g_root.l_lid );\r
+ }\r
+\r
+ /* Record the active port information. */\r
+ g_root.ca_guid = p_port_rec->p_ca_attr->ca_guid;\r
+ g_root.port_num = p_port_rec->p_port_attr->port_num;\r
+\r
+ /* Record the PKEYs available on the active port. */\r
+ size = sizeof( ib_net16_t ) * p_port_rec->p_port_attr->num_pkeys;\r
+ g_root.p_pkey_table = cl_zalloc( size );\r
+ if( !g_root.p_pkey_table )\r
+ return IB_SUCCESS;\r
+ g_root.num_pkeys = p_port_rec->p_port_attr->num_pkeys;\r
+ cl_memcpy( g_root.p_pkey_table,\r
+ p_port_rec->p_port_attr->p_pkey_table, size );\r
+\r
+ /* Set the port_guid last to indicate that we're ready. */\r
+ g_root.port_guid = p_port_rec->p_port_attr->port_guid;\r
+ return IB_SUCCESS;\r
+}\r
+\r
+\r
+/*\r
+ * Register for PnP events and wait until a port becomes active.\r
+ */\r
+static boolean_t\r
+__reg_pnp()\r
+{\r
+ ib_api_status_t status;\r
+ ib_pnp_req_t pnp_req;\r
+ ib_pnp_handle_t h_pnp;\r
+\r
+ cl_memclr( &pnp_req, sizeof( ib_pnp_req_t ) );\r
+ pnp_req.pnp_class = IB_PNP_PORT;\r
+ pnp_req.pnp_context = &g_root;\r
+ pnp_req.pfn_pnp_cb = __pnp_cb;\r
+\r
+ /* Register for PnP events. */\r
+ status = ib_reg_pnp( g_root.h_al, &pnp_req, &h_pnp );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_reg_pnp failed [%s]!\n", ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+\r
+ /* Wait until a port goes active. */\r
+ while( !g_root.port_guid )\r
+ cl_thread_suspend( 10 );\r
+\r
+ /* Deregister from PnP. */\r
+ ib_dereg_pnp( h_pnp, NULL );\r
+\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+static boolean_t\r
+__init_root()\r
+{\r
+ ib_api_status_t status;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ cl_mutex_construct( &g_root.mutex );\r
+ if( cl_mutex_init( &g_root.mutex ) != CL_SUCCESS )\r
+ return FALSE;\r
+\r
+ /* Open AL. */\r
+ status = ib_open_al( &g_root.h_al );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_open_al failed [%s]!\n", ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+\r
+ /* Register for PnP events, and wait until we have an active port. */\r
+ if( !__reg_pnp() )\r
+ return FALSE;\r
+\r
+ /* Open the CA. */\r
+ status = ib_open_ca( g_root.h_al, g_root.ca_guid,\r
+ __ca_async_event_cb, &g_root, &g_root.h_ca );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_open_ca failed [%s]!\n", ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+\r
+ /* Create a PD. */\r
+ status = ib_alloc_pd( g_root.h_ca, IB_PDT_NORMAL, &g_root, \r
+ &g_root.h_pd );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_alloc_pd failed [%s]!\n", ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+\r
+ /* Get a path record to the remote side. */\r
+ if( !__query_for_path() )\r
+ {\r
+ printf( "Unable to query for path record!\n" );\r
+ return FALSE;\r
+ }\r
+\r
+ /* Allocate and register memory for the messages. */\r
+ if( !__create_messages() )\r
+ {\r
+ printf( "Unable to create messages!\n" );\r
+ return FALSE;\r
+ }\r
+\r
+ /* Create the connection endpoints. */\r
+ g_root.p_nodes = (ib_node_t*)cl_zalloc(\r
+ sizeof(ib_node_t) * g_root.num_nodes );\r
+ if( !g_root.p_nodes )\r
+ {\r
+ printf( "Unable to allocate nodes\n" ); \r
+ return FALSE;\r
+ }\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+static void\r
+__cleanup()\r
+{\r
+ if( g_root.h_listen )\r
+ ib_cm_cancel( g_root.h_listen, __cancel_listen_cb );\r
+\r
+ /* Close AL if it was opened. */\r
+ if( g_root.h_al )\r
+ ib_close_al( g_root.h_al );\r
+\r
+ cl_mutex_destroy( &g_root.mutex );\r
+\r
+ if( g_root.p_mem )\r
+ cl_free( g_root.p_mem );\r
+\r
+ if( g_root.p_pkey_table )\r
+ cl_free( g_root.p_pkey_table );\r
+\r
+ /* Free all allocated memory. */\r
+ if( g_root.p_nodes )\r
+ cl_free( g_root.p_nodes );\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Have the server start listening for connections.\r
+ */\r
+static boolean_t\r
+__listen()\r
+{\r
+ ib_cm_listen_t cm_listen;\r
+ ib_api_status_t status;\r
+\r
+ cl_memclr( &cm_listen, sizeof( ib_cm_listen_t ) );\r
+\r
+ /* The server side listens. */\r
+ cm_listen.svc_id = CMT_BASE_SVC_ID + g_root.inst_id;\r
+\r
+ cm_listen.pfn_cm_req_cb = __req_cb;\r
+\r
+ cm_listen.qp_type = IB_QPT_RELIABLE_CONN;\r
+\r
+ status = ib_cm_listen( g_root.h_al, &cm_listen, \r
+ __cm_listen_err_cb, &g_root, &g_root.h_listen );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_cm_listen failed [%s]!\n", ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Initiate all client connection requests.\r
+ */\r
+static ib_api_status_t\r
+__conn_reqs()\r
+{\r
+ ib_api_status_t status;\r
+ int32_t i;\r
+ uint8_t pdata[IB_REQ_PDATA_SIZE];\r
+\r
+ g_root.cm_req.p_req_pdata = pdata;\r
+ g_root.cm_req.req_length = IB_REQ_PDATA_SIZE;\r
+\r
+ /* Request a connection for each client. */\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ g_root.cm_req.h_qp = g_root.p_nodes[i].h_qp;\r
+\r
+ status = ib_cm_req( &g_root.cm_req );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_cm_req failed [%s]!\n", ib_get_err_str(status) );\r
+ return status;\r
+ }\r
+ }\r
+ return IB_SUCCESS;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Send any connection replies waiting to be sent.\r
+ */\r
+static ib_api_status_t\r
+__conn_reps()\r
+{\r
+ ib_api_status_t status;\r
+ uintn_t i;\r
+ uint8_t pdata[IB_REP_PDATA_SIZE];\r
+\r
+ g_root.cm_rep.p_rep_pdata = pdata;\r
+ g_root.cm_rep.rep_length = IB_REP_PDATA_SIZE;\r
+\r
+ /* Send a reply for each connection that requires one. */\r
+ for( i = 0; i < g_root.conn_index; i++ )\r
+ {\r
+ g_root.cm_rep.h_qp = g_root.p_nodes[i].h_qp;\r
+ status = ib_cm_rep( g_root.p_nodes[i].h_cm_req, &g_root.cm_rep );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_cm_rep failed [%s]!\n", ib_get_err_str(status) );\r
+ return status;\r
+ }\r
+ }\r
+ return IB_SUCCESS;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Establish all connections.\r
+ */\r
+static ib_api_status_t\r
+__connect()\r
+{\r
+ uint64_t total_time;\r
+ ib_api_status_t status;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ printf( "Connecting...\n" );\r
+ cl_mutex_acquire( &g_root.mutex );\r
+ g_root.state = test_connecting;\r
+\r
+ /* Initiate the connections. */\r
+ if( g_root.is_server )\r
+ {\r
+ /*\r
+ * Send any replies. Note that we hold the mutex while sending the\r
+ * replies since we need to use the global cm_rep structure.\r
+ */\r
+ status = __conn_reps();\r
+ cl_mutex_release( &g_root.mutex );\r
+ }\r
+ else\r
+ {\r
+ cl_mutex_release( &g_root.mutex );\r
+ g_root.conn_start_time = cl_get_time_stamp();\r
+ status = __conn_reqs();\r
+ }\r
+\r
+ if( status != IB_SUCCESS )\r
+ return status;\r
+\r
+ /* Wait for all connections to complete. */\r
+ while( g_root.num_connected < g_root.num_nodes )\r
+ cl_thread_suspend( 0 );\r
+\r
+ /* Calculate the total connection time. */\r
+ total_time = cl_get_time_stamp() - g_root.conn_start_time;\r
+ g_root.state = test_idle;\r
+\r
+ /* Reset connection information for next test. */\r
+ g_root.conn_index = 0;\r
+ g_root.conn_start_time = 0;\r
+\r
+ printf( "Connect time: %"PRId64" ms\n", total_time/1000 );\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return status;\r
+}\r
+\r
+\r
+\r
+static void\r
+__disconnect()\r
+{\r
+ ib_api_status_t status;\r
+ int32_t i;\r
+ ib_node_t *p_node;\r
+ uint64_t total_time, start_time;\r
+\r
+ printf( "Disconnecting...\n" );\r
+\r
+ /* Initiate the disconnection process. */\r
+ cl_mutex_acquire( &g_root.mutex );\r
+ g_root.state = test_disconnecting;\r
+ start_time = cl_get_time_stamp();\r
+ cl_mutex_release( &g_root.mutex );\r
+\r
+ /* We hold the mutex to prevent calling ib_cm_drep at the same time. */\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ p_node = &g_root.p_nodes[i];\r
+\r
+ /*\r
+ * Send the DREQ. Note that some of these may fail, since the\r
+ * remote side may be disconnecting at the same time. Call DREQ\r
+ * only if we haven't called DREP yet.\r
+ */\r
+ cl_mutex_acquire( &g_root.mutex );\r
+ switch( p_node->state )\r
+ {\r
+ case node_conn:\r
+ g_root.cm_dreq.h_qp = p_node->h_qp;\r
+ status = ib_cm_dreq( &g_root.cm_dreq );\r
+ if( status == IB_SUCCESS )\r
+ p_node->state = node_dreq_sent;\r
+ break;\r
+\r
+ case node_dreq_rcvd:\r
+ status = ib_cm_drep( p_node->h_cm_dreq, &g_root.cm_drep );\r
+\r
+ /* If the DREP was successful, we're done with this connection. */\r
+ if( status == IB_SUCCESS )\r
+ {\r
+ p_node->state = node_idle;\r
+ cl_atomic_dec( &g_root.num_connected );\r
+ }\r
+ break;\r
+\r
+ default:\r
+ /* Node is already disconnected. */\r
+ break;\r
+ }\r
+ cl_mutex_release( &g_root.mutex );\r
+ }\r
+\r
+ /* Wait for all disconnections to complete. */\r
+ while( g_root.num_connected )\r
+ cl_thread_suspend( 0 );\r
+\r
+ if( g_root.h_listen )\r
+ {\r
+ ib_cm_cancel( g_root.h_listen, __cancel_listen_cb );\r
+ g_root.h_listen = NULL;\r
+ }\r
+ /* Calculate the total connection time. */\r
+ total_time = cl_get_time_stamp() - start_time;\r
+ g_root.state = test_idle;\r
+\r
+ printf( "Disconnect time: %"PRId64" ms\n", total_time/1000 );\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Send the requested number of messages on each connection.\r
+ */\r
+static boolean_t\r
+__send_msgs()\r
+{\r
+ ib_api_status_t status;\r
+ int32_t i;\r
+ uint32_t m;\r
+ ib_send_wr_t send_wr;\r
+ ib_send_wr_t *p_send_failure;\r
+ ib_local_ds_t ds_array;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ /* For each connection... */\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ /* Send the specified number of messages. */\r
+ for( m = 0; m < g_root.num_msgs; m++ )\r
+ {\r
+ /* Get the buffer for this message. */\r
+ if( g_root.per_msg_buf )\r
+ {\r
+ ds_array.vaddr = (uintn_t)(g_root.p_mem_send +\r
+ (i * g_root.num_msgs) + (m * g_root.msg_size));\r
+ }\r
+ else\r
+ {\r
+ ds_array.vaddr = (uintn_t)g_root.p_mem;\r
+ }\r
+ ds_array.length = g_root.msg_size;\r
+ ds_array.lkey = g_root.lkey;\r
+\r
+ /* Format the send WR for this message. */\r
+ send_wr.ds_array = &ds_array;\r
+ send_wr.send_opt = IB_SEND_OPT_SIGNALED | IB_SEND_OPT_SOLICITED;\r
+ send_wr.send_opt |= ((g_root.msg_size <= 4)? IB_SEND_OPT_IMMEDIATE : 0x0 );\r
+ send_wr.wr_type = WR_SEND;\r
+ send_wr.num_ds = ((g_root.msg_size <= 4)? 0 : 1 );\r
+ send_wr.p_next = NULL;\r
+ send_wr.wr_id = m;\r
+\r
+ if( g_root.msg_size < g_root.p_nodes[i].max_inline )\r
+ send_wr.send_opt |= IB_SEND_OPT_INLINE;\r
+\r
+ /* Torpedoes away! Send the message. */\r
+ CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, (".") );\r
+ status = ib_post_send( g_root.p_nodes[i].h_qp, &send_wr,\r
+ &p_send_failure );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_post_send failed [%s]!\n",\r
+ ib_get_err_str(status) );\r
+ return FALSE;\r
+ }\r
+ }\r
+ }\r
+ \r
+ CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, ("\n") );\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Remove num_msgs completions from the specified CQ.\r
+ */\r
+static boolean_t\r
+__poll_cq(\r
+ IN ib_node_t *p_node,\r
+ IN ib_cq_handle_t h_cq )\r
+{\r
+ ib_api_status_t status = IB_SUCCESS;\r
+ ib_wc_t free_wc[2];\r
+ ib_wc_t *p_free_wc, *p_done_wc;\r
+\r
+ CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+\r
+ while( status != IB_NOT_FOUND )\r
+ {\r
+ /* Get all completions. */\r
+ p_free_wc = &free_wc[0];\r
+ free_wc[0].p_next = &free_wc[1];\r
+ free_wc[1].p_next = NULL;\r
+ p_done_wc = NULL;\r
+\r
+ status = ib_poll_cq( h_cq, &p_free_wc, &p_done_wc );\r
+\r
+ /* Continue polling if nothing is done. */\r
+ if( status == IB_NOT_FOUND )\r
+ break;\r
+\r
+ /* Abort if an error occurred. */\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "Error polling status = %#x( wc_status =%s)\n", \r
+ status,\r
+ ((p_done_wc != NULL )? wc_status_text[p_done_wc->status]:"N/A") );\r
+ return FALSE;\r
+ }\r
+\r
+ while( p_done_wc )\r
+ {\r
+ switch( p_done_wc->status )\r
+ {\r
+ case IB_WCS_SUCCESS:\r
+ CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl,\r
+ ("Got a completion: \n\ttype....:%s\n\twr_id...:%"PRIx64"\n"\r
+ "status....:%s\n", wc_type_text[p_done_wc->wc_type],\r
+ p_done_wc->wr_id, wc_status_text[p_done_wc->status] ) );\r
+\r
+ if( p_done_wc->wc_type == IB_WC_RECV )\r
+ {\r
+ CL_ASSERT( p_done_wc->wr_id == p_node->recv_cnt );\r
+ if( p_done_wc->length != g_root.msg_size )\r
+ {\r
+ printf( "Error: received %d bytes, expected %d.\n",\r
+ p_done_wc->length, g_root.msg_size );\r
+ }\r
+\r
+ p_node->recv_cnt++;\r
+ g_root.total_recv++;\r
+ }\r
+ else\r
+ {\r
+ CL_ASSERT( p_done_wc->wr_id == p_node->send_cnt );\r
+ p_node->send_cnt++;\r
+ g_root.total_sent++;\r
+ }\r
+ break;\r
+\r
+ default:\r
+ printf( "[%d] Bad completion type(%s) status(%s)\n",\r
+ __LINE__, wc_type_text[p_done_wc->wc_type],\r
+ wc_status_text[p_done_wc->status] );\r
+ return FALSE;\r
+ }\r
+ p_done_wc = p_done_wc->p_next;\r
+ }\r
+ }\r
+\r
+ if( !g_root.is_polling )\r
+ {\r
+ status = ib_rearm_cq(h_cq, FALSE);\r
+ if (status != IB_SUCCESS)\r
+ {\r
+ printf("Failed to rearm CQ %p\n", h_cq );\r
+ return FALSE;\r
+ }\r
+ }\r
+\r
+ CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Remove num_msgs completions from all send CQs for all connections.\r
+ */\r
+static boolean_t\r
+__poll_send_cqs()\r
+{\r
+ ib_node_t *p_node;\r
+ int32_t i;\r
+\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ p_node = &g_root.p_nodes[i];\r
+ while( p_node->send_cnt < g_root.num_msgs )\r
+ {\r
+ if( !g_root.is_polling )\r
+ cl_thread_suspend( 0 );\r
+ else if( !__poll_cq( p_node, p_node->h_send_cq ) )\r
+ return FALSE;\r
+ }\r
+ }\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+/*\r
+ * Remove num_msgs completions from all receive CQs for all connections.\r
+ */\r
+static boolean_t\r
+__poll_recv_cqs()\r
+{\r
+ ib_node_t *p_node;\r
+ int32_t i;\r
+\r
+ for( i = 0; i < g_root.num_nodes; i++ )\r
+ {\r
+ p_node = &g_root.p_nodes[i];\r
+ while( p_node->recv_cnt < g_root.num_msgs )\r
+ {\r
+ if( !g_root.is_polling )\r
+ cl_thread_suspend( 0 );\r
+ else if( !__poll_cq( p_node, p_node->h_recv_cq ) )\r
+ return FALSE;\r
+ }\r
+ }\r
+ return TRUE;\r
+}\r
+\r
+\r
+\r
+/**********************************************************************\r
+ **********************************************************************/\r
+int __cdecl\r
+main(\r
+ int argc,\r
+ char* argv[] )\r
+{\r
+ uint64_t start_time, total_time;\r
+ uint64_t total_xfer;\r
+ uint32_t i;\r
+\r
+ cl_memclr( &g_root, sizeof(ib_root_t) );\r
+\r
+ /* Set defaults. */\r
+ if( !__parse_options( argc, argv ) )\r
+ return 1;\r
+\r
+ /* Initialize the root - open all common HCA resources. */\r
+ if( !__init_root() )\r
+ {\r
+ printf( "__init_root failed\n" );\r
+ __cleanup();\r
+ return 1;\r
+ }\r
+\r
+ /*\r
+ * Execute the test the specified number of times. Abort the test\r
+ * if any errors occur.\r
+ */\r
+ total_xfer = g_root.num_msgs * g_root.msg_size * g_root.num_nodes;\r
+ for( i = 0; i < g_root.num_iter; i++ )\r
+ {\r
+ printf( "----- Iteration: %d, %d connections -----\n",\r
+ i, g_root.num_nodes );\r
+\r
+ /* Initialize the connection parameters. */\r
+ __init_conn_info();\r
+ __init_qp_info();\r
+ __create_nodes();\r
+ /* Start listening for connections if we're the server. */\r
+ if( g_root.is_server )\r
+ __listen();\r
+\r
+ /* Allocate a new set of QPs for the connections. */\r
+ if( __create_qps() != IB_SUCCESS )\r
+ {\r
+ printf( "Unable to allocate QPs for test.\n" );\r
+ break;\r
+ }\r
+\r
+ /* Establish all connections. */\r
+ if( __connect() != IB_SUCCESS )\r
+ {\r
+ printf( "Failed to establish connections.\n" );\r
+ break;\r
+ }\r
+\r
+ printf( "Transfering data...\n" );\r
+ g_root.state = test_transfering;\r
+ start_time = cl_get_time_stamp();\r
+\r
+ if( g_root.num_msgs )\r
+ {\r
+ if( g_root.is_server )\r
+ {\r
+ /* The server initiate the sends to avoid race conditions. */\r
+ if( !__send_msgs() )\r
+ break;\r
+\r
+ /* Get all send completions. */\r
+ if( !__poll_send_cqs() )\r
+ break;\r
+\r
+ /* Get all receive completions. */\r
+ if( !__poll_recv_cqs() )\r
+ break;\r
+ }\r
+ else\r
+ {\r
+ /* Get all receive completions. */\r
+ if( !__poll_recv_cqs() )\r
+ break;\r
+\r
+ /* Reply to the sends. */\r
+ if( !__send_msgs() )\r
+ break;\r
+\r
+ /* Get all send completions. */\r
+ if( !__poll_send_cqs() )\r
+ break;\r
+ }\r
+ }\r
+\r
+ total_time = cl_get_time_stamp() - start_time;\r
+ g_root.state = test_idle;\r
+\r
+ printf( "Data transfer time: %"PRId64" ms, %d messages/conn, "\r
+ "%"PRId64" total bytes\n", total_time/1000,\r
+ g_root.num_msgs, total_xfer );\r
+\r
+ /* Disconnect all connections. */\r
+ __disconnect();\r
+ __destroy_qps();\r
+ __destroy_nodes();\r
+ }\r
+\r
+ __cleanup();\r
+ return 0;\r
+}\r
+++ /dev/null
-/*\r
- * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.\r
- * Copyright (c) 1996-2002 Intel Corporation. All rights reserved. \r
- *\r
- * This software is available to you under the OpenIB.org BSD license\r
- * below:\r
- *\r
- * Redistribution and use in source and binary forms, with or\r
- * without modification, are permitted provided that the following\r
- * conditions are met:\r
- *\r
- * - Redistributions of source code must retain the above\r
- * copyright notice, this list of conditions and the following\r
- * disclaimer.\r
- *\r
- * - Redistributions in binary form must reproduce the above\r
- * copyright notice, this list of conditions and the following\r
- * disclaimer in the documentation and/or other materials\r
- * provided with the distribution.\r
- *\r
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
- * SOFTWARE.\r
- *\r
- * $Id$\r
- */\r
-\r
-\r
-/*\r
- * Abstract:\r
- * Command line interface for cmtest.\r
- *\r
- * Environment:\r
- * User Mode\r
- */\r
-\r
-#include <stdio.h>\r
-#include <stdlib.h>\r
-#include <stdarg.h>\r
-#include <ctype.h>\r
-#include <complib/cl_atomic.h>\r
-#include <complib/cl_debug.h>\r
-#include <complib/cl_event.h>\r
-#include <complib/cl_math.h>\r
-#include <complib/cl_mutex.h>\r
-#include <complib/cl_qlist.h>\r
-#include <complib/cl_thread.h>\r
-#include <complib/cl_timer.h>\r
-#include <iba/ib_types.h>\r
-#include <iba/ib_al.h>\r
-\r
-\r
-/* Globals */\r
-#define CMT_DBG_VERBOSE 1\r
-\r
-#define CMT_BASE_SVC_ID 0xFFEE\r
-#define CMT_ACCESS_CTRL (IB_AC_LOCAL_WRITE + IB_AC_RDMA_READ + IB_AC_RDMA_WRITE)\r
-#define BAD_PKEY_INDEX 0xFFFF\r
-\r
-\r
-typedef enum _cmtest_state\r
-{\r
- test_idle, test_connecting, test_transfering, test_disconnecting\r
-\r
-} cmtest_state_t;\r
-\r
-\r
-\r
-typedef struct _ib_root\r
-{\r
- ib_al_handle_t h_al;\r
- ib_pd_handle_t h_pd;\r
-\r
- /* Input parameters to control test. */\r
- int32_t num_nodes;\r
- uint32_t num_msgs;\r
- boolean_t per_msg_buf;\r
- cl_mutex_t mutex;\r
-\r
- cmtest_state_t state;\r
- atomic32_t num_connected;\r
- uint32_t conn_index; /* current connection id */\r
- uint32_t total_sent;\r
- uint32_t total_recv;\r
-\r
- uint32_t num_iter;\r
-\r
- uint32_t msg_size;\r
-\r
- ib_ca_handle_t h_ca;\r
- ib_net16_t l_lid;\r
- ib_net16_t r_lid;\r
- ib_net64_t ca_guid;\r
- ib_net64_t port_guid;\r
- uint8_t port_num;\r
- uint16_t num_pkeys;\r
- ib_net16_t *p_pkey_table;\r
-\r
- /* cm info */\r
- boolean_t is_server;\r
- ib_listen_handle_t h_listen;\r
- ib_path_rec_t path_rec;\r
-\r
- /* CQ info. */\r
- boolean_t is_polling;\r
-\r
- struct _ib_node *p_nodes;\r
- ib_qp_create_t qp_create;\r
- ib_qp_mod_t qp_mod_reset;\r
- ib_qp_mod_t qp_mod_init;\r
-\r
- /* reg mem info */\r
- ib_mr_handle_t h_mr;\r
- uint32_t lkey;\r
- uint32_t rkey;\r
- uint8_t *p_mem;\r
- uint8_t *p_mem_recv;\r
- uint8_t *p_mem_send;\r
-\r
- uint64_t conn_start_time;\r
-\r
- /*\r
- * Connection parameters are initialized once to improve connection\r
- * establishment rate.\r
- */\r
- ib_cm_req_t cm_req;\r
- ib_cm_rep_t cm_rep;\r
- ib_cm_rtu_t cm_rtu;\r
- ib_cm_dreq_t cm_dreq;\r
- ib_cm_drep_t cm_drep;\r
-\r
- uint32_t inst_id;\r
-\r
-} ib_root_t;\r
-\r
-\r
-\r
-typedef enum _cmnode_state\r
-{\r
- node_idle, node_conn, node_dreq_sent, node_dreq_rcvd\r
-\r
-} cmnode_state_t;\r
-\r
-\r
-\r
-typedef struct _ib_node\r
-{\r
- uint64_t id;\r
-\r
- ib_cq_handle_t h_send_cq;\r
- ib_cq_handle_t h_recv_cq;\r
- ib_qp_handle_t h_qp;\r
- uint32_t max_inline;\r
-\r
- cmnode_state_t state;\r
- ib_cm_handle_t h_cm_req;\r
- ib_cm_handle_t h_cm_dreq;\r
-\r
- uint32_t send_cnt;\r
- uint32_t recv_cnt;\r
-\r
-} ib_node_t;\r
-\r
-\r
-\r
-uint32_t cmt_dbg_lvl = 0x80000000;\r
-\r
-ib_root_t g_root;\r
-\r
-\r
-static char *wc_type_text[] = {\r
- "IB_WC_SEND",\r
- "IB_WC_RDMA_WRITE",\r
- "IB_WC_RECV",\r
- "IB_WC_RDMA_READ",\r
- "IB_WC_MW_BIND",\r
- "IB_WC_FETCH_ADD",\r
- "IB_WC_COMPARE_SWAP",\r
- "IB_WC_RECV_RDMA_WRITE"\r
-};\r
-\r
-static char *wc_status_text[] = {\r
- "IB_WCS_SUCCESS",\r
- "IB_WCS_LOCAL_LEN_ERR",\r
- "IB_WCS_LOCAL_OP_ERR",\r
- "IB_WCS_LOCAL_EEC_OP_ERR",\r
- "IB_WCS_LOCAL_PROTECTION_ERR",\r
- "IB_WCS_WR_FLUSHED_ERR",\r
- "IB_WCS_MEM_WINDOW_BIND_ERR",\r
- "IB_WCS_REM_ACCESS_ERR",\r
- "IB_WCS_REM_OP_ERR",\r
- "IB_WCS_RNR_RETRY_ERR",\r
- "IB_WCS_TIMEOUT_RETRY_ERR",\r
- "IB_WCS_REM_INVALID_REQ_ERR",\r
- "IB_WCS_REM_INVALID_RD_REQ_ERR",\r
- "IB_WCS_INVALID_EECN",\r
- "IB_WCS_INVALID_EEC_STATE",\r
- "IB_WCS_UNMATCHED_RESPONSE", \r
- "IB_WCS_CANCELED" \r
-};\r
-\r
-\r
-\r
-static void\r
-__req_cb(\r
- IN ib_cm_req_rec_t *p_cm_req_rec );\r
-\r
-static void\r
-__rep_cb(\r
- IN ib_cm_rep_rec_t *p_cm_rep_rec );\r
-\r
-static void\r
-__rtu_cb(\r
- IN ib_cm_rtu_rec_t *p_cm_rtu_rec );\r
-\r
-static void\r
-__rej_cb(\r
- IN ib_cm_rej_rec_t *p_cm_rej_rec );\r
-\r
-static void\r
-__mra_cb(\r
- IN ib_cm_mra_rec_t *p_cm_mra_rec );\r
-\r
-static void\r
-__apr_cb(\r
- IN ib_cm_apr_rec_t *p_cm_apr_rec );\r
-\r
-static void\r
-__lap_cb(\r
- IN ib_cm_lap_rec_t *p_cm_lap_rec );\r
-\r
-static void\r
-__dreq_cb(\r
- IN ib_cm_dreq_rec_t *p_cm_dreq_rec );\r
-\r
-static void\r
-__drep_cb(\r
- IN ib_cm_drep_rec_t *p_cm_drep_rec );\r
-\r
-static boolean_t\r
-__poll_cq(\r
- IN ib_node_t *p_node,\r
- IN ib_cq_handle_t h_cq );\r
-\r
-\r
-/**********************************************************************\r
- **********************************************************************/\r
-static void\r
-__show_usage()\r
-{\r
- printf( "\n------- cmtest - Usage and options ----------------------\n" );\r
- printf( "Usage: cmtest [options]\n");\r
- printf( "Options:\n" );\r
- printf( "-s\n"\r
- "--server\n"\r
- " This option directs cmtest to act as a Server\n" );\r
- printf( "-l <lid>\n"\r
- "--local <lid>\n"\r
- " This option specifies the local endpoint.\n" );\r
- printf( "-r <lid>\n"\r
- "--remote <lid>\n"\r
- " This option specifies the remote endpoint.\n" );\r
- printf( "-c <number>\n"\r
- "--connect <number>\n"\r
- " This option specifies the number of connections to open.\n"\r
- " Default of 1.\n" );\r
- printf( "-m <bytes>\n"\r
- "--msize <bytes>\n"\r
- " This option specifies the byte size of each message.\n"\r
- " Default is 100 bytes.\n" );\r
- printf( "-n <number>\n"\r
- "--nmsgs <number>\n"\r
- " This option specifies the number of messages to send at a time.\n" );\r
- printf( "-p\n"\r
- "--permsg\n"\r
- " This option indicates if a separate buffer should be used per message.\n"\r
- " Default is one buffer for all messages.\n" );\r
- printf( "-i <number>\n"\r
- "--iterate <number>\n"\r
- " This option specifies the number of times to loop through 'nmsgs'.\n"\r
- " Default of 1.\n" );\r
- printf( "-v\n"\r
- "--verbose\n"\r
- " This option enables verbosity level to debug console.\n" );\r
- printf( "-h\n"\r
- "--help\n"\r
- " Display this usage info then exit.\n\n" );\r
-}\r
-\r
-\r
-/* Windows support. */\r
-struct option\r
-{\r
- const char *long_name;\r
- unsigned long flag;\r
- void *pfn_handler;\r
- char short_name;\r
-};\r
-\r
-static char *optarg;\r
-\r
-#define strtoull strtoul\r
-\r
-\r
-char\r
-getopt_long(\r
- int argc,\r
- char *argv[],\r
- const char *short_option,\r
- const struct option *long_option,\r
- void *unused )\r
-{\r
- static int i = 1;\r
- int j;\r
- char ret = 0;\r
-\r
- UNUSED_PARAM( unused );\r
-\r
- if( i == argc )\r
- return -1;\r
-\r
- if( argv[i][0] != '-' )\r
- return ret;\r
-\r
- /* find the first character of the value. */\r
- for( j = 1; isalpha( argv[i][j] ); j++ )\r
- ;\r
- optarg = &argv[i][j];\r
-\r
- if( argv[i][1] == '-' )\r
- {\r
- /* Long option. */\r
- for( j = 0; long_option[j].long_name; j++ )\r
- {\r
- if( strncmp( &argv[i][2], long_option[j].long_name,\r
- optarg - argv[i] - 2 ) )\r
- {\r
- continue;\r
- }\r
-\r
- switch( long_option[j].flag )\r
- {\r
- case 1:\r
- if( *optarg == '\0' )\r
- return 0;\r
- default:\r
- break;\r
- }\r
- ret = long_option[j].short_name;\r
- break;\r
- }\r
- }\r
- else\r
- {\r
- for( j = 0; short_option[j] != '\0'; j++ )\r
- {\r
- if( !isalpha( short_option[j] ) )\r
- return 0;\r
-\r
- if( short_option[j] == argv[i][1] )\r
- {\r
- ret = short_option[j];\r
- break;\r
- }\r
-\r
- if( short_option[j+1] == ':' )\r
- {\r
- if( *optarg == '\0' )\r
- return 0;\r
- j++;\r
- }\r
- }\r
- }\r
- i++;\r
- return ret;\r
-}\r
-\r
-\r
-static boolean_t\r
-__parse_options(\r
- int argc,\r
- char* argv[] )\r
-{\r
- uint32_t next_option;\r
- const char* const short_option = "esl:r:c:m:n:i:pvh";\r
-\r
- /*\r
- In the array below, the 2nd parameter specified the number\r
- of arguments as follows:\r
- 0: no arguments\r
- 1: argument\r
- 2: optional\r
- */\r
- const struct option long_option[] =\r
- {\r
- { "event", 2, NULL, 'e'},\r
- { "server", 2, NULL, 's'},\r
- { "local", 1, NULL, 'l'},\r
- { "remote", 1, NULL, 'r'},\r
- { "connect", 1, NULL, 'c'},\r
- { "msize", 1, NULL, 'm'},\r
- { "nmsgs", 1, NULL, 'n'},\r
- { "iterate", 1, NULL, 'i'},\r
- { "permsg", 0, NULL, 'p'},\r
- { "verbose", 0, NULL, 'v'},\r
- { "help", 0, NULL, 'h'},\r
- { NULL, 0, NULL, 0 } /* Required at end of array */\r
- };\r
-\r
- /* Set the default options. */\r
- g_root.msg_size = 100;\r
- g_root.num_nodes = 1;\r
- g_root.num_msgs = 0;\r
- g_root.num_iter = 1;\r
- g_root.is_polling = TRUE;\r
-\r
- /* parse cmd line arguments as input params */\r
- do\r
- {\r
- next_option = getopt_long( argc, argv, short_option,\r
- long_option, NULL );\r
-\r
- switch( next_option )\r
- {\r
- case 's':\r
- g_root.is_server = TRUE;\r
- printf( "\tServer mode\n" );\r
- break;\r
-\r
- case 'd':\r
- g_root.inst_id = strtoull( optarg, NULL, 0 );\r
- printf( "\tinstance_id..: %d\n", g_root.inst_id );\r
- break;\r
-\r
- case 'c':\r
- g_root.num_nodes = strtoull( optarg, NULL, 0 );\r
- printf( "\tconnections..: %d\n", g_root.num_nodes );\r
- break;\r
-\r
- case 'l':\r
- g_root.l_lid = cl_ntoh16( (uint16_t)strtoull( optarg, NULL, 0 ) );\r
- printf( "\tlocal lid....: x%x\n", g_root.l_lid );\r
- break;\r
-\r
- case 'r':\r
- g_root.r_lid = cl_ntoh16( (uint16_t)strtoull( optarg, NULL, 0 ) );\r
- printf( "\tremote lid...: x%x\n", g_root.r_lid );\r
- break;\r
-\r
- case 'm':\r
- g_root.msg_size = strtoull( optarg, NULL, 0 );\r
- printf( "\tmsg size.....: %d bytes\n", g_root.msg_size );\r
- break;\r
-\r
- case 'n':\r
- g_root.num_msgs = strtoull( optarg, NULL, 0 );\r
- printf( "\tnum msgs.....: %d\n", g_root.num_msgs );\r
- break;\r
-\r
- case 'i':\r
- g_root.num_iter = strtoull( optarg, NULL, 0 );\r
- printf( "\titerate......: %d\n", g_root.num_iter );\r
- break;\r
-\r
- case 'p':\r
- g_root.per_msg_buf = TRUE;\r
- printf( "\tper message data buffer\n" );\r
- break;\r
-\r
- case 'v':\r
- cmt_dbg_lvl = 0xFFFFFFFF;\r
- printf( "\tverbose\n" );\r
- break;\r
-\r
- case 'e':\r
- g_root.is_polling = FALSE;\r
- printf( "\tevent driven completions\n" );\r
- break;\r
-\r
- case 'h':\r
- __show_usage();\r
- return FALSE;\r
-\r
- case -1:\r
- break;\r
-\r
- default: /* something wrong */\r
- __show_usage();\r
- return FALSE;\r
- }\r
- } while( next_option != -1 );\r
-\r
- return TRUE;\r
-}\r
-\r
-\r
-/**********************************************************************\r
- **********************************************************************/\r
-static void\r
-__init_conn_info()\r
-{\r
- /* Initialize connection request parameters. */\r
- g_root.cm_req.svc_id = CMT_BASE_SVC_ID + g_root.inst_id;\r
- g_root.cm_req.max_cm_retries = 5;\r
- g_root.cm_req.p_primary_path = &g_root.path_rec;\r
- g_root.cm_req.pfn_cm_rep_cb = __rep_cb;\r
- g_root.cm_req.qp_type = IB_QPT_RELIABLE_CONN;\r
- g_root.cm_req.resp_res = 3;\r
- g_root.cm_req.init_depth = 1;\r
- g_root.cm_req.remote_resp_timeout = 20;\r
- g_root.cm_req.flow_ctrl = TRUE;\r
- g_root.cm_req.local_resp_timeout = 20;\r
- g_root.cm_req.rnr_nak_timeout = 6;\r
- g_root.cm_req.rnr_retry_cnt = 3;\r
- g_root.cm_req.retry_cnt = 5;\r
- g_root.cm_req.pfn_cm_mra_cb = __mra_cb;\r
- g_root.cm_req.pfn_cm_rej_cb = __rej_cb;\r
-\r
- /* Initialize connection reply parameters. */\r
- g_root.cm_rep.qp_type = IB_QPT_RELIABLE_CONN;\r
- g_root.cm_rep.access_ctrl = CMT_ACCESS_CTRL;\r
- g_root.cm_rep.sq_depth = 0;\r
- g_root.cm_rep.rq_depth = 0;\r
- g_root.cm_rep.init_depth = 1;\r
- g_root.cm_rep.target_ack_delay = 7;\r
- g_root.cm_rep.failover_accepted = IB_FAILOVER_ACCEPT_UNSUPPORTED;\r
- g_root.cm_rep.flow_ctrl = TRUE;\r
- g_root.cm_rep.rnr_nak_timeout = 7;\r
- g_root.cm_rep.rnr_retry_cnt = 6;\r
- g_root.cm_rep.pfn_cm_rej_cb = __rej_cb;\r
- g_root.cm_rep.pfn_cm_mra_cb = __mra_cb;\r
- g_root.cm_rep.pfn_cm_rtu_cb = __rtu_cb;\r
- g_root.cm_rep.pfn_cm_lap_cb = __lap_cb;\r
- g_root.cm_rep.pfn_cm_dreq_cb = __dreq_cb;\r
-\r
- /* Initialize connection RTU parameters. */\r
- g_root.cm_rtu.pfn_cm_apr_cb = __apr_cb;\r
- g_root.cm_rtu.pfn_cm_dreq_cb = __dreq_cb;\r
-\r
- /* Initialize disconnection request parameters. */\r
- g_root.cm_dreq.pfn_cm_drep_cb = __drep_cb;\r
- g_root.cm_dreq.qp_type = IB_QPT_RELIABLE_CONN;\r
-\r
- /* Disconnection reply parameters are all zero. */\r
-}\r
-\r
-\r
-\r
-static uint16_t\r
-__get_pkey_index()\r
-{\r
- uint16_t i;\r
-\r
- for( i = 0; i < g_root.num_pkeys; i++ )\r
- {\r
- if( g_root.p_pkey_table[i] == g_root.path_rec.pkey )\r
- return i;\r
- }\r
-\r
- return BAD_PKEY_INDEX;\r
-}\r
-\r
-\r
-\r
-static void\r
-__init_qp_info()\r
-{\r
- /* Set common QP attributes for all create calls. */\r
- g_root.qp_create.qp_type = IB_QPT_RELIABLE_CONN;\r
- if( g_root.num_msgs )\r
- {\r
- g_root.qp_create.sq_depth = g_root.num_msgs;\r
- g_root.qp_create.rq_depth = g_root.num_msgs;\r
- }\r
- else\r
- {\r
- /* Minimal queue depth of one. */\r
- g_root.qp_create.sq_depth = 1;\r
- g_root.qp_create.rq_depth = 1;\r
- }\r
-\r
- g_root.qp_create.sq_signaled = FALSE;\r
- g_root.qp_create.sq_sge = 1;\r
- g_root.qp_create.rq_sge = 1;\r
-\r
- /* Set the QP attributes when modifying the QP to the reset state. */\r
- g_root.qp_mod_reset.req_state = IB_QPS_RESET;\r
-\r
- /* Set the QP attributes when modifying the QP to the init state. */\r
- g_root.qp_mod_init.req_state = IB_QPS_INIT;\r
- g_root.qp_mod_init.state.init.access_ctrl = CMT_ACCESS_CTRL;\r
- g_root.qp_mod_init.state.init.primary_port = g_root.port_num;\r
- g_root.qp_mod_init.state.init.pkey_index = __get_pkey_index();\r
-}\r
-\r
-\r
-\r
-static ib_api_status_t\r
-__post_recvs(\r
- IN ib_node_t *p_node )\r
-{\r
- ib_api_status_t status = IB_SUCCESS;\r
- ib_recv_wr_t recv_wr;\r
- ib_recv_wr_t *p_recv_failure;\r
- ib_local_ds_t ds_array;\r
- uint32_t i;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- if( !g_root.num_msgs )\r
- {\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return IB_SUCCESS;\r
- }\r
-\r
- cl_memclr( &recv_wr, sizeof( ib_recv_wr_t ) );\r
- ds_array.length = g_root.msg_size;\r
- ds_array.lkey = g_root.lkey;\r
- recv_wr.ds_array = &ds_array;\r
- recv_wr.num_ds = (( g_root.msg_size <= 4 )? 0: 1 );\r
-\r
- for( i = 0; i < g_root.num_msgs; i++ )\r
- {\r
- CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, (".") );\r
-\r
- if( g_root.per_msg_buf )\r
- {\r
- ds_array.vaddr = (uintn_t)(g_root.p_mem_recv + (i * g_root.msg_size) +\r
- (p_node->id * g_root.num_msgs * g_root.msg_size));\r
- }\r
- else\r
- {\r
- ds_array.vaddr = (uintn_t)g_root.p_mem;\r
- }\r
-\r
- recv_wr.wr_id = i;\r
- \r
- status = ib_post_recv( p_node->h_qp, &recv_wr, &p_recv_failure );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_post_recv failed [%s]!\n", ib_get_err_str(status) );\r
- break;\r
- }\r
- }\r
-\r
- CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, ("\n") );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return status;\r
-}\r
-\r
-\r
-\r
-static void\r
-__ca_async_event_cb(\r
- ib_async_event_rec_t *p_err_rec )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- \r
- CL_TRACE( CMT_DBG_VERBOSE, cmt_dbg_lvl, \r
- ( "p_err_rec->code is %d\n", p_err_rec->code ) );\r
- \r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__cm_listen_err_cb(\r
- IN ib_listen_err_rec_t *p_listen_err_rec )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- if( !p_listen_err_rec )\r
- printf( "__listen_err_cb NULL p_listen_err_rec\n" );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__cancel_listen_cb(\r
- IN void *context )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- if( !context )\r
- printf( "__cancel_listen_cb NULL context\n" );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-/* We need to halt the test and recover from the reject error. */\r
-static void\r
-__rej_cb(\r
- IN ib_cm_rej_rec_t *p_cm_rej_rec )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- /*\r
- * Note - because this callback exits the app, any output beyond the\r
- * the first time may report junk. There have been instances where\r
- * the callback is invoked more times than there are connection requests\r
- * but that behavior disapeared if the call to exit below is removed.\r
- */\r
- printf( "Connection was rejected, status: 0x%x\n",\r
- p_cm_rej_rec->rej_status );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- exit( 1 );\r
-}\r
-\r
-\r
-\r
-static void\r
-__req_cb(\r
- IN ib_cm_req_rec_t *p_cm_req_rec )\r
-{\r
- ib_api_status_t status;\r
- ib_node_t *p_node;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- \r
- CL_ASSERT( p_cm_req_rec );\r
-\r
- /* Record the starting time for the server. */\r
- if( !g_root.conn_start_time )\r
- g_root.conn_start_time = cl_get_time_stamp( );\r
-\r
- /*\r
- * Do not send replies until the server is ready to establish all\r
- * connections.\r
- */\r
- cl_mutex_acquire( &g_root.mutex );\r
- p_node = &g_root.p_nodes[g_root.conn_index++];\r
-\r
- if( g_root.state == test_connecting )\r
- {\r
- /* Get a node for this connection and send the reply. */\r
- g_root.cm_rep.h_qp = p_node->h_qp;\r
- status = ib_cm_rep( p_cm_req_rec->h_cm_req, &g_root.cm_rep );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "Call to ib_cm_rep failed\n" );\r
- exit( 1 );\r
- }\r
- }\r
- else\r
- {\r
- p_node->h_cm_req = p_cm_req_rec->h_cm_req;\r
- }\r
- cl_mutex_release( &g_root.mutex );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__rep_cb(\r
- IN ib_cm_rep_rec_t *p_cm_rep_rec )\r
-{\r
- ib_api_status_t status;\r
- ib_node_t *p_node;\r
- uint8_t pdata[IB_RTU_PDATA_SIZE];\r
- ib_cm_mra_t mra;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_rep_rec );\r
- \r
- p_node = (ib_node_t*)p_cm_rep_rec->qp_context;\r
- CL_ASSERT( p_node );\r
-\r
- mra.p_mra_pdata = NULL;\r
- mra.mra_length = 0;\r
- mra.svc_timeout = 0xff;\r
-\r
- ib_cm_mra( p_cm_rep_rec->h_cm_rep, &mra );\r
-\r
- __post_recvs( p_node );\r
-\r
- /* Mark that we're connected before sending the RTU. */\r
- p_node->state = node_conn;\r
-\r
- g_root.cm_rtu.p_rtu_pdata = pdata;\r
- g_root.cm_rtu.rtu_length = IB_RTU_PDATA_SIZE;\r
-\r
- status = ib_cm_rtu( p_cm_rep_rec->h_cm_rep, &g_root.cm_rtu );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "Call to ib_cm_rtu returned %s\n", ib_get_err_str( status ) );\r
- exit( 1 );\r
- }\r
-\r
- cl_atomic_inc( &g_root.num_connected );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__rtu_cb(\r
- IN ib_cm_rtu_rec_t *p_cm_rtu_rec )\r
-{\r
- ib_node_t *p_node;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_rtu_rec );\r
- \r
- p_node = (ib_node_t*)p_cm_rtu_rec->qp_context;\r
- p_node->state = node_conn;\r
-\r
- __post_recvs( p_node );\r
- cl_atomic_inc( &g_root.num_connected );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__mra_cb(\r
- IN ib_cm_mra_rec_t *p_cm_mra_rec )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_mra_rec );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__apr_cb(\r
- IN ib_cm_apr_rec_t *p_cm_apr_rec )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_apr_rec );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__lap_cb(\r
- IN ib_cm_lap_rec_t *p_cm_lap_rec )\r
-{\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_lap_rec );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__dreq_cb(\r
- IN ib_cm_dreq_rec_t *p_cm_dreq_rec )\r
-{\r
- ib_node_t *p_node;\r
- ib_api_status_t status;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_dreq_rec );\r
- p_node = (ib_node_t*)p_cm_dreq_rec->qp_context;\r
- CL_ASSERT( p_node );\r
-\r
- /*\r
- * Record that we've already received a DREQ to avoid trying to\r
- * disconnect the QP a second time. Synchronize with the DREQ call\r
- * using the mutex.\r
- */\r
- cl_mutex_acquire( &g_root.mutex );\r
-\r
- /* If we need to send or receive more data, don't disconnect yet. */\r
- if( g_root.state == test_disconnecting )\r
- {\r
- /* Send the DREP. */\r
- status = ib_cm_drep( p_cm_dreq_rec->h_cm_dreq, &g_root.cm_drep );\r
-\r
- /* If the DREP was successful, we're done with this connection. */\r
- if( status == IB_SUCCESS )\r
- {\r
- p_node->state = node_idle;\r
- cl_atomic_dec( &g_root.num_connected );\r
- }\r
- }\r
- else\r
- {\r
- /* Record that we need to disconnect, but don't send the DREP yet. */\r
- p_node->state = node_dreq_rcvd;\r
- p_node->h_cm_dreq = p_cm_dreq_rec->h_cm_dreq;\r
- }\r
- cl_mutex_release( &g_root.mutex );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__drep_cb(\r
- IN ib_cm_drep_rec_t *p_cm_drep_rec )\r
-{\r
- ib_node_t *p_node;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- CL_ASSERT( p_cm_drep_rec );\r
- p_node = (ib_node_t*)p_cm_drep_rec->qp_context;\r
- CL_ASSERT( p_node );\r
-\r
- /* We're done with this connection. */\r
- cl_mutex_acquire( &g_root.mutex );\r
- p_node->state = node_idle;\r
- cl_atomic_dec( &g_root.num_connected );\r
- cl_mutex_release( &g_root.mutex );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static void\r
-__cq_cb(\r
- IN const ib_cq_handle_t h_cq,\r
- IN void* cq_context )\r
-{\r
- ib_node_t *p_node = (ib_node_t*)cq_context;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- if( !g_root.is_polling )\r
- {\r
- if( !__poll_cq( p_node, h_cq ) )\r
- exit( 1 );\r
- }\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-static ib_api_status_t\r
-__create_qp(\r
- IN ib_node_t *p_node )\r
-{\r
- ib_api_status_t status;\r
- ib_qp_attr_t attr;\r
-\r
- /* Set the per node QP attributes. */\r
- g_root.qp_create.h_sq_cq = p_node->h_send_cq; \r
- g_root.qp_create.h_rq_cq = p_node->h_recv_cq;\r
-\r
- /* Allocate the QP. */\r
- status = ib_create_qp( g_root.h_pd, &g_root.qp_create, p_node, NULL,\r
- &p_node->h_qp );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "[%d] ib_create_qp failed [%s]!\n", __LINE__, \r
- ib_get_err_str(status) );\r
- return status;\r
- }\r
-\r
- /* Store the max inline size. */\r
- status = ib_query_qp( p_node->h_qp, &attr );\r
- if( status != IB_SUCCESS )\r
- p_node->max_inline = 0;\r
- else\r
- p_node->max_inline = attr.sq_max_inline;\r
-\r
- /*\r
- * Transition the QP to the initialize state. This prevents the CM\r
- * from having to make this QP transition and improves the connection\r
- * establishment rate.\r
- */\r
- status = ib_modify_qp( p_node->h_qp, &g_root.qp_mod_reset );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_modify_qp to IB_QPS_RESET returned %s\n",\r
- ib_get_err_str(status) );\r
- return status;\r
- }\r
-\r
- status = ib_modify_qp( p_node->h_qp, &g_root.qp_mod_init );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_modify_qp to IB_QPS_INIT returned %s\n",\r
- ib_get_err_str(status) );\r
- return status;\r
- }\r
-\r
- return status;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Allocate new QPs for all nodes.\r
- */\r
-static ib_api_status_t\r
-__create_qps()\r
-{\r
- uint64_t start_time, total_time;\r
- int32_t i;\r
- ib_api_status_t status;\r
-\r
- printf( "Creating QPs...\n" );\r
- start_time = cl_get_time_stamp();\r
-\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- /* Allocate a new QP. */\r
- status = __create_qp( &g_root.p_nodes[i] );\r
- if( status != IB_SUCCESS )\r
- break;\r
- }\r
-\r
- total_time = cl_get_time_stamp() - start_time;\r
- printf( "Allocation time: %"PRId64" ms\n", total_time/1000 );\r
-\r
- return status;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Destroy all QPs for all nodes.\r
- */\r
-static void\r
-__destroy_qps()\r
-{\r
- uint64_t start_time, total_time;\r
- int32_t i;\r
-\r
- printf( "Destroying QPs...\n" );\r
- start_time = cl_get_time_stamp();\r
-\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- /* Destroy the QP. */\r
- if( g_root.p_nodes[i].h_qp )\r
- {\r
- ib_destroy_qp( g_root.p_nodes[i].h_qp, ib_sync_destroy );\r
- g_root.p_nodes[i].h_qp = NULL;\r
- }\r
- }\r
-\r
- total_time = cl_get_time_stamp() - start_time;\r
- printf( "Destruction time: %"PRId64" ms\n", total_time/1000 );\r
-}\r
-\r
-\r
-\r
-static boolean_t\r
-__init_node(\r
- IN OUT ib_node_t* p_node )\r
-{\r
- ib_api_status_t status;\r
- ib_cq_create_t cq_create;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- /* Create the CQs. */\r
- cl_memclr( &cq_create, sizeof(ib_cq_create_t) );\r
- if( g_root.num_msgs )\r
- cq_create.size = g_root.num_msgs;\r
- else\r
- cq_create.size = 1; /* minimal of one entry */\r
-\r
- cq_create.pfn_comp_cb = __cq_cb;\r
- status = ib_create_cq( g_root.h_ca, &cq_create, p_node, NULL,\r
- &p_node->h_send_cq );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_create_cq failed for send CQ [%s]!\n",\r
- ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
- if( !g_root.is_polling )\r
- {\r
- status = ib_rearm_cq( p_node->h_send_cq, FALSE );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_rearm_cq failed for send CQ [%s]!\n",\r
- ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
- }\r
-\r
- status = ib_create_cq( g_root.h_ca, &cq_create, p_node, NULL,\r
- &p_node->h_recv_cq );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_create_cq failed for recv CQ [%s]!\n",\r
- ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
- if( !g_root.is_polling )\r
- {\r
- status = ib_rearm_cq( p_node->h_recv_cq, FALSE );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_rearm_cq failed for recv CQ [%s]!\n",\r
- ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
- }\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return TRUE;\r
-}\r
-\r
-static boolean_t\r
-__destroy_node(\r
- IN OUT ib_node_t* p_node )\r
-{\r
- ib_api_status_t status = IB_SUCCESS;\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- if (!p_node )\r
- return (FALSE);\r
- if ( p_node->h_send_cq )\r
- {\r
- status = ib_destroy_cq( p_node->h_send_cq, ib_sync_destroy );\r
- p_node->h_send_cq = NULL;\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_destroy_cq failed for send CQ [%s]!\n",\r
- ib_get_err_str(status) );\r
- }\r
- }\r
- if (p_node->h_recv_cq)\r
- {\r
- status = ib_destroy_cq( p_node->h_recv_cq, ib_sync_destroy );\r
- p_node->h_recv_cq = NULL;\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_destroy_cq failed for recv CQ [%s]!\n",\r
- ib_get_err_str(status) );\r
- }\r
- }\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return (status == IB_SUCCESS);\r
-}\r
-\r
-\r
-\r
-static boolean_t\r
-__create_nodes()\r
-{\r
- int32_t i;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- g_root.p_nodes[i].id = i;\r
-\r
- CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, \r
- ("--> create connection %d of instance %d\n", i, g_root.inst_id) );\r
- \r
- if( !__init_node( &g_root.p_nodes[i] ) )\r
- return FALSE;\r
- }\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return TRUE;\r
-}\r
-\r
-static boolean_t\r
-__destroy_nodes()\r
-{\r
- int32_t i;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- if( !__destroy_node( &g_root.p_nodes[i] ) )\r
- return FALSE;\r
- }\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return TRUE;\r
-}\r
-\r
-/* query function called by ib_query() */\r
-static void\r
-__sa_query_cb(\r
- IN ib_query_rec_t *p_query_rec )\r
-{\r
- ib_path_rec_t *p_path;\r
- ib_api_status_t status;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- CL_ASSERT( p_query_rec );\r
-\r
- if( p_query_rec->status != IB_SUCCESS )\r
- {\r
- printf( "ib_query failed [%d]\n", p_query_rec->status ); \r
- return;\r
- }\r
-\r
- if( p_query_rec->query_type != IB_QUERY_PATH_REC_BY_LIDS )\r
- {\r
- printf( "Unexpected query type returned.\n" ); \r
- return;\r
- }\r
-\r
- if( !p_query_rec->p_result_mad )\r
- {\r
- printf( "No result MAD returned from ib_query.\n" ); \r
- return;\r
- }\r
-\r
- /* copy the 1st (zero'th) path record to local storage. */\r
- p_path = ib_get_query_path_rec( p_query_rec->p_result_mad, 0 );\r
- memcpy( (void*)&g_root.path_rec, (void*)p_path, \r
- sizeof(ib_path_rec_t) );\r
-\r
- CL_TRACE( CMT_DBG_VERBOSE, cmt_dbg_lvl,\r
- ( "path{ slid:0x%x, dlid:0x%x }\n", \r
- g_root.path_rec.slid, g_root.path_rec.dlid) );\r
-\r
- /* release response MAD(s) back to AL pool */\r
- if( p_query_rec->p_result_mad )\r
- {\r
- status = ib_put_mad( p_query_rec->p_result_mad );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_put_mad() failed [%s]\n",\r
- ib_get_err_str(status) );\r
- }\r
- }\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-}\r
-\r
-\r
-\r
-static boolean_t\r
-__query_for_path()\r
-{\r
- ib_api_status_t status;\r
- ib_query_req_t query_rec;\r
- ib_lid_pair_t lid_pair;\r
-\r
- /* Query the SA for a path record. */\r
- query_rec.query_type = IB_QUERY_PATH_REC_BY_LIDS;\r
-\r
- lid_pair.src_lid = g_root.l_lid;\r
- lid_pair.dest_lid = g_root.r_lid;\r
-\r
- query_rec.p_query_input = (void*)&lid_pair;\r
- query_rec.port_guid = g_root.port_guid;\r
- query_rec.timeout_ms = 5 * 1000; // seconds\r
- query_rec.retry_cnt = 2;\r
- query_rec.flags = IB_FLAGS_SYNC;\r
- query_rec.query_context = &g_root;\r
- query_rec.pfn_query_cb = __sa_query_cb;\r
-\r
- status = ib_query( g_root.h_al, &query_rec, NULL );\r
- if( ( status != IB_SUCCESS ) || ( !g_root.path_rec.dlid ) )\r
- {\r
- printf( "ib_query failed.\n" );\r
- return FALSE;\r
- }\r
-\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-static boolean_t\r
-__create_messages()\r
-{\r
- ib_mr_create_t mr_create;\r
- uint32_t buf_size;\r
- ib_api_status_t status;\r
-\r
- /* If we're not sending messages, just return. */\r
- if( !g_root.num_msgs || !g_root.msg_size )\r
- return TRUE;\r
-\r
- /* Allocate the message memory - we ignore the data, so just one buffer. */\r
- if( g_root.per_msg_buf )\r
- buf_size = (g_root.num_nodes * g_root.num_msgs * g_root.msg_size) << 1;\r
- else\r
- buf_size = g_root.msg_size;\r
- g_root.p_mem = cl_zalloc( buf_size );\r
- if( !g_root.p_mem )\r
- {\r
- printf( "Not enough memory for transfers!\n" );\r
- return FALSE;\r
- }\r
- memset (g_root.p_mem, 0xae, buf_size);\r
- g_root.p_mem_recv = g_root.p_mem;\r
- g_root.p_mem_send = g_root.p_mem + (buf_size >> 1);\r
-\r
- /* Register the memory with AL. */\r
- mr_create.vaddr = g_root.p_mem;\r
- mr_create.length = buf_size;\r
- mr_create.access_ctrl = IB_AC_LOCAL_WRITE | IB_AC_MW_BIND;\r
- status = ib_reg_mem( g_root.h_pd, &mr_create, &g_root.lkey, \r
- &g_root.rkey, &g_root.h_mr );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_reg_mem failed [%s]!\n", ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
-\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-/*\r
- * PnP callback handler. Record the port GUID of an active port.\r
- */\r
-static ib_api_status_t\r
-__pnp_cb(\r
- IN ib_pnp_rec_t *p_pnp_rec )\r
-{\r
- ib_pnp_port_rec_t* p_port_rec;\r
- uint32_t size;\r
-\r
- p_port_rec = (ib_pnp_port_rec_t*)p_pnp_rec;\r
-\r
- /*\r
- * Ignore PNP events that are not related to port active, or if\r
- * we already have an active port.\r
- */\r
- if( p_pnp_rec->pnp_event != IB_PNP_PORT_ACTIVE || g_root.port_guid )\r
- return IB_SUCCESS;\r
-\r
- /* Find the proper port for the given local LID. */\r
- if( g_root.l_lid )\r
- {\r
- if( g_root.l_lid != p_port_rec->p_port_attr->lid )\r
- return IB_SUCCESS;\r
- }\r
- else\r
- {\r
- g_root.l_lid = p_port_rec->p_port_attr->lid;\r
- printf( "\tlocal lid....: x%x\n", g_root.l_lid );\r
- }\r
-\r
- /* Record the active port information. */\r
- g_root.ca_guid = p_port_rec->p_ca_attr->ca_guid;\r
- g_root.port_num = p_port_rec->p_port_attr->port_num;\r
-\r
- /* Record the PKEYs available on the active port. */\r
- size = sizeof( ib_net16_t ) * p_port_rec->p_port_attr->num_pkeys;\r
- g_root.p_pkey_table = cl_zalloc( size );\r
- if( !g_root.p_pkey_table )\r
- return IB_SUCCESS;\r
- g_root.num_pkeys = p_port_rec->p_port_attr->num_pkeys;\r
- cl_memcpy( g_root.p_pkey_table,\r
- p_port_rec->p_port_attr->p_pkey_table, size );\r
-\r
- /* Set the port_guid last to indicate that we're ready. */\r
- g_root.port_guid = p_port_rec->p_port_attr->port_guid;\r
- return IB_SUCCESS;\r
-}\r
-\r
-\r
-/*\r
- * Register for PnP events and wait until a port becomes active.\r
- */\r
-static boolean_t\r
-__reg_pnp()\r
-{\r
- ib_api_status_t status;\r
- ib_pnp_req_t pnp_req;\r
- ib_pnp_handle_t h_pnp;\r
-\r
- cl_memclr( &pnp_req, sizeof( ib_pnp_req_t ) );\r
- pnp_req.pnp_class = IB_PNP_PORT;\r
- pnp_req.pnp_context = &g_root;\r
- pnp_req.pfn_pnp_cb = __pnp_cb;\r
-\r
- /* Register for PnP events. */\r
- status = ib_reg_pnp( g_root.h_al, &pnp_req, &h_pnp );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_reg_pnp failed [%s]!\n", ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
-\r
- /* Wait until a port goes active. */\r
- while( !g_root.port_guid )\r
- cl_thread_suspend( 10 );\r
-\r
- /* Deregister from PnP. */\r
- ib_dereg_pnp( h_pnp, NULL );\r
-\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-static boolean_t\r
-__init_root()\r
-{\r
- ib_api_status_t status;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- cl_mutex_construct( &g_root.mutex );\r
- if( cl_mutex_init( &g_root.mutex ) != CL_SUCCESS )\r
- return FALSE;\r
-\r
- /* Open AL. */\r
- status = ib_open_al( &g_root.h_al );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_open_al failed [%s]!\n", ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
-\r
- /* Register for PnP events, and wait until we have an active port. */\r
- if( !__reg_pnp() )\r
- return FALSE;\r
-\r
- /* Open the CA. */\r
- status = ib_open_ca( g_root.h_al, g_root.ca_guid,\r
- __ca_async_event_cb, &g_root, &g_root.h_ca );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_open_ca failed [%s]!\n", ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
-\r
- /* Create a PD. */\r
- status = ib_alloc_pd( g_root.h_ca, IB_PDT_NORMAL, &g_root, \r
- &g_root.h_pd );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_alloc_pd failed [%s]!\n", ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
-\r
- /* Get a path record to the remote side. */\r
- if( !__query_for_path() )\r
- {\r
- printf( "Unable to query for path record!\n" );\r
- return FALSE;\r
- }\r
-\r
- /* Allocate and register memory for the messages. */\r
- if( !__create_messages() )\r
- {\r
- printf( "Unable to create messages!\n" );\r
- return FALSE;\r
- }\r
-\r
- /* Create the connection endpoints. */\r
- g_root.p_nodes = (ib_node_t*)cl_zalloc(\r
- sizeof(ib_node_t) * g_root.num_nodes );\r
- if( !g_root.p_nodes )\r
- {\r
- printf( "Unable to allocate nodes\n" ); \r
- return FALSE;\r
- }\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-static void\r
-__cleanup()\r
-{\r
- if( g_root.h_listen )\r
- ib_cm_cancel( g_root.h_listen, __cancel_listen_cb );\r
-\r
- /* Close AL if it was opened. */\r
- if( g_root.h_al )\r
- ib_close_al( g_root.h_al );\r
-\r
- cl_mutex_destroy( &g_root.mutex );\r
-\r
- if( g_root.p_mem )\r
- cl_free( g_root.p_mem );\r
-\r
- if( g_root.p_pkey_table )\r
- cl_free( g_root.p_pkey_table );\r
-\r
- /* Free all allocated memory. */\r
- if( g_root.p_nodes )\r
- cl_free( g_root.p_nodes );\r
-}\r
-\r
-\r
-\r
-/*\r
- * Have the server start listening for connections.\r
- */\r
-static boolean_t\r
-__listen()\r
-{\r
- ib_cm_listen_t cm_listen;\r
- ib_api_status_t status;\r
-\r
- cl_memclr( &cm_listen, sizeof( ib_cm_listen_t ) );\r
-\r
- /* The server side listens. */\r
- cm_listen.svc_id = CMT_BASE_SVC_ID + g_root.inst_id;\r
-\r
- cm_listen.pfn_cm_req_cb = __req_cb;\r
-\r
- cm_listen.qp_type = IB_QPT_RELIABLE_CONN;\r
-\r
- status = ib_cm_listen( g_root.h_al, &cm_listen, \r
- __cm_listen_err_cb, &g_root, &g_root.h_listen );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_cm_listen failed [%s]!\n", ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Initiate all client connection requests.\r
- */\r
-static ib_api_status_t\r
-__conn_reqs()\r
-{\r
- ib_api_status_t status;\r
- int32_t i;\r
- uint8_t pdata[IB_REQ_PDATA_SIZE];\r
-\r
- g_root.cm_req.p_req_pdata = pdata;\r
- g_root.cm_req.req_length = IB_REQ_PDATA_SIZE;\r
-\r
- /* Request a connection for each client. */\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- g_root.cm_req.h_qp = g_root.p_nodes[i].h_qp;\r
-\r
- status = ib_cm_req( &g_root.cm_req );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_cm_req failed [%s]!\n", ib_get_err_str(status) );\r
- return status;\r
- }\r
- }\r
- return IB_SUCCESS;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Send any connection replies waiting to be sent.\r
- */\r
-static ib_api_status_t\r
-__conn_reps()\r
-{\r
- ib_api_status_t status;\r
- uintn_t i;\r
- uint8_t pdata[IB_REP_PDATA_SIZE];\r
-\r
- g_root.cm_rep.p_rep_pdata = pdata;\r
- g_root.cm_rep.rep_length = IB_REP_PDATA_SIZE;\r
-\r
- /* Send a reply for each connection that requires one. */\r
- for( i = 0; i < g_root.conn_index; i++ )\r
- {\r
- g_root.cm_rep.h_qp = g_root.p_nodes[i].h_qp;\r
- status = ib_cm_rep( g_root.p_nodes[i].h_cm_req, &g_root.cm_rep );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_cm_rep failed [%s]!\n", ib_get_err_str(status) );\r
- return status;\r
- }\r
- }\r
- return IB_SUCCESS;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Establish all connections.\r
- */\r
-static ib_api_status_t\r
-__connect()\r
-{\r
- uint64_t total_time;\r
- ib_api_status_t status;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- printf( "Connecting...\n" );\r
- cl_mutex_acquire( &g_root.mutex );\r
- g_root.state = test_connecting;\r
-\r
- /* Initiate the connections. */\r
- if( g_root.is_server )\r
- {\r
- /*\r
- * Send any replies. Note that we hold the mutex while sending the\r
- * replies since we need to use the global cm_rep structure.\r
- */\r
- status = __conn_reps();\r
- cl_mutex_release( &g_root.mutex );\r
- }\r
- else\r
- {\r
- cl_mutex_release( &g_root.mutex );\r
- g_root.conn_start_time = cl_get_time_stamp();\r
- status = __conn_reqs();\r
- }\r
-\r
- if( status != IB_SUCCESS )\r
- return status;\r
-\r
- /* Wait for all connections to complete. */\r
- while( g_root.num_connected < g_root.num_nodes )\r
- cl_thread_suspend( 0 );\r
-\r
- /* Calculate the total connection time. */\r
- total_time = cl_get_time_stamp() - g_root.conn_start_time;\r
- g_root.state = test_idle;\r
-\r
- /* Reset connection information for next test. */\r
- g_root.conn_index = 0;\r
- g_root.conn_start_time = 0;\r
-\r
- printf( "Connect time: %"PRId64" ms\n", total_time/1000 );\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return status;\r
-}\r
-\r
-\r
-\r
-static void\r
-__disconnect()\r
-{\r
- ib_api_status_t status;\r
- int32_t i;\r
- ib_node_t *p_node;\r
- uint64_t total_time, start_time;\r
-\r
- printf( "Disconnecting...\n" );\r
-\r
- /* Initiate the disconnection process. */\r
- cl_mutex_acquire( &g_root.mutex );\r
- g_root.state = test_disconnecting;\r
- start_time = cl_get_time_stamp();\r
- cl_mutex_release( &g_root.mutex );\r
-\r
- /* We hold the mutex to prevent calling ib_cm_drep at the same time. */\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- p_node = &g_root.p_nodes[i];\r
-\r
- /*\r
- * Send the DREQ. Note that some of these may fail, since the\r
- * remote side may be disconnecting at the same time. Call DREQ\r
- * only if we haven't called DREP yet.\r
- */\r
- cl_mutex_acquire( &g_root.mutex );\r
- switch( p_node->state )\r
- {\r
- case node_conn:\r
- g_root.cm_dreq.h_qp = p_node->h_qp;\r
- status = ib_cm_dreq( &g_root.cm_dreq );\r
- if( status == IB_SUCCESS )\r
- p_node->state = node_dreq_sent;\r
- break;\r
-\r
- case node_dreq_rcvd:\r
- status = ib_cm_drep( p_node->h_cm_dreq, &g_root.cm_drep );\r
-\r
- /* If the DREP was successful, we're done with this connection. */\r
- if( status == IB_SUCCESS )\r
- {\r
- p_node->state = node_idle;\r
- cl_atomic_dec( &g_root.num_connected );\r
- }\r
- break;\r
-\r
- default:\r
- /* Node is already disconnected. */\r
- break;\r
- }\r
- cl_mutex_release( &g_root.mutex );\r
- }\r
-\r
- /* Wait for all disconnections to complete. */\r
- while( g_root.num_connected )\r
- cl_thread_suspend( 0 );\r
-\r
- if( g_root.h_listen )\r
- {\r
- ib_cm_cancel( g_root.h_listen, __cancel_listen_cb );\r
- g_root.h_listen = NULL;\r
- }\r
- /* Calculate the total connection time. */\r
- total_time = cl_get_time_stamp() - start_time;\r
- g_root.state = test_idle;\r
-\r
- printf( "Disconnect time: %"PRId64" ms\n", total_time/1000 );\r
-}\r
-\r
-\r
-\r
-/*\r
- * Send the requested number of messages on each connection.\r
- */\r
-static boolean_t\r
-__send_msgs()\r
-{\r
- ib_api_status_t status;\r
- int32_t i;\r
- uint32_t m;\r
- ib_send_wr_t send_wr;\r
- ib_send_wr_t *p_send_failure;\r
- ib_local_ds_t ds_array;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- /* For each connection... */\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- /* Send the specified number of messages. */\r
- for( m = 0; m < g_root.num_msgs; m++ )\r
- {\r
- /* Get the buffer for this message. */\r
- if( g_root.per_msg_buf )\r
- {\r
- ds_array.vaddr = (uintn_t)(g_root.p_mem_send +\r
- (i * g_root.num_msgs) + (m * g_root.msg_size));\r
- }\r
- else\r
- {\r
- ds_array.vaddr = (uintn_t)g_root.p_mem;\r
- }\r
- ds_array.length = g_root.msg_size;\r
- ds_array.lkey = g_root.lkey;\r
-\r
- /* Format the send WR for this message. */\r
- send_wr.ds_array = &ds_array;\r
- send_wr.send_opt = IB_SEND_OPT_SIGNALED | IB_SEND_OPT_SOLICITED;\r
- send_wr.send_opt |= ((g_root.msg_size <= 4)? IB_SEND_OPT_IMMEDIATE : 0x0 );\r
- send_wr.wr_type = WR_SEND;\r
- send_wr.num_ds = ((g_root.msg_size <= 4)? 0 : 1 );\r
- send_wr.p_next = NULL;\r
- send_wr.wr_id = m;\r
-\r
- if( g_root.msg_size < g_root.p_nodes[i].max_inline )\r
- send_wr.send_opt |= IB_SEND_OPT_INLINE;\r
-\r
- /* Torpedoes away! Send the message. */\r
- CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, (".") );\r
- status = ib_post_send( g_root.p_nodes[i].h_qp, &send_wr,\r
- &p_send_failure );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_post_send failed [%s]!\n",\r
- ib_get_err_str(status) );\r
- return FALSE;\r
- }\r
- }\r
- }\r
- \r
- CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl, ("\n") );\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Remove num_msgs completions from the specified CQ.\r
- */\r
-static boolean_t\r
-__poll_cq(\r
- IN ib_node_t *p_node,\r
- IN ib_cq_handle_t h_cq )\r
-{\r
- ib_api_status_t status = IB_SUCCESS;\r
- ib_wc_t free_wc[2];\r
- ib_wc_t *p_free_wc, *p_done_wc;\r
-\r
- CL_ENTER( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
-\r
- while( status != IB_NOT_FOUND )\r
- {\r
- /* Get all completions. */\r
- p_free_wc = &free_wc[0];\r
- free_wc[0].p_next = &free_wc[1];\r
- free_wc[1].p_next = NULL;\r
- p_done_wc = NULL;\r
-\r
- status = ib_poll_cq( h_cq, &p_free_wc, &p_done_wc );\r
-\r
- /* Continue polling if nothing is done. */\r
- if( status == IB_NOT_FOUND )\r
- break;\r
-\r
- /* Abort if an error occurred. */\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "Error polling status = %#x( wc_status =%s)\n", \r
- status,\r
- ((p_done_wc != NULL )? wc_status_text[p_done_wc->status]:"N/A") );\r
- return FALSE;\r
- }\r
-\r
- while( p_done_wc )\r
- {\r
- switch( p_done_wc->status )\r
- {\r
- case IB_WCS_SUCCESS:\r
- CL_PRINT( CMT_DBG_VERBOSE, cmt_dbg_lvl,\r
- ("Got a completion: \n\ttype....:%s\n\twr_id...:%"PRIx64"\n"\r
- "status....:%s\n", wc_type_text[p_done_wc->wc_type],\r
- p_done_wc->wr_id, wc_status_text[p_done_wc->status] ) );\r
-\r
- if( p_done_wc->wc_type == IB_WC_RECV )\r
- {\r
- CL_ASSERT( p_done_wc->wr_id == p_node->recv_cnt );\r
- if( p_done_wc->length != g_root.msg_size )\r
- {\r
- printf( "Error: received %d bytes, expected %d.\n",\r
- p_done_wc->length, g_root.msg_size );\r
- }\r
-\r
- p_node->recv_cnt++;\r
- g_root.total_recv++;\r
- }\r
- else\r
- {\r
- CL_ASSERT( p_done_wc->wr_id == p_node->send_cnt );\r
- p_node->send_cnt++;\r
- g_root.total_sent++;\r
- }\r
- break;\r
-\r
- default:\r
- printf( "[%d] Bad completion type(%s) status(%s)\n",\r
- __LINE__, wc_type_text[p_done_wc->wc_type],\r
- wc_status_text[p_done_wc->status] );\r
- return FALSE;\r
- }\r
- p_done_wc = p_done_wc->p_next;\r
- }\r
- }\r
-\r
- if( !g_root.is_polling )\r
- {\r
- status = ib_rearm_cq(h_cq, FALSE);\r
- if (status != IB_SUCCESS)\r
- {\r
- printf("Failed to rearm CQ %p\n", h_cq );\r
- return FALSE;\r
- }\r
- }\r
-\r
- CL_EXIT( CMT_DBG_VERBOSE, cmt_dbg_lvl );\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Remove num_msgs completions from all send CQs for all connections.\r
- */\r
-static boolean_t\r
-__poll_send_cqs()\r
-{\r
- ib_node_t *p_node;\r
- int32_t i;\r
-\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- p_node = &g_root.p_nodes[i];\r
- while( p_node->send_cnt < g_root.num_msgs )\r
- {\r
- if( !g_root.is_polling )\r
- cl_thread_suspend( 0 );\r
- else if( !__poll_cq( p_node, p_node->h_send_cq ) )\r
- return FALSE;\r
- }\r
- }\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-/*\r
- * Remove num_msgs completions from all receive CQs for all connections.\r
- */\r
-static boolean_t\r
-__poll_recv_cqs()\r
-{\r
- ib_node_t *p_node;\r
- int32_t i;\r
-\r
- for( i = 0; i < g_root.num_nodes; i++ )\r
- {\r
- p_node = &g_root.p_nodes[i];\r
- while( p_node->recv_cnt < g_root.num_msgs )\r
- {\r
- if( !g_root.is_polling )\r
- cl_thread_suspend( 0 );\r
- else if( !__poll_cq( p_node, p_node->h_recv_cq ) )\r
- return FALSE;\r
- }\r
- }\r
- return TRUE;\r
-}\r
-\r
-\r
-\r
-/**********************************************************************\r
- **********************************************************************/\r
-int __cdecl\r
-main(\r
- int argc,\r
- char* argv[] )\r
-{\r
- uint64_t start_time, total_time;\r
- uint64_t total_xfer;\r
- uint32_t i;\r
-\r
- cl_memclr( &g_root, sizeof(ib_root_t) );\r
-\r
- /* Set defaults. */\r
- if( !__parse_options( argc, argv ) )\r
- return 1;\r
-\r
- /* Initialize the root - open all common HCA resources. */\r
- if( !__init_root() )\r
- {\r
- printf( "__init_root failed\n" );\r
- __cleanup();\r
- return 1;\r
- }\r
-\r
- /*\r
- * Execute the test the specified number of times. Abort the test\r
- * if any errors occur.\r
- */\r
- total_xfer = g_root.num_msgs * g_root.msg_size * g_root.num_nodes;\r
- for( i = 0; i < g_root.num_iter; i++ )\r
- {\r
- printf( "----- Iteration: %d, %d connections -----\n",\r
- i, g_root.num_nodes );\r
-\r
- /* Initialize the connection parameters. */\r
- __init_conn_info();\r
- __init_qp_info();\r
- __create_nodes();\r
- /* Start listening for connections if we're the server. */\r
- if( g_root.is_server )\r
- __listen();\r
-\r
- /* Allocate a new set of QPs for the connections. */\r
- if( __create_qps() != IB_SUCCESS )\r
- {\r
- printf( "Unable to allocate QPs for test.\n" );\r
- break;\r
- }\r
-\r
- /* Establish all connections. */\r
- if( __connect() != IB_SUCCESS )\r
- {\r
- printf( "Failed to establish connections.\n" );\r
- break;\r
- }\r
-\r
- printf( "Transfering data...\n" );\r
- g_root.state = test_transfering;\r
- start_time = cl_get_time_stamp();\r
-\r
- if( g_root.num_msgs )\r
- {\r
- if( g_root.is_server )\r
- {\r
- /* The server initiate the sends to avoid race conditions. */\r
- if( !__send_msgs() )\r
- break;\r
-\r
- /* Get all send completions. */\r
- if( !__poll_send_cqs() )\r
- break;\r
-\r
- /* Get all receive completions. */\r
- if( !__poll_recv_cqs() )\r
- break;\r
- }\r
- else\r
- {\r
- /* Get all receive completions. */\r
- if( !__poll_recv_cqs() )\r
- break;\r
-\r
- /* Reply to the sends. */\r
- if( !__send_msgs() )\r
- break;\r
-\r
- /* Get all send completions. */\r
- if( !__poll_send_cqs() )\r
- break;\r
- }\r
- }\r
-\r
- total_time = cl_get_time_stamp() - start_time;\r
- g_root.state = test_idle;\r
-\r
- printf( "Data transfer time: %"PRId64" ms, %d messages/conn, "\r
- "%"PRId64" total bytes\n", total_time/1000,\r
- g_root.num_msgs, total_xfer );\r
-\r
- /* Disconnect all connections. */\r
- __disconnect();\r
- __destroy_qps();\r
- __destroy_nodes();\r
- }\r
-\r
- __cleanup();\r
- return 0;\r
-}\r
UMTYPE=console\r
USE_CRTDLL=1\r
\r
-SOURCES=main.c\r
+SOURCES=limits_main.c\r
\r
INCLUDES=..\..\..\inc;..\..\..\inc\user;\r
\r
--- /dev/null
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id$\r
+ */\r
+\r
+\r
+/*\r
+ * Abstract:\r
+ * Test limits for:\r
+ * - memory registration\r
+ * - CQ creation\r
+ * - CQ resize\r
+ * - QP creation\r
+ *\r
+ * Environment:\r
+ * User Mode\r
+ */\r
+\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <stdarg.h>\r
+#include <ctype.h>\r
+#include <complib/cl_atomic.h>\r
+#include <complib/cl_debug.h>\r
+#include <complib/cl_event.h>\r
+#include <complib/cl_math.h>\r
+#include <complib/cl_mutex.h>\r
+#include <complib/cl_qlist.h>\r
+#include <complib/cl_thread.h>\r
+#include <complib/cl_timer.h>\r
+#include <iba/ib_types.h>\r
+#include <iba/ib_al.h>\r
+\r
+\r
+/* Globals */\r
+#define CMT_DBG_VERBOSE 1\r
+\r
+\r
+uint32_t cmt_dbg_lvl = 0x80000000;\r
+\r
+\r
+/**********************************************************************\r
+ **********************************************************************/\r
+static void\r
+__show_usage()\r
+{\r
+ printf( "\n------- ib_limits - Usage and options ----------------------\n" );\r
+ printf( "Usage: ib_limits [options]\n");\r
+ printf( "Options:\n" );\r
+ printf( "-m\n"\r
+ "--memory\n"\r
+ "\tThis option directs ib_limits to test memory registration\n" );\r
+ printf( "-c\n"\r
+ "--cq\n"\r
+ "\tThis option directs ib_limits to test CQ creation\n" );\r
+ printf( "-r\n"\r
+ "--resize_cq\n"\r
+ "\tThis option directs ib_limits to test CQ resize\n" );\r
+ printf( "-q\n"\r
+ "--qp\n"\r
+ "\tThis option directs ib_limits to test QP creation\n" );\r
+ printf( "-v\n"\r
+ "--verbose\n"\r
+ " This option enables verbosity level to debug console.\n" );\r
+ printf( "-h\n"\r
+ "--help\n"\r
+ " Display this usage info then exit.\n\n" );\r
+}\r
+\r
+\r
+/* Windows support. */\r
+struct option\r
+{\r
+ const char *long_name;\r
+ unsigned long flag;\r
+ void *pfn_handler;\r
+ char short_name;\r
+};\r
+\r
+static char *optarg;\r
+\r
+#define strtoull strtoul\r
+\r
+\r
+boolean_t test_mr, test_cq, test_resize, test_qp;\r
+\r
+\r
+char\r
+getopt_long(\r
+ int argc,\r
+ char *argv[],\r
+ const char *short_option,\r
+ const struct option *long_option,\r
+ void *unused )\r
+{\r
+ static int i = 1;\r
+ int j;\r
+ char ret = 0;\r
+\r
+ UNUSED_PARAM( unused );\r
+\r
+ if( i == argc )\r
+ return -1;\r
+\r
+ if( argv[i][0] != '-' )\r
+ return ret;\r
+\r
+ /* find the first character of the value. */\r
+ for( j = 1; isalpha( argv[i][j] ); j++ )\r
+ ;\r
+ optarg = &argv[i][j];\r
+\r
+ if( argv[i][1] == '-' )\r
+ {\r
+ /* Long option. */\r
+ for( j = 0; long_option[j].long_name; j++ )\r
+ {\r
+ if( strncmp( &argv[i][2], long_option[j].long_name,\r
+ optarg - argv[i] - 2 ) )\r
+ {\r
+ continue;\r
+ }\r
+\r
+ switch( long_option[j].flag )\r
+ {\r
+ case 1:\r
+ if( *optarg == '\0' )\r
+ return 0;\r
+ default:\r
+ break;\r
+ }\r
+ ret = long_option[j].short_name;\r
+ break;\r
+ }\r
+ }\r
+ else\r
+ {\r
+ for( j = 0; short_option[j] != '\0'; j++ )\r
+ {\r
+ if( !isalpha( short_option[j] ) )\r
+ return 0;\r
+\r
+ if( short_option[j] == argv[i][1] )\r
+ {\r
+ ret = short_option[j];\r
+ break;\r
+ }\r
+\r
+ if( short_option[j+1] == ':' )\r
+ {\r
+ if( *optarg == '\0' )\r
+ return 0;\r
+ j++;\r
+ }\r
+ }\r
+ }\r
+ i++;\r
+ return ret;\r
+}\r
+\r
+\r
+static boolean_t\r
+__parse_options(\r
+ int argc,\r
+ char* argv[] )\r
+{\r
+ uint32_t next_option;\r
+ const char* const short_option = "mcrq:vh";\r
+\r
+ /*\r
+ In the array below, the 2nd parameter specified the number\r
+ of arguments as follows:\r
+ 0: no arguments\r
+ 1: argument\r
+ 2: optional\r
+ */\r
+ const struct option long_option[] =\r
+ {\r
+ { "memory", 2, NULL, 'm'},\r
+ { "cq", 2, NULL, 'c'},\r
+ { "resize_cq",2, NULL, 'r'},\r
+ { "qp", 2, NULL, 'q'},\r
+ { "verbose", 0, NULL, 'v'},\r
+ { "help", 0, NULL, 'h'},\r
+ { NULL, 0, NULL, 0 } /* Required at end of array */\r
+ };\r
+\r
+ test_mr = FALSE;\r
+ test_cq = FALSE;\r
+ test_resize = FALSE;\r
+ test_qp = FALSE;\r
+\r
+ /* parse cmd line arguments as input params */\r
+ do\r
+ {\r
+ next_option = getopt_long( argc, argv, short_option,\r
+ long_option, NULL );\r
+\r
+ switch( next_option )\r
+ {\r
+ case 'm':\r
+ test_mr = TRUE;\r
+ printf( "\tTest Memory Registration\n" );\r
+ break;\r
+\r
+ case 'c':\r
+ test_cq = TRUE;\r
+ printf( "\tTest CQ\n" );\r
+ break;\r
+\r
+ case 'r':\r
+ test_resize = TRUE;\r
+ printf( "\tTest CQ Resize\n" );\r
+ break;\r
+\r
+ case 'q':\r
+ test_qp = TRUE;\r
+ printf( "\tTest QP\n" );\r
+ break;\r
+\r
+ case 'v':\r
+ cmt_dbg_lvl = 0xFFFFFFFF;\r
+ printf( "\tverbose\n" );\r
+ break;\r
+\r
+ case 'h':\r
+ __show_usage();\r
+ return FALSE;\r
+\r
+ case -1:\r
+ break;\r
+\r
+ default: /* something wrong */\r
+ __show_usage();\r
+ return FALSE;\r
+ }\r
+ } while( next_option != -1 );\r
+\r
+ return TRUE;\r
+}\r
+\r
+\r
+struct __mr_buf\r
+{\r
+ cl_list_item_t list_item;\r
+ ib_mr_handle_t h_mr;\r
+ char buf[8192 - sizeof(ib_mr_handle_t) - sizeof(cl_list_item_t)];\r
+};\r
+\r
+static void\r
+__test_mr(\r
+ ib_pd_handle_t h_pd )\r
+{\r
+ ib_api_status_t status = IB_SUCCESS;\r
+ struct __mr_buf *p_mr;\r
+ int i = 0;\r
+ ib_mr_create_t mr_create;\r
+ cl_qlist_t mr_list;\r
+ net32_t lkey, rkey;\r
+ int64_t reg_time, dereg_time, tmp_time, cnt;\r
+\r
+ printf( "MR testing [\n" );\r
+\r
+ cl_qlist_init( &mr_list );\r
+ reg_time = 0;\r
+ dereg_time = 0;\r
+ cnt = 0;\r
+\r
+ do\r
+ {\r
+ p_mr = cl_malloc( sizeof(struct __mr_buf) );\r
+ if( !p_mr )\r
+ {\r
+ i++;\r
+ printf( "Failed to allocate memory.\n" );\r
+ continue;\r
+ }\r
+\r
+ mr_create.vaddr = p_mr->buf;\r
+ mr_create.length = sizeof(p_mr->buf);\r
+ mr_create.access_ctrl =\r
+ IB_AC_LOCAL_WRITE | IB_AC_RDMA_READ | IB_AC_RDMA_WRITE;\r
+\r
+ tmp_time = cl_get_time_stamp();\r
+ status = ib_reg_mem( h_pd, &mr_create, &lkey, &rkey, &p_mr->h_mr );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ i++;\r
+ printf( "ib_reg_mem returned %s\n", ib_get_err_str( status ) );\r
+ cl_free( p_mr );\r
+ continue;\r
+ }\r
+ reg_time += cl_get_time_stamp() - tmp_time;\r
+ cnt++;\r
+\r
+ cl_qlist_insert_tail( &mr_list, &p_mr->list_item );\r
+\r
+ } while( status == IB_SUCCESS || i < 1000 );\r
+\r
+ while( cl_qlist_count( &mr_list ) )\r
+ {\r
+ p_mr = PARENT_STRUCT( cl_qlist_remove_head( &mr_list ),\r
+ struct __mr_buf, list_item );\r
+\r
+ tmp_time = cl_get_time_stamp();\r
+ status = ib_dereg_mr( p_mr->h_mr );\r
+ if( status != IB_SUCCESS )\r
+ printf( "ib_dereg_mr returned %s\n", ib_get_err_str( status ) );\r
+ dereg_time += cl_get_time_stamp() - tmp_time;\r
+\r
+ cl_free( p_mr );\r
+ }\r
+\r
+ printf( "reg time %f, dereg time %f\n", (double)reg_time/(double)cnt,\r
+ (double)dereg_time/(double)cnt );\r
+ printf( "MR testing ]\n" );\r
+}\r
+\r
+\r
+struct __cq\r
+{\r
+ cl_list_item_t list_item;\r
+ ib_cq_handle_t h_cq;\r
+};\r
+\r
+static void\r
+__test_cq(\r
+ ib_ca_handle_t h_ca,\r
+ boolean_t resize )\r
+{\r
+ ib_api_status_t status = IB_SUCCESS;\r
+ struct __cq *p_cq;\r
+ int i = 0, j;\r
+ ib_cq_create_t cq_create;\r
+ cl_qlist_t cq_list;\r
+ cl_waitobj_handle_t h_waitobj;\r
+ uint32_t size;\r
+\r
+ printf( "CQ %stesting [\n", resize?"resize ":"" );\r
+\r
+ cl_qlist_init( &cq_list );\r
+\r
+ if( cl_waitobj_create( FALSE, &h_waitobj ) != CL_SUCCESS )\r
+ {\r
+ printf( "Failed to allocate CQ wait object.\n" );\r
+ return;\r
+ }\r
+\r
+ do\r
+ {\r
+ p_cq = cl_malloc( sizeof(*p_cq) );\r
+ if( !p_cq )\r
+ {\r
+ i++;\r
+ printf( "Failed to allocate memory.\n" );\r
+ continue;\r
+ }\r
+\r
+ cq_create.h_wait_obj = h_waitobj;\r
+ cq_create.pfn_comp_cb = NULL;\r
+ if( resize )\r
+ cq_create.size = 32;\r
+ else\r
+ cq_create.size = 4096;\r
+\r
+ status = ib_create_cq( h_ca, &cq_create, NULL, NULL, &p_cq->h_cq );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ i++;\r
+ printf( "ib_create_cq returned %s\n", ib_get_err_str( status ) );\r
+ cl_free( p_cq );\r
+ continue;\r
+ }\r
+\r
+ if( resize )\r
+ {\r
+ size = 256;\r
+ j = 0;\r
+\r
+ do\r
+ {\r
+ status = ib_modify_cq( p_cq->h_cq, &size );\r
+ if( status == IB_SUCCESS )\r
+ {\r
+ size += 256;\r
+ }\r
+ else\r
+ {\r
+ j++;\r
+ printf( "ib_modify_cq returned %s\n",\r
+ ib_get_err_str( status ) );\r
+ }\r
+\r
+ } while( status == IB_SUCCESS || j < 100 );\r
+ }\r
+\r
+ cl_qlist_insert_tail( &cq_list, &p_cq->list_item );\r
+\r
+ } while( status == IB_SUCCESS || i < 1000 );\r
+\r
+ while( cl_qlist_count( &cq_list ) )\r
+ {\r
+ p_cq = PARENT_STRUCT( cl_qlist_remove_head( &cq_list ),\r
+ struct __cq, list_item );\r
+\r
+ status = ib_destroy_cq( p_cq->h_cq, NULL );\r
+ if( status != IB_SUCCESS )\r
+ printf( "ib_destroy_cq returned %s\n", ib_get_err_str( status ) );\r
+\r
+ cl_free( p_cq );\r
+ }\r
+\r
+ printf( "CQ %stesting ]\n", resize?"resize ":"" );\r
+}\r
+\r
+/**********************************************************************\r
+ **********************************************************************/\r
+int __cdecl\r
+main(\r
+ int argc,\r
+ char* argv[] )\r
+{\r
+ ib_api_status_t status;\r
+ ib_al_handle_t h_al;\r
+ ib_ca_handle_t h_ca;\r
+ ib_pd_handle_t h_pd;\r
+ size_t size;\r
+ net64_t *ca_guids;\r
+\r
+ /* Set defaults. */\r
+ if( !__parse_options( argc, argv ) )\r
+ return 1;\r
+\r
+ status = ib_open_al( &h_al );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_open_al returned %s\n", ib_get_err_str( status ) );\r
+ return 1;\r
+ }\r
+\r
+ size = 0;\r
+ status = ib_get_ca_guids( h_al, NULL, &size );\r
+ if( status != IB_INSUFFICIENT_MEMORY )\r
+ {\r
+ printf( "ib_get_ca_guids for array size returned %s",\r
+ ib_get_err_str( status ) );\r
+ goto done;\r
+ }\r
+\r
+ if( size == 0 )\r
+ {\r
+ printf( "No CAs installed.\n" );\r
+ goto done;\r
+ }\r
+\r
+ ca_guids = malloc( sizeof(net64_t) * size );\r
+ if( !ca_guids )\r
+ {\r
+ printf( "Failed to allocate CA GUID array.\n" );\r
+ goto done;\r
+ }\r
+\r
+ status = ib_get_ca_guids( h_al, ca_guids, &size );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_get_ca_guids for CA guids returned %s",\r
+ ib_get_err_str( status ) );\r
+ free( ca_guids );\r
+ goto done;\r
+ }\r
+\r
+ status = ib_open_ca( h_al, ca_guids[0], NULL, NULL, &h_ca );\r
+ free( ca_guids );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_open_ca returned %s", ib_get_err_str( status ) );\r
+ goto done;\r
+ }\r
+\r
+ status = ib_alloc_pd( h_ca, IB_PDT_NORMAL, NULL, &h_pd );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ printf( "ib_alloc_pd returned %s", ib_get_err_str( status ) );\r
+ goto done;\r
+ }\r
+\r
+ if( test_mr )\r
+ __test_mr( h_pd );\r
+\r
+ if( test_cq )\r
+ __test_cq( h_ca, FALSE );\r
+\r
+ if( test_resize )\r
+ __test_cq( h_ca, TRUE );\r
+\r
+ //if( test_qp )\r
+ // __test_qp( h_ca, h_pd );\r
+\r
+done:\r
+ ib_close_al( h_al );\r
+\r
+ return 0;\r
+}\r
+++ /dev/null
-/*\r
- * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.\r
- *\r
- * This software is available to you under the OpenIB.org BSD license\r
- * below:\r
- *\r
- * Redistribution and use in source and binary forms, with or\r
- * without modification, are permitted provided that the following\r
- * conditions are met:\r
- *\r
- * - Redistributions of source code must retain the above\r
- * copyright notice, this list of conditions and the following\r
- * disclaimer.\r
- *\r
- * - Redistributions in binary form must reproduce the above\r
- * copyright notice, this list of conditions and the following\r
- * disclaimer in the documentation and/or other materials\r
- * provided with the distribution.\r
- *\r
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
- * SOFTWARE.\r
- *\r
- * $Id$\r
- */\r
-\r
-\r
-/*\r
- * Abstract:\r
- * Test limits for:\r
- * - memory registration\r
- * - CQ creation\r
- * - CQ resize\r
- * - QP creation\r
- *\r
- * Environment:\r
- * User Mode\r
- */\r
-\r
-#include <stdio.h>\r
-#include <stdlib.h>\r
-#include <stdarg.h>\r
-#include <ctype.h>\r
-#include <complib/cl_atomic.h>\r
-#include <complib/cl_debug.h>\r
-#include <complib/cl_event.h>\r
-#include <complib/cl_math.h>\r
-#include <complib/cl_mutex.h>\r
-#include <complib/cl_qlist.h>\r
-#include <complib/cl_thread.h>\r
-#include <complib/cl_timer.h>\r
-#include <iba/ib_types.h>\r
-#include <iba/ib_al.h>\r
-\r
-\r
-/* Globals */\r
-#define CMT_DBG_VERBOSE 1\r
-\r
-\r
-uint32_t cmt_dbg_lvl = 0x80000000;\r
-\r
-\r
-/**********************************************************************\r
- **********************************************************************/\r
-static void\r
-__show_usage()\r
-{\r
- printf( "\n------- ib_limits - Usage and options ----------------------\n" );\r
- printf( "Usage: ib_limits [options]\n");\r
- printf( "Options:\n" );\r
- printf( "-m\n"\r
- "--memory\n"\r
- "\tThis option directs ib_limits to test memory registration\n" );\r
- printf( "-c\n"\r
- "--cq\n"\r
- "\tThis option directs ib_limits to test CQ creation\n" );\r
- printf( "-r\n"\r
- "--resize_cq\n"\r
- "\tThis option directs ib_limits to test CQ resize\n" );\r
- printf( "-q\n"\r
- "--qp\n"\r
- "\tThis option directs ib_limits to test QP creation\n" );\r
- printf( "-v\n"\r
- "--verbose\n"\r
- " This option enables verbosity level to debug console.\n" );\r
- printf( "-h\n"\r
- "--help\n"\r
- " Display this usage info then exit.\n\n" );\r
-}\r
-\r
-\r
-/* Windows support. */\r
-struct option\r
-{\r
- const char *long_name;\r
- unsigned long flag;\r
- void *pfn_handler;\r
- char short_name;\r
-};\r
-\r
-static char *optarg;\r
-\r
-#define strtoull strtoul\r
-\r
-\r
-boolean_t test_mr, test_cq, test_resize, test_qp;\r
-\r
-\r
-char\r
-getopt_long(\r
- int argc,\r
- char *argv[],\r
- const char *short_option,\r
- const struct option *long_option,\r
- void *unused )\r
-{\r
- static int i = 1;\r
- int j;\r
- char ret = 0;\r
-\r
- UNUSED_PARAM( unused );\r
-\r
- if( i == argc )\r
- return -1;\r
-\r
- if( argv[i][0] != '-' )\r
- return ret;\r
-\r
- /* find the first character of the value. */\r
- for( j = 1; isalpha( argv[i][j] ); j++ )\r
- ;\r
- optarg = &argv[i][j];\r
-\r
- if( argv[i][1] == '-' )\r
- {\r
- /* Long option. */\r
- for( j = 0; long_option[j].long_name; j++ )\r
- {\r
- if( strncmp( &argv[i][2], long_option[j].long_name,\r
- optarg - argv[i] - 2 ) )\r
- {\r
- continue;\r
- }\r
-\r
- switch( long_option[j].flag )\r
- {\r
- case 1:\r
- if( *optarg == '\0' )\r
- return 0;\r
- default:\r
- break;\r
- }\r
- ret = long_option[j].short_name;\r
- break;\r
- }\r
- }\r
- else\r
- {\r
- for( j = 0; short_option[j] != '\0'; j++ )\r
- {\r
- if( !isalpha( short_option[j] ) )\r
- return 0;\r
-\r
- if( short_option[j] == argv[i][1] )\r
- {\r
- ret = short_option[j];\r
- break;\r
- }\r
-\r
- if( short_option[j+1] == ':' )\r
- {\r
- if( *optarg == '\0' )\r
- return 0;\r
- j++;\r
- }\r
- }\r
- }\r
- i++;\r
- return ret;\r
-}\r
-\r
-\r
-static boolean_t\r
-__parse_options(\r
- int argc,\r
- char* argv[] )\r
-{\r
- uint32_t next_option;\r
- const char* const short_option = "mcrq:vh";\r
-\r
- /*\r
- In the array below, the 2nd parameter specified the number\r
- of arguments as follows:\r
- 0: no arguments\r
- 1: argument\r
- 2: optional\r
- */\r
- const struct option long_option[] =\r
- {\r
- { "memory", 2, NULL, 'm'},\r
- { "cq", 2, NULL, 'c'},\r
- { "resize_cq",2, NULL, 'r'},\r
- { "qp", 2, NULL, 'q'},\r
- { "verbose", 0, NULL, 'v'},\r
- { "help", 0, NULL, 'h'},\r
- { NULL, 0, NULL, 0 } /* Required at end of array */\r
- };\r
-\r
- test_mr = FALSE;\r
- test_cq = FALSE;\r
- test_resize = FALSE;\r
- test_qp = FALSE;\r
-\r
- /* parse cmd line arguments as input params */\r
- do\r
- {\r
- next_option = getopt_long( argc, argv, short_option,\r
- long_option, NULL );\r
-\r
- switch( next_option )\r
- {\r
- case 'm':\r
- test_mr = TRUE;\r
- printf( "\tTest Memory Registration\n" );\r
- break;\r
-\r
- case 'c':\r
- test_cq = TRUE;\r
- printf( "\tTest CQ\n" );\r
- break;\r
-\r
- case 'r':\r
- test_resize = TRUE;\r
- printf( "\tTest CQ Resize\n" );\r
- break;\r
-\r
- case 'q':\r
- test_qp = TRUE;\r
- printf( "\tTest QP\n" );\r
- break;\r
-\r
- case 'v':\r
- cmt_dbg_lvl = 0xFFFFFFFF;\r
- printf( "\tverbose\n" );\r
- break;\r
-\r
- case 'h':\r
- __show_usage();\r
- return FALSE;\r
-\r
- case -1:\r
- break;\r
-\r
- default: /* something wrong */\r
- __show_usage();\r
- return FALSE;\r
- }\r
- } while( next_option != -1 );\r
-\r
- return TRUE;\r
-}\r
-\r
-\r
-struct __mr_buf\r
-{\r
- cl_list_item_t list_item;\r
- ib_mr_handle_t h_mr;\r
- char buf[8192 - sizeof(ib_mr_handle_t) - sizeof(cl_list_item_t)];\r
-};\r
-\r
-static void\r
-__test_mr(\r
- ib_pd_handle_t h_pd )\r
-{\r
- ib_api_status_t status = IB_SUCCESS;\r
- struct __mr_buf *p_mr;\r
- int i = 0;\r
- ib_mr_create_t mr_create;\r
- cl_qlist_t mr_list;\r
- net32_t lkey, rkey;\r
- int64_t reg_time, dereg_time, tmp_time, cnt;\r
-\r
- printf( "MR testing [\n" );\r
-\r
- cl_qlist_init( &mr_list );\r
- reg_time = 0;\r
- dereg_time = 0;\r
- cnt = 0;\r
-\r
- do\r
- {\r
- p_mr = cl_malloc( sizeof(struct __mr_buf) );\r
- if( !p_mr )\r
- {\r
- i++;\r
- printf( "Failed to allocate memory.\n" );\r
- continue;\r
- }\r
-\r
- mr_create.vaddr = p_mr->buf;\r
- mr_create.length = sizeof(p_mr->buf);\r
- mr_create.access_ctrl =\r
- IB_AC_LOCAL_WRITE | IB_AC_RDMA_READ | IB_AC_RDMA_WRITE;\r
-\r
- tmp_time = cl_get_time_stamp();\r
- status = ib_reg_mem( h_pd, &mr_create, &lkey, &rkey, &p_mr->h_mr );\r
- if( status != IB_SUCCESS )\r
- {\r
- i++;\r
- printf( "ib_reg_mem returned %s\n", ib_get_err_str( status ) );\r
- cl_free( p_mr );\r
- continue;\r
- }\r
- reg_time += cl_get_time_stamp() - tmp_time;\r
- cnt++;\r
-\r
- cl_qlist_insert_tail( &mr_list, &p_mr->list_item );\r
-\r
- } while( status == IB_SUCCESS || i < 1000 );\r
-\r
- while( cl_qlist_count( &mr_list ) )\r
- {\r
- p_mr = PARENT_STRUCT( cl_qlist_remove_head( &mr_list ),\r
- struct __mr_buf, list_item );\r
-\r
- tmp_time = cl_get_time_stamp();\r
- status = ib_dereg_mr( p_mr->h_mr );\r
- if( status != IB_SUCCESS )\r
- printf( "ib_dereg_mr returned %s\n", ib_get_err_str( status ) );\r
- dereg_time += cl_get_time_stamp() - tmp_time;\r
-\r
- cl_free( p_mr );\r
- }\r
-\r
- printf( "reg time %f, dereg time %f\n", (double)reg_time/(double)cnt,\r
- (double)dereg_time/(double)cnt );\r
- printf( "MR testing ]\n" );\r
-}\r
-\r
-\r
-struct __cq\r
-{\r
- cl_list_item_t list_item;\r
- ib_cq_handle_t h_cq;\r
-};\r
-\r
-static void\r
-__test_cq(\r
- ib_ca_handle_t h_ca,\r
- boolean_t resize )\r
-{\r
- ib_api_status_t status = IB_SUCCESS;\r
- struct __cq *p_cq;\r
- int i = 0, j;\r
- ib_cq_create_t cq_create;\r
- cl_qlist_t cq_list;\r
- cl_waitobj_handle_t h_waitobj;\r
- uint32_t size;\r
-\r
- printf( "CQ %stesting [\n", resize?"resize ":"" );\r
-\r
- cl_qlist_init( &cq_list );\r
-\r
- if( cl_waitobj_create( FALSE, &h_waitobj ) != CL_SUCCESS )\r
- {\r
- printf( "Failed to allocate CQ wait object.\n" );\r
- return;\r
- }\r
-\r
- do\r
- {\r
- p_cq = cl_malloc( sizeof(*p_cq) );\r
- if( !p_cq )\r
- {\r
- i++;\r
- printf( "Failed to allocate memory.\n" );\r
- continue;\r
- }\r
-\r
- cq_create.h_wait_obj = h_waitobj;\r
- cq_create.pfn_comp_cb = NULL;\r
- if( resize )\r
- cq_create.size = 32;\r
- else\r
- cq_create.size = 4096;\r
-\r
- status = ib_create_cq( h_ca, &cq_create, NULL, NULL, &p_cq->h_cq );\r
- if( status != IB_SUCCESS )\r
- {\r
- i++;\r
- printf( "ib_create_cq returned %s\n", ib_get_err_str( status ) );\r
- cl_free( p_cq );\r
- continue;\r
- }\r
-\r
- if( resize )\r
- {\r
- size = 256;\r
- j = 0;\r
-\r
- do\r
- {\r
- status = ib_modify_cq( p_cq->h_cq, &size );\r
- if( status == IB_SUCCESS )\r
- {\r
- size += 256;\r
- }\r
- else\r
- {\r
- j++;\r
- printf( "ib_modify_cq returned %s\n",\r
- ib_get_err_str( status ) );\r
- }\r
-\r
- } while( status == IB_SUCCESS || j < 100 );\r
- }\r
-\r
- cl_qlist_insert_tail( &cq_list, &p_cq->list_item );\r
-\r
- } while( status == IB_SUCCESS || i < 1000 );\r
-\r
- while( cl_qlist_count( &cq_list ) )\r
- {\r
- p_cq = PARENT_STRUCT( cl_qlist_remove_head( &cq_list ),\r
- struct __cq, list_item );\r
-\r
- status = ib_destroy_cq( p_cq->h_cq, NULL );\r
- if( status != IB_SUCCESS )\r
- printf( "ib_destroy_cq returned %s\n", ib_get_err_str( status ) );\r
-\r
- cl_free( p_cq );\r
- }\r
-\r
- printf( "CQ %stesting ]\n", resize?"resize ":"" );\r
-}\r
-\r
-/**********************************************************************\r
- **********************************************************************/\r
-int __cdecl\r
-main(\r
- int argc,\r
- char* argv[] )\r
-{\r
- ib_api_status_t status;\r
- ib_al_handle_t h_al;\r
- ib_ca_handle_t h_ca;\r
- ib_pd_handle_t h_pd;\r
- size_t size;\r
- net64_t *ca_guids;\r
-\r
- /* Set defaults. */\r
- if( !__parse_options( argc, argv ) )\r
- return 1;\r
-\r
- status = ib_open_al( &h_al );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_open_al returned %s\n", ib_get_err_str( status ) );\r
- return 1;\r
- }\r
-\r
- size = 0;\r
- status = ib_get_ca_guids( h_al, NULL, &size );\r
- if( status != IB_INSUFFICIENT_MEMORY )\r
- {\r
- printf( "ib_get_ca_guids for array size returned %s",\r
- ib_get_err_str( status ) );\r
- goto done;\r
- }\r
-\r
- if( size == 0 )\r
- {\r
- printf( "No CAs installed.\n" );\r
- goto done;\r
- }\r
-\r
- ca_guids = malloc( sizeof(net64_t) * size );\r
- if( !ca_guids )\r
- {\r
- printf( "Failed to allocate CA GUID array.\n" );\r
- goto done;\r
- }\r
-\r
- status = ib_get_ca_guids( h_al, ca_guids, &size );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_get_ca_guids for CA guids returned %s",\r
- ib_get_err_str( status ) );\r
- free( ca_guids );\r
- goto done;\r
- }\r
-\r
- status = ib_open_ca( h_al, ca_guids[0], NULL, NULL, &h_ca );\r
- free( ca_guids );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_open_ca returned %s", ib_get_err_str( status ) );\r
- goto done;\r
- }\r
-\r
- status = ib_alloc_pd( h_ca, IB_PDT_NORMAL, NULL, &h_pd );\r
- if( status != IB_SUCCESS )\r
- {\r
- printf( "ib_alloc_pd returned %s", ib_get_err_str( status ) );\r
- goto done;\r
- }\r
-\r
- if( test_mr )\r
- __test_mr( h_pd );\r
-\r
- if( test_cq )\r
- __test_cq( h_ca, FALSE );\r
-\r
- if( test_resize )\r
- __test_cq( h_ca, TRUE );\r
-\r
- //if( test_qp )\r
- // __test_qp( h_ca, h_pd );\r
-\r
-done:\r
- ib_close_al( h_al );\r
-\r
- return 0;\r
-}\r
projects / ~shefty / rdma-win.git / commitdiff