openib_cma: fix cleanup issues in destroy_cm_id

            add macros to convert SID and PORT
            fix init/responder settings in accept

Signed-off by: Arlin Davis <ardavis@ichips.intel.com>

diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c
index e8c33f2857aef286ccf0825204b61d0207e6ce85..f7d83e0ec952bac3e1553a156caee51d7240de6d 100755 (executable)
--- a/dapl/openib_cma/dapl_ib_cm.c
+++ b/dapl/openib_cma/dapl_ib_cm.c
@@ -69,13 +69,15 @@ static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
 static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
 #endif
 
-/* cma requires 16 bit SID */
+/* cma requires 16 bit SID, in network order */
 #define IB_PORT_MOD 32001
 #define IB_PORT_BASE (65535 - IB_PORT_MOD)
-#define MAKE_PORT(SID) \
+#define SID_TO_PORT(SID) \
     (SID > 0xffff ? \
-    (unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE) :\
-    (unsigned short)SID)
+    htons((unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE)) :\
+    htons((unsigned short)SID))
+
+#define PORT_TO_SID(p) ntohs(p)
 
 static void dapli_addr_resolve(struct dapl_cm_id *conn)
 {
@@ -173,8 +175,10 @@ void dapli_destroy_conn(struct dapl_cm_id *conn)
        dapl_os_lock(&conn->lock);
        conn->destroy = 1;
        
-       if (conn->ep)
+       if (conn->ep) {
                conn->ep->cm_handle = IB_INVALID_HANDLE;
+               conn->ep->qp_handle = IB_INVALID_HANDLE;
+       }
 
        cm_id = conn->cm_id;
        conn->cm_id = NULL;
@@ -220,10 +224,10 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
                
                /* Get requesters connect data, setup for accept */
                new_conn->params.responder_resources = 
-                       DAPL_MIN(event->param.conn.initiator_depth,
+                       DAPL_MIN(event->param.conn.responder_resources,
                                 conn->hca->ib_trans.max_rdma_rd_in);
                new_conn->params.initiator_depth = 
-                       DAPL_MIN(event->param.conn.responder_resources,
+                       DAPL_MIN(event->param.conn.initiator_depth,
                                 conn->hca->ib_trans.max_rdma_rd_out);
 
                new_conn->params.flow_control = event->param.conn.flow_control;
@@ -348,8 +352,8 @@ static void dapli_cm_active_cb(struct dapl_cm_id *conn,
                        &conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr));
 
                /* setup local and remote ports for ep query */
-               conn->ep->param.remote_port_qual = rdma_get_dst_port(conn->cm_id);
-               conn->ep->param.local_port_qual = rdma_get_src_port(conn->cm_id);
+               conn->ep->param.remote_port_qual = PORT_TO_SID(rdma_get_dst_port(conn->cm_id));
+               conn->ep->param.local_port_qual = PORT_TO_SID(rdma_get_src_port(conn->cm_id));
 
                dapl_evd_connection_callback(conn, IB_CME_CONNECTED,
                                             event->param.conn.private_data, conn->ep);
@@ -526,8 +530,9 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
        if (NULL == ep_ptr) 
                return DAT_SUCCESS;
 
-       dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect: rSID %d, pdata %p, ln %d\n", 
-                    r_qual,p_data,p_size);
+       dapl_dbg_log(DAPL_DBG_TYPE_CM, 
+                    " connect: rSID 0x%llx rPort %d, pdata %p, ln %d\n", 
+                    r_qual,ntohs(SID_TO_PORT(r_qual)),p_data,p_size);
                        
        /* rdma conn and cm_id pre-bound; reference via qp_handle */
        conn = ep_ptr->cm_handle = ep_ptr->qp_handle;
@@ -549,7 +554,7 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
        dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(*r_addr));
 
        /* Resolve remote address, src already bound during QP create */
-       ((struct sockaddr_in*)&conn->r_addr)->sin_port = htons(MAKE_PORT(r_qual));
+       ((struct sockaddr_in*)&conn->r_addr)->sin_port = SID_TO_PORT(r_qual);
        ((struct sockaddr_in*)&conn->r_addr)->sin_family = AF_INET;
 
        if (rdma_resolve_addr(conn->cm_id, NULL, 
@@ -593,7 +598,7 @@ dapls_ib_disconnect(IN DAPL_EP *ep_ptr,
                     " disconnect(ep %p, conn %p, id %d flags %x)\n",
                     ep_ptr,conn, (conn?conn->cm_id:0),close_flags);
 
-       if (conn == IB_INVALID_HANDLE)
+        if ((conn == IB_INVALID_HANDLE) || (conn->cm_id == NULL))
                return DAT_SUCCESS;
 
        /* no graceful half-pipe disconnect option */
@@ -682,8 +687,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
        /* open identifies the local device; per DAT specification */
        /* Get family and address then set port to consumer's ServiceID */
        dapl_os_memcpy(&addr, &ia_ptr->hca_ptr->hca_address, sizeof(addr));
-       ((struct sockaddr_in *)&addr)->sin_port = htons(MAKE_PORT(ServiceID));
-
+       ((struct sockaddr_in *)&addr)->sin_port = SID_TO_PORT(ServiceID);
 
        if (rdma_bind_addr(conn->cm_id,(struct sockaddr *)&addr)) {
                if (errno == EBUSY)
@@ -695,8 +699,8 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
        }
 
        dapl_dbg_log(DAPL_DBG_TYPE_CM,
-               " listen(ia_ptr %p SID %d sp %p conn %p id %d)\n",
-               ia_ptr, MAKE_PORT(ServiceID), 
+               " listen(ia_ptr %p SID 0x%llx Port %d sp %p conn %p id %d)\n",
+               ia_ptr, ServiceID, ntohs(SID_TO_PORT(ServiceID)), 
                sp_ptr, conn, conn->cm_id);
 
        sp_ptr->cm_srvc_handle = conn;
@@ -841,9 +845,6 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
        }
 
        cr_ptr->param.local_ep_handle = ep_handle;
-       ep_ptr->qp_handle = cr_conn;
-       ep_ptr->cm_handle = cr_conn;
-       cr_conn->ep = ep_ptr;
        cr_conn->params.private_data = p_data;
        cr_conn->params.private_data_len = p_size;
 
@@ -854,9 +855,15 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
                goto bail;
        }
 
+       /* save accepted conn and EP reference */
+       ep_ptr->qp_handle = cr_conn;
+       ep_ptr->cm_handle = cr_conn;
+       cr_conn->ep = ep_ptr;
+
        /* setup local and remote ports for ep query */
-       ep_ptr->param.remote_port_qual = rdma_get_dst_port(cr_conn->cm_id);
-       ep_ptr->param.local_port_qual = rdma_get_src_port(cr_conn->cm_id);
+       /* Note: port qual in network order */
+       ep_ptr->param.remote_port_qual = PORT_TO_SID(rdma_get_dst_port(cr_conn->cm_id));
+       ep_ptr->param.local_port_qual = PORT_TO_SID(rdma_get_src_port(cr_conn->cm_id));
 
        return DAT_SUCCESS;
 bail: