--- /dev/null
+/* Copyright mellanox */\r
+\r
+/*\r
+ This file holds the types that are used for SDP comunication\r
+\r
+*/\r
+\r
+#ifndef H_SDP_MSGS_H\r
+#define H_SDP_MSGS_H \r
+\r
+\r
+#pragma pack(push)\r
+#pragma pack(1)\r
+\r
+\r
+/*\r
+ * Message Identifier Opcodes for BSDH\r
+ */\r
+/* Name Value Extended Header Payload */\r
+#define SDP_MID_HELLO 0x00 /* msg_hdr_hh <none> */\r
+#define SDP_MID_HELLO_ACK 0x01 /* msg_hdr_hah <none> */\r
+#define SDP_MID_DISCONNECT 0x02 /* <none> <none> */\r
+#define SDP_MID_ABORT_CONN 0x03 /* <none> <none> */\r
+#define SDP_MID_SEND_SM 0x04 /* <none> <none> */\r
+#define SDP_MID_RDMA_WR_COMP 0x05 /* msg_hdr_rwch <none> */\r
+#define SDP_MID_RDMA_RD_COMP 0x06 /* msg_hdr_rrch <none> */\r
+#define SDP_MID_MODE_CHANGE 0x07 /* msg_hdr_mch <none> */\r
+#define SDP_MID_SRC_CANCEL 0x08 /* <none> <none> */\r
+#define SDP_MID_SNK_CANCEL 0x09 /* <none> <none> */\r
+#define SDP_MID_SNK_CANCEL_ACK 0x0A /* <none> <none> */\r
+#define SDP_MID_CH_RECV_BUF 0x0B /* msg_hdr_crbh <none> */\r
+#define SDP_MID_CH_RECV_BUF_ACK 0x0C /* msg_hdr_crbah <none> */\r
+#define SDP_MID_SUSPEND 0x0D /* msg_hdr_sch <none> */\r
+#define SDP_MID_SUSPEND_ACK 0x0E /* <none> <none> */\r
+#define SDP_MID_SNK_AVAIL 0xFD /* msg_hdr_snkah <optional> */\r
+#define SDP_MID_SRC_AVAIL 0xFE /* msg_hdr_srcah <optional> */\r
+#define SDP_MID_DATA 0xFF /* <none> <optional> */\r
+/*\r
+ * shift number for BSDH Flags.\r
+ */\r
+#define SDP_MSG_FLAG_NON_FLAG (0x0) /* no flag present */\r
+#define SDP_MSG_FLAG_OOB_PRES 0 /* out-of-band data present */\r
+#define SDP_MSG_FLAG_OOB_PEND 1 /* out-of-band data pending */\r
+#define SDP_MSG_FLAG_REQ_PIPE 2 /* request change to pipelined */\r
+/*\r
+ * message type masks\r
+ */\r
+#define SDP_MID_CTRL(mid) ((0xF0 & mid) ? 0 : 1)\r
+\r
+\r
+/*\r
+ * Base Sockets Direct Header (header for all SDP messages)\r
+ */\r
+struct msg_hdr_bsdh {\r
+ uint8_t mid; /* message identifier opcode (SDP_MID_*) */\r
+ uint8_t flags; /* flags as defined by SDP_MSG_FLAG_* */\r
+ uint16_t recv_bufs; /* current number of posted private recv buffers */\r
+ uint32_t size; /* length of msg, including header(s) and data */\r
+ uint32_t seq_num; /* message sequence number */\r
+ uint32_t seq_ack; /* last received message sequence number */\r
+};// __attribute__ ((packed)); /* struct msg_hdr_bsdh */\r
+\r
+/*\r
+ * Hello Header constants (two 8-bit constants, no conversion needed)\r
+ */\r
+#ifdef _SDP_MS_APRIL_ERROR_COMPAT\r
+this bug was probably solved \r
+#define SDP_MSG_IPVER 0x04 /* (1: ipversion), (0: reserved) */\r
+#else\r
+#define SDP_MSG_IPVER 0x40 /* (1: ipversion), (0: reserved) */\r
+#endif\r
+#define SDP_MSG_VERSION 0x11 /* (1: major ), (0: minor ) */\r
+\r
+\r
+/*\r
+ * Hello Header (BSDH + HH are contained in private data of the CM REQ MAD\r
+ */\r
+struct msg_hdr_hh {\r
+ uint8_t version; /* 0-3: minor version (current spec; 0x1)\r
+ 4-7: major version (current spec; 0x1) */\r
+ uint8_t ip_ver; /* 0-3: reserved\r
+ 4-7: ip version (0x4 = ipv4, 0x6 = ipv6) */\r
+ uint8_t rsvd_1; /* reserved */\r
+ uint8_t max_adv; /* max outstanding Zcopy advertisments (>0) */\r
+ uint32_t r_rcv_size; /* requested size of each remote recv buffer */\r
+ uint32_t l_rcv_size; /* initial size of each local receive buffer */\r
+ uint16_t port; /* local port */\r
+ uint16_t rsvd_2; /* reserved */\r
+\r
+ union { /* source IP address. */\r
+ struct {\r
+ uint32_t addr3; /* ipv6 96-127 */\r
+ uint32_t addr2; /* ipv6 64-95 */\r
+ uint32_t addr1; /* ipv6 32-63 */\r
+ uint32_t addr0; /* ipv6 0-31 */\r
+ } ipv6; /* 128bit IPv6 address */\r
+ struct {\r
+ uint32_t none2; /* unused 96-127 */\r
+ uint32_t none1; /* unused 64-95 */\r
+ uint32_t none0; /* unused 32-63 */\r
+ uint32_t addr; /* ipv4 0-31 */\r
+ } ipv4; /* 32bit IPv4 address */\r
+ } src;\r
+\r
+ union { /* destination IP address. */\r
+ struct {\r
+ uint32_t addr3; /* ipv6 96-127 */\r
+ uint32_t addr2; /* ipv6 64-95 */\r
+ uint32_t addr1; /* ipv6 32-63 */\r
+ uint32_t addr0; /* ipv6 0-31 */\r
+ } ipv6; /* 128bit IPv6 address */\r
+ struct {\r
+ uint32_t none2; /* unused 96-127 */\r
+ uint32_t none1; /* unused 64-95 */\r
+ uint32_t none0; /* unused 32-63 */\r
+ uint32_t addr; /* ipv4 0-31 */\r
+ } ipv4; /* 32bit IPv4 address */\r
+ } dst;\r
+\r
+ uint8_t rsvd_3[28]; /* reserved for future use, and zero'd */\r
+}; //__attribute__ ((packed)); /* struct msg_hdr_hh */\r
+\r
+struct msg_hdr_hah {\r
+ uint8_t version; /* 0-3: minor version (current spec; 0x1)\r
+ 4-7: major version (current spec; 0x1) */\r
+ uint16_t rsvd_1; /* reserved */\r
+ uint8_t max_adv; /* max outstanding Zcopy advertisments (>0) */\r
+ uint32_t l_rcv_size; /* initial size of each local receive buffer */\r
+#if 0 /* There is a bug in the 1.1 spec. REP message grew by 8 bytes. */\r
+ __u8 rsvd_2[180]; /* reserved for future use, and zero'd (big) */\r
+#else\r
+ uint8_t rsvd_2[172]; /* reserved for future use, and zero'd (big) */\r
+#endif\r
+}; //__attribute__ ((packed)); /* struct msg_hdr_hah */\r
+\r
+struct sdp_msg_hello {\r
+ struct msg_hdr_bsdh bsdh; /* base sockets direct header */\r
+ struct msg_hdr_hh hh; /* hello message header */\r
+} ;//__attribute__ ((packed)); /* struct sdp_msg_hello */\r
+\r
+struct sdp_msg_hello_ack {\r
+ struct msg_hdr_bsdh bsdh; /* base sockets direct header */\r
+ struct msg_hdr_hah hah; /* hello ack message header */\r
+};// __attribute__ ((packed)); /* struct sdp_msg_hello_ack */\r
+\r
+static inline void sdp_msg_swap_bsdh(struct msg_hdr_bsdh *header)\r
+{\r
+ header->recv_bufs = CL_NTOH16(header->recv_bufs);\r
+ header->size = CL_NTOH32(header->size);\r
+ header->seq_num = CL_NTOH32(header->seq_num);\r
+ header->seq_ack = CL_NTOH32(header->seq_ack);\r
+}\r
+\r
+static inline void sdp_msg_swap_hh(struct msg_hdr_hh *header)\r
+{\r
+ header->r_rcv_size = CL_NTOH32(header->r_rcv_size);\r
+ header->l_rcv_size = CL_NTOH32(header->l_rcv_size);\r
+ header->port = CL_NTOH16(header->port);\r
+ header->src.ipv6.addr0 = CL_NTOH32(header->src.ipv6.addr0);\r
+ header->src.ipv6.addr1 = CL_NTOH32(header->src.ipv6.addr1);\r
+ header->src.ipv6.addr2 = CL_NTOH32(header->src.ipv6.addr2);\r
+ header->src.ipv6.addr3 = CL_NTOH32(header->src.ipv6.addr3);\r
+ header->dst.ipv6.addr0 = CL_NTOH32(header->dst.ipv6.addr0);\r
+ header->dst.ipv6.addr1 = CL_NTOH32(header->dst.ipv6.addr1);\r
+ header->dst.ipv6.addr2 = CL_NTOH32(header->dst.ipv6.addr2);\r
+ header->dst.ipv6.addr3 = CL_NTOH32(header->dst.ipv6.addr3);\r
+}\r
+\r
+/*\r
+ * sdp_msg_swap_hah - SDP header endian byte swapping\r
+ */\r
+static inline void sdp_msg_net_to_cpu_hah(struct msg_hdr_hah *header)\r
+{\r
+ header->l_rcv_size = CL_NTOH32(header->l_rcv_size);\r
+}\r
+\r
+#define BASE_LISTEN_ID (CL_CONST64(0x0000000000010000))\r
+\r
+static inline ib_net64_t\r
+get_service_id_for_port(\r
+ ib_net16_t ip_port)\r
+{\r
+ return CL_NTOH64( BASE_LISTEN_ID | ip_port );\r
+}\r
+\r
+\r
+// CM timeouts - Used for creating the sockets\r
+#define CM_MIN_LOCAL_TIMEOUT (18)\r
+#define CM_LOCAL_TIMEOUT (1)\r
+#define CM_MIN_REMOTE_TIMEOUT (18)\r
+#define CM_REMOTE_TIMEOUT (2)\r
+#define CM_RETRIES 4\r
+\r
+\r
+#pragma pack(pop)\r
+\r
+#endif H_SDP_MSGS_H\r
+\r
\r
\r
#include "Precompile.h"\r
+#pragma warning(disable: 4244 ) \r
+\r
+NTSTATUS sdp_cm_hello_ack_check(struct sdp_msg_hello_ack *hello_ack);\r
+\r
+static void AL_API\r
+cm_rej_callback(IN ib_cm_rej_rec_t *p_cm_rej_rec )\r
+{\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_rej_callback called"));\r
+}\r
+\r
+static void AL_API\r
+cm_rep_callback(\r
+ IN ib_cm_rep_rec_t *p_cm_rep_rec )\r
+{\r
+ \r
+\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_rep_callback called\n"));\r
+ SdpSocket *pSocket = (SdpSocket *) p_cm_rep_rec->qp_context;\r
+ pSocket->CmRepCallback(p_cm_rep_rec);\r
+}\r
+\r
+static void AL_API\r
+cm_req_callback(\r
+ IN ib_cm_req_rec_t *p_cm_req_rec )\r
+{\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_req_callback called"));\r
+ ASSERT(FALSE);\r
+}\r
+\r
+static void AL_API\r
+cm_mra_callback(\r
+ IN ib_cm_mra_rec_t *p_cm_mra_rec )\r
+{\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_mra_callback called"));\r
+ ASSERT(FALSE);\r
+}\r
+\r
+/*\r
+ * A user-specified callback that is invoked after receiving a load\r
+ * alternate path response message.\r
+ */\r
+static void AL_API\r
+cm_apr_callback(\r
+ IN ib_cm_apr_rec_t *p_cm_apr_rec )\r
+{\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_apr_callback called"));\r
+ ASSERT(FALSE);\r
+}\r
+\r
+static void AL_API\r
+cm_dreq_callback(IN ib_cm_dreq_rec_t *p_cm_dreq_rec )\r
+{\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_dreq_callback called"));\r
+ ASSERT(FALSE);\r
+}\r
+SdpSocket::SdpSocket()\r
+{\r
+ m_CreationFlags = 0;\r
+ m_SrcPort = 0;\r
+ m_SrcIp = 0;\r
+\r
+ mh_Ca = NULL;\r
+ m_pd = NULL;\r
+ m_rcq = NULL;\r
+ m_scq = NULL;\r
+ m_qp = NULL;\r
+\r
+ m_shutdown = false;\r
+\r
+ m_state = SS_IDLE;\r
+}\r
\r
NTSTATUS SdpSocket::Init(\r
WspSocketIn *pSocketInParam, \r
WspSocketOut *pSocketOutParam)\r
{\r
- SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Init this = 0x%x\n", this));\r
+ NTSTATUS rc = STATUS_SUCCESS;\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Init this = 0x%p\n", this));\r
\r
m_CreationFlags = pSocketInParam->dwFlags;\r
pSocketOutParam->Errno = 0;// No error\r
pSocketOutParam->pSocket = this; // give the user a handle to the socket\r
- \r
- return STATUS_SUCCESS;\r
+ KeInitializeSpinLock(&m_Lock);\r
+\r
+ return rc;\r
}\r
\r
NTSTATUS SdpSocket::WSPConnect(\r
)\r
{\r
NTSTATUS rc = STATUS_SUCCESS;\r
- SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::WSPConnect this = 0x%x remote addresses ip=%d.%d.%d.%d:%d",\r
+ ib_api_status_t ib_status;\r
+ ib_net64_t SrcPortGuid;\r
+ ib_net64_t DestPortGuid;\r
+ ib_path_rec_t path_rec;\r
+ CSpinLockWrapper Lock(m_Lock);\r
+ \r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::WSPConnect this = 0x%p remote addresses ip=%d.%d.%d.%d:%d\n",\r
this,\r
- (pWspConnectIn->IP & 0XFF),\r
- (pWspConnectIn->IP & 0XFF00) >> 8,\r
- (pWspConnectIn->IP & 0XFF0000) >> 16,\r
(pWspConnectIn->IP & 0XFF000000) >> 24,\r
- nthos(pWspConnectIn->Port)\r
+ (pWspConnectIn->IP & 0XFF0000) >> 16,\r
+ (pWspConnectIn->IP & 0XFF00) >> 8,\r
+ (pWspConnectIn->IP & 0XFF),\r
+ pWspConnectIn->Port\r
));\r
\r
if((pWspConnectIn->IP == 0) || (pWspConnectIn->Port == 0)) {\r
goto Cleanup;\r
}\r
\r
+ // check socket state\r
+ // BUGBUG: Do a better work here\r
+ Lock.Lock();\r
+ if (m_state != SS_IDLE) {\r
+ // We can not connect in this state \r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Invalid Socket state %s\n", SS2String(m_state)));\r
+ pWspConnectOut->Errno = WSAEINVAL;\r
+ goto Cleanup;\r
+ Lock.Unlock();\r
+ }\r
+\r
+ //\r
+ // Arp: this means that based on the ARP protocol we should convert the \r
+ // IP address into gid.\r
+ //\r
+ if (m_SrcIp == 0) {\r
+ // This means that we need to do an implicit bind to get the source\r
+ // address\r
+ rc = g_pSdpDriver->m_pSdpArp->SourceAddrFromDestAddr(pWspConnectIn->IP , &m_SrcIp);\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->SourceAddrFromDestAddr failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH;\r
+ Lock.Unlock();\r
+ goto Cleanup;\r
+ } \r
+ }\r
+\r
+ // Now that we know the source IP we can decide about the src port\r
+ if (m_SrcPort == 0) {\r
+ rc = g_pSdpDriver->m_pSdpArp->GetPort(m_SrcIp, &m_SrcPort);\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->SourcePortGidFromIP failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ Lock.Unlock();\r
+ goto Cleanup;\r
+ } \r
+ }\r
+ rc = g_pSdpDriver->m_pSdpArp->SourcePortGidFromIP(m_SrcIp, &SrcPortGuid, &m_CaGuid);\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->SourcePortGidFromIP failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ Lock.Unlock();\r
+ goto Cleanup;\r
+ } \r
+\r
+ rc = g_pSdpDriver->m_pSdpArp->DestPortGidFromIP(pWspConnectIn->IP, &DestPortGuid);\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->DestPortGidFromIP failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ Lock.Unlock();\r
+ goto Cleanup;\r
+ } \r
+\r
+ //\r
+ // Next step is convert the gid to a path record\r
+ //\r
+\r
+ // Since this is a function that might wait we do it without the lock\r
+ m_state = SS_CONNECTING_QPR_SENT;\r
+ Lock.Unlock();\r
+\r
+ rc = g_pSdpDriver->m_pSdpArp->QueryPathRecord( SrcPortGuid, DestPortGuid, &path_rec );\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->QueryPathRecord failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ goto Cleanup;\r
+ }\r
+\r
+\r
+ // Verify that we are in the correct state (just looking - without the lock)\r
+ if (m_state != SS_CONNECTING_QPR_SENT) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("wrong state after QueryPathRecord\n" ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ goto Cleanup;\r
+ }\r
+\r
+ rc = CreateQp();\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("CreateQp failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ goto Cleanup;\r
+ } \r
+\r
+// Lock.Lock(); // Do we really need the lock ?\r
+\r
+\r
+\r
+ // We need to prepare the hello mesage for the CM\r
+ sdp_msg_hello hello_msg;\r
+ CreateHelloHeader(&hello_msg, pWspConnectIn->IP);\r
+\r
+ // Create the CM request\r
+ ib_cm_req_t cm_req;\r
+ CreateCmRequest(&cm_req, &hello_msg, &path_rec, pWspConnectIn->Port);\r
+\r
+ // Create the event to wait on to the connection request to end:\r
+ KeInitializeEvent(&m_ConnectCmCompleteEvent, NotificationEvent, FALSE ); \r
+\r
+ m_state = SS_CONNECTING_REQ_SENT;\r
+ \r
+ ib_status = ib_cm_req( &cm_req );\r
+ if( ib_status != IB_SUCCESS ) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_open_ca failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ goto Cleanup;\r
+ }\r
+\r
+ rc = MyKeWaitForSingleObject(\r
+ &m_ConnectCmCompleteEvent,\r
+ UserRequest,\r
+ UserMode,\r
+ FALSE,\r
+ NULL);\r
+\r
+ if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) {\r
+ // BUGBUG: Think what to do here, we should be able to stop the\r
+ // connect, and quit (probably shutdown should be enough)\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted = 0x%x\n", rc ));\r
+ rc = STATUS_UNEXPECTED_IO_ERROR;\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ Shutdown();\r
+ goto Cleanup;\r
+ }\r
+\r
+ //\r
+ // We have recieved the REP, we send the RTU code\r
+ // \r
+ \r
+ m_state = SS_CONNECTING_REQ_SENT;\r
+\r
+ rc = CmSendRTU();\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("CmSendRTU failed rc = 0x%x\n", rc ));\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ goto Cleanup;\r
+ } \r
+\r
+ // we should now complete the request\r
+ Lock.Lock(); \r
+ if (m_state == SS_CONNECTED) {\r
+ pWspConnectOut->Errno = 0;\r
+ ASSERT(rc == STATUS_SUCCESS);\r
+ Lock.Unlock();\r
+ goto Cleanup;\r
+ } else {\r
+ // There probably was some error or some kind of shutdown, we \r
+ // need to return an error.\r
+ rc = STATUS_UNEXPECTED_IO_ERROR;\r
+ pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error\r
+ Lock.Unlock();\r
+ goto Cleanup;\r
+ }\r
+\r
+Cleanup:\r
+ // Make sure that success/failure is marked in both places\r
+ ASSERT(((pWspConnectOut->Errno == 0) && (NT_SUCCESS(rc))) |\r
+ ((pWspConnectOut->Errno != 0) && (!NT_SUCCESS(rc))));\r
+\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("connect is returning %s this = 0x%p\n",\r
+ NT_SUCCESS(rc) ? "SUCCESS" : "FAILURE", this )); \r
+ return rc;\r
+\r
+}\r
+\r
+\r
+NTSTATUS SdpSocket::CmSendRTU()\r
+{\r
+ // This is only valid in connect.\r
+ // We might also be in shutdown, but nothing else currently\r
+ \r
+ ib_api_status_t ib_status;\r
+ NTSTATUS rc = STATUS_SUCCESS;\r
+\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::CmSendRTU this = 0x%p \n", this));\r
+\r
+ if (m_state != SS_CONNECTING_REQ_SENT) {\r
+ // There was some error, we can release the waiting thread.\r
+ // The error will be handled there\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("SdpSocket::CmSendRTU this = 0x%p invalid state %s\n", this, SS2String(m_state)));\r
+ rc = STATUS_UNEXPECTED_IO_ERROR;\r
+ goto Cleanup;\r
+ } \r
+ \r
+ /*\r
+ * check Hello Header Ack, to determine if we want\r
+ * the connection.\r
+ */\r
+\r
+ rc = sdp_cm_hello_ack_check(&m_hello_ack);\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("sdp_cm_hello_ack_check failed rc = 0x%x\n", rc ));\r
+ goto Cleanup;\r
+ } \r
+\r
+#if 0\r
+ /*\r
+ * read remote information\r
+ */\r
+ conn->send_size = hello_ack->hah.l_rcv_size;\r
+ conn->r_max_adv = hello_ack->hah.max_adv;\r
+ conn->r_recv_bf = hello_ack->bsdh.recv_bufs;\r
+ conn->recv_seq = hello_ack->bsdh.seq_num;\r
+ conn->advt_seq = hello_ack->bsdh.seq_num;\r
+\r
+ conn->d_qpn = event->param.rep_rcvd.remote_qpn;\r
+ /*\r
+ * The maximum amount of data that can be sent to the remote\r
+ * peer is the smaller of the local and remote buffer sizes, \r
+ * minus the size of the message header.\r
+ */\r
+ conn->send_size = min((u16)sdp_buff_pool_buff_size(),\r
+ (u16)conn->send_size) - SDP_MSG_HDR_SIZE;\r
+\r
+#endif\r
+ /*\r
+ * Pop the hello message that was sent ????? I don't think that we need this\r
+ */\r
+\r
+ ib_cm_rtu_t cm_rtu;\r
+\r
+ memset( &cm_rtu, 0, sizeof(cm_rtu) );\r
+\r
+ cm_rtu.access_ctrl = IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE;\r
+\r
+#if 0 // Do we want to keep it\r
+ // Bug in TAVOR\r
+ cm_rtu.sq_depth = QP_ATTRIB_SQ_DEPTH;\r
+ cm_rtu.rq_depth = QP_ATTRIB_RQ_DEPTH;\r
+#endif\r
+\r
+ cm_rtu.pfn_cm_apr_cb = cm_apr_callback;\r
+ cm_rtu.pfn_cm_dreq_cb = cm_dreq_callback;\r
+ \r
+ ib_status = ib_cm_rtu( m_cm_handle_t, &cm_rtu );\r
+ if( ib_status != IB_SUCCESS ) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_cm_rtu failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ goto Cleanup;\r
+ }\r
+ // We are now connected\r
+ // How should this be locked ??\r
+ m_state = SS_CONNECTED;\r
+\r
Cleanup: \r
return rc;\r
+}\r
+\r
+\r
+VOID SdpSocket::CmRepCallback(IN ib_cm_rep_rec_t *p_cm_rep_rec)\r
+{\r
+ if (m_state != SS_CONNECTING_REQ_SENT) {\r
+ // This is not the state that we waited for, not much that we can\r
+ // do. (This might happen in shutdown)\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("SdpSocket::CmRepCallback Not the expacted state %s\n", SS2String(m_state)));\r
+ ASSERT(FALSE);\r
+ return;\r
+ }\r
+\r
+ // We need to store the params and signal the event\r
+ sdp_msg_hello_ack *hello_ack = (sdp_msg_hello_ack *)p_cm_rep_rec->p_rep_pdata;\r
+ m_hello_ack = *hello_ack;\r
+ m_cm_handle_t = p_cm_rep_rec->h_cm_rep;\r
\r
+ KeSetEvent(&m_ConnectCmCompleteEvent, IO_NO_INCREMENT, FALSE);\r
}\r
\r
\r
+// BUGBUG: based on __recv_cb - need to implment\r
+static void\r
+__recv_cb1(\r
+ IN const ib_cq_handle_t h_cq,\r
+ IN void *cq_context )\r
+{\r
+// status = ib_rearm_cq(\r
+// p_port->ib_mgr.h_recv_cq, FALSE );\r
+// ASSERT(exp)( status == IB_SUCCESS );\r
+ ASSERT(FALSE);\r
+}\r
+\r
+\r
+\r
+// BUGBUG: This code is based on __cq_event, find out what it realy does\r
+static void\r
+__cq_event1(\r
+ IN ib_async_event_rec_t *p_event_rec )\r
+{\r
+ ASSERT(FALSE);\r
+ UNUSED_PARAM( p_event_rec );\r
+ ASSERT( p_event_rec->context );\r
+ /* Place holder for proper error handler. */\r
+ ASSERT( p_event_rec->code == IB_AE_UNKNOWN );\r
+}\r
+\r
+\r
+// Based on __qp_event - do we need it?\r
+static void\r
+__qp_event1(\r
+ IN ib_async_event_rec_t *p_event_rec )\r
+{\r
+ UNUSED_PARAM( p_event_rec );\r
+ ASSERT( p_event_rec->context );\r
+ /* Place holder for proper error handler. */\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("Async QP event: %d\n", p_event_rec->code));\r
+ ASSERT( p_event_rec->code == IB_AE_UNKNOWN );\r
+}\r
+\r
+\r
+// This currently works only for the connect\r
+NTSTATUS SdpSocket::CreateQp()\r
+{\r
+ NTSTATUS rc = STATUS_SUCCESS;\r
+ ib_cq_create_t cq_create;\r
+ ib_qp_create_t qp_create;\r
+ ib_api_status_t ib_status;\r
+\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("CreateQp called this = 0x%p\n", this));\r
+ /* Open the CA. */\r
+ ib_status = ib_open_ca(\r
+ g_pSdpDriver->m_al_handle, \r
+ m_CaGuid,\r
+ NULL, \r
+ this, \r
+ &mh_Ca \r
+ );\r
+ if( ib_status != IB_SUCCESS ) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_open_ca failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ goto Cleanup;\r
+ }\r
+\r
+ /* Allocate the PD. */\r
+ ib_status = ib_alloc_pd(\r
+ mh_Ca, \r
+ IB_PDT_NORMAL, \r
+ this, \r
+ &m_pd \r
+ );\r
+ \r
+ if( ib_status != IB_SUCCESS )\r
+ {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_alloc_pd failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ goto Cleanup;\r
+ }\r
+\r
+ /* Allocate receive CQ. */\r
+ cq_create.size = QP_ATTRIB_RQ_DEPTH;\r
+ cq_create.pfn_comp_cb = __recv_cb1;\r
+ cq_create.h_wait_obj = NULL;\r
+\r
+ ib_status = ib_create_cq(\r
+ mh_Ca, \r
+ &cq_create, \r
+ this,\r
+ __cq_event1, \r
+ &m_rcq \r
+ );\r
+\r
+ if( ib_status != IB_SUCCESS ) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_create_cq failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ goto Cleanup;\r
+ }\r
+\r
+ /* Allocate send CQ. */\r
+ cq_create.size = QP_ATTRIB_SQ_DEPTH;\r
+ cq_create.pfn_comp_cb = __recv_cb1; // ???? We are not doing anything there ??? why bother\r
+\r
+ ib_status = ib_create_cq(\r
+ mh_Ca, \r
+ &cq_create, \r
+ this,\r
+ __cq_event1, \r
+ &m_scq \r
+ );\r
+\r
+ if( ib_status != IB_SUCCESS ) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_create_cq failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ goto Cleanup;\r
+ }\r
+\r
+ /* Allocate the QP. */\r
+ cl_memclr( &qp_create, sizeof(qp_create) );\r
+ qp_create.qp_type = IB_QPT_RELIABLE_CONN;\r
+ qp_create.rq_depth = QP_ATTRIB_RQ_DEPTH;\r
+ qp_create.rq_sge = QP_ATTRIB_RQ_SGE; /* To support buffers spanning pages. */\r
+ qp_create.h_rq_cq = m_rcq;\r
+ qp_create.sq_depth = QP_ATTRIB_SQ_DEPTH;\r
+ //TODO: Figure out the right number of SGE entries for sends.\r
+ qp_create.sq_sge = QP_ATTRIB_SQ_SGE;\r
+ qp_create.h_sq_cq = m_scq;\r
+ qp_create.sq_signaled = TRUE;\r
+ ib_status = ib_create_qp( m_pd, &qp_create, this, __qp_event1, &m_qp );\r
+ if( ib_status != IB_SUCCESS ) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_create_qp failed ib_status = 0x%d\n", ib_status ));\r
+ rc = IB2Status(ib_status);\r
+ goto Cleanup;\r
+ }\r
+#if 0 \r
+ /* Query the QP so we can get our QPN. */\r
+ status = p_port->p_adapter->p_ifc->query_qp(\r
+ p_port->ib_mgr.h_qp, &qp_attr );\r
+ if( status != IB_SUCCESS )\r
+ {\r
+ IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR,\r
+ ("ib_query_qp returned %s\n", \r
+ p_port->p_adapter->p_ifc->get_err_str( status )) );\r
+ return status;\r
+ }\r
+ p_port->ib_mgr.qpn = qp_attr.num;\r
+#endif \r
+ \r
+Cleanup:\r
+ return rc;\r
+\r
+}\r
+\r
+VOID SdpSocket::CreateHelloHeader(\r
+ sdp_msg_hello *hello_msg, \r
+ ULONG DestIp\r
+ \r
+ )\r
+{\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::CreateHelloHeader called this = 0x%p\n", this));\r
+ ASSERT(DestIp != 0);\r
+ ASSERT(m_SrcPort != 0);\r
+ ASSERT(m_SrcIp != 0);\r
+\r
+ memset(hello_msg, 0, sizeof(struct sdp_msg_hello));\r
+ hello_msg->bsdh.recv_bufs = QP_ATTRIB_RQ_DEPTH; //????conn->l_advt_bf;\r
+ hello_msg->bsdh.flags = SDP_MSG_FLAG_NON_FLAG;\r
+ hello_msg->bsdh.mid = SDP_MID_HELLO;\r
+ hello_msg->bsdh.size = sizeof(struct sdp_msg_hello);\r
+ hello_msg->bsdh.seq_num = 0;//conn->send_seq; ???\r
+ hello_msg->bsdh.seq_ack = 0;//conn->advt_seq; ???\r
+\r
+ hello_msg->hh.max_adv = QP_ATTRIB_RQ_DEPTH;// ??? conn->l_max_adv;\r
+ hello_msg->hh.ip_ver = SDP_MSG_IPVER;\r
+ hello_msg->hh.version = SDP_MSG_VERSION;\r
+ hello_msg->hh.r_rcv_size = 4096;//???conn->recv_size;\r
+ hello_msg->hh.l_rcv_size = 4096;//???conn->recv_size;\r
+ hello_msg->hh.port = m_SrcPort;\r
+ hello_msg->hh.src.ipv4.addr = m_SrcIp;\r
+ hello_msg->hh.dst.ipv4.addr = DestIp;\r
+\r
+ /*\r
+ * endian swap\r
+ */\r
+ sdp_msg_swap_bsdh(&hello_msg->bsdh);\r
+ sdp_msg_swap_hh(&hello_msg->hh);\r
+\r
+\r
+}\r
+\r
+VOID SdpSocket::CreateCmRequest(\r
+ ib_cm_req_t *cm_req,\r
+ sdp_msg_hello *hello_msg,\r
+ ib_path_rec_t *path_rec,\r
+ USHORT DestPort\r
+ )\r
+{\r
+\r
+ memset( cm_req, 0, sizeof(ib_cm_req_t) );\r
+ cm_req->qp_type = IB_QPT_RELIABLE_CONN;\r
+ cm_req->h_qp = m_qp;\r
+ cm_req->p_primary_path = path_rec;\r
+ cm_req->p_alt_path = NULL;\r
+ cm_req->svc_id = get_service_id_for_port(DestPort );\r
+ cm_req->p_req_pdata = (uint8_t *) hello_msg;\r
+ cm_req->req_length = sizeof(sdp_msg_hello);\r
+\r
+ // Caution: From here we live the linux code, as it was in: gen2 - sdp_cm_path_complete\r
+ cm_req->max_cm_retries = CM_RETRIES;\r
+ cm_req->resp_res = 4;//???????? what are this??? QP_ATTRIB_RESPONDER_RESOURCES;\r
+ cm_req->init_depth = 4;//?????QP_ATTRIB_INITIATOR_DEPTH;\r
+\r
+ cm_req->remote_resp_timeout =\r
+ ib_path_rec_pkt_life( path_rec ) + CM_REMOTE_TIMEOUT;\r
+ if( cm_req->remote_resp_timeout > 0x1F )\r
+ cm_req->remote_resp_timeout = 0x1F;\r
+ else if( cm_req->remote_resp_timeout < CM_MIN_REMOTE_TIMEOUT )\r
+ cm_req->remote_resp_timeout = CM_MIN_REMOTE_TIMEOUT;\r
+\r
+ cm_req->flow_ctrl = TRUE; /* HCAs must support end-to-end flow control. */\r
+\r
+ cm_req->local_resp_timeout =\r
+ ib_path_rec_pkt_life( path_rec ) + CM_LOCAL_TIMEOUT;\r
+ if( cm_req->local_resp_timeout > 0x1F )\r
+ cm_req->local_resp_timeout = 0x1F;\r
+ else if( cm_req->local_resp_timeout < CM_MIN_LOCAL_TIMEOUT )\r
+ cm_req->local_resp_timeout = CM_MIN_LOCAL_TIMEOUT;\r
+\r
+ cm_req->rnr_nak_timeout = 6;//???QP_ATTRIB_RNR_NAK_TIMEOUT;\r
+ cm_req->rnr_retry_cnt = 6;//????QP_ATTRIB_RNR_RETRY;\r
+ cm_req->retry_cnt = 6;//????QP_ATTRIB_RETRY_COUNT;\r
+\r
+ cm_req->pfn_cm_req_cb = cm_req_callback;\r
+ cm_req->pfn_cm_mra_cb = cm_mra_callback;\r
+ cm_req->pfn_cm_rej_cb = cm_rej_callback;\r
+ cm_req->pfn_cm_rep_cb = cm_rep_callback;\r
+}\r
\r
VOID SdpSocket::Shutdown()\r
{\r
- SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%x\n", this));\r
+ //???? locking\r
+ // if(m_shutdown) ???\r
+ m_shutdown = true;\r
+\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%p\n", this));\r
+\r
+ if (m_qp != NULL) {\r
+ ib_destroy_qp(m_qp, NULL); //?????? CALL BACK ??? IMPLMENT\r
+ }\r
+\r
+ if (m_scq != NULL) {\r
+ ib_destroy_cq(m_scq, NULL); //?????? CALL BACK ??? IMPLMENT\r
+ }\r
+\r
+ if (m_rcq != NULL) {\r
+ ib_destroy_cq(m_rcq, NULL); //?????? CALL BACK ??? IMPLMENT\r
+ }\r
+\r
+ if (m_pd != NULL) {\r
+ ib_dealloc_pd(m_pd, NULL); //?????? CALL BACK ??? IMPLMENT\r
+ }\r
+\r
+ if (mh_Ca != NULL) {\r
+ ib_close_ca(mh_Ca, NULL); //?????? CALL BACK ??? IMPLMENT\r
+ }\r
+}\r
+\r
+/*\r
+ * sdp_cm_hello_ack_check - validate the hello ack header\r
+ */\r
+NTSTATUS sdp_cm_hello_ack_check(struct sdp_msg_hello_ack *hello_ack)\r
+{\r
+ /*\r
+ * endian swap\r
+ */\r
+ sdp_msg_swap_bsdh(&hello_ack->bsdh);\r
+ sdp_msg_net_to_cpu_hah(&hello_ack->hah);\r
+ /*\r
+ * validation and consistency checks\r
+ */\r
+ if (hello_ack->bsdh.size != sizeof(struct sdp_msg_hello_ack)) {\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, size mismatch. (2) <%d:%d>",\r
+ hello_ack->bsdh.size,\r
+ sizeof(struct sdp_msg_hello_ack)));\r
+ return STATUS_UNEXPECTED_IO_ERROR;\r
+ }\r
+\r
+ if (SDP_MID_HELLO_ACK != hello_ack->bsdh.mid) {\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, unexpected message. <%d>",\r
+ hello_ack->bsdh.mid\r
+ ));\r
+ return STATUS_UNEXPECTED_IO_ERROR; \r
+ }\r
\r
+ if (hello_ack->hah.max_adv <= 0) {\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, bad zcopy advertisment. <%d>",\r
+ hello_ack->hah.max_adv\r
+ ));\r
+ return STATUS_UNEXPECTED_IO_ERROR; \r
+ }\r
\r
+ if ((0xF0 & hello_ack->hah.version) != (0xF0 & SDP_MSG_VERSION)) {\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, version mismatch. <%d:%d>",\r
+ ((0xF0 & hello_ack->hah.version) >> 4),\r
+ ((0xF0 & SDP_MSG_VERSION) >> 4)));\r
+ return STATUS_UNEXPECTED_IO_ERROR; \r
+ }\r
+\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("Hello Ack BSDH <%04x:%02x:%02x:%08x:%08x:%08x>",\r
+ hello_ack->bsdh.recv_bufs,\r
+ hello_ack->bsdh.flags,\r
+ hello_ack->bsdh.mid,\r
+ hello_ack->bsdh.size,\r
+ hello_ack->bsdh.seq_num,\r
+ hello_ack->bsdh.seq_ack));\r
+ SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("Hello Ack HAH <%02x:%02x:%08x",\r
+ hello_ack->hah.max_adv,\r
+ hello_ack->hah.version, \r
+ hello_ack->hah.l_rcv_size));\r
+\r
+ return STATUS_SUCCESS;\r
}\r
\r
+\r
+\r
projects / ~shefty / rdma-win.git / commitdiff