From 86af5ac2d0263515c820fb3da24c98d3e55a0fff Mon Sep 17 00:00:00 2001 From: tzachid Date: Mon, 21 Nov 2005 12:03:31 +0000 Subject: [PATCH] sdp dll connects to a Linux machine from kernel mode. (Locking mechanism still has bugs) (Rev 74) git-svn-id: svn://openib.tc.cornell.edu/gen1@161 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/sdp/kernel/Precompile.h | 4 + trunk/ulp/sdp/kernel/SOURCES | 9 +- trunk/ulp/sdp/kernel/SdpArp.cpp | 109 +++++ trunk/ulp/sdp/kernel/SdpArp.h | 114 +++++ trunk/ulp/sdp/kernel/SdpDriver.cpp | 45 +- trunk/ulp/sdp/kernel/SdpDriver.h | 18 +- trunk/ulp/sdp/kernel/SdpGenUtils.cpp | 61 ++- trunk/ulp/sdp/kernel/SdpGenUtils.h | 10 + trunk/ulp/sdp/kernel/SdpMsgs.h | 198 ++++++++ trunk/ulp/sdp/kernel/SdpSocket.cpp | 674 ++++++++++++++++++++++++++- trunk/ulp/sdp/kernel/SdpSocket.h | 89 +++- trunk/ulp/sdp/kernel/SdpTrace.h | 1 + 12 files changed, 1310 insertions(+), 22 deletions(-) create mode 100644 trunk/ulp/sdp/kernel/SdpArp.cpp create mode 100644 trunk/ulp/sdp/kernel/SdpArp.h create mode 100644 trunk/ulp/sdp/kernel/SdpMsgs.h diff --git a/trunk/ulp/sdp/kernel/Precompile.h b/trunk/ulp/sdp/kernel/Precompile.h index e936c55d..ca6a3bc5 100644 --- a/trunk/ulp/sdp/kernel/Precompile.h +++ b/trunk/ulp/sdp/kernel/Precompile.h @@ -11,8 +11,11 @@ extern "C" { // External classes declrations class SdpSocket; +class SdpArp; +#include "ib_al.h" +#include "sdpMsgs.h" #include "SdpTrace.h" #include "RefCount.h" #include "sdpdriver.h" @@ -20,6 +23,7 @@ class SdpSocket; #include "SdpUserFile.h" #include "SdpGenUtils.h" #include "SdpSocket.h" +#include "SdpArp.h" #endif // _SDP_PRECOMP_ diff --git a/trunk/ulp/sdp/kernel/SOURCES b/trunk/ulp/sdp/kernel/SOURCES index 667b1fb4..2767487f 100644 --- a/trunk/ulp/sdp/kernel/SOURCES +++ b/trunk/ulp/sdp/kernel/SOURCES @@ -5,15 +5,20 @@ TARGETTYPE=DRIVER SOURCES= SdpDriver.cpp \ SdpUserFile.cpp \ SdpGenUtils.cpp \ - SdpSocket.cpp + SdpSocket.cpp \ + SdpArp.cpp -INCLUDES=..\include;..\..\..\inc;..\..\..\inc\kernel; +INCLUDES=..\include;\ + ..\..\..\inc;\ + ..\..\..\inc\kernel;\ + ..\..\..\inc\iba C_DEFINES=$(C_DEFINES) -DNDIS_WDM=1 \ -DDEPRECATE_DDK_FUNCTIONS TARGETLIBS= \ $(TARGETPATH)\*\complib.lib \ + $(TARGETPATH)\*\ibal.lib \ !if !defined(DDK_TARGET_OS) || "$(DDK_TARGET_OS)"=="Win2K" # diff --git a/trunk/ulp/sdp/kernel/SdpArp.cpp b/trunk/ulp/sdp/kernel/SdpArp.cpp new file mode 100644 index 00000000..62b985d7 --- /dev/null +++ b/trunk/ulp/sdp/kernel/SdpArp.cpp @@ -0,0 +1,109 @@ +/* Copyright mellanox */ + +#include "precompile.h" + +#pragma warning(disable: 4244 ) + + +NTSTATUS +SdpArp::QueryPathRecord( + IN ib_net64_t SrcPortGuid, + IN ib_net64_t DestPortGuid, + OUT ib_path_rec_t *path_rec ) +{ + NTSTATUS rc = STATUS_SUCCESS; + ib_gid_pair_t user_query; + struct query_pr_context query_context; + ib_query_handle_t query_handle; + ib_query_req_t query_req; + ib_api_status_t ib_status; + + query_req.query_type = IB_QUERY_PATH_REC_BY_GIDS; + query_req.p_query_input = &user_query; + query_req.port_guid = SrcPortGuid; + query_req.timeout_ms = 500; + query_req.retry_cnt = 4; + query_req.flags = IB_FLAGS_SYNC; + query_req.query_context = &query_context; + query_req.pfn_query_cb = SdpArp::query_pr_callback; + + ib_gid_set_default( &user_query.src_gid, SrcPortGuid ); + ib_gid_set_default( &user_query.dest_gid, DestPortGuid ); + + query_context.path_rec = path_rec; + + SDP_PRINT( SDP_TRACE, SDP_ARP, + ("Query for path from %I64x to %I64x\n", + SrcPortGuid, DestPortGuid) ); + + ib_status = ib_query( g_pSdpDriver->m_al_handle, &query_req, &query_handle ); + + if ( ib_status != IB_SUCCESS) { + SDP_PRINT(SDP_ERR, SDP_ARP, ("ib_query failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } + + if( query_context.status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_ARP, ("ib_query failed ib_status = 0x%d\n", query_context.status )); + rc = IB2Status(query_context.status); + goto Cleanup; + } + +// BUGBUG: This code was in the orginanl function. We have to decide +// if we want to keep it. +#if 0 + if( (port->hca->dev_id == 0x5A44) && + (ib_path_rec_mtu( path_rec ) > IB_MTU_1024) ) + { + /* Local endpoint is Tavor - cap MTU to 1K for extra bandwidth. */ + path_rec->mtu &= IB_PATH_REC_SELECTOR_MASK; + path_rec->mtu |= IB_MTU_1024; + } +#endif + +Cleanup: + return rc; +} + +void AL_API +SdpArp::query_pr_callback( + IN ib_query_rec_t *p_query_rec) +{ + query_pr_context * __ptr64 query_context = + (query_pr_context * __ptr64)p_query_rec->query_context; + + + ib_api_status_t status; + + SDP_PRINT( SDP_TRACE, SDP_ARP,("query_pr_callback status is %d irql=%d\n", p_query_rec->status, KeGetCurrentIrql()) ); + + query_context->status = p_query_rec->status; + + if( p_query_rec->status == IB_SUCCESS ){ + ib_path_rec_t *path_rec; + + path_rec = ib_get_query_path_rec(p_query_rec->p_result_mad, 0 ); + + ASSERT( path_rec ); + + /* Copy the path record */ + *query_context->path_rec = *path_rec; + } + + if( p_query_rec->status == IB_SUCCESS || p_query_rec->status == IB_REMOTE_ERROR ) + { + status = ib_put_mad( p_query_rec->p_result_mad ); + + ASSERT(status == IB_SUCCESS); + if( status != IB_SUCCESS ) + { + SDP_PRINT( SDP_ERR, SDP_ARP, ("ib_put_mad failed (%d)\n", status) ); + } + } else { + // When do we free this mad ??? + ASSERT(p_query_rec->p_result_mad == NULL); + } + +} + diff --git a/trunk/ulp/sdp/kernel/SdpArp.h b/trunk/ulp/sdp/kernel/SdpArp.h new file mode 100644 index 00000000..ada64324 --- /dev/null +++ b/trunk/ulp/sdp/kernel/SdpArp.h @@ -0,0 +1,114 @@ +/* Copyright mellanox */ + +// BUGBUG: complete this mechanisms + + +class SdpArp { + /* + Look at the routing table of the machine and see which port + to use for outbound traffic + */ + +public: + + struct query_pr_context + { + ib_api_status_t status; + ib_path_rec_t *path_rec; + }; + + NTSTATUS Init() { return STATUS_SUCCESS;} + + NTSTATUS SourceAddrFromDestAddr( + IN ULONG DestIp, + OUT ULONG *SrcIp + ) + { + NTSTATUS rc = STATUS_SUCCESS; + if (DestIp == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 124) { + *SrcIp = 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 120; + return STATUS_SUCCESS; + } + ASSERT(FALSE); + *SrcIp = 0; + return STATUS_UNEXPECTED_IO_ERROR; + + } + + NTSTATUS GetPort( + IN ULONG SourceAddr, + OUT USHORT *SrcPort + ) + { + ASSERT(SourceAddr != 0); + // If the port is 0, choose your own free port. + // If the port is not 0 check if this port is already in use + *SrcPort = 5050; + return STATUS_SUCCESS; + + } + + + NTSTATUS SourcePortGidFromIP( + IN ULONG SourceAddr, + OUT ib_net64_t *SrcPortGuid, + OUT ib_net64_t *SrcCaGuid + ) + { + if (SourceAddr == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 120) { + *SrcPortGuid = CL_NTOH64(0x2c90200002002);//????? swlab120 + *SrcCaGuid = CL_NTOH64(0x2c90200002000); + return STATUS_SUCCESS; + } + ASSERT(FALSE); + *SrcPortGuid = 0; + *SrcCaGuid = 0; + return STATUS_UNEXPECTED_IO_ERROR; + + } + + + NTSTATUS DestPortGidFromIP( + IN ULONG DestAddr, + OUT ib_net64_t *DestPortGuid) + { + if (DestAddr == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 124) { + *DestPortGuid = CL_NTOH64(0x0000c900012a3a41);//????? swlab124 + return STATUS_SUCCESS; + } + ASSERT(FALSE); + *DestPortGuid = 0; + return STATUS_UNEXPECTED_IO_ERROR; + + } +/* +Synchronously query the SA for a GUID. (started from wsd - query_pr) +*/ +/* + * Get a path record from a GUID + */ + + NTSTATUS + QueryPathRecord( + IN ib_net64_t SrcPortGuid, + IN ib_net64_t DestPortGuid, + OUT ib_path_rec_t *path_rec + ); + + static void AL_API + query_pr_callback( + IN ib_query_rec_t *p_query_rec); +}; + diff --git a/trunk/ulp/sdp/kernel/SdpDriver.cpp b/trunk/ulp/sdp/kernel/SdpDriver.cpp index 3af15f2b..7166fe3d 100644 --- a/trunk/ulp/sdp/kernel/SdpDriver.cpp +++ b/trunk/ulp/sdp/kernel/SdpDriver.cpp @@ -10,17 +10,20 @@ VOID DriverUnload ( ) { SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("DriverUnload called pDriverObject = 0x%x\n", pDriverObject )); + ib_api_status_t ib_status; - SdpDriver *pSdpDriver; - UNICODE_STRING LinkName; - pSdpDriver = (SdpDriver *) pDriverObject->DeviceObject->DeviceExtension; + ib_status = ib_close_al(g_pSdpDriver->m_al_handle); + g_pSdpDriver->m_al_handle = NULL; + ASSERT( ib_status == IB_SUCCESS); + UNICODE_STRING LinkName; RtlInitUnicodeString( &LinkName, SDP_LINK_NAME ); IoDeleteSymbolicLink(&LinkName); - IoDeleteDevice( pSdpDriver->GetDeviceObject()); + IoDeleteDevice( g_pSdpDriver->GetDeviceObject()); + g_pSdpDriver = NULL; } @@ -30,9 +33,11 @@ extern "C" NTSTATUS DriverEntry ( IN PUNICODE_STRING pRegistryPath ) { NTSTATUS rc; + ib_api_status_t ib_status; PDEVICE_OBJECT pDevObj; SdpDriver *pSdpDriver; bool DeviceCreated = false; + bool LinkCreated = false; UNICODE_STRING DevName, LinkName; int i; @@ -88,9 +93,23 @@ extern "C" NTSTATUS DriverEntry ( SDP_PRINT(SDP_ERR, SDP_DRIVER, ("IoCreateSymbolicLink failed rc = 0x%x\n", rc )); goto Cleanup; } + LinkCreated = true; +#pragma warning( push ) +#pragma warning(disable: 4244 ) + ib_status = ib_open_al(&pSdpDriver->m_al_handle); +#pragma warning( pop ) + if ( ib_status != IB_SUCCESS) { + SDP_PRINT(SDP_ERR, SDP_DRIVER, ("ib_open_al failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } Cleanup: if (!NT_SUCCESS(rc)) { + if (LinkCreated) { + IoDeleteSymbolicLink(&LinkName); + + } if ( DeviceCreated ) { IoDeleteDevice(pDevObj); } @@ -285,6 +304,24 @@ if ((InputBufferLength < sizeof (InStruct)) || goto Cleanup; \ } +NTSTATUS SdpDriver::Init(PDEVICE_OBJECT pDevObj) +{ + NTSTATUS rc = STATUS_SUCCESS; + m_pDevObj = pDevObj; + m_pSdpArp = new SdpArp; + if (m_pSdpArp == NULL) { + SDP_PRINT(SDP_ERR, SDP_DRIVER, ("new SdpArp failed \n")); + rc = STATUS_NO_MEMORY; + goto Cleanup; + } + rc = m_pSdpArp->Init(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_DRIVER, ("m_pSdpArp->Init failed rc = 0x%x\n", rc )); + goto Cleanup; + } +Cleanup: + return rc; +} NTSTATUS SdpDriver::DispatchDeviceIoControl( diff --git a/trunk/ulp/sdp/kernel/SdpDriver.h b/trunk/ulp/sdp/kernel/SdpDriver.h index d820c7ce..f509fabf 100644 --- a/trunk/ulp/sdp/kernel/SdpDriver.h +++ b/trunk/ulp/sdp/kernel/SdpDriver.h @@ -6,10 +6,14 @@ class SdpDriver { public: - NTSTATUS Init(PDEVICE_OBJECT pDevObj) { - m_pDevObj = pDevObj; - return STATUS_SUCCESS; - } + SdpDriver() { + m_al_handle = NULL; + m_pDevObj = NULL; + m_pSdpArp = NULL; + } + + NTSTATUS Init(PDEVICE_OBJECT pDevObj); + PDEVICE_OBJECT GetDeviceObject() { return m_pDevObj; } @@ -34,11 +38,15 @@ public: - +public: + ib_al_handle_t m_al_handle ; + SdpArp *m_pSdpArp; private: PDEVICE_OBJECT m_pDevObj; + + }; extern SdpDriver *g_pSdpDriver; diff --git a/trunk/ulp/sdp/kernel/SdpGenUtils.cpp b/trunk/ulp/sdp/kernel/SdpGenUtils.cpp index 67cc886f..b99b18bb 100644 --- a/trunk/ulp/sdp/kernel/SdpGenUtils.cpp +++ b/trunk/ulp/sdp/kernel/SdpGenUtils.cpp @@ -3,11 +3,70 @@ #include "Precompile.h" -USHORT nthos(USHORT in) +USHORT ntohs(USHORT in) { return ((in & 0xff) << 8) | ((in & 0xff00) >> 8); } +NTSTATUS IB2Status (ib_api_status_t ib_status) +{ + if ( ib_status == IB_SUCCESS ) { + return STATUS_SUCCESS; + } if (( ib_status > IB_SUCCESS ) && (ib_status < IB_UNKNOWN_ERROR)) { +// +// Values are 32 bit values layed out as follows: +// +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +---+-+-+-----------------------+-------------------------------+ +// |Sev|C|R| Facility | Code | +// +---+-+-+-----------------------+-------------------------------+ + + return 0xcc000000 | ib_status; + } + + + // Keep translating as the error pops + ASSERT(FALSE); + return STATUS_UNEXPECTED_IO_ERROR; + +} + + +// This function is a wrapper for the KeWaitForSingleObject that adds +// assertsions to the valuas returned by it +NTSTATUS + MyKeWaitForSingleObject( + IN PVOID Object, + IN KWAIT_REASON WaitReason, + IN KPROCESSOR_MODE WaitMode, + IN BOOLEAN Alertable, + IN PLARGE_INTEGER Timeout OPTIONAL + ) +{ + NTSTATUS rc = KeWaitForSingleObject( + Object, + WaitReason, + WaitMode, + Alertable, + Timeout + ); + if (!NT_SUCCESS(rc)) { + ASSERT(FALSE); + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("KeWaitForSingleObject failed rc = 0x%x\n", rc )); + // No meter what we do the program can't continue, let's crush it + int *i = NULL; + *i = 5; + } + ASSERT((rc == STATUS_SUCCESS ) || + (rc == STATUS_ALERTED ) || + (rc == STATUS_USER_APC ) || + (rc == STATUS_TIMEOUT )); // This are simply all the return code from DDK + + ASSERT( (Timeout != NULL ) || rc != STATUS_TIMEOUT); + return rc; +} + void* __cdecl operator new(size_t n ) throw() { diff --git a/trunk/ulp/sdp/kernel/SdpGenUtils.h b/trunk/ulp/sdp/kernel/SdpGenUtils.h index 8e9bd9e5..30deb83e 100644 --- a/trunk/ulp/sdp/kernel/SdpGenUtils.h +++ b/trunk/ulp/sdp/kernel/SdpGenUtils.h @@ -39,10 +39,20 @@ public: }; +NTSTATUS IB2Status (ib_api_status_t ib_status); USHORT nthos(USHORT in); +NTSTATUS + MyKeWaitForSingleObject( + IN PVOID Object, + IN KWAIT_REASON WaitReason, + IN KPROCESSOR_MODE WaitMode, + IN BOOLEAN Alertable, + IN PLARGE_INTEGER Timeout OPTIONAL + ); + // This error codes are taken from winsock2.h (the file can not) // be included from user mode diff --git a/trunk/ulp/sdp/kernel/SdpMsgs.h b/trunk/ulp/sdp/kernel/SdpMsgs.h new file mode 100644 index 00000000..1895d67e --- /dev/null +++ b/trunk/ulp/sdp/kernel/SdpMsgs.h @@ -0,0 +1,198 @@ +/* Copyright mellanox */ + +/* + This file holds the types that are used for SDP comunication + +*/ + +#ifndef H_SDP_MSGS_H +#define H_SDP_MSGS_H + + +#pragma pack(push) +#pragma pack(1) + + +/* + * Message Identifier Opcodes for BSDH + */ +/* Name Value Extended Header Payload */ +#define SDP_MID_HELLO 0x00 /* msg_hdr_hh */ +#define SDP_MID_HELLO_ACK 0x01 /* msg_hdr_hah */ +#define SDP_MID_DISCONNECT 0x02 /* */ +#define SDP_MID_ABORT_CONN 0x03 /* */ +#define SDP_MID_SEND_SM 0x04 /* */ +#define SDP_MID_RDMA_WR_COMP 0x05 /* msg_hdr_rwch */ +#define SDP_MID_RDMA_RD_COMP 0x06 /* msg_hdr_rrch */ +#define SDP_MID_MODE_CHANGE 0x07 /* msg_hdr_mch */ +#define SDP_MID_SRC_CANCEL 0x08 /* */ +#define SDP_MID_SNK_CANCEL 0x09 /* */ +#define SDP_MID_SNK_CANCEL_ACK 0x0A /* */ +#define SDP_MID_CH_RECV_BUF 0x0B /* msg_hdr_crbh */ +#define SDP_MID_CH_RECV_BUF_ACK 0x0C /* msg_hdr_crbah */ +#define SDP_MID_SUSPEND 0x0D /* msg_hdr_sch */ +#define SDP_MID_SUSPEND_ACK 0x0E /* */ +#define SDP_MID_SNK_AVAIL 0xFD /* msg_hdr_snkah */ +#define SDP_MID_SRC_AVAIL 0xFE /* msg_hdr_srcah */ +#define SDP_MID_DATA 0xFF /* */ +/* + * shift number for BSDH Flags. + */ +#define SDP_MSG_FLAG_NON_FLAG (0x0) /* no flag present */ +#define SDP_MSG_FLAG_OOB_PRES 0 /* out-of-band data present */ +#define SDP_MSG_FLAG_OOB_PEND 1 /* out-of-band data pending */ +#define SDP_MSG_FLAG_REQ_PIPE 2 /* request change to pipelined */ +/* + * message type masks + */ +#define SDP_MID_CTRL(mid) ((0xF0 & mid) ? 0 : 1) + + +/* + * Base Sockets Direct Header (header for all SDP messages) + */ +struct msg_hdr_bsdh { + uint8_t mid; /* message identifier opcode (SDP_MID_*) */ + uint8_t flags; /* flags as defined by SDP_MSG_FLAG_* */ + uint16_t recv_bufs; /* current number of posted private recv buffers */ + uint32_t size; /* length of msg, including header(s) and data */ + uint32_t seq_num; /* message sequence number */ + uint32_t seq_ack; /* last received message sequence number */ +};// __attribute__ ((packed)); /* struct msg_hdr_bsdh */ + +/* + * Hello Header constants (two 8-bit constants, no conversion needed) + */ +#ifdef _SDP_MS_APRIL_ERROR_COMPAT +this bug was probably solved +#define SDP_MSG_IPVER 0x04 /* (1: ipversion), (0: reserved) */ +#else +#define SDP_MSG_IPVER 0x40 /* (1: ipversion), (0: reserved) */ +#endif +#define SDP_MSG_VERSION 0x11 /* (1: major ), (0: minor ) */ + + +/* + * Hello Header (BSDH + HH are contained in private data of the CM REQ MAD + */ +struct msg_hdr_hh { + uint8_t version; /* 0-3: minor version (current spec; 0x1) + 4-7: major version (current spec; 0x1) */ + uint8_t ip_ver; /* 0-3: reserved + 4-7: ip version (0x4 = ipv4, 0x6 = ipv6) */ + uint8_t rsvd_1; /* reserved */ + uint8_t max_adv; /* max outstanding Zcopy advertisments (>0) */ + uint32_t r_rcv_size; /* requested size of each remote recv buffer */ + uint32_t l_rcv_size; /* initial size of each local receive buffer */ + uint16_t port; /* local port */ + uint16_t rsvd_2; /* reserved */ + + union { /* source IP address. */ + struct { + uint32_t addr3; /* ipv6 96-127 */ + uint32_t addr2; /* ipv6 64-95 */ + uint32_t addr1; /* ipv6 32-63 */ + uint32_t addr0; /* ipv6 0-31 */ + } ipv6; /* 128bit IPv6 address */ + struct { + uint32_t none2; /* unused 96-127 */ + uint32_t none1; /* unused 64-95 */ + uint32_t none0; /* unused 32-63 */ + uint32_t addr; /* ipv4 0-31 */ + } ipv4; /* 32bit IPv4 address */ + } src; + + union { /* destination IP address. */ + struct { + uint32_t addr3; /* ipv6 96-127 */ + uint32_t addr2; /* ipv6 64-95 */ + uint32_t addr1; /* ipv6 32-63 */ + uint32_t addr0; /* ipv6 0-31 */ + } ipv6; /* 128bit IPv6 address */ + struct { + uint32_t none2; /* unused 96-127 */ + uint32_t none1; /* unused 64-95 */ + uint32_t none0; /* unused 32-63 */ + uint32_t addr; /* ipv4 0-31 */ + } ipv4; /* 32bit IPv4 address */ + } dst; + + uint8_t rsvd_3[28]; /* reserved for future use, and zero'd */ +}; //__attribute__ ((packed)); /* struct msg_hdr_hh */ + +struct msg_hdr_hah { + uint8_t version; /* 0-3: minor version (current spec; 0x1) + 4-7: major version (current spec; 0x1) */ + uint16_t rsvd_1; /* reserved */ + uint8_t max_adv; /* max outstanding Zcopy advertisments (>0) */ + uint32_t l_rcv_size; /* initial size of each local receive buffer */ +#if 0 /* There is a bug in the 1.1 spec. REP message grew by 8 bytes. */ + __u8 rsvd_2[180]; /* reserved for future use, and zero'd (big) */ +#else + uint8_t rsvd_2[172]; /* reserved for future use, and zero'd (big) */ +#endif +}; //__attribute__ ((packed)); /* struct msg_hdr_hah */ + +struct sdp_msg_hello { + struct msg_hdr_bsdh bsdh; /* base sockets direct header */ + struct msg_hdr_hh hh; /* hello message header */ +} ;//__attribute__ ((packed)); /* struct sdp_msg_hello */ + +struct sdp_msg_hello_ack { + struct msg_hdr_bsdh bsdh; /* base sockets direct header */ + struct msg_hdr_hah hah; /* hello ack message header */ +};// __attribute__ ((packed)); /* struct sdp_msg_hello_ack */ + +static inline void sdp_msg_swap_bsdh(struct msg_hdr_bsdh *header) +{ + header->recv_bufs = CL_NTOH16(header->recv_bufs); + header->size = CL_NTOH32(header->size); + header->seq_num = CL_NTOH32(header->seq_num); + header->seq_ack = CL_NTOH32(header->seq_ack); +} + +static inline void sdp_msg_swap_hh(struct msg_hdr_hh *header) +{ + header->r_rcv_size = CL_NTOH32(header->r_rcv_size); + header->l_rcv_size = CL_NTOH32(header->l_rcv_size); + header->port = CL_NTOH16(header->port); + header->src.ipv6.addr0 = CL_NTOH32(header->src.ipv6.addr0); + header->src.ipv6.addr1 = CL_NTOH32(header->src.ipv6.addr1); + header->src.ipv6.addr2 = CL_NTOH32(header->src.ipv6.addr2); + header->src.ipv6.addr3 = CL_NTOH32(header->src.ipv6.addr3); + header->dst.ipv6.addr0 = CL_NTOH32(header->dst.ipv6.addr0); + header->dst.ipv6.addr1 = CL_NTOH32(header->dst.ipv6.addr1); + header->dst.ipv6.addr2 = CL_NTOH32(header->dst.ipv6.addr2); + header->dst.ipv6.addr3 = CL_NTOH32(header->dst.ipv6.addr3); +} + +/* + * sdp_msg_swap_hah - SDP header endian byte swapping + */ +static inline void sdp_msg_net_to_cpu_hah(struct msg_hdr_hah *header) +{ + header->l_rcv_size = CL_NTOH32(header->l_rcv_size); +} + +#define BASE_LISTEN_ID (CL_CONST64(0x0000000000010000)) + +static inline ib_net64_t +get_service_id_for_port( + ib_net16_t ip_port) +{ + return CL_NTOH64( BASE_LISTEN_ID | ip_port ); +} + + +// CM timeouts - Used for creating the sockets +#define CM_MIN_LOCAL_TIMEOUT (18) +#define CM_LOCAL_TIMEOUT (1) +#define CM_MIN_REMOTE_TIMEOUT (18) +#define CM_REMOTE_TIMEOUT (2) +#define CM_RETRIES 4 + + +#pragma pack(pop) + +#endif H_SDP_MSGS_H + diff --git a/trunk/ulp/sdp/kernel/SdpSocket.cpp b/trunk/ulp/sdp/kernel/SdpSocket.cpp index ca54437b..dc74a7b6 100644 --- a/trunk/ulp/sdp/kernel/SdpSocket.cpp +++ b/trunk/ulp/sdp/kernel/SdpSocket.cpp @@ -2,18 +2,91 @@ #include "Precompile.h" +#pragma warning(disable: 4244 ) + +NTSTATUS sdp_cm_hello_ack_check(struct sdp_msg_hello_ack *hello_ack); + +static void AL_API +cm_rej_callback(IN ib_cm_rej_rec_t *p_cm_rej_rec ) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_rej_callback called")); +} + +static void AL_API +cm_rep_callback( + IN ib_cm_rep_rec_t *p_cm_rep_rec ) +{ + + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_rep_callback called\n")); + SdpSocket *pSocket = (SdpSocket *) p_cm_rep_rec->qp_context; + pSocket->CmRepCallback(p_cm_rep_rec); +} + +static void AL_API +cm_req_callback( + IN ib_cm_req_rec_t *p_cm_req_rec ) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_req_callback called")); + ASSERT(FALSE); +} + +static void AL_API +cm_mra_callback( + IN ib_cm_mra_rec_t *p_cm_mra_rec ) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_mra_callback called")); + ASSERT(FALSE); +} + +/* + * A user-specified callback that is invoked after receiving a load + * alternate path response message. + */ +static void AL_API +cm_apr_callback( + IN ib_cm_apr_rec_t *p_cm_apr_rec ) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_apr_callback called")); + ASSERT(FALSE); +} + +static void AL_API +cm_dreq_callback(IN ib_cm_dreq_rec_t *p_cm_dreq_rec ) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("cm_dreq_callback called")); + ASSERT(FALSE); +} +SdpSocket::SdpSocket() +{ + m_CreationFlags = 0; + m_SrcPort = 0; + m_SrcIp = 0; + + mh_Ca = NULL; + m_pd = NULL; + m_rcq = NULL; + m_scq = NULL; + m_qp = NULL; + + m_shutdown = false; + + m_state = SS_IDLE; +} NTSTATUS SdpSocket::Init( WspSocketIn *pSocketInParam, WspSocketOut *pSocketOutParam) { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Init this = 0x%x\n", this)); + NTSTATUS rc = STATUS_SUCCESS; + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Init this = 0x%p\n", this)); m_CreationFlags = pSocketInParam->dwFlags; pSocketOutParam->Errno = 0;// No error pSocketOutParam->pSocket = this; // give the user a handle to the socket - - return STATUS_SUCCESS; + KeInitializeSpinLock(&m_Lock); + + return rc; } NTSTATUS SdpSocket::WSPConnect( @@ -22,13 +95,19 @@ NTSTATUS SdpSocket::WSPConnect( ) { NTSTATUS rc = STATUS_SUCCESS; - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::WSPConnect this = 0x%x remote addresses ip=%d.%d.%d.%d:%d", + ib_api_status_t ib_status; + ib_net64_t SrcPortGuid; + ib_net64_t DestPortGuid; + ib_path_rec_t path_rec; + CSpinLockWrapper Lock(m_Lock); + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::WSPConnect this = 0x%p remote addresses ip=%d.%d.%d.%d:%d\n", this, - (pWspConnectIn->IP & 0XFF), - (pWspConnectIn->IP & 0XFF00) >> 8, - (pWspConnectIn->IP & 0XFF0000) >> 16, (pWspConnectIn->IP & 0XFF000000) >> 24, - nthos(pWspConnectIn->Port) + (pWspConnectIn->IP & 0XFF0000) >> 16, + (pWspConnectIn->IP & 0XFF00) >> 8, + (pWspConnectIn->IP & 0XFF), + pWspConnectIn->Port )); if((pWspConnectIn->IP == 0) || (pWspConnectIn->Port == 0)) { @@ -37,17 +116,594 @@ NTSTATUS SdpSocket::WSPConnect( goto Cleanup; } + // check socket state + // BUGBUG: Do a better work here + Lock.Lock(); + if (m_state != SS_IDLE) { + // We can not connect in this state + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Invalid Socket state %s\n", SS2String(m_state))); + pWspConnectOut->Errno = WSAEINVAL; + goto Cleanup; + Lock.Unlock(); + } + + // + // Arp: this means that based on the ARP protocol we should convert the + // IP address into gid. + // + if (m_SrcIp == 0) { + // This means that we need to do an implicit bind to get the source + // address + rc = g_pSdpDriver->m_pSdpArp->SourceAddrFromDestAddr(pWspConnectIn->IP , &m_SrcIp); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->SourceAddrFromDestAddr failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; + Lock.Unlock(); + goto Cleanup; + } + } + + // Now that we know the source IP we can decide about the src port + if (m_SrcPort == 0) { + rc = g_pSdpDriver->m_pSdpArp->GetPort(m_SrcIp, &m_SrcPort); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->SourcePortGidFromIP failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + Lock.Unlock(); + goto Cleanup; + } + } + rc = g_pSdpDriver->m_pSdpArp->SourcePortGidFromIP(m_SrcIp, &SrcPortGuid, &m_CaGuid); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->SourcePortGidFromIP failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + Lock.Unlock(); + goto Cleanup; + } + + rc = g_pSdpDriver->m_pSdpArp->DestPortGidFromIP(pWspConnectIn->IP, &DestPortGuid); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->DestPortGidFromIP failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + Lock.Unlock(); + goto Cleanup; + } + + // + // Next step is convert the gid to a path record + // + + // Since this is a function that might wait we do it without the lock + m_state = SS_CONNECTING_QPR_SENT; + Lock.Unlock(); + + rc = g_pSdpDriver->m_pSdpArp->QueryPathRecord( SrcPortGuid, DestPortGuid, &path_rec ); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_pSdpArp->QueryPathRecord failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + goto Cleanup; + } + + + // Verify that we are in the correct state (just looking - without the lock) + if (m_state != SS_CONNECTING_QPR_SENT) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("wrong state after QueryPathRecord\n" )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + goto Cleanup; + } + + rc = CreateQp(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("CreateQp failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + goto Cleanup; + } + +// Lock.Lock(); // Do we really need the lock ? + + + + // We need to prepare the hello mesage for the CM + sdp_msg_hello hello_msg; + CreateHelloHeader(&hello_msg, pWspConnectIn->IP); + + // Create the CM request + ib_cm_req_t cm_req; + CreateCmRequest(&cm_req, &hello_msg, &path_rec, pWspConnectIn->Port); + + // Create the event to wait on to the connection request to end: + KeInitializeEvent(&m_ConnectCmCompleteEvent, NotificationEvent, FALSE ); + + m_state = SS_CONNECTING_REQ_SENT; + + ib_status = ib_cm_req( &cm_req ); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_open_ca failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + goto Cleanup; + } + + rc = MyKeWaitForSingleObject( + &m_ConnectCmCompleteEvent, + UserRequest, + UserMode, + FALSE, + NULL); + + if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) { + // BUGBUG: Think what to do here, we should be able to stop the + // connect, and quit (probably shutdown should be enough) + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted = 0x%x\n", rc )); + rc = STATUS_UNEXPECTED_IO_ERROR; + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + Shutdown(); + goto Cleanup; + } + + // + // We have recieved the REP, we send the RTU code + // + + m_state = SS_CONNECTING_REQ_SENT; + + rc = CmSendRTU(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("CmSendRTU failed rc = 0x%x\n", rc )); + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + goto Cleanup; + } + + // we should now complete the request + Lock.Lock(); + if (m_state == SS_CONNECTED) { + pWspConnectOut->Errno = 0; + ASSERT(rc == STATUS_SUCCESS); + Lock.Unlock(); + goto Cleanup; + } else { + // There probably was some error or some kind of shutdown, we + // need to return an error. + rc = STATUS_UNEXPECTED_IO_ERROR; + pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error + Lock.Unlock(); + goto Cleanup; + } + +Cleanup: + // Make sure that success/failure is marked in both places + ASSERT(((pWspConnectOut->Errno == 0) && (NT_SUCCESS(rc))) | + ((pWspConnectOut->Errno != 0) && (!NT_SUCCESS(rc)))); + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("connect is returning %s this = 0x%p\n", + NT_SUCCESS(rc) ? "SUCCESS" : "FAILURE", this )); + return rc; + +} + + +NTSTATUS SdpSocket::CmSendRTU() +{ + // This is only valid in connect. + // We might also be in shutdown, but nothing else currently + + ib_api_status_t ib_status; + NTSTATUS rc = STATUS_SUCCESS; + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::CmSendRTU this = 0x%p \n", this)); + + if (m_state != SS_CONNECTING_REQ_SENT) { + // There was some error, we can release the waiting thread. + // The error will be handled there + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("SdpSocket::CmSendRTU this = 0x%p invalid state %s\n", this, SS2String(m_state))); + rc = STATUS_UNEXPECTED_IO_ERROR; + goto Cleanup; + } + + /* + * check Hello Header Ack, to determine if we want + * the connection. + */ + + rc = sdp_cm_hello_ack_check(&m_hello_ack); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("sdp_cm_hello_ack_check failed rc = 0x%x\n", rc )); + goto Cleanup; + } + +#if 0 + /* + * read remote information + */ + conn->send_size = hello_ack->hah.l_rcv_size; + conn->r_max_adv = hello_ack->hah.max_adv; + conn->r_recv_bf = hello_ack->bsdh.recv_bufs; + conn->recv_seq = hello_ack->bsdh.seq_num; + conn->advt_seq = hello_ack->bsdh.seq_num; + + conn->d_qpn = event->param.rep_rcvd.remote_qpn; + /* + * The maximum amount of data that can be sent to the remote + * peer is the smaller of the local and remote buffer sizes, + * minus the size of the message header. + */ + conn->send_size = min((u16)sdp_buff_pool_buff_size(), + (u16)conn->send_size) - SDP_MSG_HDR_SIZE; + +#endif + /* + * Pop the hello message that was sent ????? I don't think that we need this + */ + + ib_cm_rtu_t cm_rtu; + + memset( &cm_rtu, 0, sizeof(cm_rtu) ); + + cm_rtu.access_ctrl = IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE; + +#if 0 // Do we want to keep it + // Bug in TAVOR + cm_rtu.sq_depth = QP_ATTRIB_SQ_DEPTH; + cm_rtu.rq_depth = QP_ATTRIB_RQ_DEPTH; +#endif + + cm_rtu.pfn_cm_apr_cb = cm_apr_callback; + cm_rtu.pfn_cm_dreq_cb = cm_dreq_callback; + + ib_status = ib_cm_rtu( m_cm_handle_t, &cm_rtu ); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_cm_rtu failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } + // We are now connected + // How should this be locked ?? + m_state = SS_CONNECTED; + Cleanup: return rc; +} + + +VOID SdpSocket::CmRepCallback(IN ib_cm_rep_rec_t *p_cm_rep_rec) +{ + if (m_state != SS_CONNECTING_REQ_SENT) { + // This is not the state that we waited for, not much that we can + // do. (This might happen in shutdown) + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("SdpSocket::CmRepCallback Not the expacted state %s\n", SS2String(m_state))); + ASSERT(FALSE); + return; + } + + // We need to store the params and signal the event + sdp_msg_hello_ack *hello_ack = (sdp_msg_hello_ack *)p_cm_rep_rec->p_rep_pdata; + m_hello_ack = *hello_ack; + m_cm_handle_t = p_cm_rep_rec->h_cm_rep; + KeSetEvent(&m_ConnectCmCompleteEvent, IO_NO_INCREMENT, FALSE); } +// BUGBUG: based on __recv_cb - need to implment +static void +__recv_cb1( + IN const ib_cq_handle_t h_cq, + IN void *cq_context ) +{ +// status = ib_rearm_cq( +// p_port->ib_mgr.h_recv_cq, FALSE ); +// ASSERT(exp)( status == IB_SUCCESS ); + ASSERT(FALSE); +} + + + +// BUGBUG: This code is based on __cq_event, find out what it realy does +static void +__cq_event1( + IN ib_async_event_rec_t *p_event_rec ) +{ + ASSERT(FALSE); + UNUSED_PARAM( p_event_rec ); + ASSERT( p_event_rec->context ); + /* Place holder for proper error handler. */ + ASSERT( p_event_rec->code == IB_AE_UNKNOWN ); +} + + +// Based on __qp_event - do we need it? +static void +__qp_event1( + IN ib_async_event_rec_t *p_event_rec ) +{ + UNUSED_PARAM( p_event_rec ); + ASSERT( p_event_rec->context ); + /* Place holder for proper error handler. */ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("Async QP event: %d\n", p_event_rec->code)); + ASSERT( p_event_rec->code == IB_AE_UNKNOWN ); +} + + +// This currently works only for the connect +NTSTATUS SdpSocket::CreateQp() +{ + NTSTATUS rc = STATUS_SUCCESS; + ib_cq_create_t cq_create; + ib_qp_create_t qp_create; + ib_api_status_t ib_status; + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("CreateQp called this = 0x%p\n", this)); + /* Open the CA. */ + ib_status = ib_open_ca( + g_pSdpDriver->m_al_handle, + m_CaGuid, + NULL, + this, + &mh_Ca + ); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_open_ca failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } + + /* Allocate the PD. */ + ib_status = ib_alloc_pd( + mh_Ca, + IB_PDT_NORMAL, + this, + &m_pd + ); + + if( ib_status != IB_SUCCESS ) + { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_alloc_pd failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } + + /* Allocate receive CQ. */ + cq_create.size = QP_ATTRIB_RQ_DEPTH; + cq_create.pfn_comp_cb = __recv_cb1; + cq_create.h_wait_obj = NULL; + + ib_status = ib_create_cq( + mh_Ca, + &cq_create, + this, + __cq_event1, + &m_rcq + ); + + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_create_cq failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } + + /* Allocate send CQ. */ + cq_create.size = QP_ATTRIB_SQ_DEPTH; + cq_create.pfn_comp_cb = __recv_cb1; // ???? We are not doing anything there ??? why bother + + ib_status = ib_create_cq( + mh_Ca, + &cq_create, + this, + __cq_event1, + &m_scq + ); + + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_create_cq failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } + + /* Allocate the QP. */ + cl_memclr( &qp_create, sizeof(qp_create) ); + qp_create.qp_type = IB_QPT_RELIABLE_CONN; + qp_create.rq_depth = QP_ATTRIB_RQ_DEPTH; + qp_create.rq_sge = QP_ATTRIB_RQ_SGE; /* To support buffers spanning pages. */ + qp_create.h_rq_cq = m_rcq; + qp_create.sq_depth = QP_ATTRIB_SQ_DEPTH; + //TODO: Figure out the right number of SGE entries for sends. + qp_create.sq_sge = QP_ATTRIB_SQ_SGE; + qp_create.h_sq_cq = m_scq; + qp_create.sq_signaled = TRUE; + ib_status = ib_create_qp( m_pd, &qp_create, this, __qp_event1, &m_qp ); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_create_qp failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } +#if 0 + /* Query the QP so we can get our QPN. */ + status = p_port->p_adapter->p_ifc->query_qp( + p_port->ib_mgr.h_qp, &qp_attr ); + if( status != IB_SUCCESS ) + { + IPOIB_TRACE_EXIT( IPOIB_DBG_ERROR, + ("ib_query_qp returned %s\n", + p_port->p_adapter->p_ifc->get_err_str( status )) ); + return status; + } + p_port->ib_mgr.qpn = qp_attr.num; +#endif + +Cleanup: + return rc; + +} + +VOID SdpSocket::CreateHelloHeader( + sdp_msg_hello *hello_msg, + ULONG DestIp + + ) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::CreateHelloHeader called this = 0x%p\n", this)); + ASSERT(DestIp != 0); + ASSERT(m_SrcPort != 0); + ASSERT(m_SrcIp != 0); + + memset(hello_msg, 0, sizeof(struct sdp_msg_hello)); + hello_msg->bsdh.recv_bufs = QP_ATTRIB_RQ_DEPTH; //????conn->l_advt_bf; + hello_msg->bsdh.flags = SDP_MSG_FLAG_NON_FLAG; + hello_msg->bsdh.mid = SDP_MID_HELLO; + hello_msg->bsdh.size = sizeof(struct sdp_msg_hello); + hello_msg->bsdh.seq_num = 0;//conn->send_seq; ??? + hello_msg->bsdh.seq_ack = 0;//conn->advt_seq; ??? + + hello_msg->hh.max_adv = QP_ATTRIB_RQ_DEPTH;// ??? conn->l_max_adv; + hello_msg->hh.ip_ver = SDP_MSG_IPVER; + hello_msg->hh.version = SDP_MSG_VERSION; + hello_msg->hh.r_rcv_size = 4096;//???conn->recv_size; + hello_msg->hh.l_rcv_size = 4096;//???conn->recv_size; + hello_msg->hh.port = m_SrcPort; + hello_msg->hh.src.ipv4.addr = m_SrcIp; + hello_msg->hh.dst.ipv4.addr = DestIp; + + /* + * endian swap + */ + sdp_msg_swap_bsdh(&hello_msg->bsdh); + sdp_msg_swap_hh(&hello_msg->hh); + + +} + +VOID SdpSocket::CreateCmRequest( + ib_cm_req_t *cm_req, + sdp_msg_hello *hello_msg, + ib_path_rec_t *path_rec, + USHORT DestPort + ) +{ + + memset( cm_req, 0, sizeof(ib_cm_req_t) ); + cm_req->qp_type = IB_QPT_RELIABLE_CONN; + cm_req->h_qp = m_qp; + cm_req->p_primary_path = path_rec; + cm_req->p_alt_path = NULL; + cm_req->svc_id = get_service_id_for_port(DestPort ); + cm_req->p_req_pdata = (uint8_t *) hello_msg; + cm_req->req_length = sizeof(sdp_msg_hello); + + // Caution: From here we live the linux code, as it was in: gen2 - sdp_cm_path_complete + cm_req->max_cm_retries = CM_RETRIES; + cm_req->resp_res = 4;//???????? what are this??? QP_ATTRIB_RESPONDER_RESOURCES; + cm_req->init_depth = 4;//?????QP_ATTRIB_INITIATOR_DEPTH; + + cm_req->remote_resp_timeout = + ib_path_rec_pkt_life( path_rec ) + CM_REMOTE_TIMEOUT; + if( cm_req->remote_resp_timeout > 0x1F ) + cm_req->remote_resp_timeout = 0x1F; + else if( cm_req->remote_resp_timeout < CM_MIN_REMOTE_TIMEOUT ) + cm_req->remote_resp_timeout = CM_MIN_REMOTE_TIMEOUT; + + cm_req->flow_ctrl = TRUE; /* HCAs must support end-to-end flow control. */ + + cm_req->local_resp_timeout = + ib_path_rec_pkt_life( path_rec ) + CM_LOCAL_TIMEOUT; + if( cm_req->local_resp_timeout > 0x1F ) + cm_req->local_resp_timeout = 0x1F; + else if( cm_req->local_resp_timeout < CM_MIN_LOCAL_TIMEOUT ) + cm_req->local_resp_timeout = CM_MIN_LOCAL_TIMEOUT; + + cm_req->rnr_nak_timeout = 6;//???QP_ATTRIB_RNR_NAK_TIMEOUT; + cm_req->rnr_retry_cnt = 6;//????QP_ATTRIB_RNR_RETRY; + cm_req->retry_cnt = 6;//????QP_ATTRIB_RETRY_COUNT; + + cm_req->pfn_cm_req_cb = cm_req_callback; + cm_req->pfn_cm_mra_cb = cm_mra_callback; + cm_req->pfn_cm_rej_cb = cm_rej_callback; + cm_req->pfn_cm_rep_cb = cm_rep_callback; +} VOID SdpSocket::Shutdown() { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%x\n", this)); + //???? locking + // if(m_shutdown) ??? + m_shutdown = true; + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%p\n", this)); + + if (m_qp != NULL) { + ib_destroy_qp(m_qp, NULL); //?????? CALL BACK ??? IMPLMENT + } + + if (m_scq != NULL) { + ib_destroy_cq(m_scq, NULL); //?????? CALL BACK ??? IMPLMENT + } + + if (m_rcq != NULL) { + ib_destroy_cq(m_rcq, NULL); //?????? CALL BACK ??? IMPLMENT + } + + if (m_pd != NULL) { + ib_dealloc_pd(m_pd, NULL); //?????? CALL BACK ??? IMPLMENT + } + + if (mh_Ca != NULL) { + ib_close_ca(mh_Ca, NULL); //?????? CALL BACK ??? IMPLMENT + } +} + +/* + * sdp_cm_hello_ack_check - validate the hello ack header + */ +NTSTATUS sdp_cm_hello_ack_check(struct sdp_msg_hello_ack *hello_ack) +{ + /* + * endian swap + */ + sdp_msg_swap_bsdh(&hello_ack->bsdh); + sdp_msg_net_to_cpu_hah(&hello_ack->hah); + /* + * validation and consistency checks + */ + if (hello_ack->bsdh.size != sizeof(struct sdp_msg_hello_ack)) { + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, size mismatch. (2) <%d:%d>", + hello_ack->bsdh.size, + sizeof(struct sdp_msg_hello_ack))); + return STATUS_UNEXPECTED_IO_ERROR; + } + + if (SDP_MID_HELLO_ACK != hello_ack->bsdh.mid) { + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, unexpected message. <%d>", + hello_ack->bsdh.mid + )); + return STATUS_UNEXPECTED_IO_ERROR; + } + if (hello_ack->hah.max_adv <= 0) { + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, bad zcopy advertisment. <%d>", + hello_ack->hah.max_adv + )); + return STATUS_UNEXPECTED_IO_ERROR; + } + if ((0xF0 & hello_ack->hah.version) != (0xF0 & SDP_MSG_VERSION)) { + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("hello ack, version mismatch. <%d:%d>", + ((0xF0 & hello_ack->hah.version) >> 4), + ((0xF0 & SDP_MSG_VERSION) >> 4))); + return STATUS_UNEXPECTED_IO_ERROR; + } + + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("Hello Ack BSDH <%04x:%02x:%02x:%08x:%08x:%08x>", + hello_ack->bsdh.recv_bufs, + hello_ack->bsdh.flags, + hello_ack->bsdh.mid, + hello_ack->bsdh.size, + hello_ack->bsdh.seq_num, + hello_ack->bsdh.seq_ack)); + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("Hello Ack HAH <%02x:%02x:%08x", + hello_ack->hah.max_adv, + hello_ack->hah.version, + hello_ack->hah.l_rcv_size)); + + return STATUS_SUCCESS; } + + diff --git a/trunk/ulp/sdp/kernel/SdpSocket.h b/trunk/ulp/sdp/kernel/SdpSocket.h index 9dc1479f..24e402e2 100644 --- a/trunk/ulp/sdp/kernel/SdpSocket.h +++ b/trunk/ulp/sdp/kernel/SdpSocket.h @@ -12,14 +12,60 @@ It keeps a list of all the objects so we know when to remove them. #define _SDP_SOCKET_H +#define QP_ATTRIB_SQ_DEPTH 16 +#define QP_ATTRIB_SQ_SGE 1 /* Set based on inline data requirements */ + +/* + * TODO: During testing, the switch has been observed to post + * 12 receive buffers. It would be nice to know what the max is. + */ +#define QP_ATTRIB_RQ_DEPTH 16 +#define QP_ATTRIB_RQ_SGE 1 + +/* Number of entries in a CQ */ +#define IB_CQ_SIZE (QP_ATTRIB_SQ_DEPTH + QP_ATTRIB_RQ_DEPTH + 1) + +enum SocketStates { + SS_IDLE, + SS_CONNECTING_QPR_SENT, // QPR = Query path record + SS_CONNECTING_REQ_SENT, + SS_CONNECTING_RTU_SENT, + SS_CONNECTED, + SS_INSHUTDOWN +}; + + class SdpSocket : public RefCountImpl { private: + + SocketStates m_state; + // This flags are being Initiated when the socket is being // created ULONG m_CreationFlags; - + USHORT m_SrcPort; + ULONG m_SrcIp; + + + KSPIN_LOCK m_Lock; + bool m_shutdown; + + + // A handle to the ca that is being used (in connect) and its guid + ib_ca_handle_t mh_Ca; + net64_t m_CaGuid; + + ib_pd_handle_t m_pd; + ib_cq_handle_t m_rcq; + ib_cq_handle_t m_scq; + ib_qp_handle_t m_qp; + + KEVENT m_ConnectCmCompleteEvent; + public: + SdpSocket(); + NTSTATUS Init( WspSocketIn *pSocketInParam, WspSocketOut *pSocketOutParam @@ -33,7 +79,48 @@ public: VOID Shutdown(); + NTSTATUS CreateQp(); + + NTSTATUS CmSendRTU(); + + VOID CreateHelloHeader( + sdp_msg_hello *hello_msg, + ULONG DestIp + ); + + VOID CreateCmRequest( + ib_cm_req_t *cm_req, + sdp_msg_hello *hello_msg, + ib_path_rec_t *path_rec, + USHORT DestPort + ); + + VOID CmRepCallback(IN ib_cm_rep_rec_t *p_cm_rep_rec); + // Two varibales that are needed for passing REP data + struct sdp_msg_hello_ack m_hello_ack; + ib_cm_handle_t m_cm_handle_t; // BUGBUG: Check how this is used / locked + + + + // Used to allow the user file to remember us LIST_ENTRY m_UserFileList; + +#ifdef DBG + char * SS2String(SocketStates state) { + switch (state) { + case SS_IDLE : return "SS_IDLE"; + case SS_CONNECTING_REQ_SENT : return "SS_CONNECTING_REQ_SENT"; + case SS_CONNECTING_RTU_SENT : return "SS_CONNECTING_RTU_SENT"; + case SS_CONNECTED : return "SS_CONNECTED"; + case SS_INSHUTDOWN : return "SS_INSHUTDOWN"; + default : + ASSERT(FALSE); + + } + return "Unknown state"; + } +#endif + }; #endif // _SDP_SOCKET_H diff --git a/trunk/ulp/sdp/kernel/SdpTrace.h b/trunk/ulp/sdp/kernel/SdpTrace.h index 556be794..fe7b48a9 100644 --- a/trunk/ulp/sdp/kernel/SdpTrace.h +++ b/trunk/ulp/sdp/kernel/SdpTrace.h @@ -14,6 +14,7 @@ #define SDP_CONNECT 0x000002 #define SDP_DRIVER 0x000004 #define SDP_SOCKET 0x000008 +#define SDP_ARP 0x000010 // BUGBUG: CONVERT TO A FUNCTION -- 2.41.0