From b358a8517d3637c9a75a5d5c4d8fec056df76be0 Mon Sep 17 00:00:00 2001 From: "Tatyana E. Nikolova" Date: Thu, 13 Feb 2014 11:01:00 -0800 Subject: [PATCH] RDMA/nes: Fixes for OFED-3.12 Signed-off-by: Tatyana Nikolova --- ...03-nes-Fixes-for-MPA-V2-implentation.patch | 273 ++++++++++++++++++ ...04-nes-Fix-for-passing-valid-qp-addr.patch | 74 +++++ 2 files changed, 347 insertions(+) create mode 100644 linux-next-pending/0003-nes-Fixes-for-MPA-V2-implentation.patch create mode 100644 linux-next-pending/0004-nes-Fix-for-passing-valid-qp-addr.patch diff --git a/linux-next-pending/0003-nes-Fixes-for-MPA-V2-implentation.patch b/linux-next-pending/0003-nes-Fixes-for-MPA-V2-implentation.patch new file mode 100644 index 0000000..d1a993f --- /dev/null +++ b/linux-next-pending/0003-nes-Fixes-for-MPA-V2-implentation.patch @@ -0,0 +1,273 @@ +Subject: [PATCH 1/2] RDMA/nes: Fixes for the MPA V2 implementation + +Signed-off-by: Tatyana Nikolova +--- + drivers/infiniband/hw/nes/nes_cm.c | 118 +++++++++++++++++++++++++++--------- + drivers/infiniband/hw/nes/nes_cm.h | 3 + + 2 files changed, 91 insertions(+), 30 deletions(-) + +diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c +index 6b29249..5eefba4 100644 +--- a/drivers/infiniband/hw/nes/nes_cm.c ++++ b/drivers/infiniband/hw/nes/nes_cm.c +@@ -128,6 +128,7 @@ static void build_mpa_v1(struct nes_cm_node *, void *, u8); + static void build_rdma0_msg(struct nes_cm_node *, struct nes_qp **); + + static void print_core(struct nes_cm_core *core); ++static void record_ird_ord(struct nes_cm_node *, u16, u16); + + /* External CM API Interface */ + /* instance of function pointers for client API */ +@@ -317,7 +318,6 @@ static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type, + } + } + +- + if (priv_data_len + mpa_hdr_len != len) { + nes_debug(NES_DBG_CM, "The received ietf buffer was not right" + " complete (%x + %x != %x)\n", +@@ -356,25 +356,57 @@ static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type, + /* send reset */ + return -EINVAL; + } +- +- if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) { ++ if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) ++ cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; ++ ++ if (cm_node->mpav2_ird_ord != IETF_NO_IRD_ORD) { + /* responder */ +- if (cm_node->ord_size > ird_size) +- cm_node->ord_size = ird_size; +- } else { +- /* initiator */ +- if (cm_node->ord_size > ird_size) +- cm_node->ord_size = ird_size; +- +- if (cm_node->ird_size < ord_size) { +- /* no resources available */ +- /* send terminate message */ +- return -EINVAL; ++ if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) { ++ /* we are still negotiating */ ++ if (ord_size > NES_MAX_IRD) { ++ cm_node->ird_size = NES_MAX_IRD; ++ } else { ++ cm_node->ird_size = ord_size; ++ if (ord_size == 0 && ++ (rtr_ctrl_ord & IETF_RDMA0_READ)) { ++ cm_node->ird_size = 1; ++ nes_debug(NES_DBG_CM, ++ "%s: Remote peer doesn't support RDMA0_READ (ord=%u)\n", ++ __func__, ord_size); ++ } ++ } ++ if (ird_size > NES_MAX_ORD) ++ cm_node->ord_size = NES_MAX_ORD; ++ else ++ cm_node->ord_size = ird_size; ++ } else { /* initiator */ ++ if (ord_size > NES_MAX_IRD) { ++ nes_debug(NES_DBG_CM, ++ "%s: Unable to support the requested (ord =%u)\n", ++ __func__, ord_size); ++ return -EINVAL; ++ } ++ cm_node->ird_size = ord_size; ++ ++ if (ird_size > NES_MAX_ORD) { ++ cm_node->ord_size = NES_MAX_ORD; ++ } else { ++ if (ird_size == 0 && ++ (rtr_ctrl_ord & IETF_RDMA0_READ)) { ++ nes_debug(NES_DBG_CM, ++ "%s: Remote peer doesn't support RDMA0_READ (ird=%u)\n", ++ __func__, ird_size); ++ return -EINVAL; ++ } else { ++ cm_node->ord_size = ird_size; ++ } ++ } + } + } + + if (rtr_ctrl_ord & IETF_RDMA0_READ) { + cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; ++ + } else if (rtr_ctrl_ord & IETF_RDMA0_WRITE) { + cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; + } else { /* Not supported RDMA0 operation */ +@@ -514,6 +546,18 @@ static void print_core(struct nes_cm_core *core) + nes_debug(NES_DBG_CM, "-------------- end core ---------------\n"); + } + ++static void record_ird_ord(struct nes_cm_node *cm_node, u16 conn_ird, u16 conn_ord) ++{ ++ if (conn_ird > NES_MAX_IRD) ++ conn_ird = NES_MAX_IRD; ++ ++ if (conn_ord > NES_MAX_ORD) ++ conn_ord = NES_MAX_ORD; ++ ++ cm_node->ird_size = conn_ird; ++ cm_node->ord_size = conn_ord; ++} ++ + /** + * cm_build_mpa_frame - build a MPA V1 frame or MPA V2 frame + */ +@@ -557,11 +601,13 @@ static void build_mpa_v2(struct nes_cm_node *cm_node, + mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE); + + /* initialize RTR msg */ +- ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? +- IETF_NO_IRD_ORD : cm_node->ird_size; +- ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? +- IETF_NO_IRD_ORD : cm_node->ord_size; +- ++ if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { ++ ctrl_ird = IETF_NO_IRD_ORD; ++ ctrl_ord = IETF_NO_IRD_ORD; ++ } else { ++ ctrl_ird = cm_node->ird_size & IETF_NO_IRD_ORD; ++ ctrl_ord = cm_node->ord_size & IETF_NO_IRD_ORD; ++ } + ctrl_ird |= IETF_PEER_TO_PEER; + ctrl_ird |= IETF_FLPDU_ZERO_LEN; + +@@ -1410,8 +1456,9 @@ static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core, + + cm_node->mpa_frame_rev = mpa_version; + cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; +- cm_node->ird_size = IETF_NO_IRD_ORD; +- cm_node->ord_size = IETF_NO_IRD_ORD; ++ cm_node->mpav2_ird_ord = 0; ++ cm_node->ird_size = 0; ++ cm_node->ord_size = 0; + + nes_debug(NES_DBG_CM, "Make node addresses : loc = %pI4:%x, rem = %pI4:%x\n", + &cm_node->loc_addr, cm_node->loc_port, +@@ -3028,11 +3075,11 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + rem_ref_cm_node(cm_node->cm_core, cm_node); + return -ECONNRESET; + } +- + /* associate the node with the QP */ + nesqp->cm_node = (void *)cm_node; + cm_node->nesqp = nesqp; + ++ + nes_debug(NES_DBG_CM, "QP%u, cm_node=%p, jiffies = %lu listener = %p\n", + nesqp->hwqp.qp_id, cm_node, jiffies, cm_node->listener); + atomic_inc(&cm_accepts); +@@ -3055,6 +3102,11 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + if (cm_node->mpa_frame_rev == IETF_MPA_V1) + mpa_frame_offset = 4; + ++ if (cm_node->mpa_frame_rev == IETF_MPA_V1 || ++ cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { ++ record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); ++ } ++ + memcpy(mpa_v2_frame->priv_data, conn_param->private_data, + conn_param->private_data_len); + +@@ -3118,7 +3170,6 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + } + nesqp->skip_lsmm = 1; + +- + /* Cache the cm_id in the qp */ + nesqp->cm_id = cm_id; + cm_node->cm_id = cm_id; +@@ -3155,7 +3206,7 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32( + ((u32)1 << NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT)); + nesqp->nesqp_context->ird_ord_sizes |= +- cpu_to_le32((u32)conn_param->ord); ++ cpu_to_le32((u32)cm_node->ord_size); + + memset(&nes_quad, 0, sizeof(nes_quad)); + nes_quad.DstIpAdrIndex = +@@ -3195,6 +3246,9 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + cm_event.remote_addr = cm_id->remote_addr; + cm_event.private_data = NULL; + cm_event.private_data_len = 0; ++ cm_event.ird = cm_node->ird_size; ++ cm_event.ord = cm_node->ord_size; ++ + ret = cm_id->event_handler(cm_id, &cm_event); + attr.qp_state = IB_QPS_RTS; + nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); +@@ -3291,14 +3345,8 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + + /* cache the cm_id in the qp */ + nesqp->cm_id = cm_id; +- + cm_id->provider_data = nesqp; +- + nesqp->private_data_len = conn_param->private_data_len; +- nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32((u32)conn_param->ord); +- /* space for rdma0 read msg */ +- if (conn_param->ord == 0) +- nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(1); + + nes_debug(NES_DBG_CM, "requested ord = 0x%08X.\n", (u32)conn_param->ord); + nes_debug(NES_DBG_CM, "mpa private data len =%u\n", +@@ -3335,6 +3383,10 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) + return -ENOMEM; + } + ++ record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); ++ if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO && cm_node->ord_size == 0) ++ cm_node->ord_size = 1; ++ + cm_node->apbvt_set = apbvt_set; + nesqp->cm_node = cm_node; + cm_node->nesqp = nesqp; +@@ -3531,6 +3583,7 @@ static void cm_event_connected(struct nes_cm_event *event) + nesqp->nesqp_context->ird_ord_sizes |= + cpu_to_le32((u32)1 << + NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT); ++ nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32((u32)cm_node->ord_size); + + /* Adjust tail for not having a LSMM */ + /*nesqp->hwqp.sq_tail = 1;*/ +@@ -3743,8 +3796,13 @@ static void cm_event_mpa_req(struct nes_cm_event *event) + cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr); + cm_event.private_data = cm_node->mpa_frame_buf; + cm_event.private_data_len = (u8)cm_node->mpa_frame_size; ++ if (cm_node->mpa_frame_rev == IETF_MPA_V1) { ++ cm_event.ird = NES_MAX_IRD; ++ cm_event.ord = NES_MAX_ORD; ++ } else { + cm_event.ird = cm_node->ird_size; + cm_event.ord = cm_node->ord_size; ++ } + + ret = cm_id->event_handler(cm_id, &cm_event); + if (ret) +diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h +index 4646e66..522c99c 100644 +--- a/drivers/infiniband/hw/nes/nes_cm.h ++++ b/drivers/infiniband/hw/nes/nes_cm.h +@@ -58,6 +58,8 @@ + #define IETF_RDMA0_WRITE 0x8000 + #define IETF_RDMA0_READ 0x4000 + #define IETF_NO_IRD_ORD 0x3FFF ++#define NES_MAX_IRD 0x40 ++#define NES_MAX_ORD 0x7F + + enum ietf_mpa_flags { + IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */ +@@ -333,6 +335,7 @@ struct nes_cm_node { + enum mpa_frame_version mpa_frame_rev; + u16 ird_size; + u16 ord_size; ++ u16 mpav2_ird_ord; + + u16 mpa_frame_size; + struct iw_cm_id *cm_id; +-- +1.7.1 + diff --git a/linux-next-pending/0004-nes-Fix-for-passing-valid-qp-addr.patch b/linux-next-pending/0004-nes-Fix-for-passing-valid-qp-addr.patch new file mode 100644 index 0000000..ef2a0d4 --- /dev/null +++ b/linux-next-pending/0004-nes-Fix-for-passing-valid-qp-addr.patch @@ -0,0 +1,74 @@ +Subject: [PATCH 2/2] RDMA/nes: Fix for passing a valid qp pointer +to the user space library + +Signed-off-by: Tatyana Nikolova +--- + drivers/infiniband/hw/nes/nes_cm.c | 2 +- + drivers/infiniband/hw/nes/nes_user.h | 5 +++-- + drivers/infiniband/hw/nes/nes_verbs.c | 2 ++ + drivers/infiniband/hw/nes/nes_verbs.h | 1 + + 4 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c +index 5eefba4..1b8afb0 100644 +--- a/drivers/infiniband/hw/nes/nes_cm.c ++++ b/drivers/infiniband/hw/nes/nes_cm.c +@@ -656,7 +656,7 @@ static void build_rdma0_msg(struct nes_cm_node *cm_node, struct nes_qp **nesqp_a + struct nes_qp *nesqp = *nesqp_addr; + struct nes_hw_qp_wqe *wqe = &nesqp->hwqp.sq_vbase[0]; + +- u64temp = (unsigned long)nesqp; ++ u64temp = (unsigned long)nesqp->nesuqp_addr; + u64temp |= NES_SW_CONTEXT_ALIGN >> 1; + set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, u64temp); + +diff --git a/drivers/infiniband/hw/nes/nes_user.h b/drivers/infiniband/hw/nes/nes_user.h +index 4926de7..529c421 100644 +--- a/drivers/infiniband/hw/nes/nes_user.h ++++ b/drivers/infiniband/hw/nes/nes_user.h +@@ -39,8 +39,8 @@ + + #include + +-#define NES_ABI_USERSPACE_VER 1 +-#define NES_ABI_KERNEL_VER 1 ++#define NES_ABI_USERSPACE_VER 2 ++#define NES_ABI_KERNEL_VER 2 + + /* + * Make sure that all structs defined in this file remain laid out so +@@ -78,6 +78,7 @@ struct nes_create_cq_req { + + struct nes_create_qp_req { + __u64 user_wqe_buffers; ++ __u64 user_qp_buffer; + }; + + enum iwnes_memreg_type { +diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c +index 5b53ca5..9b73cb4 100644 +--- a/drivers/infiniband/hw/nes/nes_verbs.c ++++ b/drivers/infiniband/hw/nes/nes_verbs.c +@@ -1191,6 +1191,8 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd, + if (req.user_wqe_buffers) { + virt_wqs = 1; + } ++ if (req.user_qp_buffer) ++ nesqp->nesuqp_addr = req.user_qp_buffer; + if ((ibpd->uobject) && (ibpd->uobject->context)) { + nesqp->user_mode = 1; + nes_ucontext = to_nesucontext(ibpd->uobject->context); +diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h +index 0eff7c4..309b31c 100644 +--- a/drivers/infiniband/hw/nes/nes_verbs.h ++++ b/drivers/infiniband/hw/nes/nes_verbs.h +@@ -184,5 +184,6 @@ struct nes_qp { + u8 pau_busy; + u8 pau_pending; + u8 pau_state; ++ __u64 nesuqp_addr; + }; + #endif /* NES_VERBS_H */ +-- +1.7.1 + -- 2.41.0