}\r
\r
\r
-void\r
-map_mtoi_wcqe (\r
- IN VAPI_wc_desc_t *p_m_cqe,\r
- OUT ib_wc_t *p_i_cqe)\r
-{\r
- p_i_cqe->wr_id = p_m_cqe->id;\r
- p_i_cqe->status = map_mtoi_cqe_status (p_m_cqe->status);\r
- p_i_cqe->wc_type = map_mtoi_cqe_type (p_m_cqe->opcode);\r
- p_i_cqe->length = (IB_COMP_SUCCESS == p_m_cqe->status) ? \r
- p_m_cqe->byte_len : 0;\r
-\r
- CL_TRACE (MLNX_TRACE_LVL_8, mlnx_dbg_lvl,\r
- ("cqe type %d length 0x%x status %d\n",\r
- p_i_cqe->wc_type, p_i_cqe->length, p_i_cqe->status));\r
-\r
- /*\r
- * FIXME: RC case\r
- */\r
- p_i_cqe->recv.conn.recv_opt = 0;\r
-\r
- if (p_m_cqe->imm_data_valid)\r
- {\r
- p_i_cqe->recv.conn.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
- p_i_cqe->recv.conn.immediate_data = CL_NTOH32 (p_m_cqe->imm_data);\r
- }\r
-\r
- if (p_m_cqe->grh_flag)\r
- {\r
- p_i_cqe->recv.conn.recv_opt |= IB_RECV_OPT_GRH_VALID;\r
- }\r
-\r
- switch (p_m_cqe->remote_node_addr.type)\r
- {\r
- case VAPI_RNA_UD:\r
- p_i_cqe->recv.ud.remote_qp = \r
- CL_NTOH32 (p_m_cqe->remote_node_addr.qp_ety.qp);\r
- p_i_cqe->recv.ud.pkey_index = (uint16_t)p_m_cqe->pkey_ix; \r
- p_i_cqe->recv.ud.remote_lid = \r
- CL_NTOH16 (p_m_cqe->remote_node_addr.slid);\r
- p_i_cqe->recv.ud.remote_sl = p_m_cqe->remote_node_addr.sl;\r
- p_i_cqe->recv.ud.path_bits = 0;\r
- break;\r
-\r
- case VAPI_RNA_RD:\r
- CL_TRACE (MLNX_TRACE_LVL_8, mlnx_dbg_lvl, ("Unsupported RD\n"));\r
- break;\r
-\r
- case VAPI_RNA_RAW_ETY:\r
- CL_TRACE (MLNX_TRACE_LVL_8, mlnx_dbg_lvl, ("Unsupported RAW_ETY\n"));\r
- break;\r
-\r
- case VAPI_RNA_RAW_IPV6:\r
- CL_TRACE (MLNX_TRACE_LVL_8, mlnx_dbg_lvl, ("Unsupported RAW_IPV6\n"));\r
- break;\r
-\r
- default:\r
- CL_TRACE (MLNX_TRACE_LVL_8, mlnx_dbg_lvl, ("Unknown type\n"));\r
- break;\r
- }\r
-}\r
-\r
-\r
static VAPI_mrw_acl_t\r
map_itom_access_ctrl (\r
IN ib_access_t i_acl)\r
status = IB_INVALID_PARAMETER;\r
return status;\r
}\r
-#if TRUE\r
- status = THHUL_cqm_poll4wc(p_hobul->hhul_hca_hndl, p_cq_info->hhul_cq_hndl,\r
- pp_free_wclist, pp_done_wclist );\r
-#else\r
- for (p_i_cqe = *pp_free_wclist; p_i_cqe; p_i_cqe = p_i_cqe->p_next)\r
- {\r
- *pp_done_wclist = NULL;\r
-\r
- hh_ret = THHUL_cqm_poll4cqe (p_hobul->hhul_hca_hndl,\r
- p_cq_info->hhul_cq_hndl,\r
- &m_cqe);\r
- if (HH_CQ_EMPTY == hh_ret)\r
- {\r
- status = IB_NOT_FOUND;\r
- break;\r
- }\r
-\r
- /*\r
- * Errors cases\r
- */\r
- if (HH_OK != hh_ret) \r
- {\r
- CL_TRACE (MLNX_TRACE_LVL_1, mlnx_dbg_lvl,\r
- ("poll4cqe get error status %d\n", hh_ret));\r
- status = IB_ERROR;\r
- return status;\r
- }\r
\r
- map_mtoi_wcqe (&m_cqe, p_i_cqe);\r
- *pp_done_wclist = p_i_cqe;\r
- pp_done_wclist = (ib_wc_t **)&p_i_cqe->p_next;\r
- status = IB_SUCCESS;\r
- }\r
+ status = THHUL_cqm_poll4wc(p_hobul->hhul_hca_hndl, p_cq_info->hhul_cq_hndl,\r
+ pp_free_wclist, pp_done_wclist );\r
\r
- /*\r
- * Update free list to point to the first unused cqe\r
- */\r
- if (IB_NOT_FOUND == status && p_i_cqe != *pp_free_wclist)\r
- {\r
- status = IB_SUCCESS;\r
- }\r
- \r
- *pp_free_wclist = p_i_cqe;\r
-#endif\r
FUNC_EXIT;\r
return status;\r
}\r
#define CQE_OPCODE_BIT_MASK MASK32(MT_BIT_SIZE(tavorprm_completion_queue_entry_st,opcode))\r
#define CQE_OPCODE_DWORD_OFFSET MT_BYTE_OFFSET(tavorprm_completion_queue_entry_st,opcode)>>2\r
#define CQE_OPCODE_SHIFT (MT_BIT_OFFSET(tavorprm_completion_queue_entry_st,opcode) & MASK32(5))\r
-HH_ret_t THHUL_cqm_poll4cqe( \r
- /*IN*/ HHUL_hca_hndl_t hca_hndl, \r
- /*IN*/ HHUL_cq_hndl_t cq, \r
- /*OUT*/ VAPI_wc_desc_t *vapi_cqe_p \r
-) \r
-{ \r
- THHUL_cq_t *thhul_cq_p= (THHUL_cq_t*)cq;\r
- volatile u_int32_t *cur_cqe;\r
- u_int32_t wqe_addr_32lsb,next_wqe_addr_32lsb;\r
- IB_wqpn_t qpn;\r
- u_int8_t opcode;\r
- u_int8_t dbd_bit;\r
- VAPI_special_qp_t qp_type;\r
- VAPI_ts_type_t qp_ts_type;\r
- u_int32_t i,dbd_cnt;\r
- HH_ret_t rc;\r
- u_int32_t cqe_cpy[CQE_SZ>>2]; /* CQE copy */\r
- /* The CQE copy is required for 2 reasons:\r
- * 1) Hold in CPU endianess. \r
- * 2) Free real CQE as soon as possible in order to release CQ lock quickly.\r
- */\r
-\r
- if (MOSAL_EXPECT_FALSE(thhul_cq_p == NULL)) {\r
- MTL_ERROR1("THHUL_cqm_poll4cqe: NULL CQ handle.\n");\r
- return HH_EINVAL_CQ_HNDL;\r
- }\r
-\r
- MOSAL_spinlock_dpc_lock(&(thhul_cq_p->cq_lock));\r
-\r
- /* Check if CQE at consumer index is valid */\r
- cur_cqe= (volatile u_int32_t *)\r
- (thhul_cq_p->cur_buf.cqe_buf_base + (thhul_cq_p->cur_buf.consumer_index << LOG2_CQE_SZ)); \r
- if (is_cqe_hw_own(cur_cqe) && /* CQE is still in HW ownership */\r
- (!cq_transition_to_resized_buf(thhul_cq_p, &cur_cqe)) ) { \r
-\r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
-#if 0\r
- THHUL_cqm_dump_cq(cq);\r
-#endif\r
- return HH_CQ_EMPTY;\r
- }\r
-\r
- /* Remove Copy of all cqe, copy as nedded only */ \r
-#if 0 /* original code */ \r
- /* Make CQE copy in correct endianess */\r
- for (i= 0; i < (CQE_SZ>>2); i++) { \r
- cqe_cpy[i]= MOSAL_be32_to_cpu(cur_cqe[i]);\r
- }\r
- /* Extract QP/WQE context fields from the CQE */\r
- wqe_addr_32lsb= (cqe_cpy[MT_BYTE_OFFSET(tavorprm_completion_queue_entry_st,wqe_adr)>>2] & \r
- (~MASK32(CQE_WQE_ADR_BIT_SZ)) );\r
- qpn= (cqe_cpy[MT_BYTE_OFFSET(tavorprm_completion_queue_entry_st,my_qpn)>>2] & MASK32(24) );\r
- vapi_cqe_p->local_qp_num= qpn;\r
- /* new CQE: completion status is taken from "opcode" field */\r
- opcode= MT_EXTRACT_ARRAY32(cqe_cpy,\r
- MT_BIT_OFFSET(tavorprm_completion_queue_entry_st,opcode), \r
- MT_BIT_SIZE(tavorprm_completion_queue_entry_st,opcode));\r
-#endif\r
- \r
- /* Extract QP/WQE context fields from the CQE */\r
- /* Byte 6 */ \r
- wqe_addr_32lsb= MOSAL_be32_to_cpu(cur_cqe[CQE_WQE_ADDR_BYTE_OFFSET]) & \r
- CQE_WQE_ADDR_BIT_MASK;\r
- \r
- /* Byte 0*/\r
- qpn= MOSAL_be32_to_cpu(cur_cqe[CQE_MY_QPN_BYTE_OFFSET]) & CQE_MY_QPN_BYTE_BIT_MASK;\r
-\r
- /* Byte 1 */\r
- cqe_cpy[CQE_MY_EE_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_MY_EE_DWORD_OFFSET]); \r
- /* Byte 2 */\r
- cqe_cpy[CQE_RQPN_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_RQPN_DWORD_OFFSET]); \r
- /* Byte 3 */\r
- cqe_cpy[CQE_RLID_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_RLID_DWORD_OFFSET]);\r
- /* Byte 7 Convert S,Opcode,Owner fileld to Be */\r
- cqe_cpy[CQE_S_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_S_DWORD_OFFSET]); \r
- \r
- /* Byte 4 */\r
- vapi_cqe_p->imm_data = MOSAL_be32_to_cpu(cur_cqe[CQE_IMMEDIATE_DWORD_OFFSET]);\r
- /* Byte 5 */\r
- vapi_cqe_p->byte_len= MOSAL_be32_to_cpu(cur_cqe[CQE_BYTE_CNT_DWORD_OFFSET]);\r
-\r
- /* new CQE: completion status is taken from "opcode" field */\r
-\r
- opcode=(cqe_cpy[CQE_OPCODE_DWORD_OFFSET]>>CQE_OPCODE_SHIFT) & CQE_OPCODE_BIT_MASK;\r
- \r
-\r
- if (MOSAL_EXPECT_TRUE((opcode & CQE_ERROR_STATUS_MASK) != CQE_ERROR_STATUS_MASK)) { /* Completed OK */ \r
- MTPERF_TIME_START(free_cqe);\r
- free_cqe(thhul_cq_p,cur_cqe); /* Free original CQE and update consumer index */\r
- MTPERF_TIME_END(free_cqe);\r
-\r
- /* DEBUG: Sanity check that the same WQE is not used twice simultaneosly */\r
-#ifdef THHUL_CQM_DEBUG_WQE_REUSE\r
- /* Get next CQE and check if valid and NDA equals freed CQE's */\r
- cur_cqe= (volatile u_int32_t *)\r
- (thhul_cq_p->cur_buf.cqe_buf_base + (thhul_cq_p->cur_buf.consumer_index << LOG2_CQE_SZ)); \r
- if ((!is_cqe_hw_own(cur_cqe)) &&\r
- ( (MOSAL_be32_to_cpu(\r
- cur_cqe[MT_BYTE_OFFSET(tavorprm_completion_queue_entry_st,wqe_adr)>>2]) & \r
- (~MASK32(CQE_WQE_ADR_BIT_SZ)) ) == wqe_addr_32lsb) ){ \r
- MTL_ERROR1(MT_FLFMT("%s: Duplicate NDA on next CQE (NDA=0x%X , consumer index=%u,%u)"),\r
- __func__, wqe_addr_32lsb, \r
- thhul_cq_p->cur_buf.consumer_index-1, thhul_cq_p->cur_buf.consumer_index);\r
- THHUL_cqm_dump_cq(cq);\r
- }\r
-#endif \r
-\r
-#ifndef IVAPI_THH\r
- rc= THHUL_qpm_comp_ok(thhul_cq_p->qpm, qpn, wqe_addr_32lsb,\r
- &qp_type,&qp_ts_type,&(vapi_cqe_p->id),&(vapi_cqe_p->free_res_count));\r
-#else\r
- rc= THHUL_qpm_comp_ok(thhul_cq_p->qpm, qpn, wqe_addr_32lsb,\r
- &qp_type,&qp_ts_type,&(vapi_cqe_p->id),&(vapi_cqe_p->free_res_count),NULL);\r
-#endif \r
- \r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
- \r
- if (MOSAL_EXPECT_FALSE(rc != HH_OK)) {\r
- MTL_ERROR1("THHUL_cqm_poll4cqe: Failed updating associated QP.\n");\r
- for (i= 0; i < (CQE_SZ>>2); i++) { \r
- MTL_ERROR1(MT_FLFMT("CQ[0x%X][%u][%u]=0x%X"),thhul_cq_p->cq_num,\r
- (thhul_cq_p->cur_buf.consumer_index - 1) & MASK32(thhul_cq_p->cur_buf.log2_num_o_cqes),\r
- i, cqe_cpy[i]);\r
- }\r
- return HH_EFATAL; /* unexpected error */\r
- }\r
- /* Extract the rest of the CQE fields into vapi_cqe_p*/\r
- rc= extract_cqe_new(cqe_cpy,vapi_cqe_p,qp_type,qp_ts_type,opcode); \r
- vapi_cqe_p->status= VAPI_SUCCESS;\r
- \r
- } else { /* Completion with error */\r
- \r
- /* Make CQE copy in correct endianess */\r
- for (i= 0; i < (CQE_SZ>>2); i++) { \r
- cqe_cpy[i]= MOSAL_be32_to_cpu(cur_cqe[i]);\r
- }\r
- \r
- MTL_DEBUG4("THHUL_cqm_poll4cqe: completion with error: cq=%d consumer_index=%d\n",\r
- thhul_cq_p->cq_num,thhul_cq_p->cur_buf.consumer_index);\r
- DUMP_CQE(thhul_cq_p->cq_num,thhul_cq_p->cur_buf.consumer_index,cur_cqe);\r
- rc= THHUL_qpm_comp_err(thhul_cq_p->qpm, qpn, wqe_addr_32lsb,\r
- &(vapi_cqe_p->id),&(vapi_cqe_p->free_res_count),&next_wqe_addr_32lsb,&dbd_bit);\r
- if (rc != HH_OK) {\r
- MTL_ERROR1("THHUL_cqm_poll4cqe: Failed updating associated QP (QPn=0x%X , CQn=0x%X).\n",\r
- qpn, thhul_cq_p->cq_num);\r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
- return HH_EFATAL; /* unexpected error */\r
- }\r
- vapi_cqe_p->status= decode_error_syndrome((tavor_if_comp_status_t)MT_EXTRACT_ARRAY32(cqe_cpy,\r
- CQE_ERROR_SYNDROM_BIT_OFFSET, CQE_ERROR_SYNDROM_BIT_SIZE) );\r
- vapi_cqe_p->vendor_err_syndrome= MT_EXTRACT_ARRAY32(cqe_cpy,\r
- CQE_ERROR_VENDOR_SYNDROM_BIT_OFFSET, CQE_ERROR_VENDOR_SYNDROM_BIT_SIZE);\r
- dbd_cnt= MT_EXTRACT_ARRAY32(cqe_cpy,CQE_ERROR_DBDCNT_BIT_OFFSET, CQE_ERROR_DBDCNT_BIT_SIZE);\r
- if ((next_wqe_addr_32lsb == THHUL_QPM_END_OF_WQE_CHAIN) || /* End of WQE chain */\r
- ((dbd_cnt + 1 - dbd_bit) == 0) ) { /* or dbd counter reached 0 */\r
- if ((next_wqe_addr_32lsb == THHUL_QPM_END_OF_WQE_CHAIN) && (dbd_cnt > 0)) {\r
- MTL_ERROR1(MT_FLFMT("%s: CQ[0x%X]:CQE[0x%X]: Reached end of chain while dbd_cnt==%u"),\r
- __func__, thhul_cq_p->cq_num, thhul_cq_p->cur_buf.consumer_index, dbd_cnt);\r
- }\r
- MTPERF_TIME_START(free_cqe);\r
- free_cqe(thhul_cq_p,cur_cqe); /* Free original CQE and update consumer index */\r
- MTPERF_TIME_END(free_cqe);\r
- } else {\r
- recycle_cqe(cur_cqe, next_wqe_addr_32lsb, dbd_cnt - dbd_bit);\r
- } \r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
- /* Only WQE-ID, free_res_count and status are required for completion with error. \r
- * No other CQE fields are extracted (see IB-spec. 11.4.2.1). \r
- * Even though, for the sake of some legacy code:\r
- * ...putting an opcode to distinguish completion of SQ from RQ*/\r
- if (opcode == CQE_ERROR_ON_SQ) { \r
- vapi_cqe_p->opcode= VAPI_CQE_SQ_SEND_DATA; \r
- } else { /* receive queue completion */\r
- vapi_cqe_p->opcode= VAPI_CQE_RQ_SEND_DATA; \r
- }\r
- }\r
-\r
- return rc; \r
-}\r
\r
#ifdef WIN32\r
/* Successful completion */\r
\r
}\r
\r
-/* This code is mainly from poll4cqe with rearm_cqe if next_cqe available\r
- */\r
-HH_ret_t THHUL_cqm_poll_and_rearm_cq( \r
- /*IN*/ HHUL_hca_hndl_t hca_hndl, \r
- /*IN*/ HHUL_cq_hndl_t cq, \r
- /*IN*/ int solicitedNotification,\r
- /*OUT*/ VAPI_wc_desc_t *vapi_cqe_p \r
-) \r
-{ \r
-\r
- THHUL_cq_t *thhul_cq_p= (THHUL_cq_t*)cq;\r
- volatile u_int32_t *cur_cqe;\r
- u_int32_t wqe_addr_32lsb,next_wqe_addr_32lsb;\r
- IB_wqpn_t qpn;\r
- u_int8_t opcode;\r
- u_int8_t dbd_bit;\r
- VAPI_special_qp_t qp_type;\r
- VAPI_ts_type_t qp_ts_type;\r
- u_int32_t i,dbd_cnt;\r
- HH_ret_t rc;\r
- u_int32_t cqe_cpy[CQE_SZ>>2]; /* CQE copy */\r
- /* The CQE copy is required for 2 reasons:\r
- * 1) Hold in CPU endianess. \r
- * 2) Free real CQE as soon as possible in order to release CQ lock quickly.\r
- */\r
-\r
- if (MOSAL_EXPECT_FALSE(thhul_cq_p == NULL)) {\r
- MTL_ERROR1("THHUL_cqm_poll4cqe: NULL CQ handle.\n");\r
- return HH_EINVAL_CQ_HNDL;\r
- }\r
-\r
- MOSAL_spinlock_dpc_lock(&(thhul_cq_p->cq_lock));\r
-\r
- /* Check if CQE at consumer index is valid */\r
- cur_cqe= (volatile u_int32_t *)\r
- (thhul_cq_p->cur_buf.cqe_buf_base + (thhul_cq_p->cur_buf.consumer_index << LOG2_CQE_SZ)); \r
- if (is_cqe_hw_own(cur_cqe)) { /* CQE is still in HW ownership */\r
-\r
- rearm_cq(thhul_cq_p, (MT_bool)solicitedNotification);\r
- /* Tavor actually solves the race condition where the s/w may have missed the\r
- next valid entry being written, just prior to rearming the CQ. So we really\r
- don't need to repoll the cq entry, since Tavor guarantees that we will see\r
- an interrupt if it happened to go valid before or after rearming the CQ\r
- */\r
- \r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
-#if 0\r
- THHUL_cqm_dump_cq(cq);\r
-#endif\r
- return HH_CQ_EMPTY;\r
- }\r
-\r
-\r
- /* Remove Copy of all cqe, copy as nedded only */ \r
-#if 0 \r
- /* Make CQE copy in correct endianess */\r
- for (i= 0; i < (CQE_SZ>>2); i++) { \r
- cqe_cpy[i]= MOSAL_be32_to_cpu(cur_cqe[i]);\r
- }\r
-#endif\r
- \r
- /* Extract QP/WQE context fields from the CQE */\r
- \r
- wqe_addr_32lsb= MOSAL_be32_to_cpu(cur_cqe[CQE_WQE_ADDR_BYTE_OFFSET]) & \r
- CQE_WQE_ADDR_BIT_MASK;\r
- \r
- qpn= MOSAL_be32_to_cpu(cur_cqe[CQE_MY_QPN_BYTE_OFFSET]) & CQE_MY_QPN_BYTE_BIT_MASK;\r
-\r
- cqe_cpy[CQE_MY_EE_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_MY_EE_DWORD_OFFSET]); \r
- cqe_cpy[CQE_RQPN_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_RQPN_DWORD_OFFSET]); \r
- cqe_cpy[CQE_RLID_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_RLID_DWORD_OFFSET]); \r
- cqe_cpy[CQE_S_DWORD_OFFSET] = MOSAL_be32_to_cpu(cur_cqe[CQE_S_DWORD_OFFSET]); \r
- \r
- vapi_cqe_p->imm_data = MOSAL_be32_to_cpu(cur_cqe[CQE_IMMEDIATE_DWORD_OFFSET]);\r
- vapi_cqe_p->byte_len= MOSAL_be32_to_cpu(cur_cqe[CQE_BYTE_CNT_DWORD_OFFSET]);\r
-\r
- /* new CQE: completion status is taken from "opcode" field */\r
-\r
- opcode=(cqe_cpy[CQE_OPCODE_DWORD_OFFSET]>>CQE_OPCODE_SHIFT) & CQE_OPCODE_BIT_MASK;\r
- \r
-\r
- if (MOSAL_EXPECT_TRUE((opcode & CQE_ERROR_STATUS_MASK) != CQE_ERROR_STATUS_MASK)) { /* Completed OK */ \r
- HH_ret_t rc_rearm; \r
- volatile u_int32_t *next_cqe;\r
-\r
- MTPERF_TIME_START(free_cqe);\r
- free_cqe(thhul_cq_p,cur_cqe); /* Free original CQE and update consumer index */\r
- MTPERF_TIME_END(free_cqe);\r
-\r
- // form pointer to next cqe that would be serviced next \r
- next_cqe= (volatile u_int32_t *)\r
- (thhul_cq_p->cur_buf.cqe_buf_base + (thhul_cq_p->cur_buf.consumer_index << LOG2_CQE_SZ)); \r
- \r
- \r
- if (is_cqe_hw_own(next_cqe)) { /* CQE is still in HW ownership */\r
- rearm_cq(thhul_cq_p, (MT_bool)solicitedNotification);\r
- // check again\r
- next_cqe= (volatile u_int32_t *)\r
- (thhul_cq_p->cur_buf.cqe_buf_base + (thhul_cq_p->cur_buf.consumer_index << LOG2_CQE_SZ)); \r
- if (is_cqe_hw_own(next_cqe)) { /* CQE is still in HW ownership */\r
- rc_rearm = HH_COMPLETED; \r
- } else {\r
- rc_rearm = HH_POLL_NEEDED; \r
- }\r
- }\r
- else\r
- {\r
- rc_rearm = HH_OK;\r
- } \r
- \r
-\r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
-#ifndef IVAPI_THH\r
- rc= THHUL_qpm_comp_ok(thhul_cq_p->qpm, qpn, wqe_addr_32lsb,\r
- &qp_type,&qp_ts_type,&(vapi_cqe_p->id),&(vapi_cqe_p->free_res_count));\r
-#else\r
- rc= THHUL_qpm_comp_ok(thhul_cq_p->qpm, qpn, wqe_addr_32lsb,\r
- &qp_type,&qp_ts_type,&(vapi_cqe_p->id),&(vapi_cqe_p->free_res_count),NULL);\r
-#endif \r
- \r
- if (MOSAL_EXPECT_FALSE(rc != HH_OK)) {\r
- MTL_ERROR1("THHUL_cqm_poll4cqe: Failed updating associated QP.\n");\r
- DUMP_CQE(thhul_cq_p->cq_num,thhul_cq_p->cur_buf.consumer_index,cur_cqe);\r
- return rc;\r
- }\r
- /* Extract the rest of the CQE fields into vapi_cqe_p*/\r
- \r
- rc= extract_cqe_new(cqe_cpy,vapi_cqe_p,qp_type,qp_ts_type,opcode); \r
- vapi_cqe_p->status= VAPI_SUCCESS;\r
- if(MOSAL_EXPECT_FALSE(rc != HH_OK)) {\r
- return rc;\r
- }\r
- else\r
- return rc_rearm;\r
- \r
- \r
- } else { /* Completion with error */\r
- /* Make CQE copy in correct endianess */\r
- for (i= 0; i < (CQE_SZ>>2); i++) { \r
- cqe_cpy[i]= MOSAL_be32_to_cpu(cur_cqe[i]);\r
- } \r
- MTL_DEBUG4("THHUL_cqm_poll4cqe: completion with error: cq=%d consumer_index=%d\n",\r
- thhul_cq_p->cq_num,thhul_cq_p->cur_buf.consumer_index);\r
- DUMP_CQE(thhul_cq_p->cq_num,thhul_cq_p->cur_buf.consumer_index,cur_cqe);\r
- rc= THHUL_qpm_comp_err(thhul_cq_p->qpm, qpn, wqe_addr_32lsb,\r
- &(vapi_cqe_p->id),&(vapi_cqe_p->free_res_count),&next_wqe_addr_32lsb,&dbd_bit);\r
- if (rc != HH_OK) {\r
- MTL_ERROR1("THHUL_cqm_poll4cqe: Failed updating associated QP.\n");\r
- return rc;\r
- }\r
- vapi_cqe_p->status= decode_error_syndrome(MT_EXTRACT_ARRAY32(cqe_cpy,\r
- CQE_ERROR_SYNDROM_BIT_OFFSET, CQE_ERROR_SYNDROM_BIT_SIZE) );\r
- vapi_cqe_p->vendor_err_syndrome= MT_EXTRACT_ARRAY32(cqe_cpy,\r
- CQE_ERROR_VENDOR_SYNDROM_BIT_OFFSET, CQE_ERROR_VENDOR_SYNDROM_BIT_SIZE);\r
- dbd_cnt= MT_EXTRACT_ARRAY32(cqe_cpy,CQE_ERROR_DBDCNT_BIT_OFFSET, CQE_ERROR_DBDCNT_BIT_SIZE);\r
- if ((next_wqe_addr_32lsb == THHUL_QPM_END_OF_WQE_CHAIN) || /* End of WQE chain */\r
- ((dbd_cnt + 1 - dbd_bit) == 0) ) { /* or dbd counter reached 0 */\r
- MTPERF_TIME_START(free_cqe);\r
- free_cqe(thhul_cq_p,cur_cqe); /* Free original CQE and update consumer index */\r
- MTPERF_TIME_END(free_cqe);\r
- } else {\r
- recycle_cqe(cur_cqe, next_wqe_addr_32lsb, dbd_cnt - dbd_bit);\r
- } \r
- MOSAL_spinlock_unlock(&(thhul_cq_p->cq_lock)); \r
- /* Only WQE-ID, free_res_count and status are required for completion with error. \r
- * No other CQE fields are extracted (see IB-spec. 11.4.2.1). \r
- * Even though, for the sake of some legacy code:\r
- * ...putting an opcode to distinguish completion of SQ from RQ*/\r
- if (opcode == CQE_ERROR_ON_SQ) { \r
- vapi_cqe_p->opcode= VAPI_CQE_SQ_SEND_DATA; \r
- } else { /* receive queue completion */\r
- vapi_cqe_p->opcode= VAPI_CQE_RQ_SEND_DATA; \r
- }\r
- }\r
-\r
- return rc; \r
- \r
-}\r
-\r
\r
HH_ret_t THHUL_cqm_req_comp_notif( \r
/*IN*/ HHUL_hca_hndl_t hca_hndl, \r