#include <dat2/dat_ib_extensions.h>
#endif
-#define DEFAULT_DS_ENTRIES 8
-
STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p);
/*
IN DAT_COUNT segments,
IN DAT_LMR_TRIPLET *local_iov )
{
- ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
- ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
struct ibv_recv_wr wr;
struct ibv_recv_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
DAT_COUNT i, total_len;
int ret;
" post_rcv: ep %p cookie %p segs %d l_iov %p\n",
ep_ptr, cookie, segments, local_iov);
- if (segments <= DEFAULT_DS_ENTRIES)
- ds_array_p = ds_array;
- else
- ds_array_start_p = ds_array_p =
- dapl_os_alloc(segments * sizeof(ib_data_segment_t));
-
- if (NULL == ds_array_p)
- return (DAT_INSUFFICIENT_RESOURCES);
-
/* setup work request */
total_len = 0;
wr.next = 0;
- wr.num_sge = 0;
+ wr.num_sge = segments;
wr.wr_id = (uint64_t)(uintptr_t)cookie;
- wr.sg_list = ds_array_p;
-
- for (i = 0; i < segments; i++) {
- if (!local_iov[i].segment_length)
- continue;
-
- ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
- ds_array_p->length = local_iov[i].segment_length;
- ds_array_p->lkey = local_iov[i].lmr_context;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_rcv: l_key 0x%x va %p len %d\n",
- ds_array_p->lkey, ds_array_p->addr,
- ds_array_p->length );
-
- total_len += ds_array_p->length;
- wr.num_sge++;
- ds_array_p++;
- }
-
- if (cookie != NULL)
+ wr.sg_list = ds;
+
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++) {
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_rcv: l_key 0x%x va %p len %d\n",
+ ds->lkey, ds->addr, ds->length);
+ total_len += ds->length;
+ ds++;
+ }
cookie->val.dto.size = total_len;
+ }
ret = ibv_post_recv(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr);
- if (ds_array_start_p != NULL)
- dapl_os_free(ds_array_start_p, segments * sizeof(ib_data_segment_t));
-
if (ret)
return( dapl_convert_errno(errno,"ibv_recv") );
IN const DAT_RMR_TRIPLET *remote_iov,
IN DAT_COMPLETION_FLAGS completion_flags)
{
- ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
- ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
struct ibv_send_wr wr;
struct ibv_send_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
ib_hca_transport_t *ibt_ptr =
&ep_ptr->header.owner_ia->hca_ptr->ib_trans;
DAT_COUNT i, total_len;
ep_ptr, op_type, cookie, segments, local_iov,
remote_iov, completion_flags);
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: ep %p cookie %p segs %d l_iov %p\n",
- ep_ptr, cookie, segments, local_iov);
-
- if(segments <= DEFAULT_DS_ENTRIES)
- ds_array_p = ds_array;
- else
- ds_array_start_p = ds_array_p =
- dapl_os_alloc(segments * sizeof(ib_data_segment_t));
-
- if (NULL == ds_array_p)
- return (DAT_INSUFFICIENT_RESOURCES);
-
/* setup the work request */
wr.next = 0;
wr.opcode = op_type;
- wr.num_sge = 0;
+ wr.num_sge = segments;
wr.send_flags = 0;
wr.wr_id = (uint64_t)(uintptr_t)cookie;
- wr.sg_list = ds_array_p;
+ wr.sg_list = ds;
total_len = 0;
- for (i = 0; i < segments; i++ ) {
- if ( !local_iov[i].segment_length )
- continue;
-
- ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
- ds_array_p->length = local_iov[i].segment_length;
- ds_array_p->lkey = local_iov[i].lmr_context;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: lkey 0x%x va %p len %d\n",
- ds_array_p->lkey, ds_array_p->addr,
- ds_array_p->length );
-
- total_len += ds_array_p->length;
- wr.num_sge++;
- ds_array_p++;
- }
-
- if (cookie != NULL)
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++ ) {
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: lkey 0x%x va %p len %d\n",
+ ds->lkey, ds->addr, ds->length );
+ total_len += ds->length;
+ ds++;
+ }
cookie->val.dto.size = total_len;
+ }
if (wr.num_sge &&
(op_type == OP_RDMA_WRITE || op_type == OP_RDMA_READ)) {
wr.wr.rdma.rkey, wr.wr.rdma.remote_addr);
}
-
/* inline data for send or write ops */
if ((total_len <= ibt_ptr->max_inline_send) &&
((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE)))
ret = ibv_post_send(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr);
- if (ds_array_start_p != NULL)
- dapl_os_free(ds_array_start_p, segments * sizeof(ib_data_segment_t));
-
if (ret)
return( dapl_convert_errno(errno,"ibv_send") );
IN DAT_UINT64 swap,
IN DAT_COMPLETION_FLAGS completion_flags)
{
- ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
- ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
struct ibv_send_wr wr;
struct ibv_send_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
DAT_COUNT i, total_len;
int ret;
dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: ep %p op %d ck %p sgs",
+ " post_ext_snd: ep %p op %d ck %p sgs",
"%d l_iov %p r_iov %p f %d\n",
ep_ptr, op_type, cookie, segments, local_iov,
remote_iov, completion_flags);
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: ep %p cookie %p segs %d l_iov %p\n",
- ep_ptr, cookie, segments, local_iov);
-
- if(segments <= DEFAULT_DS_ENTRIES)
- ds_array_p = ds_array;
- else
- ds_array_start_p = ds_array_p =
- dapl_os_alloc(segments * sizeof(ib_data_segment_t));
-
- if (NULL == ds_array_p)
- return (DAT_INSUFFICIENT_RESOURCES);
-
/* setup the work request */
wr.next = 0;
wr.opcode = op_type;
- wr.num_sge = 0;
+ wr.num_sge = segments;
wr.send_flags = 0;
wr.wr_id = (uint64_t)(uintptr_t)cookie;
- wr.sg_list = ds_array_p;
+ wr.sg_list = ds;
total_len = 0;
- for (i = 0; i < segments; i++ ) {
- if ( !local_iov[i].segment_length )
- continue;
-
- ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
- ds_array_p->length = local_iov[i].segment_length;
- ds_array_p->lkey = local_iov[i].lmr_context;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: lkey 0x%x va %p len %d\n",
- ds_array_p->lkey, ds_array_p->addr,
- ds_array_p->length );
-
- total_len += ds_array_p->length;
- wr.num_sge++;
- ds_array_p++;
- }
-
- if (cookie != NULL)
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++ ) {
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_ext_snd: lkey 0x%x va %p ln %d\n",
+ ds->lkey, ds->addr, ds->length);
+ total_len += ds->length;
+ ds++;
+ }
cookie->val.dto.size = total_len;
+ }
switch (op_type) {
case OP_RDMA_WRITE_IMM:
ret = ibv_post_send(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr);
- if (segments > DEFAULT_DS_ENTRIES)
- dapl_os_free(ds_array_start_p, segments * sizeof(ib_data_segment_t));
-
if (ret)
return( dapl_convert_errno(errno,"ibv_send") );
typedef struct dapl_cm_id *ib_cm_srvc_handle_t;
/* Operation and state mappings */
-typedef enum ibv_send_flags ib_send_op_type_t;
+typedef int ib_send_op_type_t;
typedef struct ibv_sge ib_data_segment_t;
typedef enum ibv_qp_state ib_qp_state_t;
typedef enum ibv_event_type ib_async_event_type;
IN DAT_COUNT segments,
IN DAT_LMR_TRIPLET *local_iov )
{
- ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
- ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
struct ibv_recv_wr wr;
struct ibv_recv_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
DAT_COUNT i, total_len;
int ret;
" post_rcv: ep %p cookie %p segs %d l_iov %p\n",
ep_ptr, cookie, segments, local_iov);
- if (segments <= DEFAULT_DS_ENTRIES)
- ds_array_p = ds_array;
- else
- ds_array_start_p = ds_array_p =
- dapl_os_alloc(segments * sizeof(ib_data_segment_t));
-
- if (NULL == ds_array_p)
- return (DAT_INSUFFICIENT_RESOURCES);
-
/* setup work request */
total_len = 0;
wr.next = 0;
- wr.num_sge = 0;
+ wr.num_sge = segments;
wr.wr_id = (uint64_t)(uintptr_t)cookie;
- wr.sg_list = ds_array_p;
-
- for (i = 0; i < segments; i++) {
- if (!local_iov[i].segment_length)
- continue;
-
- ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
- ds_array_p->length = local_iov[i].segment_length;
- ds_array_p->lkey = local_iov[i].lmr_context;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_rcv: l_key 0x%x va %p len %d\n",
- ds_array_p->lkey, ds_array_p->addr,
- ds_array_p->length );
-
- total_len += ds_array_p->length;
- wr.num_sge++;
- ds_array_p++;
- }
-
- if (cookie != NULL)
+ wr.sg_list = ds;
+
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++) {
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_rcv: l_key 0x%x va %p len %d\n",
+ ds->lkey, ds->addr, ds->length );
+ total_len += ds->length;
+ ds++;
+ }
cookie->val.dto.size = total_len;
+ }
ret = ibv_post_recv(ep_ptr->qp_handle, &wr, &bad_wr);
- if (ds_array_start_p != NULL)
- dapl_os_free(ds_array_start_p, segments * sizeof(ib_data_segment_t));
-
if (ret)
- return( dapl_convert_errno(errno,"ibv_recv") );
+ return(dapl_convert_errno(errno,"ibv_recv"));
DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV);
DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len);
IN const DAT_RMR_TRIPLET *remote_iov,
IN DAT_COMPLETION_FLAGS completion_flags)
{
- ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
- ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
struct ibv_send_wr wr;
struct ibv_send_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
ib_hca_transport_t *ibt_ptr =
&ep_ptr->header.owner_ia->hca_ptr->ib_trans;
DAT_COUNT i, total_len;
ep_ptr, op_type, cookie, segments, local_iov,
remote_iov, completion_flags);
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: ep %p cookie %p segs %d l_iov %p\n",
- ep_ptr, cookie, segments, local_iov);
-
- if(segments <= DEFAULT_DS_ENTRIES)
- ds_array_p = ds_array;
- else
- ds_array_start_p = ds_array_p =
- dapl_os_alloc(segments * sizeof(ib_data_segment_t));
-
- if (NULL == ds_array_p)
- return (DAT_INSUFFICIENT_RESOURCES);
-
#ifdef DAT_EXTENSIONS
if (ep_ptr->qp_handle->qp_type != IBV_QPT_RC)
return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP));
/* setup the work request */
wr.next = 0;
wr.opcode = op_type;
- wr.num_sge = 0;
+ wr.num_sge = segments;
wr.send_flags = 0;
wr.wr_id = (uint64_t)(uintptr_t)cookie;
- wr.sg_list = ds_array_p;
+ wr.sg_list = ds;
total_len = 0;
- for (i = 0; i < segments; i++ ) {
- if ( !local_iov[i].segment_length )
- continue;
-
- ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
- ds_array_p->length = local_iov[i].segment_length;
- ds_array_p->lkey = local_iov[i].lmr_context;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: lkey 0x%x va %p len %d\n",
- ds_array_p->lkey, ds_array_p->addr,
- ds_array_p->length );
-
- total_len += ds_array_p->length;
- wr.num_sge++;
- ds_array_p++;
- }
-
- if (cookie != NULL)
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++ ) {
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: lkey 0x%x va %p len %d\n",
+ ds->lkey, ds->addr, ds->length );
+ total_len += ds->length;
+ ds++;
+ }
cookie->val.dto.size = total_len;
+ }
if (wr.num_sge &&
(op_type == OP_RDMA_WRITE || op_type == OP_RDMA_READ)) {
ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr);
- if (ds_array_start_p != NULL)
- dapl_os_free(ds_array_start_p, segments * sizeof(ib_data_segment_t));
-
if (ret)
- return( dapl_convert_errno(errno,"ibv_send") );
+ return(dapl_convert_errno(errno,"ibv_send"));
#ifdef DAPL_COUNTERS
switch (op_type) {
IN DAT_COMPLETION_FLAGS completion_flags,
IN DAT_IB_ADDR_HANDLE *remote_ah)
{
- ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
- ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
struct ibv_send_wr wr;
struct ibv_send_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
DAT_COUNT i, total_len;
int ret;
ep_ptr, op_type, cookie, segments, local_iov,
remote_iov, completion_flags, remote_ah);
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: ep %p cookie %p segs %d l_iov %p\n",
- ep_ptr, cookie, segments, local_iov);
-
- if(segments <= DEFAULT_DS_ENTRIES)
- ds_array_p = ds_array;
- else
- ds_array_start_p = ds_array_p =
- dapl_os_alloc(segments * sizeof(ib_data_segment_t));
-
- if (NULL == ds_array_p)
- return (DAT_INSUFFICIENT_RESOURCES);
-
/* setup the work request */
wr.next = 0;
wr.opcode = op_type;
- wr.num_sge = 0;
+ wr.num_sge = segments;
wr.send_flags = 0;
wr.wr_id = (uint64_t)(uintptr_t)cookie;
- wr.sg_list = ds_array_p;
+ wr.sg_list = ds;
total_len = 0;
- for (i = 0; i < segments; i++ ) {
- if ( !local_iov[i].segment_length )
- continue;
-
- ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
- ds_array_p->length = local_iov[i].segment_length;
- ds_array_p->lkey = local_iov[i].lmr_context;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " post_snd: lkey 0x%x va %p len %d\n",
- ds_array_p->lkey, ds_array_p->addr,
- ds_array_p->length );
-
- total_len += ds_array_p->length;
- wr.num_sge++;
- ds_array_p++;
- }
-
- if (cookie != NULL)
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++ ) {
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: lkey 0x%x va %p len %d\n",
+ ds->lkey, ds->addr, ds->length );
+ total_len += ds->length;
+ ds++;
+ }
cookie->val.dto.size = total_len;
+ }
switch (op_type) {
case OP_RDMA_WRITE_IMM:
ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr);
- if (segments > DEFAULT_DS_ENTRIES)
- dapl_os_free(ds_array_start_p, segments * sizeof(ib_data_segment_t));
-
if (ret)
return( dapl_convert_errno(errno,"ibv_send") );
} ib_cm_events_t;
/* Operation and state mappings */
-typedef enum ibv_send_flags ib_send_op_type_t;
+typedef int ib_send_op_type_t;
typedef struct ibv_sge ib_data_segment_t;
typedef enum ibv_qp_state ib_qp_state_t;
typedef enum ibv_event_type ib_async_event_type;