From: Sean Hefty Date: Sat, 15 Sep 2012 00:20:08 +0000 (-0700) Subject: refresh (create temporary patch) X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=a5358d72041581bdd46ca3c742ed2241ff9c8d39;p=~shefty%2Flibibverbs.git refresh (create temporary patch) --- diff --git a/meta b/meta index 2e98ff3..1ddab4d 100644 --- a/meta +++ b/meta @@ -1,9 +1,10 @@ Version: 1 -Previous: 4cf54aa0723cb70066c584da5566a46f220a02b7 -Head: dac865c51d6d752031ff009dd55e9f8c697bcb35 +Previous: 74399df36aaeb2ef2b113bfc9684edb18de50579 +Head: 1acfd2e60e6c72bf40867b6ebb6d417ab7b95796 Applied: verbs-ext: 6fbf2b29ca68fc10b7ee47dc0c09f54db0b32320 compat-ex: dac865c51d6d752031ff009dd55e9f8c697bcb35 + refresh-temp: 1acfd2e60e6c72bf40867b6ebb6d417ab7b95796 Unapplied: xrcd: 0e21cac2fc216f26a141a34cfb9e6ce5adfcd9f0 Hidden: diff --git a/patches/refresh-temp b/patches/refresh-temp new file mode 100644 index 0000000..d0bdfa6 --- /dev/null +++ b/patches/refresh-temp @@ -0,0 +1,479 @@ +Bottom: fe1e8b7156b1f23065427ed1a4d4e41446ffd0d5 +Top: ba930f22a8ee143a9890c200008b7265f0f24e5b +Author: Sean Hefty +Date: 2012-09-14 17:20:08 -0700 + +Refresh of compat-ex + +--- + +diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h +index 40bb4e6..4671231 100644 +--- a/include/infiniband/verbs.h ++++ b/include/infiniband/verbs.h +@@ -668,6 +668,7 @@ struct ibv_ah { + + struct ibv_device; + struct ibv_context; ++struct verbs_context; + + struct ibv_device_ops { + struct ibv_context * (*alloc_context)(struct ibv_device *device, int cmd_fd); +@@ -698,9 +699,9 @@ struct verbs_device { + size_t sz; + size_t size_of_context; + int (*init_context)(struct verbs_device *device, +- struct ibv_context *ctx, int cmd_fd); ++ struct verbs_context *ctx, int cmd_fd); + void (*uninit_context)(struct verbs_device *device, +- struct ibv_context *ctx); ++ struct verbs_context *ctx); + /* future fields added here */ + }; + +@@ -784,17 +785,12 @@ struct verbs_context { + struct ibv_context context;/* Must be last field in the struct */ + }; + +-static inline struct verbs_context *verbs_get_ctx( +- const struct ibv_context *ctx) ++static inline struct verbs_context *verbs_get_ctx(const struct ibv_context *ctx) + { +- if (ctx->abi_compat != ((uint8_t *)NULL)-1) +- return NULL; +- + return container_of(ctx, struct verbs_context, context); + } + +-static inline struct verbs_device *verbs_get_device( +- const struct ibv_device *dev) ++static inline struct verbs_device *verbs_get_device(const struct ibv_device *dev) + { + return container_of(dev, struct verbs_device, device); + } +diff --git a/src/compat-ex.c b/src/compat-ex.c +index 5b15be0..9aede93 100644 +--- a/src/compat-ex.c ++++ b/src/compat-ex.c +@@ -73,16 +73,19 @@ struct _ibv_ah_ex { + struct ibv_ah *real_ah; + }; + +-struct _ibv_context_ex { +- struct ibv_context context; +- struct ibv_context *real_context; +-}; ++struct ibv_context *_ibv_real_context(struct ibv_context *context) ++{ ++ struct verbs_context_private *priv_ctx; ++ struct verbs_context *context_ex = verbs_get_ctx(context); ++ priv_ctx = container_of(context_ex, struct verbs_context_private, context_ex); ++ return priv_ctx->real_context; ++} + + struct ibv_srq *_ibv_real_srq(struct ibv_srq *srq) + { + struct _ibv_srq_ex *srq_ex; + +- if (verbs_get_ctx(srq)) ++ if (ibv_support_ex(srq->context)) + return srq; + + srq_ex = container_of(srq, struct _ibv_srq_ex, srq); +@@ -93,7 +96,7 @@ struct ibv_qp *_ibv_real_qp(struct ibv_qp *qp) + { + struct _ibv_qp_ex *qp_ex; + +- if (verbs_get_ctx(qp)) ++ if (ibv_support_ex(qp->context)) + return qp; + + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); +@@ -104,7 +107,7 @@ struct ibv_cq *_ibv_real_cq(struct ibv_cq *cq) + { + struct _ibv_cq_ex *cq_ex; + +- if (verbs_get_ctx(cq)) ++ if (ibv_support_ex(cq->context)) + return cq; + + cq_ex = container_of(cq, struct _ibv_cq_ex, cq); +@@ -113,46 +116,40 @@ struct ibv_cq *_ibv_real_cq(struct ibv_cq *cq) + + struct ibv_srq *_ibv_verbs_srq(struct ibv_srq *srq) + { +- return verbs_get_ctx(srq) ? srq : (struct ibv_srq *) srq->srq_context; ++ return ibv_support_ex(srq->context) ? srq : (struct ibv_srq *) srq->srq_context; + } + + struct ibv_qp *_ibv_verbs_qp(struct ibv_qp *qp) + { +- return verbs_get_ctx(qp) ? qp : (struct ibv_qp *) qp->qp_context; ++ return ibv_support_ex(qp->context) ? qp : (struct ibv_qp *) qp->qp_context; + } + + struct ibv_cq *_ibv_verbs_cq(struct ibv_cq *cq) + { +- return verbs_get_ctx(cq) ? cq : (struct ibv_cq *) cq->cq_context; ++ return ibv_support_ex(cq->context) ? cq : (struct ibv_cq *) cq->cq_context; + } + + static int _ibv_query_device_ex(struct ibv_context *context, + struct ibv_device_attr *device_attr) + { +- struct _ibv_context_ex *context_ex; +- context_ex = container_of(context, struct _ibv_context_ex, context); +- return ibv_query_device(context_ex->real_context, device_attr); ++ return ibv_query_device(_ibv_real_context(context), device_attr); + } + + static int _ibv_query_port_ex(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr) + { +- struct _ibv_context_ex *context_ex; +- context_ex = container_of(context, struct _ibv_context_ex, context); +- return ibv_query_port(context_ex->real_context, port_num, port_attr); ++ return ibv_query_port(_ibv_real_context(context), port_num, port_attr); + } + + static struct ibv_pd *_ibv_alloc_pd_ex(struct ibv_context *context) + { +- struct _ibv_context_ex *context_ex; + struct _ibv_pd_ex *pd_ex; + +- context_ex = container_of(context, struct _ibv_context_ex, context); + pd_ex = calloc(1, sizeof *pd_ex); + if (!pd_ex) + return NULL; + +- pd_ex->real_pd = ibv_alloc_pd(context_ex->real_context); ++ pd_ex->real_pd = ibv_alloc_pd(_ibv_real_context(context)); + if (!pd_ex->real_pd) { + free(pd_ex); + return NULL; +@@ -217,22 +214,21 @@ struct ibv_cq *_ibv_create_cq_ex(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector) + { +- struct _ibv_context_ex *context_ex; ++ struct ibv_context *real_context= _ibv_real_context(context); + struct _ibv_cq_ex *cq_ex; + +- context_ex = container_of(context, struct _ibv_context_ex, context); + cq_ex = calloc(1, sizeof *cq_ex); + if (!cq_ex) + return NULL; + +- cq_ex->real_cq = context_ex->real_context->ops.create_cq(context_ex->real_context, +- cqe, channel, comp_vector); ++ cq_ex->real_cq = real_context->ops.create_cq(real_context, cqe, ++ channel, comp_vector); + if (!cq_ex->real_cq) { + free(cq_ex); + return NULL; + } + +- cq_ex->real_cq->context = context_ex->real_context; ++ cq_ex->real_cq->context = real_context; + cq_ex->real_cq->channel = channel; + if (channel) + ++channel->refcnt; +@@ -248,21 +244,12 @@ struct ibv_cq *_ibv_create_cq_ex(struct ibv_context *context, int cqe, + + static int _ibv_poll_cq_ex(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) + { +- struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); +- return ibv_poll_cq(cq_ex->real_cq, num_entries, wc); +-} +- +-void _ibv_cq_event_ex(struct ibv_cq **cq, void **cq_context) +-{ +- struct _ibv_cq_ex *cq_ex = container_of(*cq_context, struct _ibv_cq_ex, cq); +- *cq = &cq_ex->cq; +- *cq_context = cq_ex->cq.cq_context; ++ return ibv_poll_cq(_ibv_real_cq(cq), num_entries, wc); + } + + static int _ibv_req_notify_cq_ex(struct ibv_cq *cq, int solicited_only) + { +- struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); +- return ibv_req_notify_cq(cq_ex->real_cq, solicited_only); ++ return ibv_req_notify_cq(_ibv_real_cq(cq), solicited_only); + } + + static int _ibv_resize_cq_ex(struct ibv_cq *cq, int cqe) +@@ -321,15 +308,13 @@ static int _ibv_modify_srq_ex(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr, + int srq_attr_mask) + { +- struct _ibv_srq_ex *srq_ex = container_of(srq, struct _ibv_srq_ex, srq); +- return ibv_modify_srq(srq_ex->real_srq, srq_attr, srq_attr_mask); ++ return ibv_modify_srq(_ibv_real_srq(srq), srq_attr, srq_attr_mask); + } + + static int _ibv_query_srq_ex(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr) + { +- struct _ibv_srq_ex *srq_ex = container_of(srq, struct _ibv_srq_ex, srq); +- return ibv_query_srq(srq_ex->real_srq, srq_attr); ++ return ibv_query_srq(_ibv_real_srq(srq), srq_attr); + } + + static int _ibv_destroy_srq_ex(struct ibv_srq *srq) +@@ -357,22 +342,16 @@ static struct ibv_qp *_ibv_create_qp_ex(struct ibv_pd *pd, struct ibv_qp_init_at + { + struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + struct _ibv_qp_ex *qp_ex; +- struct _ibv_cq_ex *scq_ex, *rcq_ex; +- struct _ibv_srq_ex *srq_ex; + struct ibv_qp_init_attr real_attr; + + qp_ex = calloc(1, sizeof *qp_ex); + if (!qp_ex) + return NULL; + +- scq_ex = container_of(attr->send_cq, struct _ibv_cq_ex, cq); +- rcq_ex = container_of(attr->recv_cq, struct _ibv_cq_ex, cq); +- srq_ex = attr->srq ? container_of(attr->srq, struct _ibv_srq_ex, srq) : NULL; +- + real_attr.qp_context = qp_ex; +- real_attr.send_cq = scq_ex->real_cq; +- real_attr.recv_cq = rcq_ex->real_cq; +- real_attr.srq = srq_ex->real_srq; ++ real_attr.send_cq = attr->send_cq ? _ibv_real_cq(attr->send_cq) : NULL; ++ real_attr.recv_cq = attr->recv_cq ? _ibv_real_cq(attr->recv_cq) : NULL; ++ real_attr.srq = attr->srq ? _ibv_real_srq(srq) : NULL; + real_attr.cap = attr->cap; + real_attr.qp_type = attr->qp_type; + real_attr.sq_sig_all = attr->sq_sig_all; +@@ -406,8 +385,7 @@ static int _ibv_query_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, + static int _ibv_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask) + { +- struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); +- return ibv_modify_qp(qp_ex->real_qp, attr, attr_mask); ++ return ibv_modify_qp(_ibv_real_qp(qp), attr, attr_mask); + } + + static int _ibv_destroy_qp_ex(struct ibv_qp *qp) +@@ -486,36 +464,49 @@ static int _ibv_destroy_ah_ex(struct ibv_ah *ah) + static int _ibv_attach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid, + uint16_t lid) + { +- struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); +- return ibv_attach_mcast(qp_ex->real_qp, gid, lid); ++ return ibv_attach_mcast(_ibv_real_qp(qp), gid, lid); + } + + static int _ibv_detach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid, + uint16_t lid) + { +- struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); +- return ibv_detach_mcast(qp_ex->real_qp, gid, lid); +-} +- +-int _ibv_get_async_event_ex(struct ibv_context *context, +- struct ibv_async_event *event) +-{ +- +-} +- +-void _ibv_ack_async_event_ex(struct ibv_async_event *event) +-{ +- ibv_ack_async_event_ex() +-} +- +-struct ibv_context *_ibv_open_device_ex(struct ibv_device *device) +-{ +- //XXX +- +-} +- +-int _ibv_close_device_ex(struct ibv_context *context) +-{ +- //XXX +- ++ return ibv_detach_mcast(_ibv_real_qp(qp), gid, lid); ++} ++ ++void _ibv_init_context(struct ibv_device *device, ++ struct verbs_context_private *priv_ctx, int cmd_fd) ++{ ++ struct ibv_context_ops *ops = &priv_ctx->context_ex.context.ops; ++ ++ priv_ctx->real_context.device = device; ++ priv_ctx->real_context.cmd_fd = cmd_fd; ++ pthread_mutex_init(&priv_ctx->real_context.mutex, NULL); ++ priv_ctx->real_context.abi_compat = NULL; ++ ++ ops->query_device = _ibv_query_device_ex; ++ ops->query_port = _ibv_query_port_ex; ++ ops->alloc_pd = _ibv_alloc_pd_ex; ++ ops->dealloc_pd = _ibv_dealloc_pd_ex; ++ ops->reg_mr = _ibv_reg_mr_ex; ++ ops->dereg_mr = _ibv_dereg_mr_ex; ++ ops->create_cq = _ibv_create_cq_ex; ++ ops->poll_cq = _ibv_poll_cq_ex; ++ ops->req_notify_cq = _ibv_req_notify_cq_ex; ++ ops->resize_cq = _ibv_resize_cq_ex; ++ ops->destroy_cq = _ibv_destroy_cq_ex; ++ ops->create_srq = _ibv_create_srq_ex; ++ ops->modify_srq = _ibv_modify_srq_ex; ++ ops->query_srq = _ibv_query_srq_ex; ++ ops->destroy_srq = _ibv_destroy_srq_ex; ++ ops->post_srq_recv = _ibv_post_srq_recv_ex; ++ ops->create_qp = _ibv_create_qp_ex; ++ ops->query_qp = _ibv_query_qp_ex; ++ ops->modify_qp = _ibv_modify_qp_ex; ++ ops->destroy_qp = _ibv_destroy_qp_ex; ++ ops->post_send = _ibv_post_send_ex; ++ ops->post_recv = _ibv_post_recv_ex; ++ ops->create_ah = _ibv_create_ah_ex; ++ ops->destroy_ah = _ibv_destroy_ah_ex; ++ ops->attach_mcast = _ibv_attach_mcast_ex; ++ ops->detach_mcast = _ibv_detach_mcast_ex; + } +diff --git a/src/device.c b/src/device.c +index 5b598fd..bae1384 100644 +--- a/src/device.c ++++ b/src/device.c +@@ -127,7 +127,9 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) + char *devpath; + int cmd_fd; + struct ibv_context *context; +- struct verbs_context *context_ex; ++ struct verbs_context_private *priv_ctx; ++ struct verbs_device *vdev; ++ size_t prov_ctx_size; + + if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0) + return NULL; +@@ -144,49 +146,44 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) + + context = device->ops.alloc_context(device, cmd_fd); + if (!context) +- goto err; +- if (context == (struct ibv_context *)(((uint8_t *)NULL)-1)) { +- /* New provider that supports verbs extension was detected */ +- struct verbs_device *verbs_device = +- verbs_get_device(device); +- int ret; +- +- /* Library now allocates the context */ +- context_ex = calloc(1, sizeof(*context_ex) + +- verbs_device->size_of_context); +- +- if (!context_ex) { +- errno = ENOMEM; +- goto err; +- } +- context = &context_ex->context; +- /* Init new verbs_context */ +- context_ex->context.abi_compat = ((uint8_t *)NULL)-1; +- context_ex->sz = sizeof(*context_ex); +- +- /* Call provider to initialize its calls first */ +- ret = verbs_device->init_context(verbs_device, +- &context_ex->context, cmd_fd); +- if (ret) +- goto verbs_err; +- /* initialize *all* library ops to either lib calls or +- * directly to provider calls. +- context_ex-> lib_new_func1= __verbs_new_func1; +- context_ex-> lib_new_func2= __verbs_new_func2; +- */ ++ goto err1; ++ ++ if (context == IBV_EXTENDED_VERBS) { ++ vdev = verbs_get_device(device); ++ prov_ctx_size = vdev->size_of_context; ++ } else { ++ vdev = NULL; ++ prov_ctx_size = 0; + } + +- context->device = device; +- context->cmd_fd = cmd_fd; +- pthread_mutex_init(&context->mutex, NULL); ++ priv_ctx = calloc(1, sizeof(*priv_ctx) + prov_ctx_size); ++ if (!priv_ctx) { ++ errno = ENOMEM; ++ goto err1; ++ } + +- return context; ++ priv_ctx->context_ex.sz = sizeof(struct verbs_context); ++ if (vdev) { ++ if (vdev->init_context(vdev, &priv_ctx->context_ex, cmd_fd)) ++ goto err2; ++ ++ priv_ctx->real_context = &priv_ctx->context_ex.context; ++ priv_ctx->context_ex.context.abi_compat = IBV_EXTENDED_VERBS; ++ } else { ++ priv_ctx->real_context = context; ++ priv_ctx->context_ex.context.abi_compat = NULL; ++ _ibv_init_context(device, &priv_ctx->context_ex, cmd_fd); ++ } + +-verbs_err: +- free(context_ex); +-err: +- close(cmd_fd); ++ priv_ctx->context_ex.context.device = device; ++ priv_ctx->context_ex.context.cmd_fd = cmd_fd; ++ pthread_mutex_init(&priv_ctx->context_ex.context.mutex, NULL); ++ return &priv_ctx->context_ex.context; + ++err2: ++ free(priv_ctx); ++err1: ++ close(cmd_fd); + return NULL; + } + default_symver(__ibv_open_device, ibv_open_device); +@@ -199,14 +196,13 @@ int __ibv_close_device(struct ibv_context *context) + struct verbs_context *context_ex; + + context_ex = verbs_get_ctx(context); +- if (context_ex) { +- struct verbs_device *verbs_device = +- verbs_get_device(context->device); +- /* Provider supports verbs extension */ ++ if (ibv_support_ex(context)) { ++ struct verbs_device *verbs_device = verbs_get_device(context->device); + verbs_device->uninit_context(verbs_device, context); +- free(context_ex); +- } else +- context->device->ops.free_context(context); ++ } else { ++ context->device->ops.free_context(_ibv_real_context(context)); ++ } ++ free(context_ex); + + close(async_fd); + close(cmd_fd); +diff --git a/src/ibverbs.h b/src/ibverbs.h +index 3ad0803..7f8f7ba 100644 +--- a/src/ibverbs.h ++++ b/src/ibverbs.h +@@ -102,6 +102,20 @@ HIDDEN int ibverbs_init(struct ibv_device ***list); + (cmd)->response = (uintptr_t) (out); \ + } while (0) + ++struct verbs_context_private { ++ struct ibv_context *real_context; ++ /* place internal data before verbs_context */ ++ struct verbs_context context_ex; ++}; ++ ++#define IBV_EXTENDED_VERBS (((void *) NULL) - 1) ++ ++static inline int ibv_support_ex(struct ibv_context *context) ++{ ++ return ctx->abi_compat == IBV_EXTENDED_VERBS; ++} ++ ++void _ibv_init_contextex(struct verbs_context_private *priv_ctx); + struct ibv_srq *_ibv_real_srq(struct ibv_srq *srq); + struct ibv_qp *_ibv_real_qp(struct ibv_qp *qp); + struct ibv_cq *_ibv_real_cq(struct ibv_cq *cq);