From: Yishai Hadas Date: Sun, 26 Aug 2012 13:43:30 +0000 (+0300) Subject: Infrastructure to support verbs extensions X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=b3d744f488bc97442df7e6cef8536d3b0352a786;p=~shefty%2Flibibverbs.git Infrastructure to support verbs extensions Infrastructure to support extended verbs capabilities in a forward/backward manner. The general operation as shown in the following pseudo-code: ibv_open_device() { context = device->ops.alloc_context(); if (context == -1) { context_ex = malloc(verbs_context + verbs_device->context_size); verbs_device->init_context(context_ex); context_ex->context.abi_compat = -1; } } If the underlying provider supports extensions, it returns -1 from its alloc_context() call. Ibverbs then allocates the ibv_context structure and calls into the provider to finish initializing it. When extensions are supported, the ibv_device structure is embedded in a larger verbs_device structure. Similarly, ibv_context is embedded inside a larger verbs_context structure. Signed-off-by: Yishai Hadas Signed-off-by: Tzahi Oved --- diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h index 9a81416..5af0d7f 100644 --- a/include/infiniband/driver.h +++ b/include/infiniband/driver.h @@ -57,6 +57,7 @@ typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path, int abi_version); void ibv_register_driver(const char *name, ibv_driver_init_func init_func); +void verbs_register_driver(const char *name, ibv_driver_init_func init_func); int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, size_t cmd_size, struct ibv_get_context_resp *resp, size_t resp_size); diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index 6acfc81..a2577d8 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -38,6 +38,7 @@ #include #include +#include #ifdef __cplusplus # define BEGIN_C_DECLS extern "C" { @@ -63,6 +64,19 @@ union ibv_gid { } global; }; +#ifndef container_of +/** + * container_of - cast a member of a structure out to the containing structure + * @ptr: the pointer to the member. + * @type: the type of the container struct this is embedded in. + * @member: the name of the member within the struct. + * + */ +#define container_of(ptr, type, member) ({\ + const typeof(((type *)0)->member) * __mptr = (ptr);\ + (type *)((char *)__mptr - offsetof(type, member)); }) +#endif + enum ibv_node_type { IBV_NODE_UNKNOWN = -1, IBV_NODE_CA = 1, @@ -634,6 +648,17 @@ struct ibv_device { char ibdev_path[IBV_SYSFS_PATH_MAX]; }; +struct verbs_device { + struct ibv_device device; /* Must be first */ + size_t sz; + size_t size_of_context; + int (*init_context)(struct verbs_device *device, + struct ibv_context *ctx, int cmd_fd); + void (*uninit_context)(struct verbs_device *device, + struct ibv_context *ctx); + /* future fields added here */ +}; + struct ibv_context_ops { int (*query_device)(struct ibv_context *context, struct ibv_device_attr *device_attr); @@ -702,6 +727,33 @@ struct ibv_context { void *abi_compat; }; +struct verbs_context { + + /* "grows up" - new fields go here + int (*drv_new_func1) (); new corresponding provider call of func1 + int (*lib_new_func1) (); New library call func1 + */ + size_t sz; /* Set by library on struct allocation,must be + * located right before struct ibv_context + */ + struct ibv_context context;/* Must be last field in the struct */ +}; + +static inline struct verbs_context *verbs_get_ctx( + const struct ibv_context *ctx) +{ + if (ctx->abi_compat != ((uint8_t *)NULL)-1) + return NULL; + + return container_of(ctx, struct verbs_context, context); +} + +static inline struct verbs_device *verbs_get_device( + const struct ibv_device *dev) +{ + return container_of(dev, struct verbs_device, device); +} + /** * ibv_get_device_list - Get list of IB devices currently available * @num_devices: optional. if non-NULL, set to the number of devices diff --git a/src/cmd.c b/src/cmd.c index 9789092..dab8930 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -45,52 +45,13 @@ #include "ibverbs.h" -static int ibv_cmd_get_context_v2(struct ibv_context *context, - struct ibv_get_context *new_cmd, - size_t new_cmd_size, - struct ibv_get_context_resp *resp, - size_t resp_size) -{ - struct ibv_abi_compat_v2 *t; - struct ibv_get_context_v2 *cmd; - size_t cmd_size; - uint32_t cq_fd; - - t = malloc(sizeof *t); - if (!t) - return ENOMEM; - pthread_mutex_init(&t->in_use, NULL); - - cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd; - cmd = alloca(cmd_size); - memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd); - - IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size); - cmd->cq_fd_tab = (uintptr_t) &cq_fd; - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) { - free(t); - return errno; - } - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - context->async_fd = resp->async_fd; - context->num_comp_vectors = 1; - t->channel.context = context; - t->channel.fd = cq_fd; - t->channel.refcnt = 0; - context->abi_compat = t; - - return 0; -} int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, size_t cmd_size, struct ibv_get_context_resp *resp, size_t resp_size) { if (abi_ver <= 2) - return ibv_cmd_get_context_v2(context, cmd, cmd_size, resp, resp_size); + return ENOSYS; IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size); diff --git a/src/device.c b/src/device.c index 5798895..9e43138 100644 --- a/src/device.c +++ b/src/device.c @@ -127,6 +127,7 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) char *devpath; int cmd_fd; struct ibv_context *context; + struct verbs_context *context_ex; if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0) return NULL; @@ -144,6 +145,36 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) context = device->ops.alloc_context(device, cmd_fd); if (!context) goto err; + if (context == (struct ibv_context *)(((uint8_t *)NULL)-1)) { + /* New provider that supports verbs extension was detected */ + struct verbs_device *verbs_device = + verbs_get_device(device); + int ret; + + /* Library now allocates the context */ + context_ex = calloc(1, sizeof(*context_ex) + + verbs_device->size_of_context); + + if (!context_ex) { + errno = ENOMEM; + goto err; + } + context = &context_ex->context; + /* Init new verbs_context */ + context_ex->context.abi_compat = ((uint8_t *)NULL)-1; + context_ex->sz = sizeof(*context_ex); + + /* Call provider to initialize its calls first */ + ret = verbs_device->init_context(verbs_device, + &context_ex->context, cmd_fd); + if (ret) + goto verbs_err; + /* initialize *all* library ops to either lib calls or + * directly to provider calls. + context_ex-> lib_new_func1= __verbs_new_func1; + context_ex-> lib_new_func2= __verbs_new_func2; + */ + } context->device = device; context->cmd_fd = cmd_fd; @@ -151,6 +182,8 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) return context; +verbs_err: + free(context_ex); err: close(cmd_fd); @@ -163,14 +196,17 @@ int __ibv_close_device(struct ibv_context *context) int async_fd = context->async_fd; int cmd_fd = context->cmd_fd; int cq_fd = -1; - - if (abi_ver <= 2) { - struct ibv_abi_compat_v2 *t = context->abi_compat; - cq_fd = t->channel.fd; - free(context->abi_compat); - } - - context->device->ops.free_context(context); + struct verbs_context *context_ex; + + context_ex = verbs_get_ctx(context); + if (context_ex) { + struct verbs_device *verbs_device = + verbs_get_device(context->device); + /* Provider supports verbs extension */ + verbs_device->uninit_context(verbs_device, context); + free(context_ex); + } else + context->device->ops.free_context(context); close(async_fd); close(cmd_fd); diff --git a/src/init.c b/src/init.c index 8d6786e..a1b0905 100644 --- a/src/init.c +++ b/src/init.c @@ -174,6 +174,14 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func) tail_driver = driver; } +/* New registration symbol with same functionality - used by providers to + * validate that library supports verbs extension. + */ +void verbs_register_driver(const char *name, ibv_driver_init_func init_func) +{ + ibv_register_driver(name, init_func); +} + static void load_driver(const char *name) { char *so_name; diff --git a/src/libibverbs.map b/src/libibverbs.map index 1827da0..ee9adea 100644 --- a/src/libibverbs.map +++ b/src/libibverbs.map @@ -91,6 +91,7 @@ IBVERBS_1.1 { ibv_dontfork_range; ibv_dofork_range; ibv_register_driver; + verbs_register_driver; ibv_node_type_str; ibv_port_state_str;