Infrastructure to support verbs extensions

Infrastructure to support extended verbs capabilities in a forward/backward
manner.

The general operation as shown in the following pseudo-code:

ibv_open_device()
{
	context = device->ops.alloc_context();
	if (context == -1) {
		context_ex = malloc(verbs_context + verbs_device->context_size);
		verbs_device->init_context(context_ex);
		context_ex->context.abi_compat = -1;
	}
}

If the underlying provider supports extensions, it returns -1 from its
alloc_context() call.  Ibverbs then allocates the ibv_context structure and
calls into the provider to finish initializing it.

When extensions are supported, the ibv_device structure is embedded in a
larger verbs_device structure.  Similarly, ibv_context is embedded inside
a larger verbs_context structure.

Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Tzahi Oved <tzahio@mellanox.com>

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 9a81416febb54852784faf677d5eeb65c7f89204..5af0d7fac2701d8b8f4eaf7b3ca59c35d6d665c2 100644 (file)
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -57,6 +57,7 @@ typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
                                                   int abi_version);
 
 void ibv_register_driver(const char *name, ibv_driver_init_func init_func);
+void verbs_register_driver(const char *name, ibv_driver_init_func init_func);
 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
                        size_t cmd_size, struct ibv_get_context_resp *resp,
                        size_t resp_size);

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 6acfc8131f42641bfc48b848806befaa9dade56a..a2577d87ddc9a86973367675f546ffc8eb6e836e 100644 (file)
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -38,6 +38,7 @@
 
 #include <stdint.h>
 #include <pthread.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 #  define BEGIN_C_DECLS extern "C" {
@@ -63,6 +64,19 @@ union ibv_gid {
        } global;
 };
 
+#ifndef container_of
+/**
+  * container_of - cast a member of a structure out to the containing structure
+  * @ptr:        the pointer to the member.
+  * @type:       the type of the container struct this is embedded in.
+  * @member:     the name of the member within the struct.
+  *
+ */
+#define container_of(ptr, type, member) ({\
+       const typeof(((type *)0)->member) * __mptr = (ptr);\
+       (type *)((char *)__mptr - offsetof(type, member)); })
+#endif
+
 enum ibv_node_type {
        IBV_NODE_UNKNOWN        = -1,
        IBV_NODE_CA             = 1,
@@ -634,6 +648,17 @@ struct ibv_device {
        char                    ibdev_path[IBV_SYSFS_PATH_MAX];
 };
 
+struct verbs_device {
+       struct ibv_device device; /* Must be first */
+       size_t  sz;
+       size_t  size_of_context;
+       int     (*init_context)(struct verbs_device *device,
+                               struct ibv_context *ctx, int cmd_fd);
+       void    (*uninit_context)(struct verbs_device *device,
+                               struct ibv_context *ctx);
+       /* future fields added here */
+};
+
 struct ibv_context_ops {
        int                     (*query_device)(struct ibv_context *context,
                                              struct ibv_device_attr *device_attr);
@@ -702,6 +727,33 @@ struct ibv_context {
        void                   *abi_compat;
 };
 
+struct verbs_context {
+
+       /*  "grows up" - new fields go here
+       int (*drv_new_func1) ();        new corresponding provider call of func1
+       int (*lib_new_func1) ();        New library call func1
+       */
+       size_t sz;      /* Set by library on struct allocation,must be
+                       * located right before struct ibv_context
+                       */
+       struct ibv_context context;/* Must be last field in the struct */
+};
+
+static inline struct verbs_context *verbs_get_ctx(
+                                       const struct ibv_context *ctx)
+{
+       if (ctx->abi_compat != ((uint8_t *)NULL)-1)
+               return NULL;
+
+       return container_of(ctx, struct verbs_context, context);
+}
+
+static inline struct verbs_device *verbs_get_device(
+                                       const struct ibv_device *dev)
+{
+       return container_of(dev, struct verbs_device, device);
+}
+
 /**
  * ibv_get_device_list - Get list of IB devices currently available
  * @num_devices: optional.  if non-NULL, set to the number of devices

diff --git a/src/cmd.c b/src/cmd.c
index 9789092489140c680c5feebae3bbcd9a72218115..dab8930790041c6ec39394caacd0e02aa1bf0ef6 100644 (file)
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -45,52 +45,13 @@
 
 #include "ibverbs.h"
 
-static int ibv_cmd_get_context_v2(struct ibv_context *context,
-                                 struct ibv_get_context *new_cmd,
-                                 size_t new_cmd_size,
-                                 struct ibv_get_context_resp *resp,
-                                 size_t resp_size)
-{
-       struct ibv_abi_compat_v2 *t;
-       struct ibv_get_context_v2 *cmd;
-       size_t cmd_size;
-       uint32_t cq_fd;
-
-       t = malloc(sizeof *t);
-       if (!t)
-               return ENOMEM;
-       pthread_mutex_init(&t->in_use, NULL);
-
-       cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
-       cmd      = alloca(cmd_size);
-       memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
-
-       IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
-       cmd->cq_fd_tab = (uintptr_t) &cq_fd;
-
-       if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) {
-               free(t);
-               return errno;
-       }
-
-       (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
-       context->async_fd         = resp->async_fd;
-       context->num_comp_vectors = 1;
-       t->channel.context        = context;
-       t->channel.fd             = cq_fd;
-       t->channel.refcnt         = 0;
-       context->abi_compat       = t;
-
-       return 0;
-}
 
 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
                        size_t cmd_size, struct ibv_get_context_resp *resp,
                        size_t resp_size)
 {
        if (abi_ver <= 2)
-               return ibv_cmd_get_context_v2(context, cmd, cmd_size, resp, resp_size);
+               return ENOSYS;
 
        IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
 

diff --git a/src/device.c b/src/device.c
index 57988955c84839a35385515aaa486b7b6f4625e9..9e43138e2b8397085993ccdf275f10c6504eb27e 100644 (file)
--- a/src/device.c
+++ b/src/device.c
@@ -127,6 +127,7 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
        char *devpath;
        int cmd_fd;
        struct ibv_context *context;
+       struct verbs_context *context_ex;
 
        if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0)
                return NULL;
@@ -144,6 +145,36 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
        context = device->ops.alloc_context(device, cmd_fd);
        if (!context)
                goto err;
+       if (context == (struct ibv_context *)(((uint8_t *)NULL)-1)) {
+               /* New provider that supports verbs extension was detected */
+               struct verbs_device *verbs_device =
+                                       verbs_get_device(device);
+               int ret;
+
+               /* Library now allocates the context */
+               context_ex = calloc(1, sizeof(*context_ex) +
+                       verbs_device->size_of_context);
+
+               if (!context_ex) {
+                       errno = ENOMEM;
+                       goto err;
+               }
+               context = &context_ex->context;
+               /* Init new verbs_context */
+               context_ex->context.abi_compat  = ((uint8_t *)NULL)-1;
+               context_ex->sz = sizeof(*context_ex);
+
+               /* Call provider to initialize its calls first */
+               ret = verbs_device->init_context(verbs_device,
+                                       &context_ex->context, cmd_fd);
+               if (ret)
+                       goto verbs_err;
+               /* initialize *all* library ops to either lib calls or
+                 * directly to provider calls.
+               context_ex-> lib_new_func1= __verbs_new_func1;
+               context_ex-> lib_new_func2= __verbs_new_func2;
+               */
+       }
 
        context->device = device;
        context->cmd_fd = cmd_fd;
@@ -151,6 +182,8 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 
        return context;
 
+verbs_err:
+       free(context_ex);
 err:
        close(cmd_fd);
 
@@ -163,14 +196,17 @@ int __ibv_close_device(struct ibv_context *context)
        int async_fd = context->async_fd;
        int cmd_fd   = context->cmd_fd;
        int cq_fd    = -1;
-
-       if (abi_ver <= 2) {
-               struct ibv_abi_compat_v2 *t = context->abi_compat;
-               cq_fd = t->channel.fd;
-               free(context->abi_compat);
-       }
-
-       context->device->ops.free_context(context);
+       struct verbs_context *context_ex;
+
+       context_ex = verbs_get_ctx(context);
+       if (context_ex) {
+               struct verbs_device *verbs_device =
+                                       verbs_get_device(context->device);
+               /* Provider supports verbs extension */
+               verbs_device->uninit_context(verbs_device, context);
+               free(context_ex);
+       } else
+               context->device->ops.free_context(context);
 
        close(async_fd);
        close(cmd_fd);

diff --git a/src/init.c b/src/init.c
index 8d6786e509cb578d684388fb23de7117dc4487ec..a1b0905ca1f8f88ce327f6da78ede08a90666b21 100644 (file)
--- a/src/init.c
+++ b/src/init.c
@@ -174,6 +174,14 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
        tail_driver = driver;
 }
 
+/* New registration symbol with same functionality - used by providers to
+  * validate that library supports verbs extension.
+  */
+void verbs_register_driver(const char *name, ibv_driver_init_func init_func)
+{
+       ibv_register_driver(name, init_func);
+}
+
 static void load_driver(const char *name)
 {
        char *so_name;

diff --git a/src/libibverbs.map b/src/libibverbs.map
index 1827da058331a7b1de10681ea8e1ef18ed53ad3b..ee9adea7105d2fb268d6fbf7b136a1ec21b3bc01 100644 (file)
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -91,6 +91,7 @@ IBVERBS_1.1 {
                ibv_dontfork_range;
                ibv_dofork_range;
                ibv_register_driver;
+               verbs_register_driver;
 
                ibv_node_type_str;
                ibv_port_state_str;