Fix static linking of libibverbs-1.0

Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/ChangeLog b/ChangeLog
index 971474ab8c9dede204d28abcdb476a3c26cf8684..2ccfda05a0c760834adfa50a3602d356d6bcee69 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2006-10-27  Roland Dreier  <rdreier@cisco.com>
+
+       * src/init.c: Revise initialization order to fix static linking.
+       Using dlopen() on a device-specific driver from a statically
+       linked copy of libibverbs will crash, because the driver will
+       bring in dynamic copies of libibverbs and libdl that clash with
+       the copies already linked statically.
+
+       To fix this, we change the way we search for drivers: first we
+       find all uverbs devices and try the driver (if any) that is
+       linked in directly.  If all devices are handled by that driver,
+       then we don't proceed any further.  If not, then we try dynamic
+       loading of drivers and match them against any remaining devices.
+
 2006-10-17  Roland Dreier  <rdreier@cisco.com>
 
        * include/infiniband/arch.h: Update i386 and x86_64 memory barrier

diff --git a/README b/README
index 063a3341b9add87e641f27cbc8765f05b3a60bc2..7b5780e11abc028e26da76b1633dad2fef6ea3e5 100644 (file)
--- a/README
+++ b/README
@@ -60,6 +60,23 @@ via the file /etc/security/limits.conf.  More configuration may be
 necessary if you are logging in via OpenSSH and your sshd is
 configured to use privilege separation.
 
+Static linking
+--------------
+
+In almost all cases it is better to dynamically link libibverbs into
+an application.  However, if you are forced to use static linking for
+libibverbs, then you will also have to link a device-specific
+userspace driver (such as libmthca, libipathverbs, libehca, etc)
+statically into your application.  This is because of limitations on
+dynamically loading new modules into a static executable.
+
+In particular, a static application can only be linked against a
+single device-specific driver, which means that the application will
+only work with a single type of device.  This limitation will be
+removed in future libibverbs releases, but this will require a change
+to the libibverbs ABI, so it cannot be done as part of the libibverbs
+1.0 release series.
+
 Valgrind support
 ----------------
 

diff --git a/src/ibverbs.h b/src/ibverbs.h
index d6b9fe7788d5c5a7a3101d77c8ce912820bb9b28..61148d5b98f504e0728fbc2ce603a0a72c68f883 100644 (file)
--- a/src/ibverbs.h
+++ b/src/ibverbs.h
@@ -60,6 +60,12 @@
 
 #define PFX            "libibverbs: "
 
+struct ibv_sysfs_dev {
+       struct sysfs_class_device      *verbs_dev;
+       struct ibv_sysfs_dev           *next;
+       int                             have_driver;
+};
+
 struct ibv_driver {
        ibv_driver_init_func    init_func;
        struct ibv_driver      *next;

diff --git a/src/init.c b/src/init.c
index 2e45a55e2b45c4f1d1ef2a49ac1d972e02b0c55d..36040e7361ca988152764d800792b3a6cabfbaa1 100644 (file)
--- a/src/init.c
+++ b/src/init.c
@@ -52,11 +52,52 @@
 
 HIDDEN int abi_ver;
 
-static char default_path[] = DRIVER_PATH;
+static const char default_path[] = DRIVER_PATH;
 static const char *user_path;
 
+static struct ibv_sysfs_dev *sysfs_dev_list;
 static struct ibv_driver *driver_list;
 
+static void find_sysfs_devs(void)
+{
+       struct sysfs_class *cls;
+       struct dlist *verbs_dev_list;
+       struct sysfs_class_device *verbs_dev;
+       struct ibv_sysfs_dev *dev;
+
+       cls = sysfs_open_class("infiniband_verbs");
+       if (!cls) {
+               fprintf(stderr, PFX "Fatal: couldn't open sysfs class 'infiniband_verbs'.\n");
+               return;
+       }
+
+       verbs_dev_list = sysfs_get_class_devices(cls);
+       if (!verbs_dev_list) {
+               fprintf(stderr, PFX "Fatal: no infiniband class devices found.\n");
+               return;
+       }
+
+       dlist_for_each_data(verbs_dev_list, verbs_dev, struct sysfs_class_device) {
+               dev = malloc(sizeof *dev);
+               if (!dev) {
+                       fprintf(stderr, PFX "Warning: couldn't allocate device for %s\n",
+                               verbs_dev->name);
+                       continue;
+               }
+
+               dev->verbs_dev   = verbs_dev;
+               dev->next        = sysfs_dev_list;
+               dev->have_driver = 0;
+               sysfs_dev_list   = dev;
+       }
+}
+
+__attribute__((weak))
+struct ibv_device *openib_driver_init(struct sysfs_class_device *dev)
+{
+        return NULL;
+}
+
 static void load_driver(char *so_path)
 {
        void *dlhandle;
@@ -79,7 +120,7 @@ static void load_driver(char *so_path)
 
        driver = malloc(sizeof *driver);
        if (!driver) {
-               fprintf(stderr, PFX "Fatal: couldn't allocate driver for %s\n", so_path);
+               fprintf(stderr, PFX "Warning: couldn't allocate driver for %s\n", so_path);
                dlclose(dlhandle);
                return;
        }
@@ -89,7 +130,7 @@ static void load_driver(char *so_path)
        driver_list       = driver;
 }
 
-static void find_drivers(char *dir)
+static void find_drivers(const char *dir)
 {
        size_t len = strlen(dir);
        glob_t so_glob;
@@ -101,9 +142,9 @@ static void find_drivers(char *dir)
                return;
 
        while (len && dir[len - 1] == '/')
-               dir[--len] = '\0';
+               --len;
 
-       asprintf(&pat, "%s/*.so", dir);
+       asprintf(&pat, "%.*s/*.so", (int) len, dir);
 
        ret = glob(pat, 0, NULL, &so_glob);
        free(pat);
@@ -120,10 +161,10 @@ static void find_drivers(char *dir)
        globfree(&so_glob);
 }
 
-static struct ibv_device *init_drivers(struct sysfs_class_device *verbs_dev)
+static struct ibv_device *try_driver(ibv_driver_init_func init_func,
+                                    struct sysfs_class_device *verbs_dev)
 {
        struct sysfs_class_device *ib_dev;
-       struct ibv_driver *driver;
        struct ibv_device *dev;
        char ibdev_name[64];
 
@@ -141,24 +182,14 @@ static struct ibv_device *init_drivers(struct sysfs_class_device *verbs_dev)
                return NULL;
        }
 
-       for (driver = driver_list; driver; driver = driver->next) {
-               dev = driver->init_func(verbs_dev);
-               if (dev) {
-                       dev->dev    = verbs_dev;
-                       dev->ibdev  = ib_dev;
-                       dev->driver = driver;
-
-                       return dev;
-               }
+       dev = init_func(verbs_dev);
+       if (dev) {
+               dev->dev    = verbs_dev;
+               dev->ibdev  = ib_dev;
+               dev->driver = NULL;
        }
 
-       fprintf(stderr, PFX "Warning: no userspace device-specific driver found for %s\n"
-               "       driver search path: ", verbs_dev->name);
-       if (user_path)
-               fprintf(stderr, "%s:", user_path);
-       fprintf(stderr, "%s\n", default_path);
-
-       return NULL;
+       return dev;
 }
 
 static int check_abi_version(void)
@@ -191,26 +222,87 @@ static int check_abi_version(void)
        return 0;
 }
 
+static void add_device(struct ibv_device *dev,
+                      struct ibv_device ***dev_list,
+                      int *num_devices,
+                      int *list_size)
+{
+       struct ibv_device **new_list;
+
+       if (*list_size <= *num_devices) {
+               *list_size = *list_size ? *list_size * 2 : 1;
+               new_list = realloc(*dev_list, *list_size * sizeof (struct ibv_device *));
+               if (!new_list)
+                       return;
+               *dev_list = new_list;
+       }
+
+       *dev_list[*num_devices++] = dev;
+}
+
 HIDDEN int ibverbs_init(struct ibv_device ***list)
 {
        char *wr_path, *dir;
-       struct sysfs_class *cls;
-       struct dlist *verbs_dev_list;
-       struct sysfs_class_device *verbs_dev;
+       struct ibv_sysfs_dev *sysfs_dev, *next_dev;
        struct ibv_device *device;
-       struct ibv_device **new_list;
+       struct ibv_driver *driver;
        int num_devices = 0;
        int list_size = 0;
+       int no_driver = 0;
+       int statically_linked = 0;
 
        *list = NULL;
 
+       if (check_abi_version())
+               return 0;
+
        if (ibv_init_mem_map())
                return 0;
 
+       find_sysfs_devs();
+
+       /*
+        * First check if a driver statically linked in can support
+        * all the devices.  This is needed to avoid dlopen() in the
+        * all-static case (which will break because we end up with
+        * both a static and dynamic copy of libdl).
+        */
+       for (sysfs_dev = sysfs_dev_list; sysfs_dev; sysfs_dev = sysfs_dev->next) {
+               device = try_driver(openib_driver_init, sysfs_dev->verbs_dev);
+               if (device) {
+                       add_device(device, list, &num_devices, &list_size);
+                       sysfs_dev->have_driver = 1;
+               } else
+                       ++no_driver;
+       }
+
+       if (!no_driver)
+               goto out;
+
+       /*
+        * Check if we can dlopen() ourselves.  If this fails,
+        * libibverbs is probably statically linked into the
+        * executable, and we should just give up, since trying to
+        * dlopen() a driver module will fail spectacularly (loading a
+        * driver .so will bring in dynamic copies of libibverbs and
+        * libdl to go along with the static copies the executable
+        * has, which quickly leads to a crash.
+        */
+       {
+               void *hand = dlopen(NULL, RTLD_NOW);
+               if (!hand) {
+                       fprintf(stderr, PFX "Warning: dlopen(NULL) failed, "
+                               "assuming static linking.\n");
+                       statically_linked = 1;
+                       goto out;
+               }
+               dlclose(hand);
+       }
+
        find_drivers(default_path);
 
        /*
-        * Only follow use path passed in through the calling user's
+        * Only use path passed in through the calling user's
         * environment if we're not running SUID.
         */
        if (getuid() == geteuid()) {
@@ -222,42 +314,37 @@ HIDDEN int ibverbs_init(struct ibv_device ***list)
                }
        }
 
-       /*
-        * Now check if a driver is statically linked.  Since we push
-        * drivers onto our driver list, the last driver we find will
-        * be the first one we try.
-        */
-       load_driver(NULL);
-
-       cls = sysfs_open_class("infiniband_verbs");
-       if (!cls) {
-               fprintf(stderr, PFX "Fatal: couldn't open sysfs class 'infiniband_verbs'.\n");
-               return 0;
-       }
-
-       if (check_abi_version())
-               return 0;
-
-       verbs_dev_list = sysfs_get_class_devices(cls);
-       if (!verbs_dev_list) {
-               fprintf(stderr, PFX "Fatal: no infiniband class devices found.\n");
-               return 0;
+       for (sysfs_dev = sysfs_dev_list; sysfs_dev; sysfs_dev = sysfs_dev->next) {
+               if (sysfs_dev->have_driver)
+                       continue;
+               for (driver = driver_list; driver; driver = driver->next) {
+                       device = try_driver(driver->init_func, sysfs_dev->verbs_dev);
+                       if (device) {
+                               add_device(device, list, &num_devices, &list_size);
+                               sysfs_dev->have_driver = 1;
+                       }
+               }
        }
 
-       dlist_for_each_data(verbs_dev_list, verbs_dev, struct sysfs_class_device) {
-               device = init_drivers(verbs_dev);
-               if (device) {
-                       if (list_size <= num_devices) {
-                               list_size = list_size ? list_size * 2 : 1;
-                               new_list = realloc(*list, list_size * sizeof (struct ibv_device *));
-                               if (!new_list)
-                                       goto out;
-                               *list = new_list;
+out:
+       for (sysfs_dev = sysfs_dev_list, next_dev = sysfs_dev->next;
+            sysfs_dev;
+            sysfs_dev = next_dev, next_dev = sysfs_dev ? sysfs_dev->next : NULL) {
+               if (!sysfs_dev->have_driver) {
+                       fprintf(stderr, PFX "Warning: no userspace device-specific "
+                               " driver found for %s\n", sysfs_dev->verbs_dev->name);
+                       if (statically_linked)
+                               fprintf(stderr, "       When linking libibverbs statically, "
+                                       "driver must be statically linked too.\n");
+                       else {
+                               fprintf(stderr, "       driver search path: ");
+                               if (user_path)
+                                       fprintf(stderr, "%s:", user_path);
+                               fprintf(stderr, "%s\n", default_path);
                        }
-                       (*list)[num_devices++] = device;
                }
+               free(sysfs_dev);
        }
 
-out:
        return num_devices;
 }