ibacm: Add functions to load/unload provider.

Signed-off-by: Kaike Wan <kaike.wan@intel.com>

diff --git a/Makefile.am b/Makefile.am
index aec90fab5b542c36e0b867c937e386a206b1e960..4029c635735e0e15453a005ac47a152d72c08a40 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 AM_CPPFLAGS = -I$(srcdir)/include -I$(srcdir)/linux -I$(srcdir)/src
 
-AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DSYSCONFDIR=\"$(sysconfdir)\" -DBINDIR=\"$(bindir)\" -DRDMADIR=\"@rdmadir@\"
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DSYSCONFDIR=\"$(sysconfdir)\" -DBINDIR=\"$(bindir)\" -DRDMADIR=\"@rdmadir@\" -DLIBDIR=\"$(libdir)\"
 
 bin_PROGRAMS = util/ib_acme
 sbin_PROGRAMS = svc/ibacm

diff --git a/include/infiniband/acm_prov.h b/include/infiniband/acm_prov.h
index 52385b146b11fd28d66c951662af5f35b48553fc..9ec407e43678c47febcaf8d1223758fb88cd99ca 100644 (file)
--- a/include/infiniband/acm_prov.h
+++ b/include/infiniband/acm_prov.h
@@ -32,6 +32,8 @@
 
 #include <infiniband/acm.h>
 
+#define ACM_PROV_VERSION          1
+
 struct acm_device {
        struct ibv_context      *verbs;
        uint64_t                dev_guid;
@@ -55,6 +57,9 @@ struct acm_address {
 };
 
 struct acm_provider {
+       size_t    size; 
+       uint32_t  version;
+       char      *name;
        int     (*open_device)(const struct acm_device *device, 
                        void **dev_context);
        void    (*close_device)(void *dev_context);
@@ -76,6 +81,8 @@ struct acm_provider {
 extern atomic_t counter[ACM_MAX_COUNTER];
 extern char *opts_file;
 
+int provider_query(struct acm_provider **info, uint32_t *version);
+
 /* Functions exported from core */
 #define acm_log(level, format, ...) \
        acm_write(level, "%s: "format, __func__, ## __VA_ARGS__)

diff --git a/src/acm.c b/src/acm.c
index d4869924098dd9b987c58c469ad1d57a7ee86d27..e1a4d9200cba17d1c85600cf4dd6c4ae07dda473 100644 (file)
--- a/src/acm.c
+++ b/src/acm.c
@@ -41,11 +41,13 @@
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <fcntl.h>
+#include <dirent.h>
 #include <infiniband/acm.h>
 #include <infiniband/acm_prov.h>
 #include <infiniband/umad.h>
 #include <infiniband/verbs.h>
 #include <dlist.h>
+#include <dlfcn.h> 
 #include <search.h>
 #include <net/if.h>
 #include <sys/ioctl.h>
@@ -98,6 +100,12 @@ enum acmp_addr_preload {
        ACMP_ADDR_PRELOAD_HOSTS
 };
 
+struct acmc_prov {
+       struct acm_provider    *prov;
+       void                   *handle; 
+       DLIST_ENTRY            entry;   
+};
+
 /*
  * Nested locking order: dest -> ep, dest -> port
  */
@@ -283,6 +291,9 @@ static int acmp_query(void *addr_context, struct acm_msg *msg, uint64_t id);
 static int acmp_handle_event(void *port_context, enum ibv_event_type type);
 
 static struct acm_provider def_prov = {
+       .size = sizeof(struct acm_provider),
+       .version = ACM_PROV_VERSION,
+       .name = "ibacmp",
        .open_device = acmp_open_dev,
        .close_device = acmp_close_dev,
        .open_port = acmp_open_port,
@@ -302,6 +313,10 @@ union socket_addr {
        struct sockaddr_in6 sin6;
 };
 
+static char *def_prov_name = "ibacmp";
+static DLIST_ENTRY provider_list;
+static struct acmc_prov *def_provider = NULL;
+
 static DLIST_ENTRY dev_list;
 static DLIST_ENTRY acmp_dev_list;
 static lock_t acmp_dev_lock;
@@ -357,6 +372,7 @@ static uint8_t min_rate = IBV_RATE_10_GBPS;
 static enum acmp_route_preload route_preload;
 static enum acmp_addr_preload addr_preload;
 static int support_ips_in_addr_cfg = 0;
+static char *prov_lib_path = LIBDIR "/ibacm";
 
 void acm_write(int level, const char *format, ...)
 {
@@ -4589,6 +4605,106 @@ static int acm_open_devices(void)
        return 0;
 }
 
+static int acm_open_providers(void)
+{
+       DIR *shlib_dir;
+       struct dirent *dent;
+       char file_name[256];
+       struct stat buf;
+       void *handle;
+       struct acmc_prov *prov;
+       struct acm_provider *provider;
+       uint32_t version;
+       char *err_str;
+       int (*query)(struct acm_provider **, uint32_t *);
+
+       acm_log(1, "\n");
+       shlib_dir = opendir(prov_lib_path);
+       if (!shlib_dir) {
+               acm_log(0, "ERROR - could not open provider lib dir: %s\n",
+                       prov_lib_path);
+               return -1;
+       }
+
+       while ((dent = readdir(shlib_dir))) {
+               if (!strstr(dent->d_name, ".so"))  
+                       continue;
+
+               snprintf(file_name, sizeof(file_name), "%s/%s", prov_lib_path,
+                        dent->d_name);
+               if (lstat(file_name, &buf)) {
+                       acm_log(0, "Error - could not stat: %s\n", file_name);
+                       continue;
+               }
+               if (!S_ISREG(buf.st_mode))
+                       continue;
+
+               acm_log(2, "Loading provider %s...\n", file_name);
+               if (!(handle = dlopen(file_name, RTLD_LAZY))) {
+                       acm_log(0, "Error - could not load provider %s (%s)\n",
+                               file_name, dlerror());
+                       continue;
+               }
+
+               query = dlsym(handle, "provider_query");
+               if ((err_str = dlerror()) != NULL) {
+                       acm_log(0, "Error -provider_query not found in %s (%s)\n",
+                               file_name, err_str);
+                       dlclose(handle);
+                       continue;
+               }
+
+               if (query(&provider, &version)) {
+                       acm_log(0, "Error - provider_query failed to %s\n", file_name);
+                       dlclose(handle);
+                       continue;
+               }
+
+               if (version != ACM_PROV_VERSION ||
+                   provider->size != sizeof(struct acm_provider)) {
+                       acm_log(0, "Error -unmatched provider version 0x%08x (size %d)"
+                               " core 0x%08x (size %d)\n", version, provider->size,
+                               ACM_PROV_VERSION, sizeof(struct acm_provider));
+                       dlclose(handle);
+                       continue;
+               }
+
+               acm_log(1, "Provider %s (%s) loaded\n", provider->name, file_name);
+
+               prov = calloc(1, sizeof(*prov));
+               if (!prov) {
+                       acm_log(0, "Error -failed to allocate provider %s\n", file_name);
+                       dlclose(handle);
+                       continue;
+               }
+
+               prov->prov = provider;
+               prov->handle = handle;
+               DListInsertTail(&prov->entry, &provider_list);
+               if (!strcasecmp(provider->name, def_prov_name)) 
+                       def_provider = prov;
+       }
+
+       closedir(shlib_dir);
+       return 0;
+}
+
+static void acm_close_providers(void)
+{
+       struct acmc_prov *prov;
+       DLIST_ENTRY *entry;
+
+       acm_log(1, "\n");
+       def_provider = NULL;
+       while (!DListEmpty(&provider_list)) {
+               entry = provider_list.Next;
+               DListRemove(entry);
+               prov = container_of(entry, struct acmc_prov, entry);
+               dlclose(prov->handle);
+               free(prov);
+       }
+}
+
 static void acm_set_options(void)
 {
        FILE *f;
@@ -4613,6 +4729,8 @@ static void acm_set_options(void)
                        strcpy(lock_file, value);
                else if (!stricmp("server_port", opt))
                        server_port = (short) atoi(value);
+               else if (!stricmp("provider_lib_path", opt))
+                       strcpy(prov_lib_path, value);
        }
 
        fclose(f);
@@ -4681,6 +4799,7 @@ static void acm_log_options(void)
        acm_log(0, "log level %d\n", log_level);
        acm_log(0, "lock file %s\n", lock_file);
        acm_log(0, "server_port %d\n", server_port);
+       acm_log(0, "provider lib path %s\n", prov_lib_path);
 }
 
 static void acmp_log_options(void)
@@ -4822,6 +4941,7 @@ int CDECL_FUNC main(int argc, char **argv)
 
        atomic_init(&tid);
        atomic_init(&wait_cnt);
+       DListInit(&provider_list);
        DListInit(&dev_list);
        DListInit(&acmp_dev_list);
        lock_init(&acmp_dev_lock);
@@ -4831,6 +4951,12 @@ int CDECL_FUNC main(int argc, char **argv)
                atomic_init(&counter[i]);
 
        umad_init();
+
+       if (acm_open_providers()) {
+               acm_log(0, "ERROR - unable to open any providers\n");
+               return -1;
+       }
+
        if (acm_open_devices()) {
                acm_log(0, "ERROR - unable to open any devices\n");
                return -1;
@@ -4858,6 +4984,7 @@ int CDECL_FUNC main(int argc, char **argv)
                        acm_log(0, "Error: failed to join the retry thread\n");
                retry_thread_started = 0;
        }
+       acm_close_providers();
        fclose(flog);
        return 0;
 }