From 283e7162528ce45ac4ca9a3be0298ddfacec29c8 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Thu, 27 Jun 2013 22:33:43 -0700 Subject: [PATCH] Refresh of add-the-ability-to-preload-the --- src/acm.c | 72 ++++++++++++++++++++++++++++-------------------------- src/acme.c | 26 +++++++++++--------- 2 files changed, 53 insertions(+), 45 deletions(-) diff --git a/src/acm.c b/src/acm.c index 1df3ed5..c640132 100644 --- a/src/acm.c +++ b/src/acm.c @@ -77,9 +77,9 @@ enum acm_loopback_prot { ACM_LOOPBACK_PROT_LOCAL }; -enum acm_path_rec_fmt { - ACM_PATH_REV_FMT_NONE, - ACM_PATH_REV_FMT_OSM_FULL_V1 +enum acm_route_preload { + ACM_ROUTE_PRELOAD_NONE, + ACM_ROUTE_PRELOAD_OSM_FULL_V1 }; /* @@ -213,12 +213,12 @@ PER_THREAD char log_data[ACM_MAX_ADDRESS]; static atomic_t counter[ACM_MAX_COUNTER]; /* - * Service options - may be set through acm_opts file. + * Service options - may be set through ibacm_opts.cfg file. */ static char *acme = BINDIR "/ib_acme -A"; static char *opts_file = ACM_CONF_DIR "/" ACM_OPTS_FILE; static char *addr_file = ACM_CONF_DIR "/" ACM_ADDR_FILE; -static char path_rec_file[128] = ACM_CONF_DIR "/" ACM_PATH_REC_FILE; +static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data"; static char log_file[128] = "/var/log/ibacm.log"; static int log_level = 0; static char lock_file[128] = "/var/run/ibacm.pid"; @@ -236,7 +236,7 @@ static int send_depth = 1; static int recv_depth = 1024; static uint8_t min_mtu = IBV_MTU_2048; static uint8_t min_rate = IBV_RATE_10_GBPS; -static enum acm_path_rec_fmt path_rec_fmt = ACM_PATH_REV_FMT_NONE; +static enum acm_route_preload route_preload; #define acm_log(level, format, ...) \ acm_write(level, "%s: "format, __func__, ## __VA_ARGS__) @@ -2454,13 +2454,14 @@ static enum acm_loopback_prot acm_convert_loopback_prot(char *param) return loopback_prot; } -static enum acm_path_rec_fmt acm_convert_path_rec_fmt(char *param) +static enum acm_route_preload acm_convert_route_preload(char *param) { - if (!stricmp("none", param)) - return ACM_PATH_REV_FMT_NONE; - else if (!stricmp("full_opensm_v1", param)) - return ACM_PATH_REV_FMT_OSM_FULL_V1; - return path_rec_fmt; + if (!stricmp("none", param) || !stricmp("no", param)) + return ACM_ROUTE_PRELOAD_NONE; + else if (!stricmp("opensm_full_v1", param)) + return ACM_ROUTE_PRELOAD_OSM_FULL_V1; + + return route_preload; } static enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed) @@ -2572,14 +2573,14 @@ static FILE *acm_open_addr_file(void) return fopen(addr_file, "r"); } -static void acm_parse_path_records_pass1(FILE *f, uint64_t *lid2guid) +/* Parse "opensm full v1" file to build LID to GUID table */ +static void acm_parse_osm_fullv1_lid2guid(FILE *f, uint64_t *lid2guid) { char s[128]; char *p, *ptr, *p_guid, *p_lid; uint64_t guid; uint16_t lid; - /* Pass 1 - LID to GUID table */ while (fgets(s, sizeof s, f)) { if (s[0] == '#') continue; @@ -2621,8 +2622,8 @@ static void acm_parse_path_records_pass1(FILE *f, uint64_t *lid2guid) } } -static int acm_parse_path_records_pass2(FILE *f, uint64_t *lid2guid, - struct acm_ep *ep) +/* Parse 'opensm full v1' file to populate PR cache */ +static int acm_parse_osm_fullv1_paths(FILE *f, uint64_t *lid2guid, struct acm_ep *ep) { union ibv_gid sgid, dgid; struct ibv_port_attr attr = { 0 }; @@ -2638,8 +2639,7 @@ static int acm_parse_path_records_pass2(FILE *f, uint64_t *lid2guid, ibv_query_gid(ep->port->dev->verbs, ep->port->port_num, 0, &sgid); - /* Pass 2 - Path records for source to all destinations */ - + /* Search for endpoint's SLID */ while (fgets(s, sizeof s, f)) { if (s[0] == '#') continue; @@ -2676,6 +2676,7 @@ static int acm_parse_path_records_pass2(FILE *f, uint64_t *lid2guid, lid = (uint16_t) strtoul(p_lid, NULL, 0); if (lid != ep->port->lid) continue; + ibv_query_port(ep->port->dev->verbs, ep->port->port_num, &attr); ret = 0; break; @@ -2761,14 +2762,14 @@ static int acm_parse_path_records_pass2(FILE *f, uint64_t *lid2guid, return ret; } -static int acm_parse_path_records(struct acm_ep *ep) +static int acm_parse_osm_fullv1(struct acm_ep *ep) { FILE *f; uint64_t *lid2guid; int ret = 1; - if (!(f = fopen(path_rec_file, "r"))) { - acm_log(0, "ERROR - couldn't open %s\n", path_rec_file); + if (!(f = fopen(route_data_file, "r"))) { + acm_log(0, "ERROR - couldn't open %s\n", route_data_file); return ret; } @@ -2778,12 +2779,9 @@ static int acm_parse_path_records(struct acm_ep *ep) goto err; } - if (path_rec_fmt == ACM_PATH_REV_FMT_OSM_FULL_V1) { - acm_parse_path_records_pass1(f, lid2guid); - rewind(f); - ret = acm_parse_path_records_pass2(f, lid2guid, ep); - } - + acm_parse_osm_fullv1_lid2guid(f, lid2guid); + rewind(f); + ret = acm_parse_osm_fullv1_paths(f, lid2guid, ep); free(lid2guid); err: fclose(f); @@ -2858,6 +2856,18 @@ static int acm_assign_ep_names(struct acm_ep *ep) return !index; } +static void acm_ep_preload(struct acm_ep *ep) +{ + switch (route_preload) { + case ACM_ROUTE_PRELOAD_OSM_FULL_V1: + if (acm_parse_osm_fullv1(ep)) + acm_log(0, "ERROR - failed to preload EP\n"); + break; + default: + break; + } +} + static int acm_init_ep_loopback(struct acm_ep *ep) { struct acm_dest *dest; @@ -3039,13 +3049,7 @@ static void acm_ep_up(struct acm_port *port, uint16_t pkey_index) lock_acquire(&port->lock); DListInsertHead(&ep->entry, &port->ep_list); lock_release(&port->lock); - - if (path_rec_fmt == ACM_PATH_REV_FMT_OSM_FULL_V1) { - ret = acm_parse_path_records(ep); - if (ret) - acm_log(1, "unable to find ep in path records\n"); - } - + acm_ep_preload(ep); return; err2: diff --git a/src/acme.c b/src/acme.c index 7c87aa1..8bdb938 100644 --- a/src/acme.c +++ b/src/acme.c @@ -244,17 +244,21 @@ static void gen_opts_temp(FILE *f) fprintf(f, "\n"); fprintf(f, "min_rate 10\n"); fprintf(f, "\n"); - fprintf(f, "# path_rec_fmt:\n"); - fprintf(f, "# Indicates format of optional path records file for preloading ACM cache.\n"); - fprintf(f, "# Supported formats are:\n"); - fprintf(f, "# none - No path record file preloading (default)\n"); - fprintf(f, "# full_opensm_v1 - OpenSM \"full\" path records dump file format (version 1)\n"); - fprintf(f, "\n"); - fprintf(f, "path_rec_fmt none\n"); - fprintf(f, "\n"); - fprintf(f, "# path_rec_file:\n"); - fprintf(f, "# If path_rec_fmt is other than \"none\", full pathname of path records file\n"); - fprintf(f, "# to use for preloading the ACM cache. Default is ACM_CONF_DIR/ibacm_path_records.dump\n"); + fprintf(f, "# route_preload:\n"); + fprintf(f, "# Specifies if the ACM routing cache should be preloaded, or built on demand.\n"); + fprintf(f, "# If preloaded, indicates the method used to build the cache.\n"); + fprintf(f, "# Supported preload values are:\n"); + fprintf(f, "# none - The routing cache is not pre-built (default)\n"); + fprintf(f, "# opensm_full_v1 - OpenSM 'full' path records dump file format (version 1)\n"); + fprintf(f, "\n"); + fprintf(f, "route_preload: none\n"); + fprintf(f, "\n"); + fprintf(f, "# route_data_file:\n"); + fprintf(f, "# Specifies the location of the route data file to use when preloading\n"); + fprintf(f, "# the ACM cache. This option is only valid if route_preload\n"); + fprintf(f, "# indicates that routing data should be read from a file.\n"); + fprintf(f, "# Default is ACM_CONF_DIR/ibacm_route.data\n"); + fprintf(f, "# /etc/rdma/ibacm_route.data\n"); fprintf(f, "\n"); } -- 2.46.0