mpxyd: change MIC cpu_mask to per numa node instead of adapter

The proxy processing threads for multiple cards in same socket will overlap
same cpu cores with existing cpumask per adapter. Change thread affinity
and cpumask to a per socket method.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c
index 4dfd558b5c93affa3a06b0de8d1de931520548ac..d28406095e82dabaa8d83cea298cc3b1df3aad03 100644 (file)
--- a/dapl/svc/mpxyd.c
+++ b/dapl/svc/mpxyd.c
@@ -52,7 +52,11 @@ extern FILE *logfile;
 extern mpxy_lock_t flock;
 extern char *gid_str;
 
+#define MCM_NNODES 8
+
 static mcm_client_t mcm_client_list[MCM_CLIENT_MAX];
+static int mcm_cpumask[MCM_NNODES];
+static mpxy_lock_t mcm_cplock;
 
 void mcm_check_io();
 
@@ -73,6 +77,10 @@ static int init_scif()
        mcm_client_t *mc;
        mcm_ib_dev_t *md;
 
+       mpxy_lock_init(&mcm_cplock, NULL);
+       for (i=0;i<MCM_NNODES;i++)
+               mcm_cpumask[i] = 0;
+
        /* initialize client list and associated devices to init state */
        for (i=0; i<MCM_CLIENT_MAX; i++) {
                mc = &mcm_client_list[i];
@@ -91,7 +99,6 @@ static int init_scif()
                mcm_config_fd(mc->rx_pipe[0]);
                mcm_config_fd(mc->rx_pipe[1]);
 
-               mpxy_lock_init(&mc->cplock, NULL);
                mpxy_lock_init(&mc->oplock, NULL);
                mpxy_lock_init(&mc->txlock, NULL);
                mpxy_lock_init(&mc->cmlock, NULL);
@@ -660,11 +667,14 @@ mcm_scif_dev_t *mix_open_device(dat_mix_open_t *msg, scif_epd_t op_ep, scif_epd_
                if (!rd_dev_file(path, "numa_node", value, sizeof value))
                        mc->numa_node = atoi(value);
 
+               if (mc->numa_node < 0 || mc->numa_node > MCM_NNODES)
+                       mc->numa_node = 0;
+
                if (!rd_dev_file(path, "local_cpulist", value, sizeof value))
-                       mc->cpu_mask = atoi(value);
+                       mcm_cpumask[mc->numa_node] = atoi(value);
 
                mlog(0, " New MIC device - %s, numa_node %d, cpu %d - %s\n",
-                       path, mc->numa_node, mc->cpu_mask, value);
+                       path, mc->numa_node, mcm_cpumask[mc->numa_node], value);
 
                if (pthread_create(&mc->op_thread, NULL,
                                   (void *(*)(void *))mpxy_op_thread, (void*)mc)) {
@@ -825,17 +835,17 @@ void mpxy_tx_thread(void *mic_client)
        char rbuf[2];
 
        if (mcm_affinity) {
-               mpxy_lock(&mc->cplock);
+               mpxy_lock(&mcm_cplock);
                CPU_ZERO( &mc->op_mask );
                if (mcm_affinity_base_mic) /* static config settings */
                        cpu_id = mcm_affinity_base_mic++;
                else
-                       cpu_id = mc->cpu_mask++;
+                       cpu_id = mcm_cpumask[mc->numa_node]++;
 
                CPU_SET(cpu_id, &mc->op_mask );
                if(sched_setaffinity( 0, sizeof(mc->op_mask), &mc->op_mask) == -1)
                      mlog(0, "WARNING: could not set CPU Affinity (%s), continuing...\n", strerror(errno));
-               mpxy_unlock(&mc->cplock);
+               mpxy_unlock(&mcm_cplock);
        }
        mlog(0, "TX thread (%x) MIC node_id %d bound to numa_node %d and cpu_id=%d\n",
                pthread_self(), mc->scif_id, mc->numa_node, mcm_affinity ? cpu_id:0 );
@@ -913,17 +923,17 @@ void mpxy_op_thread(void *mic_client)
        int i, ret, time_ms, cpu_id;
 
        if (mcm_affinity) {
-               mpxy_lock(&mc->cplock);
+               mpxy_lock(&mcm_cplock);
                CPU_ZERO( &mc->op_mask );
                if (mcm_affinity_base_mic) /* static config settings */
                        cpu_id = mcm_affinity_base_mic++;
                else
-                       cpu_id = mc->cpu_mask++;
+                       cpu_id = mcm_cpumask[mc->numa_node]++;
 
                CPU_SET(cpu_id, &mc->op_mask );
                if(sched_setaffinity( 0, sizeof(mc->op_mask), &mc->op_mask) == -1)
                      mlog(0, "WARNING: could not set CPU Affinity (%s), continuing...\n", strerror(errno));
-               mpxy_unlock(&mc->cplock);
+               mpxy_unlock(&mcm_cplock);
        }
        mlog(0, "OP thread (%x) MIC node_id %d bound to numa_node %d and cpu_id=%d\n",
                pthread_self(), mc->scif_id, mc->numa_node, mcm_affinity ? cpu_id:0 );
@@ -997,17 +1007,17 @@ void mpxy_cm_thread(void *mic_client)
        char rbuf[2];
 
        if (mcm_affinity) {
-               mpxy_lock(&mc->cplock);
+               mpxy_lock(&mcm_cplock);
                CPU_ZERO( &mc->op_mask );
                if (mcm_affinity_base_mic) /* static config settings */
                        cpu_id = mcm_affinity_base_mic++;
                else
-                       cpu_id = mc->cpu_mask++;
+                       cpu_id = mcm_cpumask[mc->numa_node]++;
 
                CPU_SET(cpu_id, &mc->op_mask );
                if(sched_setaffinity( 0, sizeof(mc->op_mask), &mc->op_mask) == -1)
                      mlog(0, "WARNING: could not set CPU Affinity (%s), continuing...\n", strerror(errno));
-               mpxy_unlock(&mc->cplock);
+               mpxy_unlock(&mcm_cplock);
        }
        mlog(0, "CM thread (%x) MIC node_id %d bound to numa_node %d and cpu_id=%d\n",
                pthread_self(), mc->scif_id, mc->numa_node, mcm_affinity ? cpu_id:0 );
@@ -1093,17 +1103,17 @@ void mpxy_rx_thread(void *mic_client)
        int i, data = 0, cpu_id, time_ms;
 
        if (mcm_affinity) {
-               mpxy_lock(&mc->cplock);
+               mpxy_lock(&mcm_cplock);
                CPU_ZERO( &mc->op_mask );
                if (mcm_affinity_base_mic)  /* static config settings */
                        cpu_id = mcm_affinity_base_mic++;
                else
-                       cpu_id = mc->cpu_mask++;
+                       cpu_id = mcm_cpumask[mc->numa_node]++;
 
                CPU_SET(cpu_id, &mc->op_mask );
                if(sched_setaffinity( 0, sizeof(mc->op_mask), &mc->op_mask) == -1)
                      mlog(0, "WARNING: could not set CPU Affinity (%s), continuing...\n", strerror(errno));
-               mpxy_unlock(&mc->cplock);
+               mpxy_unlock(&mcm_cplock);
        }
        mlog(0, "RX thread (%x) MIC node_id %d bound to numa_node %d and cpu_id=%d\n",
                pthread_self(), mc->scif_id, mc->numa_node, mcm_affinity ? cpu_id:0 );

diff --git a/dapl/svc/mpxyd.h b/dapl/svc/mpxyd.h
index 13cc3288bebaea280f44e24f053c65689a677027..5909cc924d9ba45b517fb353b0543fe162ff6880 100644 (file)
--- a/dapl/svc/mpxyd.h
+++ b/dapl/svc/mpxyd.h
@@ -356,7 +356,6 @@ typedef struct mcm_client {
        uint16_t ver;
        uint16_t scif_id;
        int numa_node;
-       int cpu_mask;
        int op_pipe[2];
        int tx_pipe[2];
        int rx_pipe[2];
@@ -365,7 +364,6 @@ typedef struct mcm_client {
        cpu_set_t tx_mask;
        cpu_set_t rx_mask;
        cpu_set_t cm_mask;
-       mpxy_lock_t cplock;
        mpxy_lock_t oplock;
        mpxy_lock_t txlock;
        mpxy_lock_t rxlock;