mcm: implement proxy mix_prov_attr function, add fields CPU model and family

Provide MIC consumers with a provider specific query for proxy CPU model and family
to identify platform type from MIC side. Supported in MCM provider only.

The following provider specific name attributes were added to MCM:

DAT_IB_PROXY_CPU_FAMILY
DAT_IB_PROXY_CPU_MODEL

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/openib_common/dapl_mic_common.h b/dapl/openib_common/dapl_mic_common.h
index dc61eb59b68aee832a2f4207bc43b6628945cf3a..1f5ab86d52499937ee1808e6da9cd5e3788b2497 100755 (executable)
--- a/dapl/openib_common/dapl_mic_common.h
+++ b/dapl/openib_common/dapl_mic_common.h
@@ -431,7 +431,9 @@ typedef struct dat_mix_prov_attr
        dat_mix_dev_attr_t      dev_attr;
        uint64_t                system_guid;
        uint8_t                 gid_idx;
-       uint8_t                 resv[39];
+       uint32_t                cpu_model;
+       uint32_t                cpu_family;
+       uint8_t                 resv[31];
 
 }  __attribute__((packed)) dat_mix_prov_attr_t;
 

diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c
index 772174069add0fe1e647775feae71bfa7d4e4983..6e8597d4877cd772933f2f5e3b2682687b3523dd 100644 (file)
--- a/dapl/openib_common/util.c
+++ b/dapl/openib_common/util.c
@@ -424,6 +424,12 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
                else
                        hca_ptr->ib_trans.na.mode = "DIRECT";
                hca_ptr->ib_trans.na.read = "FALSE";
+
+               if (!hca_ptr->ib_trans.pr_attr.cpu_family) {
+                       dapli_mix_get_attr(&hca_ptr->ib_trans, &hca_ptr->ib_trans.pr_attr);
+                       sprintf(hca_ptr->ib_trans.fam_str, "%d", hca_ptr->ib_trans.pr_attr.cpu_family);
+                       sprintf(hca_ptr->ib_trans.mod_str, "%d", hca_ptr->ib_trans.pr_attr.cpu_model);
+               }
 #else
                hca_ptr->ib_trans.na.mode = "DIRECT";
                hca_ptr->ib_trans.na.read = "TRUE";
@@ -717,6 +723,14 @@ DAT_NAMED_ATTR ib_attrs[] = {
        {
         "DAT_IB_PORT_STATUS", "UNKNOWN"}
        ,
+#ifdef _OPENIB_MCM_
+       {
+        "DAT_IB_PROXY_CPU_FAMILY", "UNKNOWN"}
+       ,
+       {
+        "DAT_IB_PROXY_CPU_MODEL", "UNKNOWN"}
+       ,
+#endif
 #ifdef DAT_EXTENSIONS
        {
         "DAT_EXTENSION_INTERFACE", "TRUE"}
@@ -784,7 +798,10 @@ void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr,
        ib_attrs[4].value = ia_ptr->hca_ptr->ib_trans.guid_str;
        ib_attrs[5].value = ia_ptr->hca_ptr->ib_trans.na.mtu;
        ib_attrs[6].value = ia_ptr->hca_ptr->ib_trans.na.port;
-
+#ifdef _OPENIB_MCM_
+       ib_attrs[7].value = ia_ptr->hca_ptr->ib_trans.fam_str;
+       ib_attrs[8].value = ia_ptr->hca_ptr->ib_trans.mod_str;
+#endif
 }
 
 /*

diff --git a/dapl/openib_mcm/dapl_ib_util.h b/dapl/openib_mcm/dapl_ib_util.h
index a5e1c1bc54bf35feac664a9987748b096be3b624..dad8212adb8214e95681542474255b4109a40a95 100644 (file)
--- a/dapl/openib_mcm/dapl_ib_util.h
+++ b/dapl/openib_mcm/dapl_ib_util.h
@@ -148,7 +148,10 @@ typedef struct _ib_hca_transport
        uint64_t                sys_guid;       /* system image guid, network order */
        uint64_t                guid;           /* host order */
        char                    guid_str[32];
+       char                    fam_str[8];
+       char                    mod_str[8];
        ib_named_attr_t         na;
+       dat_mix_prov_attr_t     pr_attr;        /* attributes from proxy */
 
 } ib_hca_transport_t;
 
@@ -178,6 +181,7 @@ void mcm_destroy_pi_cq(struct dcm_ib_qp *m_qp);
 /* MIC eXchange (MIX) operations, mix.c */
 int  dapli_mix_open(ib_hca_transport_t *tp, char *name, int port, int query);
 void dapli_mix_close(ib_hca_transport_t *tp);
+int  dapli_mix_get_attr(ib_hca_transport_t *tp, dat_mix_prov_attr_t *pr_attr);
 int  dapli_mix_listen(dp_ib_cm_handle_t cm, uint16_t sid);
 int  dapli_mix_listen_free(dp_ib_cm_handle_t cm);
 int  dapli_mix_qp_create(ib_qp_handle_t m_qp, struct ibv_qp_init_attr *attr,

diff --git a/dapl/openib_mcm/mix.c b/dapl/openib_mcm/mix.c
index c9c9672ea98027cc81a24efbc08b7cd6be0ee1f9..d518b0cffd7371d3c239be51b4717f3d7bcf7759 100644 (file)
--- a/dapl/openib_mcm/mix.c
+++ b/dapl/openib_mcm/mix.c
@@ -203,6 +203,57 @@ void dapli_mix_close(ib_hca_transport_t *tp)
        }
 }
 
+/* MIX_PROV_ATTR */
+int dapli_mix_get_attr(ib_hca_transport_t *tp, dat_mix_prov_attr_t *pr_attr)
+{
+       dat_mix_attr_t msg;
+       scif_epd_t mix_ep = tp->scif_ep;
+       int ret, len;
+
+       dapl_log(DAPL_DBG_TYPE_EXTENSION, " MIX_GET_ATTR tp = %p\n", tp);
+
+       /* get attr request */
+       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.op = MIX_PROV_ATTR;
+       msg.hdr.status = 0;
+       msg.hdr.flags = MIX_OP_REQ;
+
+       len = sizeof(dat_mix_hdr_t);
+       ret = scif_send(mix_ep, &msg, len, SCIF_SEND_BLOCK);
+       if (ret != len) {
+               dapl_log(1, " ERR: %s msg %p send on %d, ret %d, exp %d, error %s\n",
+                        mix_op_str(msg.hdr.op), &msg, mix_ep, ret, len, strerror(errno));
+               return -1;
+       }
+       dapl_log(DAPL_DBG_TYPE_EXTENSION," Sent %s request on SCIF EP %d\n", mix_op_str(msg.hdr.op), mix_ep);
+
+       /* get attr response */
+       len = sizeof(dat_mix_attr_t);
+       ret = scif_recv(mix_ep, &msg, len, SCIF_RECV_BLOCK);
+       if (ret != len) {
+               dapl_log(1, " ERR: rcv on new_ep %d, ret %d, exp %d, error %s\n", mix_ep, ret, len, strerror(errno));
+               return -1;
+       }
+       dapl_log(DAPL_DBG_TYPE_EXTENSION," Recv'd %s reply on SCIF EP %d for dev_id %d\n",
+                mix_op_str(msg.hdr.op), mix_ep, msg.hdr.req_id);
+
+       if (msg.hdr.ver != DAT_MIX_VER || msg.hdr.op != MIX_PROV_ATTR ||
+           msg.hdr.flags != MIX_OP_RSP || msg.hdr.status != MIX_SUCCESS) {
+               dapl_log(1, " ERR: MIX_PROV_ATTR ver %d, op %s, flgs %d, st %d dev_id %d\n",
+                            msg.hdr.ver, mix_op_str(msg.hdr.op),
+                            msg.hdr.flags, msg.hdr.status, msg.hdr.req_id);
+               if (msg.hdr.status != MIX_SUCCESS)
+                       return msg.hdr.status;
+               else
+                       return -1;
+       }
+
+       memcpy(pr_attr, &msg.attr, sizeof(dat_mix_prov_attr_t));
+
+       dapl_log(DAPL_DBG_TYPE_EXTENSION," MIX_PROV_ATTR successful on SCIF EP %d\n", mix_ep);
+       return 0;
+}
+
 /* MIX_LISTEN */
 int dapli_mix_listen(dp_ib_cm_handle_t cm, uint16_t sid)
 {
@@ -324,7 +375,6 @@ int dapli_mix_mr_create(ib_hca_transport_t *tp, DAPL_LMR * lmr)
        msg.ib_rkey = lmr->param.rmr_context;
        msg.ctx = (uint64_t)lmr;
 
-
        len = sizeof(dat_mix_mr_t);
        ret = scif_send(mix_ep, &msg, len, SCIF_SEND_BLOCK);
        if (ret != len) {

diff --git a/dapl/svc/mcm.c b/dapl/svc/mcm.c
index b29e109f0769fd0e9abae554ab64aa98360a80fa..3287073e8e4249da711284e40f4d25332d37d8cd 100644 (file)
--- a/dapl/svc/mcm.c
+++ b/dapl/svc/mcm.c
@@ -54,6 +54,16 @@ extern int mcm_rx_entries;
 extern uint64_t system_guid;
 extern char *gid_str;
 
+void mcm_get_attr(dat_mix_prov_attr_t *pr_attr)
+{
+       pr_attr->cm_retry = mcm_retry;
+       pr_attr->cm_disc_retry = mcm_disc_retry;
+       pr_attr->cm_rep_time_ms = mcm_rep_ms;
+       pr_attr->cm_rtu_time_ms = mcm_rtu_ms;
+       pr_attr->cm_drep_time_ms = mcm_dreq_ms;
+}
+
+
 /* Create address handle for remote QP, info in network order */
 struct ibv_ah *mcm_create_ah(mcm_ib_dev_t *md,
                             struct ibv_pd *pd,

diff --git a/dapl/svc/mix.c b/dapl/svc/mix.c
index da8f3c731db285aa62468ee1802e873f83717459..2ad93cd76938440f13096d4fdac4bb3879af7874 100644 (file)
--- a/dapl/svc/mix.c
+++ b/dapl/svc/mix.c
@@ -55,10 +55,31 @@ int mcm_rx_cq_size = MCM_WRC_QLEN;
 int mcm_tx_cq_size = MCM_WRC_QLEN;
 int mcm_buf_wc_size = MCM_WRC_QLEN;
 
+extern int mix_buffer_mb;
 extern int mix_buffer_sg_po2;
 extern uint64_t system_guid;
 extern int mcm_profile;
 extern int log_level;
+extern int mcm_cpu_family;
+extern int mcm_cpu_model;
+
+static void mix_get_prov_attr(mcm_scif_dev_t *smd, dat_mix_prov_attr_t *pr_attr)
+{
+       memset(pr_attr, 0, sizeof(dat_mix_prov_attr_t));
+       memcpy(&pr_attr->dev_attr, &smd->md->dev_attr, sizeof(dat_mix_dev_attr_t));
+
+       mcm_get_attr(pr_attr);  /* CM attributes */
+       pr_attr->max_msg_sz = mix_max_msg_mb * 1024 * 1024;
+       pr_attr->max_tx_dtos = mcm_tx_entries;
+       pr_attr->max_rx_dtos = mcm_rx_entries;
+       pr_attr->max_tx_pool = mix_buffer_mb * 1024 * 1024;
+       pr_attr->max_rx_pool = mix_buffer_mb * 1024 * 1024;
+       pr_attr->tx_segment_sz = (mix_buffer_sg_po2 << 1);
+       pr_attr->rx_segment_sz = (mix_buffer_sg_po2 << 1);
+       pr_attr->system_guid = system_guid;
+       pr_attr->cpu_model = mcm_cpu_model;
+       pr_attr->cpu_family = mcm_cpu_family;
+}
 
 /* close MCM device, MIC client, md->slock held */
 void mix_close_device(mcm_ib_dev_t *md, mcm_scif_dev_t *smd)
@@ -197,6 +218,42 @@ out_close_op_ep:
        scif_close(op_ep);
 }
 
+static int mix_prov_attr(mcm_scif_dev_t *smd, dat_mix_attr_t *pmsg)
+{
+       int len, ret;
+
+       if (pmsg->hdr.flags & MIX_OP_SET) {
+               len = sizeof(dat_mix_attr_t) - sizeof(dat_mix_hdr_t);
+               ret = scif_recv(smd->scif_op_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
+               if (ret != len) {
+                       mlog(0, " ERR: ret %d, exp %d\n", ret, len);
+                       return ret;
+               }
+
+               /* SET not supported */
+               len = sizeof(dat_mix_hdr_t);
+               pmsg->hdr.flags = MIX_OP_RSP;
+               pmsg->hdr.status = MIX_EINVAL;
+
+               /* send back response */
+               return (scif_send_msg(smd->scif_op_ep, pmsg, len));
+
+       } else {
+               dat_mix_attr_t msg;
+
+               /* GET attributes and send back response and data */
+               mix_get_prov_attr(smd, &msg.attr);
+
+               len = sizeof(dat_mix_attr_t);
+               msg.hdr.ver = DAT_MIX_VER;
+               msg.hdr.op = MIX_PROV_ATTR;
+               msg.hdr.flags = MIX_OP_RSP;
+               msg.hdr.status = MIX_SUCCESS;
+
+               return (scif_send_msg(smd->scif_op_ep, &msg, len));
+       }
+}
+
 static int mix_listen_free(mcm_scif_dev_t *smd, dat_mix_hdr_t *pmsg)
 {
        int len;
@@ -2173,6 +2230,9 @@ int mix_scif_recv(mcm_scif_dev_t *smd, scif_epd_t scif_ep)
        case MIX_LISTEN_FREE:
                ret = mix_listen_free(smd, phdr);
                break;
+       case MIX_PROV_ATTR:
+               ret = mix_prov_attr(smd, (dat_mix_attr_t *)phdr);
+               break;
        case MIX_CM_REQ:
                ret = mix_cm_req_out(smd, (dat_mix_cm_t *)phdr, scif_ep);
                break;

diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c
index 428e6f60d491d5cf55864d9f3b255fe60f7d0d4a..2c5ba9d7532b1677117495e20b04032b119451c7 100644 (file)
--- a/dapl/svc/mpxyd.c
+++ b/dapl/svc/mpxyd.c
@@ -40,6 +40,8 @@ int mcm_affinity = 1;
 int mcm_affinity_base_mic = 0;
 int mcm_affinity_base_hca = 0;
 int mcm_counters = 0;
+int mcm_cpu_model = 0;
+int mcm_cpu_family = 0;
 uint64_t system_guid = 0; /* network order */
 
 extern int mix_max_msg_mb;
@@ -891,7 +893,6 @@ void mpxy_tx_thread(void *mic_client)
                                while (m_qp) {
                                        if (m_qp->r_entry.tid)
                                                m_pi_pending_wc(m_qp, &events);
-
                                        m_po_pending_wr(m_qp, &data, &events); /* proxy-out WR's */
                                        m_qp = get_next_entry(&m_qp->t_entry, &smd->qptlist);
                                }
@@ -1262,6 +1263,8 @@ static void show_usage(char *program)
 int main(int argc, char **argv)
 {
        int op, mdaemon = 1, kill = 0, debug_mode = 0;
+       const char *fam_str = "cpu family";
+       const char *mod_str = "model";
 
        while ((op = getopt(argc, argv, "dkDPO:")) != -1) {
                switch (op) {
@@ -1315,6 +1318,12 @@ int main(int argc, char **argv)
                return -1;
        }
 
+       mcm_cpu_family = mpxy_cpuinfo_atoi(fam_str);
+       mcm_cpu_model  = mpxy_cpuinfo_atoi(mod_str);
+
+       mlog(1, "Host CPU Family = %d\n", mcm_cpu_family);
+       mlog(1, "Host CPU Model  = %d\n", mcm_cpu_model);
+
        mlog(0, "Starting server\n");
        mpxy_server();
        mlog(0, "Shutting down\n");

diff --git a/dapl/svc/mpxyd.h b/dapl/svc/mpxyd.h
index 5909cc924d9ba45b517fb353b0543fe162ff6880..24a33aede038f332bef202083f398053fb0c66cb 100644 (file)
--- a/dapl/svc/mpxyd.h
+++ b/dapl/svc/mpxyd.h
@@ -504,6 +504,7 @@ void mcm_check_io();
 #endif
 
 /* mcm.c, cm services */
+void mcm_get_attr(dat_mix_prov_attr_t *pr_attr);
 int mcm_init_cm_service(mcm_ib_dev_t *md);
 mcm_cm_t *m_cm_create(mcm_scif_dev_t *smd, mcm_qp_t *m_qp, dat_mcm_addr_t *dst);
 void m_cm_free(mcm_cm_t *cm);
@@ -547,6 +548,7 @@ int mpxy_open_lock_file(void);
 void mpxyd_release_lock_file( void );
 void mpxy_write(int level, const char *format, ...);
 void mpxy_pr_addrs(int lvl, struct dat_mcm_msg *msg, int state, int in);
+int mpxy_cpuinfo_atoi(const char *v_str);
 #ifdef MCM_PROFILE
 void mcm_qp_prof_pr(struct mcm_qp *m_qp, int type);
 void mcm_qp_prof_ts(struct mcm_qp *m_qp, int type, uint32_t start, uint32_t qcnt, uint32_t ccnt);

diff --git a/dapl/svc/util.c b/dapl/svc/util.c
index c2a3693195d9ec5293454630b35f4e6f155a3442..323595f7de249da4f25c8843fb18c7f3ba8831b4 100644 (file)
--- a/dapl/svc/util.c
+++ b/dapl/svc/util.c
@@ -536,6 +536,44 @@ int rd_dev_file(char *path, char *file, char *v_str, int len)
        return 0;
 }
 
+int mpxy_cpuinfo_atoi(const char *v_str)
+{
+       char r_buf[500];
+       char *f_path = "/proc/cpuinfo";
+       char *token = ":";
+       int i, ii, fd, len, v_len, r_len;
+       int val = 0;
+
+       fd = open(f_path, O_RDONLY);
+       if (fd < 0)
+               return val;
+
+       v_len = strlen(v_str);
+       r_len = sizeof(r_buf) - 1;
+       len = read(fd, r_buf, r_len);
+
+       if (len < 1)
+               return val;
+
+       /* get value pattern followed by : followed by value */
+       for (i=0; i < len; i++) {
+               for (ii=0; ii < v_len && i < len; ii++, i++) {
+                       if ((v_str[ii] == r_buf[i]) && (ii == v_len-1))
+                               for (; i < len; i++) {
+                                       if (!strncmp(&r_buf[i], token, 1)) {
+                                               val = atoi(&r_buf[i+1]);
+                                               i = len;
+                                       }
+                               }
+                       else if (v_str[ii] != r_buf[i])
+                               break;
+               }
+       }
+
+       close(fd);
+       return val;
+}
+
 #ifdef MCM_PROFILE
 void mcm_qp_prof_pr(struct mcm_qp *m_qp, int type)
 {