From b79bd5432699b9f1824024275237490a43533c1a Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Thu, 22 May 2014 15:28:23 -0700 Subject: [PATCH] Release dapl-2.0.42.1-1 Signed-off-by: Arlin Davis --- ChangeLog | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++ README.mcm | 139 +++++++++++++++++++++++++++++++++++-------------- configure.in | 4 +- dapl.spec.in | 2 +- 4 files changed, 245 insertions(+), 43 deletions(-) diff --git a/ChangeLog b/ChangeLog index c940884..3b1f768 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,146 @@ +commit a507b6da8bfce6310264eca76efa34b3be7e2094 +Author: Arlin Davis +Date: Tue Apr 15 14:48:54 2014 -0700 + + dapltest: increase DTO evd size to prevent CQ overflow on limit_rpost test + + Signed-off-by: Arlin Davis + +commit a6646dbe0a5a1d740e10bbcd09127c7b618b6365 +Author: Arlin Davis +Date: Tue Apr 15 13:44:16 2014 -0700 + + Creation of reserved SP moves EP state to DAT_EP_STATE_RESERVED even in failure cases. Reserve EP after successfully binding the listening port. + + Signed-off-by: Arlin Davis + +commit 0d675f741493ee1af054d21fce55f6302bb734e3 +Author: Dave Goodell +Date: Mon Mar 24 14:07:37 2014 -0700 + + dapl: fix string bug in dapls_dto_op_str + + This led to indexing off the end of the array and gave surprising + results for OP_RECV_UD. + +commit 785b4e3532978eb681c8a1a30c786bb17200f846 +Author: Arlin Davis +Date: Thu May 22 14:56:07 2014 -0700 + + mpxyd: change affinity to avoid overlapping cores with 2 MICs in same socket + + Signed-off-by: Arlin Davis + +commit d8fec14a41d4a65634c883e0b99fdbd1fa76f933 +Author: Arlin Davis +Date: Thu May 22 14:42:28 2014 -0700 + + mpxyd: add mpxyd.conf option to disable proxy-in service + + Signed-off-by: Arlin Davis + +commit 6358b4c17b1d380f9132904b3d2ea4f8df40313e +Author: Arlin Davis +Date: Thu May 22 13:57:42 2014 -0700 + + mpxyd: set default seg size to 128KB, down from 256KB + + Signed-off-by: Arlin Davis + +commit 18829ab3672b35d7ea2517bb7b341b7cf5cc7807 +Author: Arlin Davis +Date: Thu May 22 13:16:31 2014 -0700 + + install: base package, without mpss, incorrectly tries to install /usr/sbin as file + + modify specfile and makefile to only install /usr/sbin/mpxyd + if built with mpss present. If files don't exist, mpss not + installed, make sure /usr/sbin is not incorrectly installed as a file. + + Signed-off-by: Patrick Mccormick + Ack-by: Arlin Davis + +commit 616d3b5e5480dfabfb0ec5ebec5672517dc04450 +Author: Arlin Davis +Date: Thu May 22 09:30:59 2014 -0700 + + mpxyd: scale-up with MPI dapl:dapl hits low mem issue with 1 byte traffic patterns + + PI and PO buffer management use last byte offset of work requests + and assume non-zero value. However, in the case where a 1 byte + rdma occurs at offset 0 will result in m_idx being set to zero + as a valid offset and the buffer never being marked complete. + + Clean-up buffer management, add error reporting on setting, + serialization on PO by setting in post_send op thread, setting + start location at cacheline when buffer ring wraps instead of 0. + + Signed-off-by: Arlin Davis + +commit a396523e1651fae6271b6d6af252550f401cd909 +Author: Arlin Davis +Date: Thu May 22 09:30:52 2014 -0700 + + mpxyd: add MIC client and device id to logging + + Signed-off-by: Arlin Davis + +commit 58c37bdf30fa2956ccb26aee7ce8227446dcedd4 +Author: Arlin Davis +Date: Thu May 22 09:29:00 2014 -0700 + + add new po-pi rdma write perf profile + + Signed-off-by: Arlin Davis + +commit 64022f6708152131e3d83221cc655d58a70f07b4 +Author: Arlin Davis +Date: Thu May 22 09:27:35 2014 -0700 + + change default wrc entries from 1024 to 512 + + Signed-off-by: Arlin Davis + +commit 4aff960ae9882b206295168a3181b21af6fcc49e +Author: Arlin Davis +Date: Fri May 16 10:04:21 2014 -0700 + + mpxyd: MIC scale-up issue with MPI gather workloads, I_MPI_FABRICS=dapl:dapl + + issue with shared proxy-in buffer pool when rdma reads complete + out of order across QP's. The tail adjustment when read completes + fails to walk entire queue and process head entry. + + Signed-off-by: Arlin Davis + +commit 413a451f799dbebc8e8838b8941f91f228d24ac7 +Author: Arlin Davis +Date: Fri May 16 09:26:35 2014 -0700 + + mpxyd: remove unnecessary logging + + Signed-off-by: Arlin Davis + +commit 5e5d84eaa5486f599a3fb56c390eee02c5429876 +Author: Arlin Davis +Date: Fri May 16 09:21:26 2014 -0700 + + mcm: mpxyd error event of m_pi_prep_rcv_q: ERR: ib_qp == 0 + + When incorrect ep_mode is provided by consumer the proxy-in + service modes get setup incorrectly. Validate remote address + ep mode, set to unknown if out of range. + + Signed-off-by: Arlin Davis + +commit 1cbd6d1c9d66ef1e58b8d49f8bce6fd9509c1824 +Author: Arlin Davis +Date: Thu May 1 13:08:04 2014 -0700 + + Release dapl-2.0.41.2-1 + + Signed-off-by: Arlin Davis + commit b85ade15899d1936e42dfa7df332a89a05d27639 Author: Arlin Davis Date: Thu May 1 12:47:43 2014 -0700 diff --git a/README.mcm b/README.mcm index 003be4c..600fe3f 100644 --- a/README.mcm +++ b/README.mcm @@ -1,7 +1,6 @@ uDAPL MCM Provider and MPXYD Daemon (CCL-proxy) - intel-mic-ofed-dapl-2.0.36.8 - March 2013 - + intel-mic-ofed-dapl-2.0.42.1 + May 2014 MCM is a new uDAPL provider that is an extension to standard DAT 2.0 libraries. The purpose of this service is to proxy RDMA writes from the MIC to the HOST to improve large IO performance. The provider will support @@ -10,8 +9,7 @@ It requires a new MPXYD daemon service when clients are running on a MIC KNC ada host side libraries and daemon service. The MIC libraries must be built and moved over to MIC adapter. This verion is currently included with MPSS and all libraries and services will be installed by default. -Current release package for MPSS: intel-mic-ofed-dapl-2.0.36.8.tar.gz - +Current release package for MPSS: dapl-2.0.42.1.tar.gz * Sample host build from source package (ofed must installed) @@ -24,13 +22,27 @@ Current release package for MPSS: intel-mic-ofed-dapl-2.0.36.8.tar.gz make sudo make install -* Sample MIC build from source package (MPSS must be installed) +* Sample /home/user1 MIC build from source package for MPSS 3.x (MPSS must be installed) + source /opt/mpss/3.x/environment-setup-k1om-mpss-linux + ./autogen.sh + ./configure \ + --enable-mcm + --host=x86_64-k1om-linux \ + --prefix=/home/user1/dapl-mic-install \ + CC=/usr/linux-k1om-4.7/bin/x86_64-k1om-linux-gcc \ + CFLAGS="-I/opt/mpss/3.x/sysroots/k1om-mpss-linux/usr/include + LDFLAGS="-L/opt/mpss/3.x/sysroots/k1om-mpss-linux/usr/lib64" + make + sudo make install + +* Sample /home/user1 MIC build from source package for MPSS 2.x (MPSS must be installed) + export PATH=$PATH:/usr/linux-k1om-4.7/bin ./autogen.sh ./configure \ --enable-mcm \ - --prefix=/opt/intel/mic/ofed/card/usr \ + --prefix=/home/user1/dapl-mic-install \ --libdir=/opt/intel/mic/ofed/card/usr/lib64 \ --sysconfdir=/opt/intel/mic/ofed/card/etc \ --host=x86_64-k1om-linux \ @@ -59,27 +71,16 @@ Current release package for MPSS: intel-mic-ofed-dapl-2.0.36.8.tar.gz sudo /usr/sbin/mpxyd -* Use the MCM provider with Intel MPI +* Use the MCM provider with Intel MPI 4.1.3 or greater for best out of box experiences. (1) Recommended settings: export I_MPI_MIC=1 export I_MPI_DEBUG=2 export I_MPI_FALLBACK=0 - export I_MPI_FABRICS=shm:dapl - export I_MPI_DAPL_PROVIDER=ofa-v2-mcm-1 - export I_MPI_DAPL_RDMA_RNDV_WRITE=1 - export I_MPI_DYNAMIC_CONNECTION=1 - export DAPL_DBG_TYPE=1 - export DAPL_UCM_RETRY=7 - - When using the engineering build with dual DAPL provider support, don't set I_MPI_DAPL_PROVIDER. Instead, - add these settings: - - export I_MPI_DAPL_PROVIDER_LIST=ofa-v2-mlx4_0-1u,ofa-v2-mcm-1 - export I_MPI_DAPL_DIRECT_COPY_THRESHOLD=8192,262144 + export I_MPI_MIC_DAPL_DIRECT_COPY_THRESHOLD=8192,262144 - With these settings, messages less than 8192 bytes will be sent via pre-registered buffers; messages + With these settings on MIC, messages less than 8192 bytes will be sent via pre-registered buffers; messages between 8192 and 262144 bytes will be sent via the Rendezvous protocol throught the first provider; and larger messages will be sent via the Rendezvous protocol through the second provider. Fine tune these two sizes for the best performance. @@ -107,8 +108,7 @@ Current release package for MPSS: intel-mic-ofed-dapl-2.0.36.8.tar.gz Note: override default port id using following environment variable: export DAPL_MCM_PORT_ID=1068 - - + * Notes (1) Modify "/etc/mpxyd.conf" to change the settings for the proxy. Especially, try different values @@ -124,23 +124,82 @@ Current release package for MPSS: intel-mic-ofed-dapl-2.0.36.8.tar.gz ChangeLog: -Release 2.0.36.8 - - config: set eager_completion default setting to disabled - config: add mlx5 entries to dat.conf - mpxyd: only dump cm_lists with debug build - mpxyd: CM optimizations for MIC clients, improved checking on inbound CM messages - mcm: scif_recv err on mpxyd when scaling up on MPI IMB scatter benchmark - mcm: add inline data threshold definition, 256 bytes - common: change debug level of EP free warning - common: increase DCM RTU time from 400 to 800ms - common: increase DCM retry from 1 to 5 - common: increase level of ASYNC error logging, include in free builds - mcm: improved logging with CM retry/poll errors - mpxyd: MPI IMB scatter on 12 ranks, 2 KNCs + 2 Hosts, fails - mcm: segfault with counters, no CM object when not found and duplicate case - Enable ccl-proxy support if possible by default: yes, if nothing specified and scif.h is present no, if scif.h is not present - clean up configure.in and make consistent patch +Release dapl-2.0.42.1-1 (more bug fixing for MPSS 3.3, merge with master branch 2.0.42) + +dapltest: increase DTO evd size to prevent CQ overflow on limit_rpost test +Creation of reserved SP moves EP state to DAT_EP_STATE_RESERVED even in failure cases. Reserv +dapl: fix string bug in dapls_dto_op_str +mpxyd: change affinity to avoid overlapping cores with 2 MICs in same socket +mpxyd: add mpxyd.conf option to disable proxy-in service +mpxyd: set default seg size to 128KB, down from 256KB +install: base package, without mpss, incorrectly tries to install /usr/sbin as file +mpxyd: scale-up with MPI dapl:dapl hits low mem issue with 1 byte traffic patterns +mpxyd: add MIC client and device id to logging +add new po-pi rdma write perf profile +change default wrc entries from 1024 to 512 +mpxyd: MIC scale-up issue with MPI gather workloads, I_MPI_FABRICS=dapl:dapl +mpxyd: remove unnecessary logging +mcm: mpxyd error event of m_pi_prep_rcv_q: ERR: ib_qp == 0 + +Release dapl-2.0.41.2-1 (bug fixing for MPSS 3.3) + +mpxyd,mcm: increase default CM timers for better out of box scaling +mpxyd,mcm: changes for backward compatibility with older v4 MIC clients +mpxyd: allow proxy_out WR stalls instead of immediate error +mpxyd: increase max open files limit for proxy service + +Release dapl-2.0.41.1-1 (added proxy-in support) + +dapltest: change server port, from 45278 to 62000, out of registered IANA range +dapltest: set default limit max to 1000 +dapltest: update scripts for regression testing purposes +dapltest: Add final send/recv "sync" for transaction tests. +mpxyd: append _free to dqconn/dqlisten for readability, improve logging +mcm: check for shared CQs in PI mode – MXS-HST scaling issues +mcm: add dev_id to all mpxyd commands +mpxyd: scale-up improvements to support 200-300 processes per MIC +mpxyd: serialize MD cm port space usage, add swap to rej call, resend dropped rej +mcm: serialize CM cmds on ev_ep, add dev_id, increase dev_open listen backlog +openib: return open failure if port not active +mpxyd: disable inside box support via scif only, use IB +mcm: add host to mic cross socket support to proxy-in service to improve performance +dat: reduce log level on library load failures +mic: take cm_msg rsvd byte for segment size exchange, power of 2 +mpxyd: init proxy WC buffer queues for new queue management +mpxyd: simplify WR and WC queue management and fix inline post send +mpxyd: 64KB segment sizes hang with MPI IMB pingpong cross socket +mpxyd: add new M_READ_FROM_DONE state for send WR's and add more profiling options +mpxyd: purge posted send data only if inline +mpxyd: move to CONN state immediately on RTU_IN +dtest: fix options on query times printf +mpxyd: fix ibctx leak with device open +dtest: add new -Q option to get provider list and query and time each +openib: cleanup init/fini and device close on all providers +dat: dat_ia_open should close provider after failure +mpxyd: sync PI WC trigger to PO MP_SIG +mpxyd: improve QP destruction to manage QP1 and QP2 variations +mpxyd: proxy out doesn't release proxy buffer as quickly as necessary +dtest: add times for open_query, remove sleep +common: add provider name and len to DTO error logging +new lightweight open_query/close_query IB extension for fast attribute query +mpxyd: need CM to QP linking with CM references +dist: ib collective and MIC extension include files missing +dapltest: the quit command is missing changes for -n option. +NULL undefined on Fedora, incorrectly using kernel stddef.h +Merge branch 'proxy' of ssh://beany.openfabrics.org/home/ardavis/scm/dapl into proxy +ucm: fix CM service, initial rcv msg posts incorrect +ucm: add/cleanup debug log information +scm: add/cleanup debug log information +makefile: update for MCM proxy-in changes +dtest: update for ep_mode on MCM providers +mpxyd.conf: updated for proxy-in parameters +mpxyd: proxy-in added to proxy-out service to increase cross socket performance +mcm: add proxy in support to MCM provider and MPXYD interface +open_ib common: qp, cq, and post_recv changes for proxy-in +commom: add lmr support for proxy in service +new definitions and states for CCL Proxy-in support + + diff --git a/configure.in b/configure.in index dab39f8..cafdcb6 100644 --- a/configure.in +++ b/configure.in @@ -1,12 +1,12 @@ dnl Process this file with autoconf to produce a configure script. AC_PREREQ(2.57) -AC_INIT(dapl, 2.0.41.2, linux-rdma@vger.kernel.org) +AC_INIT(dapl, 2.0.42.1, linux-rdma@vger.kernel.org) AC_CONFIG_SRCDIR([dat/udat/udat.c]) AC_CONFIG_AUX_DIR(config) AC_CONFIG_MACRO_DIR([m4]) AM_CONFIG_HEADER(config.h) -AM_INIT_AUTOMAKE(dapl, 2.0.41.2) +AM_INIT_AUTOMAKE(dapl, 2.0.42.1) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AM_PROG_LIBTOOL diff --git a/dapl.spec.in b/dapl.spec.in index 6972c55..584883d 100644 --- a/dapl.spec.in +++ b/dapl.spec.in @@ -40,7 +40,7 @@ %define debug_package %{nil} Name: dapl -Version: 2.0.41.2 +Version: 2.0.42.1 Release: 1%{?dist} Summary: A Library for userspace access to RDMA devices using OS Agnostic DAT APIs, proxy daemon for offloading RDMA -- 2.46.0