From: Sean Hefty <sean.hefty@intel.com>
Date: Mon, 4 Jun 2012 21:51:41 +0000 (-0700)
Subject: rsocket: Spin before blocking on an rsocket
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=069439b0d4b85103b5b5b7f6066144bc7be7bd80;p=~shefty%2Flibrdmacm.git

rsocket: Spin before blocking on an rsocket

The latency cost of blocking is significant compared to round
trip ping-pong time.  Spin briefly on rsockets before calling
into the kernel and blocking.

The time to spin before blocking is read from an rsocket
configuration file rdma/rsocket/polling_time.  This is user
adjustable or may be set automatically by ibacm.

As a completely unintentional side effect, this just happens to
improve application performance in benchmarks, like netpipe.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
---

diff --git a/Makefile.am b/Makefile.am
index cbd874dc..1dc61e15 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3,7 +3,7 @@ INCLUDES = -I$(srcdir)/include
 lib_LTLIBRARIES = src/librdmacm.la
 
 ACLOCAL_AMFLAGS = -I config
-AM_CFLAGS = -g -Wall -D_GNU_SOURCE
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DSYSCONFDIR=\"$(sysconfdir)\" -DRDMADIR=\"@rdmadir@\"
 
 src_librdmacm_la_CFLAGS = $(AM_CFLAGS)
 
diff --git a/configure.in b/configure.in
index dec6064c..3ee7f9be 100644
--- a/configure.in
+++ b/configure.in
@@ -87,5 +87,10 @@ AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
 
 AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes")
 
+AC_ARG_VAR(rdmadir, [Directory for configuration files])
+if test "x$rdmadir" = "x"; then
+   AC_SUBST(rdmadir, rdma)
+fi
+
 AC_CONFIG_FILES([Makefile librdmacm.spec])
 AC_OUTPUT
diff --git a/src/cma.c b/src/cma.c
index f688c4c2..140e22c9 100755
--- a/src/cma.c
+++ b/src/cma.c
@@ -276,6 +276,7 @@ int ucma_init(void)
 
 	cma_dev_cnt = dev_cnt;
 	ucma_set_af_ib_support();
+	rs_configure();
 	pthread_mutex_unlock(&mut);
 	ibv_free_device_list(dev_list);
 	return 0;
diff --git a/src/cma.h b/src/cma.h
index 2ee47675..7cd1f041 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -114,6 +114,7 @@ static inline int ERR(int err)
 }
 
 int ucma_init();
+void rs_configure();
 extern int af_ib_support;
 
 #define RAI_ROUTEONLY		0x01000000
@@ -166,4 +167,13 @@ struct ibv_path_data
 };
 #endif
 
+#ifndef SYSCONFDIR
+#define SYSCONFDIR "/etc"
+#endif
+#ifndef RDMADIR
+#define RDMADIR "rdma"
+#endif
+#define RDMA_CONF_DIR  SYSCONFDIR "/" RDMADIR
+#define RS_CONF_DIR RDMA_CONF_DIR "/rsockets"
+
 #endif /* CMA_H */
diff --git a/src/rsocket.c b/src/rsocket.c
index e899e8ae..3b16913b 100644
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -197,6 +197,11 @@ struct rsocket {
 	uint8_t		  *sbuf;
 };
 
+void rs_configure(void)
+{
+
+}
+
 /*
  * We currently generate a completion per send.  sqe_count = 1
  */