From: Sean Hefty <sean.hefty@intel.com>
Date: Mon, 4 Jun 2012 21:51:41 +0000 (-0700)
Subject: rsocket: Spin before blocking on an rsocket
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=5a6b08ccada2d236f5405bcf5914221d68f52343;p=~shefty%2Flibrdmacm.git

rsocket: Spin before blocking on an rsocket

The latency cost of blocking is significant compared to round
trip ping-pong time.  Spin briefly on rsockets before calling
into the kernel and blocking.

The time to spin before blocking is read from an rsocket
configuration file rdma/rsocket/polling_time.  This is user
adjustable or may be set automatically by ibacm.

As a completely unintentional side effect, this just happens to
improve application performance in benchmarks, like netpipe.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
---

diff --git a/Makefile.am b/Makefile.am
index cbd874dc..1dc61e15 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3,7 +3,7 @@ INCLUDES = -I$(srcdir)/include
 lib_LTLIBRARIES = src/librdmacm.la
 
 ACLOCAL_AMFLAGS = -I config
-AM_CFLAGS = -g -Wall -D_GNU_SOURCE
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DSYSCONFDIR=\"$(sysconfdir)\" -DRDMADIR=\"@rdmadir@\"
 
 src_librdmacm_la_CFLAGS = $(AM_CFLAGS)
 
diff --git a/configure.in b/configure.in
index dec6064c..3ee7f9be 100644
--- a/configure.in
+++ b/configure.in
@@ -87,5 +87,10 @@ AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
 
 AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes")
 
+AC_ARG_VAR(rdmadir, [Directory for configuration files])
+if test "x$rdmadir" = "x"; then
+   AC_SUBST(rdmadir, rdma)
+fi
+
 AC_CONFIG_FILES([Makefile librdmacm.spec])
 AC_OUTPUT
diff --git a/src/cma.c b/src/cma.c
index f688c4c2..140e22c9 100755
--- a/src/cma.c
+++ b/src/cma.c
@@ -276,6 +276,7 @@ int ucma_init(void)
 
 	cma_dev_cnt = dev_cnt;
 	ucma_set_af_ib_support();
+	rs_configure();
 	pthread_mutex_unlock(&mut);
 	ibv_free_device_list(dev_list);
 	return 0;
diff --git a/src/cma.h b/src/cma.h
index 2ee47675..7cd1f041 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -114,6 +114,7 @@ static inline int ERR(int err)
 }
 
 int ucma_init();
+void rs_configure();
 extern int af_ib_support;
 
 #define RAI_ROUTEONLY		0x01000000
@@ -166,4 +167,13 @@ struct ibv_path_data
 };
 #endif
 
+#ifndef SYSCONFDIR
+#define SYSCONFDIR "/etc"
+#endif
+#ifndef RDMADIR
+#define RDMADIR "rdma"
+#endif
+#define RDMA_CONF_DIR  SYSCONFDIR "/" RDMADIR
+#define RS_CONF_DIR RDMA_CONF_DIR "/rsockets"
+
 #endif /* CMA_H */
diff --git a/src/rsocket.c b/src/rsocket.c
index e899e8ae..156a6fbf 100644
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -37,6 +37,7 @@
 
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 #include <stdarg.h>
 #include <netdb.h>
 #include <unistd.h>
@@ -64,6 +65,8 @@
 static struct index_map idm;
 static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
 
+static long long polling_time;
+
 /*
  * Immediate data format is determined by the upper bits
  * bit 31: message type, 0 - data, 1 - control
@@ -197,6 +200,16 @@ struct rsocket {
 	uint8_t		  *sbuf;
 };
 
+void rs_configure(void)
+{
+	FILE *f;
+
+	if ((f = fopen(RS_CONF_DIR "/polling_time", "r"))) {
+		fscanf(f, "%L", &polling_time);
+		fclose(f);
+	}
+}
+
 /*
  * We currently generate a completion per send.  sqe_count = 1
  */
@@ -1469,12 +1482,23 @@ static int rs_poll_events(struct pollfd *rfds, struct pollfd *fds, nfds_t nfds)
  */
 int rpoll(struct pollfd *fds, nfds_t nfds, int timeout)
 {
+	struct timeval s, e;
 	struct pollfd *rfds;
+	long long poll_time = 0;
 	int ret;
 
-	ret = rs_poll_check(fds, nfds);
-	if (ret || !timeout)
-		return ret;
+	do {
+		ret = rs_poll_check(fds, nfds);
+		if (ret || !timeout)
+			return ret;
+
+		if (!poll_time)
+			gettimeofday(&s, NULL);
+
+		gettimeofday(&e, NULL);
+		poll_time = (e.tv_sec - s.tv_sec) * 1000000 +
+			    (e.tv_usec - s.tv_usec) + 1;
+	} while (poll_time < polling_time);
 
 	rfds = rs_fds_alloc(nfds);
 	if (!rfds)