From 3967a69df87df87492e83642404699c8487a5295 Mon Sep 17 00:00:00 2001
From: Sean Hefty <sean.hefty@intel.com>
Date: Tue, 21 Aug 2012 13:49:59 -0700
Subject: [PATCH] debug fork issue

---
 src/preload.c | 23 ++++++++++++++++++++++-
 src/rsocket.c |  9 +++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/preload.c b/src/preload.c
index 0645f6dd..d1d8b06c 100644
--- a/src/preload.c
+++ b/src/preload.c
@@ -34,6 +34,7 @@
 #if HAVE_CONFIG_H
 #  include <config.h>
 #endif /* HAVE_CONFIG_H */
+#include <stdio.h>
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -239,7 +240,7 @@ void getenv_options(void)
 	if (var)
 		sq_inline = atoi(var);
 
-	var = getenv("RDMAV_FORK_SAFE");
+	var = getenv("RS_FORK_SUPPORT");
 	if (var)
 		fork_support = atoi(var);
 }
@@ -412,12 +413,19 @@ int socket(int domain, int type, int protocol)
 	if (index < 0)
 		return index;
 
+	if ((domain == PF_INET || domain == PF_INET6) &&
+	    (type == SOCK_STREAM) && (!protocol || protocol == IPPROTO_TCP) && fork_support) {
+		printf("skipping rsocket call\n");
+		goto realsock;
+	}
+
 	recursive = 1;
 	ret = rsocket(domain, type, protocol);
 	recursive = 0;
 	if (ret >= 0) {
 		if (fork_support) {
 			rclose(ret);
+realsock:
 			ret = real.socket(domain, type, protocol);
 			if (ret < 0)
 				return ret;
@@ -502,24 +510,30 @@ static void fork_active(int socket)
 	uint32_t msg;
 	long flags;
 
+	printf("fork_active\n");
 	sfd = fd_getd(socket);
 
 	flags = real.fcntl(sfd, F_GETFL);
 	real.fcntl(sfd, F_SETFL, 0);
+	printf("fork_active - recv\n");
 	ret = real.recv(sfd, &msg, sizeof msg, MSG_PEEK);
+	printf("fork_active - recv %d\n", ret);
 	real.fcntl(sfd, F_SETFL, flags);
 	if ((ret != sizeof msg) || msg)
 		goto err1;
 
 	len = sizeof addr;
 	ret = real.getpeername(sfd, (struct sockaddr *) &addr, &len);
+	printf("fork_active - getpeername %d\n", ret);
 	if (ret)
 		goto err1;
 
+	printf("fork_active - create rsocket\n");
 	dfd = rsocket(addr.ss_family, SOCK_STREAM, 0);
 	if (dfd < 0)
 		goto err1;
 
+	printf("fork_active - rconnect\n");
 	ret = rconnect(dfd, (struct sockaddr *) &addr, len);
 	if (ret)
 		goto err2;
@@ -551,6 +565,7 @@ static void fork_passive(int socket)
 	socklen_t len;
 	uint32_t msg;
 
+	printf("fork_passive\n");
 	sfd = fd_getd(socket);
 
 	len = sizeof sin6;
@@ -566,6 +581,7 @@ static void fork_passive(int socket)
 		ret = -1;
 		goto out;
 	}
+	printf("fork_passive - create rsocket\n");
 
 	lfd = rsocket(sin6.sin6_family, SOCK_STREAM, 0);
 	if (lfd < 0) {
@@ -578,19 +594,24 @@ static void fork_passive(int socket)
 
 	sem_wait(sem);
 	ret = rbind(lfd, (struct sockaddr *) &sin6, sizeof sin6);
+	printf("fork_passive bind %d\n", ret);
 	if (ret)
 		goto lclose;
 
 	ret = rlisten(lfd, 1);
+	printf("fork_passive listen %d\n", ret);
 	if (ret)
 		goto lclose;
 
 	msg = 0;
 	len = real.write(sfd, &msg, sizeof msg);
+	printf("fork_passive write %d\n", len);
 	if (len != sizeof msg)
 		goto lclose;
+	printf("fork_passive - raccept\n");
 
 	dfd = raccept(lfd, NULL, NULL);
+	printf("fork_passive accept %d\n", dfd);
 	if (dfd < 0) {
 		ret  = dfd;
 		goto lclose;
diff --git a/src/rsocket.c b/src/rsocket.c
index 4c4821bc..52476a27 100644
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -156,6 +156,10 @@ struct rsocket {
 	fastlock_t	  cq_lock;
 	fastlock_t	  cq_wait_lock;
 
+	struct rs_msg	  last_ctrl;
+	uint64_t	  bytes_sent;
+	uint64_t	  bytes_received;
+
 	int		  opts;
 	long		  fd_flags;
 	uint64_t	  so_opts;
@@ -816,6 +820,7 @@ static void rs_send_credits(struct rsocket *rs)
 
 	rs->ctrl_avail--;
 	rs->rseq_comp = rs->rseq_no + (rs->rq_size >> 1);
+	rs->last_ctrl.op = rs->rseq_no + rs->rq_size;
 	if (rs->rbuf_bytes_avail >= (rs->rbuf_size >> 1)) {
 		if (!(rs->opts & RS_OPT_SWAP_SGL)) {
 			sge.addr = (uintptr_t) &rs->rbuf[rs->rbuf_free_offset];
@@ -830,6 +835,7 @@ static void rs_send_credits(struct rsocket *rs)
 		ibsge.addr = (uintptr_t) &sge;
 		ibsge.lkey = 0;
 		ibsge.length = sizeof(sge);
+		rs->last_ctrl.data = sge.length;
 
 		rs_post_write(rs, &ibsge, 1,
 			      rs_msg_set(RS_OP_SGL, rs->rseq_no + rs->rq_size),
@@ -845,6 +851,7 @@ static void rs_send_credits(struct rsocket *rs)
 		if (++rs->remote_sge == rs->remote_sgl.length)
 			rs->remote_sge = 0;
 	} else {
+		rs->last_ctrl.data = 0;
 		rs_post_write(rs, NULL, 0,
 			      rs_msg_set(RS_OP_SGL, rs->rseq_no + rs->rq_size), 0, 0, 0);
 	}
@@ -1173,6 +1180,7 @@ ssize_t rrecv(int socket, void *buf, size_t len, int flags)
 
 	} while (left && (flags & MSG_WAITALL) && (rs->state & rs_connect_rd));
 
+	rs->bytes_received += (len - left);
 	fastlock_release(&rs->rlock);
 	return ret ? ret : len - left;
 }
@@ -1289,6 +1297,7 @@ ssize_t rsend(int socket, const void *buf, size_t len, int flags)
 		if (ret)
 			break;
 	}
+	rs->bytes_sent += (len - left);
 	fastlock_release(&rs->slock);
 
 	return (ret && left == len) ? ret : len - left;
-- 
2.45.2