debug fork

diff --git a/examples/rstream.c b/examples/rstream.c
index befb7c667465872c6025b967f350d9f9be989c47..545328bf5e03226752370f2a860e34c232d36b7b 100644 (file)
--- a/examples/rstream.c
+++ b/examples/rstream.c
@@ -182,6 +182,7 @@ static void show_perf(void)
 
 static int size_to_count(int size)
 {
+       return 100;
        if (size >= 1000000)
                return 100;
        else if (size >= 100000)

diff --git a/src/cma.c b/src/cma.c
index 5280620e934e2251f9dba787c85d73641ca1a5ed..54b804e1da73a1c68ad43baf63e7e5c1359ce14e 100755 (executable)
--- a/src/cma.c
+++ b/src/cma.c
@@ -48,6 +48,7 @@
 #include <byteswap.h>
 #include <stddef.h>
 #include <netdb.h>
+#include <stdio.h>
 
 #include "cma.h"
 #include <infiniband/driver.h>
@@ -141,6 +142,7 @@ static void ucma_cleanup(void)
                free(cma_dev_array);
                cma_dev_cnt = 0;
        }
+       printf("ucma_cleanup\n");
 }
 
 static int check_abi_version(void)
@@ -214,6 +216,7 @@ int ucma_init(void)
        if (cma_dev_cnt)
                return 0;
 
+       printf("ucma_init\n");
        pthread_mutex_lock(&mut);
        if (cma_dev_cnt) {
                pthread_mutex_unlock(&mut);

diff --git a/src/preload.c b/src/preload.c
index 52eaf1a52b31f083e542d1e6b88fd56e313e39f3..370e8bc8070838da57843980142a53cad7ad0357 100644 (file)
--- a/src/preload.c
+++ b/src/preload.c
@@ -48,6 +48,7 @@
 #include <netinet/tcp.h>
 #include <unistd.h>
 #include <semaphore.h>
+#include <stdio.h>
 
 #include <rdma/rdma_cma.h>
 #include <rdma/rdma_verbs.h>
@@ -382,6 +383,7 @@ int socket(int domain, int type, int protocol)
        if (index < 0)
                return index;
 
+       printf("socket\n");
        recursive = 1;
        ret = rsocket(domain, type, protocol);
        recursive = 0;
@@ -389,6 +391,7 @@ int socket(int domain, int type, int protocol)
                if (fork_support) {
                        rclose(ret);
                        ret = real.socket(domain, type, protocol);
+                       printf("socket - fork support - real socket %d\n", ret);
                        if (ret < 0)
                                return ret;
                        fd_store(index, ret, fd_fork);
@@ -396,6 +399,7 @@ int socket(int domain, int type, int protocol)
                        fd_store(index, ret, fd_rsocket);
                        set_rsocket_options(ret);
                }
+               printf("socket - return %d\n", index);
                return index;
        }
        fd_close(index, &ret);
@@ -406,6 +410,7 @@ real:
 int bind(int socket, const struct sockaddr *addr, socklen_t addrlen)
 {
        int fd;
+       printf("bind %d\n", socket);
        return (fd_get(socket, &fd) == fd_rsocket) ?
                rbind(fd, addr, addrlen) : real.bind(fd, addr, addrlen);
 }
@@ -413,6 +418,7 @@ int bind(int socket, const struct sockaddr *addr, socklen_t addrlen)
 int listen(int socket, int backlog)
 {
        int fd;
+       printf("listen %d\n", socket);
        return (fd_get(socket, &fd) == fd_rsocket) ?
                rlisten(fd, backlog) : real.listen(fd, backlog);
 }
@@ -422,6 +428,7 @@ int accept(int socket, struct sockaddr *addr, socklen_t *addrlen)
        int fd, index, ret;
        enum fd_type type;
 
+       printf("accept %d\n", socket);
        type = fd_get(socket, &fd);
        if (type == fd_rsocket || type == fd_fork) {
                index = fd_open();
@@ -430,11 +437,14 @@ int accept(int socket, struct sockaddr *addr, socklen_t *addrlen)
 
                ret = (type == fd_rsocket) ? raccept(fd, addr, addrlen) :
                                             real.accept(fd, addr, addrlen);
+               printf("accept %d, new index %d new socket %d err %s\n",
+                       socket, index, ret, strerror(errno));
                if (ret < 0) {
                        fd_close(index, &fd);
                        return ret;
                }
 
+               printf("accept %d, new index %d new socket %d\n", socket, index, ret);
                fd_store(index, ret, type);
                return index;
        } else {
@@ -455,25 +465,33 @@ static int fork_active(int socket, const struct sockaddr *addr, socklen_t addrle
        uint32_t msg;
        long flags;
 
+       printf("connect_fork\n");
        fd = fd_getd(socket);
        flags = real.fcntl(fd, F_GETFL);
        real.fcntl(fd, F_SETFL, 0);
        ret = real.connect(fd, addr, addrlen);
+       printf("connect_fork - real connect %d %s\n", ret, strerror(errno));
        if (ret)
                return ret;
 
-       ret = real.recv(fd, &msg, sizeof msg, MSG_PEEK);
+       do {
+               ret = real.recv(fd, &msg, sizeof msg, MSG_PEEK);
+       } while (!ret);
+       printf("connect_fork - real recv %d msg %d\n", ret, msg);
        if ((ret != sizeof msg) || msg) {
+               printf("connect_fork - falling back to normal socket\n");
                fd_store(socket, fd, fd_normal);
                return 0;
        }
 
        real.fcntl(fd, F_SETFL, flags);
        ret = transpose_socket(socket, fd_rsocket);
+       printf("connect_fork - transpose socket %d\n", ret);
        if (ret < 0)
                return ret;
 
        real.close(fd);
+       printf("connect_fork - connecting rsocket");
        return rconnect(ret, addr, addrlen);
 }
 
@@ -485,10 +503,12 @@ static void fork_passive(int socket)
        socklen_t len;
        uint32_t msg;
 
+       printf("fork_passive\n");
        fd_get(socket, &sfd);
 
        len = sizeof sin6;
        ret = real.getsockname(sfd, (struct sockaddr *) &sin6, &len);
+       printf("fork_passive - getsockname %d (%s)\n", ret, strerror(errno));
        if (ret)
                goto out;
        sin6.sin6_flowinfo = sin6.sin6_scope_id = 0;
@@ -496,12 +516,14 @@ static void fork_passive(int socket)
 
        sem = sem_open("/rsocket_fork", O_CREAT | O_RDWR,
                       S_IRWXU | S_IRWXG, 1);
+       printf("fork_passive - sem_open %p (%s)\n", (void *) sem, strerror(errno));
        if (sem == SEM_FAILED) {
                ret = -1;
                goto out;
        }
 
        lfd = rsocket(sin6.sin6_family, SOCK_STREAM, 0);
+       printf("fork_passive - rsocket %d (%s)\n", lfd, strerror(errno));
        if (lfd < 0) {
                ret  = lfd;
                goto sclose;
@@ -512,6 +534,7 @@ static void fork_passive(int socket)
 
        sem_wait(sem);
        ret = rbind(lfd, (struct sockaddr *) &sin6, sizeof sin6);
+       printf("fork_passive - rbind %d (%s)\n", ret, strerror(errno));
        if (ret)
                goto lclose;
 
@@ -570,6 +593,7 @@ int connect(int socket, const struct sockaddr *addr, socklen_t addrlen)
 {
        int fd, ret;
 
+       printf("conect %d\n", socket);
        switch (fd_get(socket, &fd)) {
        case fd_fork:
                return fork_active(socket, addr, addrlen);

diff --git a/src/rsocket.c b/src/rsocket.c
index ed125b3b3a382dafc7ed64b3a4fd65a3ff0c06c9..1d293f75e2caffa70958378afe32abf2b7635c61 100644 (file)
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -361,7 +361,9 @@ static int rs_init_bufs(struct rsocket *rs)
        if (!rs->sbuf)
                return -1;
 
+       printf("rs init bufs cm_id %p pd %p\n", rs->cm_id, rs->cm_id->pd);
        rs->smr = rdma_reg_msgs(rs->cm_id, rs->sbuf, rs->sbuf_size);
+       printf("rs init bufs - after reg %p\n", rs->smr);
        if (!rs->smr)
                return -1;
 
@@ -425,10 +427,12 @@ static int rs_create_ep(struct rsocket *rs)
 
        rs_set_qp_size(rs);
        ret = rs_init_bufs(rs);
+       printf("rs create ep init bufs %d\n", ret);
        if (ret)
                return ret;
 
        ret = rs_create_cq(rs);
+       printf("rs create ep - create cq %d\n", ret);
        if (ret)
                return ret;
 
@@ -445,6 +449,7 @@ static int rs_create_ep(struct rsocket *rs)
        qp_attr.cap.max_inline_data = rs->sq_inline;
 
        ret = rdma_create_qp(rs->cm_id, NULL, &qp_attr);
+       printf("rs create ep - create qp %d\n", ret);
        if (ret)
                return ret;
 
@@ -602,16 +607,19 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen)
        struct rs_conn_data *creq, cresp;
        int ret;
 
+       printf("raccept %d\n", socket);
        rs = idm_at(&idm, socket);
        new_rs = rs_alloc(rs);
        if (!new_rs)
                return ERR(ENOMEM);
 
        ret = rdma_get_request(rs->cm_id, &new_rs->cm_id);
+       printf("raccept get request %d\n", ret);
        if (ret)
                goto err;
 
        ret = rs_insert(new_rs);
+       printf("raccept insert %d\n", ret);
        if (ret < 0)
                goto err;
 
@@ -625,6 +633,7 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen)
                rs_set_nonblocking(new_rs, O_NONBLOCK);
 
        ret = rs_create_ep(new_rs);
+       printf("raccept create ep %d\n", ret);
        if (ret)
                goto err;
 
@@ -632,6 +641,7 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen)
        param = new_rs->cm_id->event->param.conn;
        rs_set_conn_data(new_rs, &param, &cresp);
        ret = rdma_accept(new_rs->cm_id, &param);
+       printf("raccept rdma accept %d\n", ret);
        if (!ret)
                new_rs->state = rs_connect_rdwr;
        else if (errno == EAGAIN || errno == EWOULDBLOCK)