From 0c269855776d3001e37da8c8afe283c20e1d6cd6 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Wed, 18 Jul 2012 09:01:43 -0700 Subject: [PATCH] debug fork --- examples/rstream.c | 1 + src/cma.c | 3 +++ src/preload.c | 24 ++++++++++++++++++++++++ src/rsocket.c | 10 ++++++++++ 4 files changed, 38 insertions(+) diff --git a/examples/rstream.c b/examples/rstream.c index befb7c66..545328bf 100644 --- a/examples/rstream.c +++ b/examples/rstream.c @@ -182,6 +182,7 @@ static void show_perf(void) static int size_to_count(int size) { + return 100; if (size >= 1000000) return 100; else if (size >= 100000) diff --git a/src/cma.c b/src/cma.c index 5280620e..54b804e1 100755 --- a/src/cma.c +++ b/src/cma.c @@ -48,6 +48,7 @@ #include #include #include +#include #include "cma.h" #include @@ -141,6 +142,7 @@ static void ucma_cleanup(void) free(cma_dev_array); cma_dev_cnt = 0; } + printf("ucma_cleanup\n"); } static int check_abi_version(void) @@ -214,6 +216,7 @@ int ucma_init(void) if (cma_dev_cnt) return 0; + printf("ucma_init\n"); pthread_mutex_lock(&mut); if (cma_dev_cnt) { pthread_mutex_unlock(&mut); diff --git a/src/preload.c b/src/preload.c index 52eaf1a5..455b41ef 100644 --- a/src/preload.c +++ b/src/preload.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -95,6 +96,7 @@ static int sq_size; static int rq_size; static int sq_inline; static int fork_support; +static FILE *log; enum fd_type { fd_normal, @@ -278,6 +280,7 @@ static void init_preload(void) rs.fcntl = dlsym(RTLD_DEFAULT, "rfcntl"); getenv_options(); + log = fopen("/var/log/rs.log", "w"); init = 1; out: pthread_mutex_unlock(&mut); @@ -382,6 +385,7 @@ int socket(int domain, int type, int protocol) if (index < 0) return index; + fprintf(log, "socket\n");fflush(log); recursive = 1; ret = rsocket(domain, type, protocol); recursive = 0; @@ -389,6 +393,7 @@ int socket(int domain, int type, int protocol) if (fork_support) { rclose(ret); ret = real.socket(domain, type, protocol); + fprintf(log, "socket - fork support - real socket %d\n", ret);fflush(log); if (ret < 0) return ret; fd_store(index, ret, fd_fork); @@ -396,6 +401,7 @@ int socket(int domain, int type, int protocol) fd_store(index, ret, fd_rsocket); set_rsocket_options(ret); } + fprintf(log, "socket - return %d\n", index);fflush(log); return index; } fd_close(index, &ret); @@ -406,6 +412,7 @@ real: int bind(int socket, const struct sockaddr *addr, socklen_t addrlen) { int fd; + fprintf(log, "bind %d\n", socket);fflush(log); return (fd_get(socket, &fd) == fd_rsocket) ? rbind(fd, addr, addrlen) : real.bind(fd, addr, addrlen); } @@ -413,6 +420,7 @@ int bind(int socket, const struct sockaddr *addr, socklen_t addrlen) int listen(int socket, int backlog) { int fd; + fprintf(log, "listen %d\n", socket);fflush(log); return (fd_get(socket, &fd) == fd_rsocket) ? rlisten(fd, backlog) : real.listen(fd, backlog); } @@ -422,6 +430,7 @@ int accept(int socket, struct sockaddr *addr, socklen_t *addrlen) int fd, index, ret; enum fd_type type; + fprintf(log, "accept %d\n", socket);fflush(log); type = fd_get(socket, &fd); if (type == fd_rsocket || type == fd_fork) { index = fd_open(); @@ -430,11 +439,14 @@ int accept(int socket, struct sockaddr *addr, socklen_t *addrlen) ret = (type == fd_rsocket) ? raccept(fd, addr, addrlen) : real.accept(fd, addr, addrlen); + fprintf(log, "accept %d, new index %d new socket %d err %s\n", + socket, index, ret, strerror(errno));fflush(log); if (ret < 0) { fd_close(index, &fd); return ret; } + fprintf(log, "accept %d, new index %d new socket %d\n", socket, index, ret);fflush(log); fd_store(index, ret, type); return index; } else { @@ -455,25 +467,31 @@ static int fork_active(int socket, const struct sockaddr *addr, socklen_t addrle uint32_t msg; long flags; + fprintf(log, "connect_fork\n");fflush(log); fd = fd_getd(socket); flags = real.fcntl(fd, F_GETFL); real.fcntl(fd, F_SETFL, 0); ret = real.connect(fd, addr, addrlen); + fprintf(log, "connect_fork - real connect %d %s\n", ret, strerror(errno));fflush(log); if (ret) return ret; ret = real.recv(fd, &msg, sizeof msg, MSG_PEEK); + fprintf(log, "connect_fork - real recv %d msg %d\n", ret, msg);fflush(log); if ((ret != sizeof msg) || msg) { + fprintf(log, "connect_fork - falling back to normal socket\n");fflush(log); fd_store(socket, fd, fd_normal); return 0; } real.fcntl(fd, F_SETFL, flags); ret = transpose_socket(socket, fd_rsocket); + fprintf(log, "connect_fork - transpose socket %d\n", ret);fflush(log); if (ret < 0) return ret; real.close(fd); + fprintf(log, "connect_fork - connecting rsocket");fflush(log); return rconnect(ret, addr, addrlen); } @@ -485,10 +503,12 @@ static void fork_passive(int socket) socklen_t len; uint32_t msg; + fprintf(log, "fork_passive\n");fflush(log); fd_get(socket, &sfd); len = sizeof sin6; ret = real.getsockname(sfd, (struct sockaddr *) &sin6, &len); + fprintf(log, "fork_passive - getsockname %d (%s)\n", ret, strerror(errno));fflush(log); if (ret) goto out; sin6.sin6_flowinfo = sin6.sin6_scope_id = 0; @@ -496,12 +516,14 @@ static void fork_passive(int socket) sem = sem_open("/rsocket_fork", O_CREAT | O_RDWR, S_IRWXU | S_IRWXG, 1); + fprintf(log, "fork_passive - sem_open %p (%s)\n", (void *) sem, strerror(errno));fflush(log); if (sem == SEM_FAILED) { ret = -1; goto out; } lfd = rsocket(sin6.sin6_family, SOCK_STREAM, 0); + fprintf(log, "fork_passive - rsocket %d (%s)\n", lfd, strerror(errno));fflush(log); if (lfd < 0) { ret = lfd; goto sclose; @@ -512,6 +534,7 @@ static void fork_passive(int socket) sem_wait(sem); ret = rbind(lfd, (struct sockaddr *) &sin6, sizeof sin6); + fprintf(log, "fork_passive - rbind %d (%s)\n", ret, strerror(errno));fflush(log); if (ret) goto lclose; @@ -570,6 +593,7 @@ int connect(int socket, const struct sockaddr *addr, socklen_t addrlen) { int fd, ret; + fprintf(log, "conect %d\n", socket); switch (fd_get(socket, &fd)) { case fd_fork: return fork_active(socket, addr, addrlen); diff --git a/src/rsocket.c b/src/rsocket.c index ed125b3b..1d293f75 100644 --- a/src/rsocket.c +++ b/src/rsocket.c @@ -361,7 +361,9 @@ static int rs_init_bufs(struct rsocket *rs) if (!rs->sbuf) return -1; + printf("rs init bufs cm_id %p pd %p\n", rs->cm_id, rs->cm_id->pd); rs->smr = rdma_reg_msgs(rs->cm_id, rs->sbuf, rs->sbuf_size); + printf("rs init bufs - after reg %p\n", rs->smr); if (!rs->smr) return -1; @@ -425,10 +427,12 @@ static int rs_create_ep(struct rsocket *rs) rs_set_qp_size(rs); ret = rs_init_bufs(rs); + printf("rs create ep init bufs %d\n", ret); if (ret) return ret; ret = rs_create_cq(rs); + printf("rs create ep - create cq %d\n", ret); if (ret) return ret; @@ -445,6 +449,7 @@ static int rs_create_ep(struct rsocket *rs) qp_attr.cap.max_inline_data = rs->sq_inline; ret = rdma_create_qp(rs->cm_id, NULL, &qp_attr); + printf("rs create ep - create qp %d\n", ret); if (ret) return ret; @@ -602,16 +607,19 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen) struct rs_conn_data *creq, cresp; int ret; + printf("raccept %d\n", socket); rs = idm_at(&idm, socket); new_rs = rs_alloc(rs); if (!new_rs) return ERR(ENOMEM); ret = rdma_get_request(rs->cm_id, &new_rs->cm_id); + printf("raccept get request %d\n", ret); if (ret) goto err; ret = rs_insert(new_rs); + printf("raccept insert %d\n", ret); if (ret < 0) goto err; @@ -625,6 +633,7 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen) rs_set_nonblocking(new_rs, O_NONBLOCK); ret = rs_create_ep(new_rs); + printf("raccept create ep %d\n", ret); if (ret) goto err; @@ -632,6 +641,7 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen) param = new_rs->cm_id->event->param.conn; rs_set_conn_data(new_rs, ¶m, &cresp); ret = rdma_accept(new_rs->cm_id, ¶m); + printf("raccept rdma accept %d\n", ret); if (!ret) new_rs->state = rs_connect_rdwr; else if (errno == EAGAIN || errno == EWOULDBLOCK) -- 2.45.2