Refresh of rs-ddp

diff --git a/src/rsocket.c b/src/rsocket.c
index cc5effe1aba45a9b560cf786c5272055ee8e51d1..40d9440221d8d64f89e1dfcfd1841fc80a123a29 100644 (file)
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -59,9 +59,11 @@
 #define RS_QP_CTRL_SIZE 4
 #define RS_CONN_RETRIES 6
 #define RS_SGL_SIZE 2
+#define RS_MAX_IOMAP 128
 static struct index_map idm;
 static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
 
+static uint16_t def_iomap_size = 0;
 static uint16_t def_inline = 64;
 static uint16_t def_sqsize = 384;
 static uint16_t def_rqsize = 384;
@@ -76,7 +78,7 @@ static uint32_t polling_time = 10;
  * bit 29: more data, 0 - end of transfer, 1 - more data available
  *
  * for data transfers:
- * bits [28:0]: bytes transfered, 0 = 1 GB
+ * bits [28:0]: bytes transfered
  * for control messages:
  * bits [28-0]: receive credits granted
  */
@@ -111,15 +113,22 @@ struct rs_sge {
        uint32_t length;
 };
 
-#define RS_MIN_INLINE    (sizeof(struct rs_sge))
-#define rs_host_is_net() (1 == htonl(1))
-#define RS_CONN_FLAG_NET 1
+struct rs_iomap {
+       uint64_t offset;
+       struct rs_sge sge;
+};
+
+#define RS_MIN_INLINE      (sizeof(struct rs_sge))
+#define rs_host_is_net()   (1 == htonl(1))
+#define RS_CONN_FLAG_NET   (1 << 0)
+#define RS_CONN_FLAG_IOMAP (1 << 1)
 
 struct rs_conn_data {
        uint8_t           version;
        uint8_t           flags;
        uint16_t          credits;
-       uint32_t          reserved2;
+       uint8_t           reserved[3];
+       uint8_t           target_iomap_size;
        struct rs_sge     target_sgl;
        struct rs_sge     data_buf;
 };
@@ -186,10 +195,15 @@ struct rsocket {
 
        int               remote_sge;
        struct rs_sge     remote_sgl;
+       struct rs_sge     remote_iomap;
+       struct rs_iomap   *remote_iomappings;
 
        struct ibv_mr    *target_mr;
        int               target_sge;
-       volatile struct rs_sge    target_sgl[RS_SGL_SIZE];
+       int               target_iomap_size;
+       void             *target_buffer_list;
+       volatile struct rs_sge    *target_sgl;
+       volatile struct rs_iomap  *target_iomap;
 
        uint32_t          rbuf_size;
        struct ibv_mr    *rmr;
@@ -201,6 +215,18 @@ struct rsocket {
        uint8_t           *sbuf;
 };
 
+static int rs_value_to_scale(int value, int bits)
+{
+       return value <= (1 << (bits - 1)) ?
+              value : (1 << (bits - 1)) | (value >> bits);
+}
+
+static int rs_scale_to_value(int value, int bits)
+{
+       return value <= (1 << (bits - 1)) ?
+              value : (value & ~(1 << (bits - 1))) << bits;
+}
+
 void rs_configure(void)
 {
        FILE *f;
@@ -251,6 +277,15 @@ void rs_configure(void)
                if (def_wmem < 1)
                        def_wmem = 1;
        }
+
+       if ((f = fopen(RS_CONN_DIR "/iomap_size", "r"))) {
+               fscanf(f, "%hu", &def_iomap_size);
+               fclose(f);
+
+               /* round to supported values */
+               def_iomap_size = (uint8_t) rs_value_to_scale(def_iomap_size, 8);
+               def_iomap_size = (uint16_t) rs_scale_to_value(def_iomap_size, 8);
+       }
        init = 1;
 out:
        pthread_mutex_unlock(&mut);
@@ -287,6 +322,7 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs)
                rs->sq_size = inherited_rs->sq_size;
                rs->rq_size = inherited_rs->rq_size;
                rs->ctrl_avail = inherited_rs->ctrl_avail;
+               rs->target_iomap_size = inherited_rs->target_iomap_size;
        } else {
                rs->sbuf_size = def_wmem;
                rs->rbuf_size = def_mem;
@@ -294,6 +330,7 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs)
                rs->sq_size = def_sqsize;
                rs->rq_size = def_rqsize;
                rs->ctrl_avail = RS_QP_CTRL_SIZE;
+               rs->target_iomap_size = def_iomap_size;
        }
        fastlock_init(&rs->slock);
        fastlock_init(&rs->rlock);
@@ -336,6 +373,8 @@ static void rs_set_qp_size(struct rsocket *rs)
 
 static int rs_init_bufs(struct rsocket *rs)
 {
+       size_t len;
+
        rs->rmsg = calloc(rs->rq_size + 1, sizeof(*rs->rmsg));
        if (!rs->rmsg)
                return -1;
@@ -348,11 +387,21 @@ static int rs_init_bufs(struct rsocket *rs)
        if (!rs->smr)
                return -1;
 
-       rs->target_mr = rdma_reg_write(rs->cm_id, (void *) rs->target_sgl,
-                                      sizeof(rs->target_sgl));
+       len = sizeof(*rs->target_sgl) * RS_SGL_SIZE +
+             sizeof(*rs->target_iomap) * rs->target_iomap_size;
+       rs->target_buffer_list = malloc(len);
+       if (!rs->target_buffer_list)
+               return -1;
+
+       rs->target_mr = rdma_reg_write(rs->cm_id, rs->target_buffer_list, len);
        if (!rs->target_mr)
                return -1;
 
+       memset(rs->target_buffer_list, 0, len);
+       rs->target_sgl = rs->target_buffer_list;
+       if (rs->target_iomap_size)
+               rs->target_iomap = (struct rs_iomap *) (rs->target_sgl + RS_SGL_SIZE);
+
        rs->rbuf = calloc(rs->rbuf_size, sizeof(*rs->rbuf));
        if (!rs->rbuf)
                return -1;
@@ -472,8 +521,11 @@ static void rs_free(struct rsocket *rs)
                free(rs->rbuf);
        }
 
-       if (rs->target_mr)
-               rdma_dereg_mr(rs->target_mr);
+       if (rs->target_buffer_list) {
+               if (rs->target_mr)
+                       rdma_dereg_mr(rs->target_mr);
+               free(rs->target_buffer_list);
+       }
 
        if (rs->cm_id) {
                if (rs->cm_id->qp)
@@ -492,9 +544,11 @@ static void rs_set_conn_data(struct rsocket *rs, struct rdma_conn_param *param,
                             struct rs_conn_data *conn)
 {
        conn->version = 1;
-       conn->flags = rs_host_is_net() ? RS_CONN_FLAG_NET : 0;
+       conn->flags = RS_CONN_FLAG_IOMAP |
+                     (rs_host_is_net() ? RS_CONN_FLAG_NET : 0);
        conn->credits = htons(rs->rq_size);
-       conn->reserved2 = 0;
+       memset(conn->reserved, 0, sizeof conn->reserved);
+       conn->target_iomap_size = (uint8_t) rs_value_to_scale(rs->target_iomap_size, 8);
 
        conn->target_sgl.addr = htonll((uintptr_t) rs->target_sgl);
        conn->target_sgl.length = htonl(RS_SGL_SIZE);
@@ -518,6 +572,13 @@ static void rs_save_conn_data(struct rsocket *rs, struct rs_conn_data *conn)
            (!rs_host_is_net() && (conn->flags & RS_CONN_FLAG_NET)))
                rs->opts = RS_OPT_SWAP_SGL;
 
+       if (conn->flags & RS_CONN_FLAG_IOMAP) {
+               rs->remote_iomap.addr = rs->remote_sgl.addr +
+                                       sizeof(rs->remote_sgl) * rs->remote_sgl.length;
+               rs->remote_iomap.length = rs_scale_to_value(conn->target_iomap_size, 8);
+               rs->remote_iomap.key = rs->remote_sgl.key;
+       }
+
        rs->target_sgl[0].addr = ntohll(conn->data_buf.addr);
        rs->target_sgl[0].length = ntohl(conn->data_buf.length);
        rs->target_sgl[0].key = ntohl(conn->data_buf.key);