From: Roland Dreier Date: Mon, 21 May 2007 03:12:15 +0000 (-0700) Subject: Pass send queue sizes from userspace to kernel X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=20e2b7e6d66cfd43210a00fa0f19690f29de9ca4;p=~shefty%2Flibmlx4.git Pass send queue sizes from userspace to kernel Update to handle kernel mlx4 ABI version 2: pass log_2 of send queue WQE basic block size and log_2 of number of send queue basic blocks to the kernel to avoid bugs caused by the kernel calculating a different send queue WQE size. This will also allow us to use multiple BBs per WQE if we want to someday. Signed-off-by: Roland Dreier --- diff --git a/src/mlx4-abi.h b/src/mlx4-abi.h index 7810913..97f5dcd 100644 --- a/src/mlx4-abi.h +++ b/src/mlx4-abi.h @@ -35,7 +35,8 @@ #include -#define MLX4_UVERBS_ABI_VERSION 1 +#define MLX4_UVERBS_MIN_ABI_VERSION 2 +#define MLX4_UVERBS_MAX_ABI_VERSION 2 struct mlx4_alloc_ucontext_resp { struct ibv_get_context_resp ibv_resp; @@ -83,6 +84,9 @@ struct mlx4_create_qp { struct ibv_create_qp ibv_cmd; __u64 buf_addr; __u64 db_addr; + __u8 log_sq_bb_count; + __u8 log_sq_stride; + __u8 reserved[6]; }; #endif /* MLX4_ABI_H */ diff --git a/src/mlx4.c b/src/mlx4.c index 23577c1..ac522a7 100644 --- a/src/mlx4.c +++ b/src/mlx4.c @@ -250,9 +250,13 @@ static struct ibv_device *mlx4_driver_init(const char *uverbs_sys_path, return NULL; found: - if (abi_version > MLX4_UVERBS_ABI_VERSION) { - fprintf(stderr, PFX "Fatal: ABI version %d of %s is too new (expected %d)\n", - abi_version, uverbs_sys_path, MLX4_UVERBS_ABI_VERSION); + if (abi_version < MLX4_UVERBS_MIN_ABI_VERSION || + abi_version > MLX4_UVERBS_MAX_ABI_VERSION) { + fprintf(stderr, PFX "Fatal: ABI version %d of %s is not supported " + "(min supported %d, max supported %d)\n", + abi_version, uverbs_sys_path, + MLX4_UVERBS_MIN_ABI_VERSION, + MLX4_UVERBS_MAX_ABI_VERSION); return NULL; } diff --git a/src/mlx4.h b/src/mlx4.h index 1e92b88..7978f9f 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -343,6 +343,8 @@ int mlx4_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr); int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap, enum ibv_qp_type type, struct mlx4_qp *qp); +void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap, + enum ibv_qp_type type); struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn); int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp); void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn); diff --git a/src/qp.c b/src/qp.c index a4384f9..bff3f33 100644 --- a/src/qp.c +++ b/src/qp.c @@ -469,6 +469,32 @@ int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap, return 0; } +void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap, + enum ibv_qp_type type) +{ + int wqe_size; + + wqe_size = 1 << qp->sq.wqe_shift; + switch (type) { + case IBV_QPT_UD: + wqe_size -= sizeof (struct mlx4_wqe_datagram_seg); + break; + + case IBV_QPT_UC: + case IBV_QPT_RC: + wqe_size -= sizeof (struct mlx4_wqe_raddr_seg); + break; + + default: + break; + } + + qp->sq.max_gs = wqe_size / sizeof (struct mlx4_wqe_data_seg); + cap->max_send_sge = qp->sq.max_gs; + qp->max_inline_data = wqe_size - sizeof (struct mlx4_wqe_inline_seg); + cap->max_inline_data = qp->max_inline_data; +} + struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn) { int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift; diff --git a/src/verbs.c b/src/verbs.c index fa9674b..41914ac 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -390,8 +390,14 @@ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) *qp->db = 0; - cmd.buf_addr = (uintptr_t) qp->buf.buf; - cmd.db_addr = (uintptr_t) qp->db; + cmd.buf_addr = (uintptr_t) qp->buf.buf; + cmd.db_addr = (uintptr_t) qp->db; + cmd.log_sq_stride = qp->sq.wqe_shift; + for (cmd.log_sq_bb_count = 0; + qp->sq.max > 1 << cmd.log_sq_bb_count; + ++cmd.log_sq_bb_count) + ; /* nothing */ + memset(cmd.reserved, 0, sizeof cmd.reserved); ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd, &resp, sizeof resp); @@ -402,11 +408,9 @@ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) if (ret) goto err_destroy; - qp->sq.max = attr->cap.max_send_wr; - qp->rq.max = attr->cap.max_recv_wr; - qp->sq.max_gs = attr->cap.max_send_sge; - qp->rq.max_gs = attr->cap.max_recv_sge; - qp->max_inline_data = attr->cap.max_inline_data; + qp->rq.max = attr->cap.max_recv_wr; + qp->rq.max_gs = attr->cap.max_recv_sge; + mlx4_set_sq_sizes(qp, &attr->cap, attr->qp_type); qp->doorbell_qpn = htonl(qp->ibv_qp.qp_num << 8); if (attr->sq_sig_all)