Add IBoE support

Modify libmlx4 to support IBoE.  The only user space piece to handle
is the creation of UD address handles - the L2 Ethernet attributes
have to be resolved from the DGID.  Derived from work by Eli Cohen
<eli@mellanox.co.il>

Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>

diff --git a/src/mlx4.h b/src/mlx4.h
index 44459981f06a2e5e4170465a1c0827eb51ea0cc8..b277b062882b86d0ac2299aef365b816ab7117c9 100644 (file)
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -241,6 +241,7 @@ struct mlx4_av {
 struct mlx4_ah {
        struct ibv_ah                   ibv_ah;
        struct mlx4_av                  av;
+       uint8_t                         mac[6];
 };
 
 static inline unsigned long align(unsigned long val, unsigned long align)

diff --git a/src/qp.c b/src/qp.c
index ec138cddd97163f4cc584cd82a5b383626892f08..4d79e3881a1195295b57feaba89baa1599cad945 100644 (file)
--- a/src/qp.c
+++ b/src/qp.c
@@ -144,6 +144,7 @@ static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
        memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
        dseg->dqpn = htonl(wr->wr.ud.remote_qpn);
        dseg->qkey = htonl(wr->wr.ud.remote_qkey);
+       memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6);
 }
 
 static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg)

diff --git a/src/verbs.c b/src/verbs.c
index 1ac1362f2f415b46a0c194a896773ae00a911f4e..389801c8ecf7fd8da0e873facb32f02e4b8f5581 100644 (file)
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -614,9 +614,44 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp)
        return 0;
 }
 
+static int link_local_gid(const union ibv_gid *gid)
+{
+       uint32_t hi = *(uint32_t *)(gid->raw);
+       uint32_t lo = *(uint32_t *)(gid->raw + 4);
+       if (hi == htonl(0xfe800000) && lo == 0)
+               return 1;
+
+       return 0;
+}
+
+static uint16_t get_vlan_id(union ibv_gid *gid)
+{
+       uint16_t vid;
+       vid = gid->raw[11] << 8 | gid->raw[12];
+       return vid < 0x1000 ? vid : 0xffff;
+}
+
+static int mlx4_resolve_grh_to_l2(struct mlx4_ah *ah, struct ibv_ah_attr *attr)
+{
+       if (get_vlan_id(&attr->grh.dgid) != 0xffff)
+               return 1;
+
+       if (link_local_gid(&attr->grh.dgid)) {
+               memcpy(ah->mac, &attr->grh.dgid.raw[8], 3);
+               memcpy(ah->mac + 3, &attr->grh.dgid.raw[13], 3);
+               ah->mac[0] ^= 2;
+               return 0;
+       } else
+               return 1;
+}
+
 struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
 {
        struct mlx4_ah *ah;
+       struct ibv_port_attr port_attr;
+
+       if (ibv_query_port(pd->context, attr->port_num, &port_attr))
+               return NULL;
 
        ah = malloc(sizeof *ah);
        if (!ah)
@@ -625,8 +660,12 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
        memset(&ah->av, 0, sizeof ah->av);
 
        ah->av.port_pd   = htonl(to_mpd(pd)->pdn | (attr->port_num << 24));
-       ah->av.g_slid    = attr->src_path_bits;
-       ah->av.dlid      = htons(attr->dlid);
+
+       if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
+               ah->av.g_slid = attr->src_path_bits;
+               ah->av.dlid   = htons(attr->dlid);
+       }
+
        if (attr->static_rate) {
                ah->av.stat_rate = attr->static_rate + MLX4_STAT_RATE_OFFSET;
                /* XXX check rate cap? */
@@ -642,6 +681,12 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
                memcpy(ah->av.dgid, attr->grh.dgid.raw, 16);
        }
 
+       if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET)
+               if (mlx4_resolve_grh_to_l2(ah, attr)) {
+                       free(ah);
+                       return NULL;
+               }
+
        return &ah->ibv_ah;
 }
 

diff --git a/src/wqe.h b/src/wqe.h
index 6f7f3091457e9ace4b6cacd63f1d8fc406ef726c..043f0dae8175101b5936c4316878a9d65145e18f 100644 (file)
--- a/src/wqe.h
+++ b/src/wqe.h
@@ -78,7 +78,8 @@ struct mlx4_wqe_datagram_seg {
        uint32_t                av[8];
        uint32_t                dqpn;
        uint32_t                qkey;
-       uint32_t                reserved[2];
+       uint16_t                reserved;
+       uint8_t                 mac[6];
 };
 
 struct mlx4_wqe_data_seg {