Bottom: fb1d0cc95bf413473d6098cbb67824845146864e
-Top: 5d77ccab732b248e36bffb1e4801f9c26c3fe7e2
+Top: 19610f5ed6d90b58d8fe662ad4c233e4cb5efe8c
Author: Sean Hefty <sean.hefty@intel.com>
Date: 2011-01-20 17:06:28 -0800
---
diff --git a/src/acm.c b/src/acm.c
-index f21f702..6515185 100644
+index f21f702..add7247 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -294,15 +294,22 @@ static int acm_compare_dest(const void *dest1, const void *dest2)
}
static struct acm_dest *
-@@ -2647,37 +2654,7 @@ err1:
- port->state = IBV_PORT_NOP;
+@@ -1477,10 +1484,9 @@ out:
+ free(umad);
}
--static int acm_activate_dev(struct acm_device *dev)
+-static void acm_port_join(void *context)
++static int acm_port_join(struct acm_port *port)
+ {
+ struct acm_device *dev;
+- struct acm_port *port = (struct acm_port *) context;
+ struct acm_ep *ep;
+ union ibv_gid port_gid;
+ DLIST_ENTRY *ep_entry;
+@@ -1494,7 +1500,7 @@ static void acm_port_join(void *context)
+ if (ret) {
+ acm_log(0, "ERROR - ibv_query_gid %d device %s port %d\n",
+ ret, dev->verbs->device->name, port->port_num);
+- return;
++ return ret;
+ }
+
+ for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
+@@ -1513,33 +1519,6 @@ static void acm_port_join(void *context)
+ port->port_num);
+ }
+
+-static void acm_join_groups(void)
-{
+- struct acm_device *dev;
+- struct acm_port *port;
+- DLIST_ENTRY *dev_entry;
- int i;
-
+- acm_log(1, "initiating multicast joins for all ports\n");
+- for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+- dev_entry = dev_entry->Next) {
+-
+- dev = container_of(dev_entry, struct acm_device, entry);
+-
+- for (i = 0; i < dev->port_cnt; i++) {
+- port = &dev->port[i];
+- if (port->state != IBV_PORT_ACTIVE)
+- continue;
+-
+- acm_log(1, "starting join for device %s, port %d\n",
+- dev->verbs->device->name, port->port_num);
+- // TODO: handle dynamic changes
+- //beginthread(acm_port_join, port);
+- acm_port_join(port);
+- }
+- }
+-}
+-
+ static void acm_process_timeouts(void)
+ {
+ DLIST_ENTRY *entry;
+@@ -1730,6 +1709,37 @@ acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path)
+ }
+
+ static struct acm_ep *
++acm_get_port_ep(struct acm_port *port, struct acm_ep_addr_data *data)
++{
++ struct acm_ep *ep;
++ DLIST_ENTRY *ep_entry;
++
++ if (port->state != IBV_PORT_ACTIVE)
++ return NULL;
++
++ if (data->type == ACM_EP_INFO_PATH &&
++ !acm_is_path_from_port(port, &data->info.path))
++ return NULL;
++
++ for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
++ ep_entry = ep_entry->Next) {
++
++ ep = container_of(ep_entry, struct acm_ep, entry);
++ if (ep->state != ACM_READY)
++ continue;
++
++ if ((data->type == ACM_EP_INFO_PATH) &&
++ (!data->info.path.pkey || (ntohs(data->info.path.pkey) == ep->pkey)))
++ return ep;
++
++ if (acm_addr_index(ep, data->info.addr, (uint8_t) data->type) >= 0)
++ return ep;
++ }
++
++ return NULL;
++}
++
++static struct acm_ep *
+ acm_get_ep(struct acm_ep_addr_data *data)
+ {
+ struct acm_device *dev;
+@@ -1746,28 +1756,11 @@ acm_get_ep(struct acm_ep_addr_data *data)
+
+ dev = container_of(dev_entry, struct acm_device, entry);
+ for (i = 0; i < dev->port_cnt; i++) {
+- port = &dev->port[i];
+-
+- if (data->type == ACM_EP_INFO_PATH &&
+- !acm_is_path_from_port(port, &data->info.path))
+- continue;
+-
+- for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
+- ep_entry = ep_entry->Next) {
+-
+- ep = container_of(ep_entry, struct acm_ep, entry);
+- if (ep->state != ACM_READY)
+- continue;
+-
+- if ((data->type == ACM_EP_INFO_PATH) &&
+- (!data->info.path.pkey ||
+- (ntohs(data->info.path.pkey) == ep->pkey)))
+- return ep;
+-
+- if (acm_addr_index(ep, data->info.addr,
+- (uint8_t) data->type) >= 0)
+- return ep;
+- }
++ lock_acquire(&dev->port[i].lock);
++ ep = acm_get_port_ep(&dev->port[i], data);
++ lock_release(&dev->port[i].lock);
++ if (ep)
++ return ep;
+ }
+ }
+
+@@ -2490,7 +2483,7 @@ static int acm_init_ep_loopback(struct acm_ep *ep)
+ return 0;
+ }
+
+-static int acm_activate_ep(struct acm_port *port, struct acm_ep *ep, uint16_t pkey_index)
++static int acm_ep_up(struct acm_port *port, struct acm_ep *ep, uint16_t pkey_index)
+ {
+ struct ibv_qp_init_attr init_attr;
+ struct ibv_qp_attr attr;
+@@ -2593,24 +2586,61 @@ err1:
+ return -1;
+ }
+
+-static void acm_activate_port(struct acm_port *port)
++static void acm_port_up(struct acm_port *port)
+ {
++ struct ibv_port_attr attr;
++ union ibv_gid gid;
++ uint16_t pkey;
+ struct acm_ep *ep;
+ int i, ret;
+
+- acm_log(1, "%s %d\n", port->dev->verbs->device->name,
+- port->port_num);
++ acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
++ ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
++ if (ret) {
++ acm_log(0, "ERROR - unable to get port state\n");
++ return;
++ }
++ if (attr.state != IBV_PORT_ACTIVE) {
++ acm_log(1, "port not active\n");
++ return;
++ }
++
++ port->mtu = attr.active_mtu;
++ port->rate = acm_get_rate(attr.active_width, attr.active_speed);
++ port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
++ for (;; port->gid_cnt++) {
++ ret = ibv_query_gid(port->dev->verbs, port->port_num, port->gid_cnt, &gid);
++ if (ret || !gid.global.interface_id)
++ break;
++ }
++
++ for (;; port->pkey_cnt++) {
++ ret = ibv_query_pkey(port->dev->verbs, port->port_num, port->pkey_cnt, &pkey);
++ if (ret || !pkey)
++ break;
++ }
++ port->lid = attr.lid;
++ port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
++
++ acm_set_dest_addr(&port->sa_dest, ACM_ADDRESS_LID,
++ (uint8_t *) &attr.sm_lid, sizeof(attr.sm_lid));
++ port->sa_dest.av.src_path_bits = 0;
++ port->sa_dest.av.dlid = attr.sm_lid;
++ port->sa_dest.av.sl = attr.sm_sl;
++ port->sa_dest.av.port_num = port->port_num;
++ port->sa_dest.remote_qpn = 1;
+
+ port->sa_dest.ah = ibv_create_ah(port->dev->pd, &port->sa_dest.av);
+ if (!port->sa_dest.ah)
+- goto err1;
++ return;
+
+ for (i = 0; i < port->pkey_cnt; i++) {
++ /* TODO: Check if endpoint already exists in port list */
+ ep = calloc(1, sizeof *ep);
+ if (!ep)
+ break;
+
+- ret = acm_activate_ep(port, ep, (uint16_t) i);
++ ret = acm_ep_up(port, ep, (uint16_t) i);
+ if (!ret) {
+ DListInsertHead(&ep->entry, &port->ep_list);
+ } else {
+@@ -2619,103 +2649,70 @@ static void acm_activate_port(struct acm_port *port)
+ }
+ }
+
+- if (DListEmpty(&port->ep_list))
+- goto err2;
+-
+- port->mad_portid = umad_open_port(port->dev->verbs->device->name, port->port_num);
+- if (port->mad_portid < 0) {
+- acm_log(0, "ERROR - unable to open MAD port\n");
+- goto err3;
+- }
+-
+- port->mad_agentid = umad_register(port->mad_portid,
+- IB_MGMT_CLASS_SA, 1, 1, NULL);
+- if (port->mad_agentid < 0) {
+- acm_log(0, "ERROR - unable to register MAD client\n");
+- goto err4;
+- }
+-
+- return;
+-
+-err4:
+- umad_close_port(port->mad_portid);
+-err3:
+- /* TODO: cleanup ep list */
+-err2:
+- ibv_destroy_ah(port->sa_dest.ah);
+-err1:
+- port->state = IBV_PORT_NOP;
++ acm_port_join(port);
++ lock_acquire(&port->lock);
++ port->state = attr.state;
++ lock_release(&port->lock);
+ }
+
+-static int acm_activate_dev(struct acm_device *dev)
++/*
++ * There is one event handler thread per device. This is the only thread that
++ * modifies the port state or a port endpoint list. Other threads which access
++ * those must synchronize against changes accordingly, but this thread only
++ * needs to lock when making modifications.
++ */
++static void CDECL_FUNC acm_event_handler(void *context)
+ {
++ struct acm_device *dev = (struct acm_device *) context;
+ int i;
+
- acm_log(1, "%s\n", dev->verbs->device->name);
- dev->pd = ibv_alloc_pd(dev->verbs);
- if (!dev->pd)
- goto err;
- }
-
-- for (i = 0; i < dev->port_cnt; i++) {
+ for (i = 0; i < dev->port_cnt; i++) {
- acm_log(2, "checking port %d\n", dev->port[i].port_num);
- if (dev->port[i].state == IBV_PORT_ACTIVE)
- acm_activate_port(&dev->port[i]);
-- }
--
++ acm_port_up(&dev->port[i]);
+ }
++ /* TODO: wait for port up/down events */
++}
+
- acm_log(1, "starting completion thread\n");
- beginthread(acm_comp_handler, dev);
- return 0;
--
--err:
-- ibv_dealloc_pd(dev->pd);
-- return -1;
--}
--
--static void acm_init_port(struct acm_port *port)
-+static void acm_open_port(struct acm_port *port)
- {
- struct ibv_port_attr attr;
- union ibv_gid gid;
-@@ -2685,10 +2662,8 @@ static void acm_init_port(struct acm_port *port)
- int ret;
-
- acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
-- lock_init(&port->lock);
-- DListInit(&port->ep_list);
- ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
-- if (ret)
-+ if (ret || attr.state != IBV_PORT_ACTIVE)
- return;
-
- port->state = attr.state;
-@@ -2709,7 +2684,7 @@ static void acm_init_port(struct acm_port *port)
- port->lid = attr.lid;
- port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
-
-- acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID,
-+ acm_set_dest_addr(&port->sa_dest, ACM_ADDRESS_LID,
- (uint8_t *) &attr.sm_lid, sizeof(attr.sm_lid));
- port->sa_dest.av.src_path_bits = 0;
- port->sa_dest.av.dlid = attr.sm_lid;
-@@ -2718,6 +2693,54 @@ static void acm_init_port(struct acm_port *port)
- port->sa_dest.remote_qpn = 1;
- }
-
-+static void CDECL_FUNC acm_event_handler(void *context)
-+{
-+ struct acm_device *dev = (struct acm_device *) context;
-+}
-+
-+static void acm_join_groups(void)
++static void acm_activate_devices()
+{
+ struct acm_device *dev;
-+ struct acm_port *port;
+ DLIST_ENTRY *dev_entry;
-+ int i;
-+
-+ acm_log(1, "initiating multicast joins for all ports\n");
+
+-err:
+- ibv_dealloc_pd(dev->pd);
+- return -1;
++ acm_log(1, "\n");
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
-+ dev_entry = dev_entry->Next) {
++ dev_entry = dev_entry->Next) {
+
+ dev = container_of(dev_entry, struct acm_device, entry);
-+
-+ for (i = 0; i < dev->port_cnt; i++) {
-+ port = &dev->port[i];
-+ if (port->state != IBV_PORT_ACTIVE)
-+ continue;
-+
-+ acm_log(1, "starting join for device %s, port %d\n",
-+ dev->verbs->device->name, port->port_num);
-+ // TODO: handle dynamic changes
-+ //beginthread(acm_port_join, port);
-+ acm_port_join(port);
-+ }
++ beginthread(acm_event_handler, dev);
++ beginthread(acm_comp_handler, dev);
+ }
-+}
-+static void acm_activate_dev(struct acm_device *dev)
-+{
-+ acm_log(1, "\n");
-+ beginthread(acm_event_handler, dev);
-+ beginthread(acm_comp_handler, dev);
-+}
-+
-+static void acm_init_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
-+{
+ }
+
+-static void acm_init_port(struct acm_port *port)
++static void acm_open_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
+ {
+- struct ibv_port_attr attr;
+- union ibv_gid gid;
+- uint16_t pkey;
+- int ret;
+-
+- acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
+ acm_log(1, "%s %d\n", dev->verbs->device->name, port_num);
+ port->dev = dev;
+ port->port_num = port_num;
-+ lock_init(&port->lock);
-+ DListInit(&port->ep_list);
+ lock_init(&port->lock);
+ DListInit(&port->ep_list);
+- ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
+- if (ret)
+- return;
+ acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID, NULL, 0);
-+}
-+
+
+- port->state = attr.state;
+- port->mtu = attr.active_mtu;
+- port->rate = acm_get_rate(attr.active_width, attr.active_speed);
+- port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
+- for (;; port->gid_cnt++) {
+- ret = ibv_query_gid(port->dev->verbs, port->port_num, port->gid_cnt, &gid);
+- if (ret || !gid.global.interface_id)
+- break;
++ port->mad_portid = umad_open_port(dev->verbs->device->name, port->port_num);
++ if (port->mad_portid < 0) {
++ acm_log(0, "ERROR - unable to open MAD port\n");
++ return;
+ }
+
+- for (;; port->pkey_cnt++) {
+- ret = ibv_query_pkey(port->dev->verbs, port->port_num, port->pkey_cnt, &pkey);
+- if (ret || !pkey)
+- break;
++ port->mad_agentid = umad_register(port->mad_portid,
++ IB_MGMT_CLASS_SA, 1, 1, NULL);
++ if (port->mad_agentid < 0) {
++ acm_log(0, "ERROR - unable to register MAD client\n");
++ goto err;
+ }
+- port->lid = attr.lid;
+- port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
+
+- acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID,
+- (uint8_t *) &attr.sm_lid, sizeof(attr.sm_lid));
+- port->sa_dest.av.src_path_bits = 0;
+- port->sa_dest.av.dlid = attr.sm_lid;
+- port->sa_dest.av.sl = attr.sm_sl;
+- port->sa_dest.av.port_num = port->port_num;
+- port->sa_dest.remote_qpn = 1;
++ port->state = IBV_PORT_DOWN;
++ return;
++err:
++ umad_close_port(port->mad_portid);
+ }
+
static void acm_open_dev(struct ibv_device *ibdev)
- {
- struct acm_device *dev;
-@@ -2748,25 +2771,59 @@ static void acm_open_dev(struct ibv_device *ibdev)
+@@ -2748,25 +2745,59 @@ static void acm_open_dev(struct ibv_device *ibdev)
dev->guid = ibv_get_device_guid(ibdev);
dev->port_cnt = attr.phys_port_cnt;
+ }
+
+ for (i = 0; i < dev->port_cnt; i++)
-+ acm_init_port(&dev->port[i], dev, i + 1);
++ acm_open_port(&dev->port[i], dev, i + 1);
- acm_log(1, "%s opened\n", ibdev->name);
DListInsertHead(&dev->entry, &dev_list);
static void acm_set_options(void)
{
FILE *f;
-@@ -2965,25 +3022,15 @@ int CDECL_FUNC main(int argc, char **argv)
+@@ -2963,27 +2994,13 @@ int CDECL_FUNC main(int argc, char **argv)
+ DListInit(&dev_list);
+ DListInit(&timeout_list);
event_init(&timeout_event);
-
+-
umad_init();
- ibdev = ibv_get_device_list(&dev_cnt);
- if (!ibdev) {
- acm_log(1, "initiating multicast joins\n");
- acm_join_groups();
- acm_log(1, "multicast joins done\n");
-+// acm_log(1, "initiating multicast joins\n");
+ acm_activate_devices();
-+// acm_join_groups();
-+// acm_log(1, "multicast joins done\n");
acm_log(1, "starting timeout/retry thread\n");
beginthread(acm_retry_handler, NULL);
acm_log(1, "starting server\n");
+++ /dev/null
-Bottom: 5d77ccab732b248e36bffb1e4801f9c26c3fe7e2
-Top: 19610f5ed6d90b58d8fe662ad4c233e4cb5efe8c
-Author: Sean Hefty <sean.hefty@intel.com>
-Date: 2011-01-26 13:48:26 -0800
-
-Refresh of ep_active
-
----
-
-diff --git a/src/acm.c b/src/acm.c
-index 6515185..add7247 100644
---- a/src/acm.c
-+++ b/src/acm.c
-@@ -1484,10 +1484,9 @@ out:
- free(umad);
- }
-
--static void acm_port_join(void *context)
-+static int acm_port_join(struct acm_port *port)
- {
- struct acm_device *dev;
-- struct acm_port *port = (struct acm_port *) context;
- struct acm_ep *ep;
- union ibv_gid port_gid;
- DLIST_ENTRY *ep_entry;
-@@ -1501,7 +1500,7 @@ static void acm_port_join(void *context)
- if (ret) {
- acm_log(0, "ERROR - ibv_query_gid %d device %s port %d\n",
- ret, dev->verbs->device->name, port->port_num);
-- return;
-+ return ret;
- }
-
- for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
-@@ -1520,33 +1519,6 @@ static void acm_port_join(void *context)
- port->port_num);
- }
-
--static void acm_join_groups(void)
--{
-- struct acm_device *dev;
-- struct acm_port *port;
-- DLIST_ENTRY *dev_entry;
-- int i;
--
-- acm_log(1, "initiating multicast joins for all ports\n");
-- for (dev_entry = dev_list.Next; dev_entry != &dev_list;
-- dev_entry = dev_entry->Next) {
--
-- dev = container_of(dev_entry, struct acm_device, entry);
--
-- for (i = 0; i < dev->port_cnt; i++) {
-- port = &dev->port[i];
-- if (port->state != IBV_PORT_ACTIVE)
-- continue;
--
-- acm_log(1, "starting join for device %s, port %d\n",
-- dev->verbs->device->name, port->port_num);
-- // TODO: handle dynamic changes
-- //beginthread(acm_port_join, port);
-- acm_port_join(port);
-- }
-- }
--}
--
- static void acm_process_timeouts(void)
- {
- DLIST_ENTRY *entry;
-@@ -1737,6 +1709,37 @@ acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path)
- }
-
- static struct acm_ep *
-+acm_get_port_ep(struct acm_port *port, struct acm_ep_addr_data *data)
-+{
-+ struct acm_ep *ep;
-+ DLIST_ENTRY *ep_entry;
-+
-+ if (port->state != IBV_PORT_ACTIVE)
-+ return NULL;
-+
-+ if (data->type == ACM_EP_INFO_PATH &&
-+ !acm_is_path_from_port(port, &data->info.path))
-+ return NULL;
-+
-+ for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
-+ ep_entry = ep_entry->Next) {
-+
-+ ep = container_of(ep_entry, struct acm_ep, entry);
-+ if (ep->state != ACM_READY)
-+ continue;
-+
-+ if ((data->type == ACM_EP_INFO_PATH) &&
-+ (!data->info.path.pkey || (ntohs(data->info.path.pkey) == ep->pkey)))
-+ return ep;
-+
-+ if (acm_addr_index(ep, data->info.addr, (uint8_t) data->type) >= 0)
-+ return ep;
-+ }
-+
-+ return NULL;
-+}
-+
-+static struct acm_ep *
- acm_get_ep(struct acm_ep_addr_data *data)
- {
- struct acm_device *dev;
-@@ -1753,28 +1756,11 @@ acm_get_ep(struct acm_ep_addr_data *data)
-
- dev = container_of(dev_entry, struct acm_device, entry);
- for (i = 0; i < dev->port_cnt; i++) {
-- port = &dev->port[i];
--
-- if (data->type == ACM_EP_INFO_PATH &&
-- !acm_is_path_from_port(port, &data->info.path))
-- continue;
--
-- for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
-- ep_entry = ep_entry->Next) {
--
-- ep = container_of(ep_entry, struct acm_ep, entry);
-- if (ep->state != ACM_READY)
-- continue;
--
-- if ((data->type == ACM_EP_INFO_PATH) &&
-- (!data->info.path.pkey ||
-- (ntohs(data->info.path.pkey) == ep->pkey)))
-- return ep;
--
-- if (acm_addr_index(ep, data->info.addr,
-- (uint8_t) data->type) >= 0)
-- return ep;
-- }
-+ lock_acquire(&dev->port[i].lock);
-+ ep = acm_get_port_ep(&dev->port[i], data);
-+ lock_release(&dev->port[i].lock);
-+ if (ep)
-+ return ep;
- }
- }
-
-@@ -2497,7 +2483,7 @@ static int acm_init_ep_loopback(struct acm_ep *ep)
- return 0;
- }
-
--static int acm_activate_ep(struct acm_port *port, struct acm_ep *ep, uint16_t pkey_index)
-+static int acm_ep_up(struct acm_port *port, struct acm_ep *ep, uint16_t pkey_index)
- {
- struct ibv_qp_init_attr init_attr;
- struct ibv_qp_attr attr;
-@@ -2600,73 +2586,25 @@ err1:
- return -1;
- }
-
--static void acm_activate_port(struct acm_port *port)
--{
-- struct acm_ep *ep;
-- int i, ret;
--
-- acm_log(1, "%s %d\n", port->dev->verbs->device->name,
-- port->port_num);
--
-- port->sa_dest.ah = ibv_create_ah(port->dev->pd, &port->sa_dest.av);
-- if (!port->sa_dest.ah)
-- goto err1;
--
-- for (i = 0; i < port->pkey_cnt; i++) {
-- ep = calloc(1, sizeof *ep);
-- if (!ep)
-- break;
--
-- ret = acm_activate_ep(port, ep, (uint16_t) i);
-- if (!ret) {
-- DListInsertHead(&ep->entry, &port->ep_list);
-- } else {
-- acm_log(0, "ERROR - failed to activate EP\n");
-- free(ep);
-- }
-- }
--
-- if (DListEmpty(&port->ep_list))
-- goto err2;
--
-- port->mad_portid = umad_open_port(port->dev->verbs->device->name, port->port_num);
-- if (port->mad_portid < 0) {
-- acm_log(0, "ERROR - unable to open MAD port\n");
-- goto err3;
-- }
--
-- port->mad_agentid = umad_register(port->mad_portid,
-- IB_MGMT_CLASS_SA, 1, 1, NULL);
-- if (port->mad_agentid < 0) {
-- acm_log(0, "ERROR - unable to register MAD client\n");
-- goto err4;
-- }
--
-- return;
--
--err4:
-- umad_close_port(port->mad_portid);
--err3:
-- /* TODO: cleanup ep list */
--err2:
-- ibv_destroy_ah(port->sa_dest.ah);
--err1:
-- port->state = IBV_PORT_NOP;
--}
--
--static void acm_open_port(struct acm_port *port)
-+static void acm_port_up(struct acm_port *port)
- {
- struct ibv_port_attr attr;
- union ibv_gid gid;
- uint16_t pkey;
-- int ret;
-+ struct acm_ep *ep;
-+ int i, ret;
-
- acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
- ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
-- if (ret || attr.state != IBV_PORT_ACTIVE)
-+ if (ret) {
-+ acm_log(0, "ERROR - unable to get port state\n");
- return;
-+ }
-+ if (attr.state != IBV_PORT_ACTIVE) {
-+ acm_log(1, "port not active\n");
-+ return;
-+ }
-
-- port->state = attr.state;
- port->mtu = attr.active_mtu;
- port->rate = acm_get_rate(attr.active_width, attr.active_speed);
- port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
-@@ -2691,47 +2629,65 @@ static void acm_open_port(struct acm_port *port)
- port->sa_dest.av.sl = attr.sm_sl;
- port->sa_dest.av.port_num = port->port_num;
- port->sa_dest.remote_qpn = 1;
-+
-+ port->sa_dest.ah = ibv_create_ah(port->dev->pd, &port->sa_dest.av);
-+ if (!port->sa_dest.ah)
-+ return;
-+
-+ for (i = 0; i < port->pkey_cnt; i++) {
-+ /* TODO: Check if endpoint already exists in port list */
-+ ep = calloc(1, sizeof *ep);
-+ if (!ep)
-+ break;
-+
-+ ret = acm_ep_up(port, ep, (uint16_t) i);
-+ if (!ret) {
-+ DListInsertHead(&ep->entry, &port->ep_list);
-+ } else {
-+ acm_log(0, "ERROR - failed to activate EP\n");
-+ free(ep);
-+ }
-+ }
-+
-+ acm_port_join(port);
-+ lock_acquire(&port->lock);
-+ port->state = attr.state;
-+ lock_release(&port->lock);
- }
-
-+/*
-+ * There is one event handler thread per device. This is the only thread that
-+ * modifies the port state or a port endpoint list. Other threads which access
-+ * those must synchronize against changes accordingly, but this thread only
-+ * needs to lock when making modifications.
-+ */
- static void CDECL_FUNC acm_event_handler(void *context)
- {
- struct acm_device *dev = (struct acm_device *) context;
-+ int i;
-+
-+ for (i = 0; i < dev->port_cnt; i++) {
-+ acm_port_up(&dev->port[i]);
-+ }
-+ /* TODO: wait for port up/down events */
- }
-
--static void acm_join_groups(void)
-+static void acm_activate_devices()
- {
- struct acm_device *dev;
-- struct acm_port *port;
- DLIST_ENTRY *dev_entry;
-- int i;
-
-- acm_log(1, "initiating multicast joins for all ports\n");
-+ acm_log(1, "\n");
- for (dev_entry = dev_list.Next; dev_entry != &dev_list;
-- dev_entry = dev_entry->Next) {
-+ dev_entry = dev_entry->Next) {
-
- dev = container_of(dev_entry, struct acm_device, entry);
--
-- for (i = 0; i < dev->port_cnt; i++) {
-- port = &dev->port[i];
-- if (port->state != IBV_PORT_ACTIVE)
-- continue;
--
-- acm_log(1, "starting join for device %s, port %d\n",
-- dev->verbs->device->name, port->port_num);
-- // TODO: handle dynamic changes
-- //beginthread(acm_port_join, port);
-- acm_port_join(port);
-- }
-+ beginthread(acm_event_handler, dev);
-+ beginthread(acm_comp_handler, dev);
- }
- }
--static void acm_activate_dev(struct acm_device *dev)
--{
-- acm_log(1, "\n");
-- beginthread(acm_event_handler, dev);
-- beginthread(acm_comp_handler, dev);
--}
-
--static void acm_init_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
-+static void acm_open_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
- {
- acm_log(1, "%s %d\n", dev->verbs->device->name, port_num);
- port->dev = dev;
-@@ -2739,6 +2695,24 @@ static void acm_init_port(struct acm_port *port, struct acm_device *dev, uint8_t
- lock_init(&port->lock);
- DListInit(&port->ep_list);
- acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID, NULL, 0);
-+
-+ port->mad_portid = umad_open_port(dev->verbs->device->name, port->port_num);
-+ if (port->mad_portid < 0) {
-+ acm_log(0, "ERROR - unable to open MAD port\n");
-+ return;
-+ }
-+
-+ port->mad_agentid = umad_register(port->mad_portid,
-+ IB_MGMT_CLASS_SA, 1, 1, NULL);
-+ if (port->mad_agentid < 0) {
-+ acm_log(0, "ERROR - unable to register MAD client\n");
-+ goto err;
-+ }
-+
-+ port->state = IBV_PORT_DOWN;
-+ return;
-+err:
-+ umad_close_port(port->mad_portid);
- }
-
- static void acm_open_dev(struct ibv_device *ibdev)
-@@ -2784,7 +2758,7 @@ static void acm_open_dev(struct ibv_device *ibdev)
- }
-
- for (i = 0; i < dev->port_cnt; i++)
-- acm_init_port(&dev->port[i], dev, i + 1);
-+ acm_open_port(&dev->port[i], dev, i + 1);
-
- DListInsertHead(&dev->entry, &dev_list);
-
-@@ -3020,17 +2994,13 @@ int CDECL_FUNC main(int argc, char **argv)
- DListInit(&dev_list);
- DListInit(&timeout_list);
- event_init(&timeout_event);
--
- umad_init();
- if (acm_open_devices()) {
- acm_log(0, "ERROR - unable to open any devices\n");
- return -1;
- }
-
--// acm_log(1, "initiating multicast joins\n");
- acm_activate_devices();
--// acm_join_groups();
--// acm_log(1, "multicast joins done\n");
- acm_log(1, "starting timeout/retry thread\n");
- beginthread(acm_retry_handler, NULL);
- acm_log(1, "starting server\n");
projects / ~shefty / ibacm.git / commitdiff