--- /dev/null
+Bottom: 5d77ccab732b248e36bffb1e4801f9c26c3fe7e2
+Top: 19610f5ed6d90b58d8fe662ad4c233e4cb5efe8c
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: 2011-01-26 13:48:26 -0800
+
+Refresh of ep_active
+
+---
+
+diff --git a/src/acm.c b/src/acm.c
+index 6515185..add7247 100644
+--- a/src/acm.c
++++ b/src/acm.c
+@@ -1484,10 +1484,9 @@ out:
+ free(umad);
+ }
+
+-static void acm_port_join(void *context)
++static int acm_port_join(struct acm_port *port)
+ {
+ struct acm_device *dev;
+- struct acm_port *port = (struct acm_port *) context;
+ struct acm_ep *ep;
+ union ibv_gid port_gid;
+ DLIST_ENTRY *ep_entry;
+@@ -1501,7 +1500,7 @@ static void acm_port_join(void *context)
+ if (ret) {
+ acm_log(0, "ERROR - ibv_query_gid %d device %s port %d\n",
+ ret, dev->verbs->device->name, port->port_num);
+- return;
++ return ret;
+ }
+
+ for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
+@@ -1520,33 +1519,6 @@ static void acm_port_join(void *context)
+ port->port_num);
+ }
+
+-static void acm_join_groups(void)
+-{
+- struct acm_device *dev;
+- struct acm_port *port;
+- DLIST_ENTRY *dev_entry;
+- int i;
+-
+- acm_log(1, "initiating multicast joins for all ports\n");
+- for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+- dev_entry = dev_entry->Next) {
+-
+- dev = container_of(dev_entry, struct acm_device, entry);
+-
+- for (i = 0; i < dev->port_cnt; i++) {
+- port = &dev->port[i];
+- if (port->state != IBV_PORT_ACTIVE)
+- continue;
+-
+- acm_log(1, "starting join for device %s, port %d\n",
+- dev->verbs->device->name, port->port_num);
+- // TODO: handle dynamic changes
+- //beginthread(acm_port_join, port);
+- acm_port_join(port);
+- }
+- }
+-}
+-
+ static void acm_process_timeouts(void)
+ {
+ DLIST_ENTRY *entry;
+@@ -1737,6 +1709,37 @@ acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path)
+ }
+
+ static struct acm_ep *
++acm_get_port_ep(struct acm_port *port, struct acm_ep_addr_data *data)
++{
++ struct acm_ep *ep;
++ DLIST_ENTRY *ep_entry;
++
++ if (port->state != IBV_PORT_ACTIVE)
++ return NULL;
++
++ if (data->type == ACM_EP_INFO_PATH &&
++ !acm_is_path_from_port(port, &data->info.path))
++ return NULL;
++
++ for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
++ ep_entry = ep_entry->Next) {
++
++ ep = container_of(ep_entry, struct acm_ep, entry);
++ if (ep->state != ACM_READY)
++ continue;
++
++ if ((data->type == ACM_EP_INFO_PATH) &&
++ (!data->info.path.pkey || (ntohs(data->info.path.pkey) == ep->pkey)))
++ return ep;
++
++ if (acm_addr_index(ep, data->info.addr, (uint8_t) data->type) >= 0)
++ return ep;
++ }
++
++ return NULL;
++}
++
++static struct acm_ep *
+ acm_get_ep(struct acm_ep_addr_data *data)
+ {
+ struct acm_device *dev;
+@@ -1753,28 +1756,11 @@ acm_get_ep(struct acm_ep_addr_data *data)
+
+ dev = container_of(dev_entry, struct acm_device, entry);
+ for (i = 0; i < dev->port_cnt; i++) {
+- port = &dev->port[i];
+-
+- if (data->type == ACM_EP_INFO_PATH &&
+- !acm_is_path_from_port(port, &data->info.path))
+- continue;
+-
+- for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
+- ep_entry = ep_entry->Next) {
+-
+- ep = container_of(ep_entry, struct acm_ep, entry);
+- if (ep->state != ACM_READY)
+- continue;
+-
+- if ((data->type == ACM_EP_INFO_PATH) &&
+- (!data->info.path.pkey ||
+- (ntohs(data->info.path.pkey) == ep->pkey)))
+- return ep;
+-
+- if (acm_addr_index(ep, data->info.addr,
+- (uint8_t) data->type) >= 0)
+- return ep;
+- }
++ lock_acquire(&dev->port[i].lock);
++ ep = acm_get_port_ep(&dev->port[i], data);
++ lock_release(&dev->port[i].lock);
++ if (ep)
++ return ep;
+ }
+ }
+
+@@ -2497,7 +2483,7 @@ static int acm_init_ep_loopback(struct acm_ep *ep)
+ return 0;
+ }
+
+-static int acm_activate_ep(struct acm_port *port, struct acm_ep *ep, uint16_t pkey_index)
++static int acm_ep_up(struct acm_port *port, struct acm_ep *ep, uint16_t pkey_index)
+ {
+ struct ibv_qp_init_attr init_attr;
+ struct ibv_qp_attr attr;
+@@ -2600,73 +2586,25 @@ err1:
+ return -1;
+ }
+
+-static void acm_activate_port(struct acm_port *port)
+-{
+- struct acm_ep *ep;
+- int i, ret;
+-
+- acm_log(1, "%s %d\n", port->dev->verbs->device->name,
+- port->port_num);
+-
+- port->sa_dest.ah = ibv_create_ah(port->dev->pd, &port->sa_dest.av);
+- if (!port->sa_dest.ah)
+- goto err1;
+-
+- for (i = 0; i < port->pkey_cnt; i++) {
+- ep = calloc(1, sizeof *ep);
+- if (!ep)
+- break;
+-
+- ret = acm_activate_ep(port, ep, (uint16_t) i);
+- if (!ret) {
+- DListInsertHead(&ep->entry, &port->ep_list);
+- } else {
+- acm_log(0, "ERROR - failed to activate EP\n");
+- free(ep);
+- }
+- }
+-
+- if (DListEmpty(&port->ep_list))
+- goto err2;
+-
+- port->mad_portid = umad_open_port(port->dev->verbs->device->name, port->port_num);
+- if (port->mad_portid < 0) {
+- acm_log(0, "ERROR - unable to open MAD port\n");
+- goto err3;
+- }
+-
+- port->mad_agentid = umad_register(port->mad_portid,
+- IB_MGMT_CLASS_SA, 1, 1, NULL);
+- if (port->mad_agentid < 0) {
+- acm_log(0, "ERROR - unable to register MAD client\n");
+- goto err4;
+- }
+-
+- return;
+-
+-err4:
+- umad_close_port(port->mad_portid);
+-err3:
+- /* TODO: cleanup ep list */
+-err2:
+- ibv_destroy_ah(port->sa_dest.ah);
+-err1:
+- port->state = IBV_PORT_NOP;
+-}
+-
+-static void acm_open_port(struct acm_port *port)
++static void acm_port_up(struct acm_port *port)
+ {
+ struct ibv_port_attr attr;
+ union ibv_gid gid;
+ uint16_t pkey;
+- int ret;
++ struct acm_ep *ep;
++ int i, ret;
+
+ acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
+ ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
+- if (ret || attr.state != IBV_PORT_ACTIVE)
++ if (ret) {
++ acm_log(0, "ERROR - unable to get port state\n");
+ return;
++ }
++ if (attr.state != IBV_PORT_ACTIVE) {
++ acm_log(1, "port not active\n");
++ return;
++ }
+
+- port->state = attr.state;
+ port->mtu = attr.active_mtu;
+ port->rate = acm_get_rate(attr.active_width, attr.active_speed);
+ port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
+@@ -2691,47 +2629,65 @@ static void acm_open_port(struct acm_port *port)
+ port->sa_dest.av.sl = attr.sm_sl;
+ port->sa_dest.av.port_num = port->port_num;
+ port->sa_dest.remote_qpn = 1;
++
++ port->sa_dest.ah = ibv_create_ah(port->dev->pd, &port->sa_dest.av);
++ if (!port->sa_dest.ah)
++ return;
++
++ for (i = 0; i < port->pkey_cnt; i++) {
++ /* TODO: Check if endpoint already exists in port list */
++ ep = calloc(1, sizeof *ep);
++ if (!ep)
++ break;
++
++ ret = acm_ep_up(port, ep, (uint16_t) i);
++ if (!ret) {
++ DListInsertHead(&ep->entry, &port->ep_list);
++ } else {
++ acm_log(0, "ERROR - failed to activate EP\n");
++ free(ep);
++ }
++ }
++
++ acm_port_join(port);
++ lock_acquire(&port->lock);
++ port->state = attr.state;
++ lock_release(&port->lock);
+ }
+
++/*
++ * There is one event handler thread per device. This is the only thread that
++ * modifies the port state or a port endpoint list. Other threads which access
++ * those must synchronize against changes accordingly, but this thread only
++ * needs to lock when making modifications.
++ */
+ static void CDECL_FUNC acm_event_handler(void *context)
+ {
+ struct acm_device *dev = (struct acm_device *) context;
++ int i;
++
++ for (i = 0; i < dev->port_cnt; i++) {
++ acm_port_up(&dev->port[i]);
++ }
++ /* TODO: wait for port up/down events */
+ }
+
+-static void acm_join_groups(void)
++static void acm_activate_devices()
+ {
+ struct acm_device *dev;
+- struct acm_port *port;
+ DLIST_ENTRY *dev_entry;
+- int i;
+
+- acm_log(1, "initiating multicast joins for all ports\n");
++ acm_log(1, "\n");
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+- dev_entry = dev_entry->Next) {
++ dev_entry = dev_entry->Next) {
+
+ dev = container_of(dev_entry, struct acm_device, entry);
+-
+- for (i = 0; i < dev->port_cnt; i++) {
+- port = &dev->port[i];
+- if (port->state != IBV_PORT_ACTIVE)
+- continue;
+-
+- acm_log(1, "starting join for device %s, port %d\n",
+- dev->verbs->device->name, port->port_num);
+- // TODO: handle dynamic changes
+- //beginthread(acm_port_join, port);
+- acm_port_join(port);
+- }
++ beginthread(acm_event_handler, dev);
++ beginthread(acm_comp_handler, dev);
+ }
+ }
+-static void acm_activate_dev(struct acm_device *dev)
+-{
+- acm_log(1, "\n");
+- beginthread(acm_event_handler, dev);
+- beginthread(acm_comp_handler, dev);
+-}
+
+-static void acm_init_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
++static void acm_open_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
+ {
+ acm_log(1, "%s %d\n", dev->verbs->device->name, port_num);
+ port->dev = dev;
+@@ -2739,6 +2695,24 @@ static void acm_init_port(struct acm_port *port, struct acm_device *dev, uint8_t
+ lock_init(&port->lock);
+ DListInit(&port->ep_list);
+ acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID, NULL, 0);
++
++ port->mad_portid = umad_open_port(dev->verbs->device->name, port->port_num);
++ if (port->mad_portid < 0) {
++ acm_log(0, "ERROR - unable to open MAD port\n");
++ return;
++ }
++
++ port->mad_agentid = umad_register(port->mad_portid,
++ IB_MGMT_CLASS_SA, 1, 1, NULL);
++ if (port->mad_agentid < 0) {
++ acm_log(0, "ERROR - unable to register MAD client\n");
++ goto err;
++ }
++
++ port->state = IBV_PORT_DOWN;
++ return;
++err:
++ umad_close_port(port->mad_portid);
+ }
+
+ static void acm_open_dev(struct ibv_device *ibdev)
+@@ -2784,7 +2758,7 @@ static void acm_open_dev(struct ibv_device *ibdev)
+ }
+
+ for (i = 0; i < dev->port_cnt; i++)
+- acm_init_port(&dev->port[i], dev, i + 1);
++ acm_open_port(&dev->port[i], dev, i + 1);
+
+ DListInsertHead(&dev->entry, &dev_list);
+
+@@ -3020,17 +2994,13 @@ int CDECL_FUNC main(int argc, char **argv)
+ DListInit(&dev_list);
+ DListInit(&timeout_list);
+ event_init(&timeout_event);
+-
+ umad_init();
+ if (acm_open_devices()) {
+ acm_log(0, "ERROR - unable to open any devices\n");
+ return -1;
+ }
+
+-// acm_log(1, "initiating multicast joins\n");
+ acm_activate_devices();
+-// acm_join_groups();
+-// acm_log(1, "multicast joins done\n");
+ acm_log(1, "starting timeout/retry thread\n");
+ beginthread(acm_retry_handler, NULL);
+ acm_log(1, "starting server\n");
projects / ~shefty / ibacm.git / commitdiff