[MLX4] fixed mechanism of mlx4_hca's asynchronous events notification.

git-svn-id: svn://openib.tc.cornell.edu/gen1@1480 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/hw/mlx4/kernel/bus/ib/main.c b/trunk/hw/mlx4/kernel/bus/ib/main.c
index 5faa22b6636b542004a2fcf8b1216c4b94225f8b..923134c0a6703a8817266eb3b8e21845af351a44 100644 (file)
--- a/trunk/hw/mlx4/kernel/bus/ib/main.c
+++ b/trunk/hw/mlx4/kernel/bus/ib/main.c
@@ -561,6 +561,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
        ibdev->ib_dev.x.find_cached_pkey        = ib_find_cached_pkey;
        ibdev->ib_dev.x.get_cached_gid          = ib_get_cached_gid;
        ibdev->ib_dev.x.get_cached_pkey         = ib_get_cached_pkey;
+       ibdev->ib_dev.x.register_ev_cb          = mlx4_reset_cb_register;
+       ibdev->ib_dev.x.unregister_ev_cb        = mlx4_reset_cb_unregister;
 
        if (mlx4_is_livefish(ibdev->dev))
                return ibdev;

diff --git a/trunk/hw/mlx4/kernel/bus/inc/ib_verbs.h b/trunk/hw/mlx4/kernel/bus/inc/ib_verbs.h
index d05558da3dbd62b8e566ef1251c39bf8cb14fa07..e2512d9b571aed5939cfbe239e3a9d1298adcb51 100644 (file)
--- a/trunk/hw/mlx4/kernel/bus/inc/ib_verbs.h
+++ b/trunk/hw/mlx4/kernel/bus/inc/ib_verbs.h
@@ -255,27 +255,27 @@ struct ib_port_modify {
 };\r
 \r
 enum ib_event_type {\r
-       IB_EVENT_CQ_ERR                                                                 = IB_AE_CQ_ERROR,\r
-       IB_EVENT_QP_FATAL                                                               = IB_AE_QP_FATAL,\r
-       IB_EVENT_QP_REQ_ERR                                                     = IB_AE_WQ_REQ_ERROR,\r
-       IB_EVENT_QP_ACCESS_ERR                                  = IB_AE_WQ_ACCESS_ERROR,\r
-       IB_EVENT_COMM_EST                                                       = IB_AE_QP_COMM,\r
-       IB_EVENT_SQ_DRAINED                                             = IB_AE_SQ_DRAINED,\r
-       IB_EVENT_PATH_MIG                                                               = IB_AE_QP_APM,\r
-       IB_EVENT_PATH_MIG_ERR                                   = IB_AE_QP_APM_ERROR,\r
-       IB_EVENT_DEVICE_FATAL                                           = IB_AE_LOCAL_FATAL,\r
-       IB_EVENT_PORT_ACTIVE                                            = IB_AE_PORT_ACTIVE,\r
-       IB_EVENT_PORT_ERR                                                               = IB_AE_PORT_DOWN,\r
-       IB_EVENT_SRQ_LIMIT_REACHED                              = IB_AE_SRQ_LIMIT_REACHED,\r
-       IB_EVENT_SRQ_ERR                                                = IB_AE_SRQ_CATAS_ERROR,\r
-       IB_EVENT_QP_LAST_WQE_REACHED                    = IB_AE_SRQ_QP_LAST_WQE_REACHED,\r
-       IB_EVENT_LID_CHANGE                                                     = IB_AE_UNKNOWN + 1,\r
+       IB_EVENT_CQ_ERR                                         = IB_AE_CQ_ERROR,\r
+       IB_EVENT_QP_FATAL                                       = IB_AE_QP_FATAL,\r
+       IB_EVENT_QP_REQ_ERR                                     = IB_AE_WQ_REQ_ERROR,\r
+       IB_EVENT_QP_ACCESS_ERR                          = IB_AE_WQ_ACCESS_ERROR,\r
+       IB_EVENT_COMM_EST                                       = IB_AE_QP_COMM,\r
+       IB_EVENT_SQ_DRAINED                                     = IB_AE_SQ_DRAINED,\r
+       IB_EVENT_PATH_MIG                                       = IB_AE_QP_APM,\r
+       IB_EVENT_PATH_MIG_ERR                           = IB_AE_QP_APM_ERROR,\r
+       IB_EVENT_DEVICE_FATAL                           = IB_AE_LOCAL_FATAL,\r
+       IB_EVENT_PORT_ACTIVE                            = IB_AE_PORT_ACTIVE,\r
+       IB_EVENT_PORT_ERR                                       = IB_AE_PORT_DOWN,\r
+       IB_EVENT_SRQ_LIMIT_REACHED                      = IB_AE_SRQ_LIMIT_REACHED,\r
+       IB_EVENT_SRQ_ERR                                        = IB_AE_SRQ_CATAS_ERROR,\r
+       IB_EVENT_QP_LAST_WQE_REACHED            = IB_AE_SRQ_QP_LAST_WQE_REACHED,\r
+       IB_EVENT_RESET_DRIVER                           = IB_AE_RESET_DRIVER,   // device will be reset upon fatal error\r
+       IB_EVENT_RESET_CLIENT                           = IB_AE_RESET_CLIENT,   // device will be reset upon client request\r
+       IB_EVENT_RESET_END                                      = IB_AE_RESET_END,              // device has been reset \r
+       IB_EVENT_LID_CHANGE                                     = IB_AE_UNKNOWN + 1,\r
        IB_EVENT_PKEY_CHANGE,\r
        IB_EVENT_SM_CHANGE,\r
-       IB_EVENT_CLIENT_REREGISTER,\r
-       IB_EVENT_RESET_DRIVER,          // device will be reset upon fatal error\r
-       IB_EVENT_RESET_CLIENT,          // device will be upon client request\r
-       IB_EVENT_RESET_END                      // device has been reset \r
+       IB_EVENT_CLIENT_REREGISTER\r
 };\r
 \r
 struct ib_event {\r

diff --git a/trunk/hw/mlx4/kernel/bus/inc/ib_verbs_ex.h b/trunk/hw/mlx4/kernel/bus/inc/ib_verbs_ex.h
index 49692cc402e795a724a3b85d0f664abe1baf58c5..0bfd7313a91806dca97b363314e175e366ab9930 100644 (file)
--- a/trunk/hw/mlx4/kernel/bus/inc/ib_verbs_ex.h
+++ b/trunk/hw/mlx4/kernel/bus/inc/ib_verbs_ex.h
@@ -36,6 +36,7 @@
 
 typedef struct _FDO_DEVICE_DATA *PFDO_DEVICE_DATA;
 struct ib_cq;
+struct ib_event_handler;
 
 /* extension for ib_device */
 struct ib_device_ex 
@@ -49,6 +50,8 @@ struct ib_device_ex
                u8 port_num, int index, __be16 *pkey);
        int (*find_cached_pkey)(struct ib_device *device,
                u8 port_num, __be16 pkey, u16 *index);
+       int (*register_ev_cb) (struct ib_event_handler *event_handler);
+       int (*unregister_ev_cb) (struct ib_event_handler *event_handler);
 };
 
 

diff --git a/trunk/hw/mlx4/kernel/hca/data.c b/trunk/hw/mlx4/kernel/hca/data.c
index a3cea329310b3836bfd86d002f14b8e0679d3373..19791df07ef4d96ba7e0bf2b51f7711284191c04 100644 (file)
--- a/trunk/hw/mlx4/kernel/hca/data.c
+++ b/trunk/hw/mlx4/kernel/hca/data.c
@@ -104,13 +104,58 @@ mlnx_hcas_init( void )
 \r
 /////////////////////////////////////////////////////////\r
 /////////////////////////////////////////////////////////\r
+\r
+void ca_event_handler(struct ib_event *ev, void *context)\r
+{\r
+       mlnx_hca_t *p_hca = (mlnx_hca_t *)context;\r
+       ib_event_rec_t event_rec;\r
+       LIST_ENTRY *entry;\r
+       ci_event_handler_t *event_handler;\r
+\r
+       // prepare parameters\r
+       event_rec.type = ev->event;\r
+       event_rec.port_number = ev->element.port_num;\r
+       if (event_rec.type > IB_AE_UNKNOWN) {\r
+               // CL_ASSERT(0); // This shouldn't happen\r
+               HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_SHIM,("Unmapped E_EV_CA event of type 0x%x. Replaced by 0x%x (IB_AE_LOCAL_FATAL)\n", \r
+                       event_rec.type, IB_AE_LOCAL_FATAL));\r
+               event_rec.type = IB_AE_LOCAL_FATAL;\r
+       }\r
+\r
+       // call the user callback\r
+       KeAcquireSpinLockAtDpcLevel(&p_hca->event_list_lock);\r
+       for (entry = p_hca->event_list.Flink; entry != &p_hca->event_list;\r
+                entry = entry->Flink) {\r
+\r
+               event_handler = CONTAINING_RECORD(entry, ci_event_handler_t, entry);\r
+               event_rec.context = (void *) event_handler;\r
+               event_handler->pfn_async_event_cb(&event_rec);\r
+       }\r
+       KeReleaseSpinLockFromDpcLevel(&p_hca->event_list_lock);\r
+\r
+       if (p_hca && p_hca->async_cb_p) {\r
+               event_rec.context = (void *)p_hca->ca_context;\r
+               (p_hca->async_cb_p)(&event_rec);\r
+       } else {\r
+               HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_SHIM ,("Incorrect context. Async callback was not invoked\n"));\r
+       }\r
+}\r
+\r
+\r
+void event_handler( struct ib_event_handler *handler, struct ib_event *event )\r
+{\r
+       ca_event_handler( event, handler->ctx );\r
+}\r
+\r
 ib_api_status_t\r
 mlnx_set_cb(\r
        IN                              mlnx_hca_t                              *       p_hca, \r
        IN                              ci_async_event_cb_t                     async_cb_p,\r
        IN              const   void* const                                     ib_context)\r
 {\r
+       int err;\r
        cl_status_t             cl_status;\r
+       struct ib_device *ibdev = hca2ibdev(p_hca);\r
 \r
        // Setup the callbacks\r
        if (!p_hca->async_proc_mgr_p)\r
@@ -133,6 +178,17 @@ mlnx_set_cb(
 \r
        p_hca->async_cb_p = async_cb_p;\r
        p_hca->ca_context = ib_context; // This is the context our CB forwards to IBAL\r
+\r
+       // register callback with bus driver\r
+       INIT_IB_EVENT_HANDLER( &p_hca->ib_event_handler, ibdev, \r
+               event_handler, p_hca, NULL, 0 );\r
+               \r
+       err = ibdev->x.register_ev_cb(&p_hca->ib_event_handler);\r
+       if (err) {\r
+               RtlZeroMemory( &p_hca->ib_event_handler, sizeof(p_hca->ib_event_handler) );\r
+               return IB_ERROR;\r
+       }\r
+\r
        return IB_SUCCESS;\r
 }\r
 \r
@@ -143,10 +199,14 @@ mlnx_reset_cb(
        IN                              mlnx_hca_t                              *       p_hca)\r
 {\r
        cl_async_proc_t *p_async_proc;\r
-\r
+       struct ib_device *ibdev = hca2ibdev(p_hca);\r
 \r
        cl_spinlock_acquire( &hca_lock );\r
 \r
+       // unregister callback with bus driver\r
+       if ( p_hca->ib_event_handler.handler )\r
+               ibdev->x.unregister_ev_cb(&p_hca->ib_event_handler);\r
+\r
        p_async_proc = p_hca->async_proc_mgr_p;\r
        p_hca->async_proc_mgr_p = NULL;\r
 \r
@@ -295,42 +355,6 @@ from_hca_cap(
        }\r
 }\r
 \r
-void ca_event_handler(struct ib_event *ev, void *context)\r
-{\r
-       mlnx_hca_t *p_hca = (mlnx_hca_t *)context;\r
-       ib_event_rec_t event_rec;\r
-       LIST_ENTRY *entry;\r
-       ci_event_handler_t *event_handler;\r
-\r
-       // prepare parameters\r
-       event_rec.type = ev->event;\r
-       event_rec.port_number = ev->element.port_num;\r
-       if (event_rec.type > IB_AE_UNKNOWN) {\r
-               // CL_ASSERT(0); // This shouldn't happen\r
-               HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_SHIM,("Unmapped E_EV_CA event of type 0x%x. Replaced by 0x%x (IB_AE_LOCAL_FATAL)\n", \r
-                       event_rec.type, IB_AE_LOCAL_FATAL));\r
-               event_rec.type = IB_AE_LOCAL_FATAL;\r
-       }\r
-\r
-       // call the user callback\r
-       KeAcquireSpinLockAtDpcLevel(&p_hca->event_list_lock);\r
-       for (entry = p_hca->event_list.Flink; entry != &p_hca->event_list;\r
-                entry = entry->Flink) {\r
-\r
-               event_handler = CONTAINING_RECORD(entry, ci_event_handler_t, entry);\r
-               event_rec.context = (void *) event_handler;\r
-               event_handler->pfn_async_event_cb(&event_rec);\r
-       }\r
-       KeReleaseSpinLockFromDpcLevel(&p_hca->event_list_lock);\r
-\r
-       if (p_hca && p_hca->async_cb_p) {\r
-               event_rec.context = (void *)p_hca->ca_context;\r
-               (p_hca->async_cb_p)(&event_rec);\r
-       } else {\r
-               HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_SHIM ,("Incorrect context. Async callback was not invoked\n"));\r
-       }\r
-}\r
-\r
 enum ib_rate to_rate(uint8_t rate)\r
 {\r
        if (rate == IB_PATH_RECORD_RATE_2_5_GBS) return IB_RATE_2_5_GBPS;\r

diff --git a/trunk/hw/mlx4/kernel/hca/data.h b/trunk/hw/mlx4/kernel/hca/data.h
index 2d0aabec8efbe7d2edbf8b9ecdaefecfcdf0301a..f02b178790c00fc507f7453b547f49c36d4bc938 100644 (file)
--- a/trunk/hw/mlx4/kernel/hca/data.h
+++ b/trunk/hw/mlx4/kernel/hca/data.h
@@ -163,6 +163,7 @@ typedef struct _mlnx_hca_t {
        void                *cl_device_h;\r
        uint32_t           index;\r
        cl_async_proc_t     *async_proc_mgr_p;\r
+       struct ib_event_handler ib_event_handler;\r
 } mlnx_hca_t;\r
 \r
 // Functions\r

diff --git a/trunk/inc/iba/ib_types.h b/trunk/inc/iba/ib_types.h
index 651dae9f64b91e239070d63b39d52ae6d426c2e7..c5c3dcf91b78063b4f0c1e2020f781cfe2c442d9 100644 (file)
--- a/trunk/inc/iba/ib_types.h
+++ b/trunk/inc/iba/ib_types.h
@@ -8792,6 +8792,9 @@ typedef enum _ib_async_event_t
        IB_AE_SRQ_LIMIT_REACHED,\r
        IB_AE_SRQ_CATAS_ERROR,\r
        IB_AE_SRQ_QP_LAST_WQE_REACHED,\r
+       IB_AE_RESET_DRIVER,\r
+       IB_AE_RESET_CLIENT,\r
+       IB_AE_RESET_END,\r
        IB_AE_UNKNOWN           /* ALWAYS LAST ENUM VALUE */\r
 \r
 }      ib_async_event_t;\r
@@ -8896,6 +8899,15 @@ typedef enum _ib_async_event_t
 *                      a CQE is generated for the last WQE, or\r
 *                      the QP gets in the Error State and there are no more WQEs on the RQ.\r
 *\r
+*      IB_AE_RESET_DRIVER\r
+*              Device will be reset upon fatal error.\r
+*\r
+*      IB_AE_RESET_CLIENT\r
+*              Device will be reset upon client request.\r
+*\r
+*      IB_AE_RESET_END\r
+*              Device has been reset.\r
+*\r
 *      IB_AE_UNKNOWN\r
 *              An unknown error occurred which cannot be attributed to any\r
 *              resource; behavior is indeterminate.\r