From 9bfc9cfc65e435537e726c62f4839855746a0d2e Mon Sep 17 00:00:00 2001 From: leonidk Date: Mon, 14 Apr 2008 07:59:31 +0000 Subject: [PATCH] [TOOLS] add support to ConnectX HCAs git-svn-id: svn://openib.tc.cornell.edu/gen1@1055 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/tools/mtcr/user/mtcr.c | 199 +++++++++++++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 11 deletions(-) diff --git a/trunk/tools/mtcr/user/mtcr.c b/trunk/tools/mtcr/user/mtcr.c index 77d58b48..4b2eeea4 100644 --- a/trunk/tools/mtcr/user/mtcr.c +++ b/trunk/tools/mtcr/user/mtcr.c @@ -21,33 +21,82 @@ #define USB_DEV_NAME "mtusb-1" #define CLEAR(st) memset(&(st), 0, sizeof(st)) - +// HERMON ORDERING WA: +#define HERMON_WA_BASE 0xf0384 // SEM BASE ADDR. SEM 0xf0380 is reserved for external tools usage. +#define HERMON_WA_SIZE 3 // Size in entries #define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL" + +/* + * DLL global variables + */ +#pragma data_seg(".sdata") + +/* Process / thread count */ +DWORD g_Slots[HERMON_WA_SIZE]; /* 3 slots */ + #ifdef DBG ULONG g_DebugLevel = DEBUG_LEVEL_MID; #else ULONG g_DebugLevel = DEBUG_LEVEL_LOW; #endif +#pragma data_seg() + + //----------------------------------------------------- #define MAX_HCA_NUM 16 +// flags in below structure +#define FIX_ORDERING_BUG 1 typedef struct mfile_ibal_t { mfile s; ib_al_handle_t h_al; ib_ca_handle_t h_ca; map_crspace cr_map; + + int bugs; + int slot_num; + u_int32_t hermon_wa_slot; /* apply hermon cr write workaround */ + int hermon_wa_last_op_write; + u_int64_t hermon_wa_max_retries; + u_int64_t hermon_wa_num_of_writes; + u_int64_t hermon_wa_num_of_retry_writes; } mfile_ibal; +// Return slot_num or -1 when no free slots +static int __get_slot() +{ + int i, prev_val; + + for (i = 0; i < HERMON_WA_SIZE; ++i) { + prev_val = InterlockedCompareExchange( + (LONG volatile* )&g_Slots[i], 0, 1 ); + if ( prev_val ) + return i; + } + return -1; +} + +static void __put_slot(int slot_num) +{ + int prev_val; + prev_val = InterlockedExchange( + (LONG volatile* )&g_Slots[slot_num], 1 ); + if ( !prev_val ) + DPRINT1(("Wrong semaphore %d value \n", slot_num)); + +} + BOOL APIENTRY DllMain( HANDLE hModule, DWORD ul_reason_for_call, LPVOID lpReserved ) { + int i; char* pszDbgLevel; switch (ul_reason_for_call) { @@ -59,6 +108,9 @@ BOOL APIENTRY DllMain( HANDLE hModule, if (pszDbgLevel) { g_DebugLevel = atol(pszDbgLevel); } + // set slot data + for (i = 0; i < HERMON_WA_SIZE; ++i) + g_Slots[i] = 1; break; @@ -85,13 +137,20 @@ BOOL APIENTRY DllMain( HANDLE hModule, #define SINAI_4X_CONF_DEV_ID 24205 #define SINAI_8X_DEV_ID 25204 #define SINAI_8X_CONF_DEV_ID 25205 +#define HERMON_SDR_DEV_ID 25408 +#define HERMON_DDR_DEV_ID 25418 +#define HERMON_QDR_DEV_ID 25428 +#define HERMON_DDR_PCI5_DEV_ID 26418 +#define HERMON_QDR_PCI5_DEV_ID 26428 +#define HERMON_CONF_DEV_ID 401 #define IS_CONF_DEV(dev_id) \ - ((dev_id == TAVOR_CONF_DEV_ID) || \ - (dev_id == ARBEL_TM_CONF_DEV_ID) || \ - (dev_id == ARBEL_CONF_DEV_ID) || \ - (dev_id == SINAI_4X_CONF_DEV_ID) || \ - (dev_id == SINAI_8X_CONF_DEV_ID)) + ((dev_id == TAVOR_CONF_DEV_ID) || \ + (dev_id == ARBEL_TM_CONF_DEV_ID) || \ + (dev_id == ARBEL_CONF_DEV_ID) || \ + (dev_id == SINAI_4X_CONF_DEV_ID) || \ + (dev_id == SINAI_8X_CONF_DEV_ID) || \ + (dev_id == HERMON_CONF_DEV_ID)) #define MAX_DEV_NAME 32 typedef struct { @@ -115,7 +174,15 @@ static DEVICE_DB_T db[] = { { SINAI_4X_DEV_ID, "InfiniHost_III_Lx", MDEVS_TAVOR }, { SINAI_4X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR }, { SINAI_8X_DEV_ID, "InfiniHost_III_Lx", MDEVS_TAVOR }, - { SINAI_8X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR }, + { SINAI_8X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR }, + + { HERMON_SDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_DDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_QDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_DDR_PCI5_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_QDR_PCI5_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + + { HERMON_CONF_DEV_ID, "ConnectXBd", MDEVS_TAVOR_CR }, }; #define DEVICE_DB_SIZE (sizeof(db) / sizeof(DEVICE_DB_T)) @@ -123,6 +190,14 @@ Mdevs dmasks[] = { MDEVS_TAVOR_CR, MDEVS_TAVOR_CR, MDEVS_TAVOR_UAR, MDEVS_TAVOR_ char *dsuffix[] = { "conf", "_cr", "_uar", "_ddr"}; #define MASKS_SIZE (sizeof(dmasks) / sizeof(Mdevs)) +static int is_hermon(USHORT dev_id) +{ + return (dev_id == HERMON_SDR_DEV_ID) || + (dev_id == HERMON_DDR_DEV_ID) || + (dev_id == HERMON_QDR_DEV_ID) || + (dev_id == HERMON_DDR_PCI5_DEV_ID) || + (dev_id == HERMON_QDR_PCI5_DEV_ID); +} // Return: < 0 - Error. > 0 - Numbern of characters written (including last '\0') int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name, int name_len, int *cnt) @@ -160,7 +235,8 @@ int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name, } // names generation - for (j=0; jhermon_wa_num_of_writes++; + + // clear sem + *((u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)) = 0; + + while (retries < max_retries) { + u_int32_t slot_value ; + + slot_value = __be32_to_cpu(*((volatile u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot))); + if (slot_value == 0) { + + // Good - The written value was read + if (retries) { + // NOT REALLY NEEDED - just to collect statistics + + // printf("-D- Hermon WA addr %06x took %d retries\n", addr, retries); + mfi->hermon_wa_num_of_retry_writes++; + if (retries > mfi->hermon_wa_max_retries) { + mfi->hermon_wa_max_retries = retries; + } + } + return 4; + } else { + // sem is still locked from previous read - read passed the write + retries++; + } + } + + DPRINT1(("-D- Hermon WA addr %06x failed after %ld retries\n", addr, retries)); + return 0; +} + + // // // List devices in their MST compatible names. @@ -628,7 +766,7 @@ MTCR_API mfile *mopend(const char *name, DType dtype) } } else { - int bar_num; + int bar_num, slot_num; // Type of file mf->s.tp = MST_PCI; @@ -648,8 +786,36 @@ MTCR_API mfile *mopend(const char *name, DType dtype) } mf->s.ptr = (void*)(ULONG_PTR)mf->cr_map.va; - } - + + // check whether we need to perform the workaround + if (is_hermon(dev_id)) { + char* hermon_wa_env = getenv("MTCR_HERMON_WA"); + // if there is no env variable or it = "0" - we perform the workaround + if (hermon_wa_env == NULL || strcmp(hermon_wa_env, "0")) { + // perform the workaround only for A0 revision + if (access_type == MDEVS_TAVOR_CR) { + u_int32_t rev = 0; + mread4((mfile*)mf, 0xf0014, &rev); + if (rev == 0xa00190) { // TODO: Should I ignore REV here ? + slot_num = __get_slot(); + if (slot_num < 0) // no free slot + goto ErrExit; + mf->bugs |= FIX_ORDERING_BUG; + mf->slot_num = slot_num; + } + } + } + } + + // prepare for the workaround + if (mf->bugs & FIX_ORDERING_BUG) { // Hermon WA setup + DPRINT1(("-D- Hermon WA setup\n")); + mf->hermon_wa_slot = HERMON_WA_BASE + 4*slot_num; + hermon_wa_write_confirm_sem((mfile*)mf, mf->hermon_wa_slot); + } + + } + } else if (dev_id == DEVASYS_DEV_ID) { // Type of file h = usb_open(); @@ -701,6 +867,16 @@ MTCR_API int mclose(mfile *mf) if (mf->tp == MST_PCICONF) { ibal_access(mfi->h_ca, 0x0, &stub, 4, FW_CLOSE_IF ); } else if (mf->tp = MST_PCI) { + + if (mfi->bugs & FIX_ORDERING_BUG) { + __put_slot( mfi->slot_num ); + DPRINT1(("-D- Hermon WA stats:\n")); + DPRINT1(("-D- : num of write flushes: %8ld\n", mfi->hermon_wa_num_of_writes)); + DPRINT1(("-D- : num of retry flushes: %8ld\n", mfi->hermon_wa_num_of_retry_writes)); + DPRINT1(("-D- : max_retries: %8ld\n", mfi->hermon_wa_max_retries)); + } + + if (mfi->cr_map.size) { if (ibal_access(mfi->h_ca, 0x0, NULL, 0, FW_UNMAP_CRSPACE ) != IB_SUCCESS) { @@ -1029,3 +1205,4 @@ unsigned char mset_i2c_slave(mfile *mf, unsigned char new_i2c_slave) return ret; } + -- 2.41.0