From: Adrian Chiris Date: Thu, 11 Dec 2014 16:01:30 +0000 (+0200) Subject: mtcr- new pci capability performance fix X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=76cf654176c47243539f83b261e86618cfdd500f;p=~adrianc%2Fmstflint.git mtcr- new pci capability performance fix --- diff --git a/include/mtcr_ul/mtcr.h b/include/mtcr_ul/mtcr.h index ca584cf..daa32cd 100644 --- a/include/mtcr_ul/mtcr.h +++ b/include/mtcr_ul/mtcr.h @@ -195,7 +195,6 @@ typedef enum mtcr_access_method { MTCR_ACCESS_MEMORY = 0x1, MTCR_ACCESS_CONFIG = 0x2, MTCR_ACCESS_INBAND = 0x3, - MTCR_ACCESS_AUTO = 0x4, } mtcr_access_method_t; /* * Read 4 bytes, return number of succ. read bytes or -1 on failure @@ -241,8 +240,6 @@ int mget_mdevs_type(mfile *mf, u_int32_t *mtype); */ mfile *mopen(const char *name); -mfile *mopen_adv(const char *name, mtcr_access_method_t access_method); - mfile *mopend(const char *name, int type); mfile *mopen_fw_ctx(void* fw_cmd_context, void* fw_cmd_func); diff --git a/mtcr_ul/mtcr_ul.c b/mtcr_ul/mtcr_ul.c index 3b36be5..cfacc7d 100644 --- a/mtcr_ul/mtcr_ul.c +++ b/mtcr_ul/mtcr_ul.c @@ -740,7 +740,6 @@ int mtcr_pciconf_cap9_sem(mfile* mf, int state) u_int32_t lock_val; u_int32_t counter = 0; int retries = 0; - if (!state) {// unlock WRITE4_PCI(mf, 0, CAP9_ADDR + PCI_SEMAPHORE_OFFSET, "unlock semaphore", return ME_PCI_WRITE_ERROR); } else { // lock @@ -752,7 +751,7 @@ int mtcr_pciconf_cap9_sem(mfile* mf, int state) READ4_PCI(mf, &lock_val, CAP9_ADDR + PCI_SEMAPHORE_OFFSET, "read counter", return ME_PCI_READ_ERROR); if (lock_val) { //semaphore is taken retries++; - msleep(3); // wait for current op to end + msleep(1); // wait for current op to end continue; } //read ticket @@ -778,6 +777,9 @@ int mtcr_pciconf_wait_on_flag(mfile* mf, u_int8_t expected_val) READ4_PCI(mf, &flag, CAP9_ADDR + PCI_ADDR_OFFSET, "read flag", return ME_PCI_READ_ERROR); flag = EXTRACT(flag, PCI_FLAG_BIT_OFFS, 1); retries++; + if ((retries & 0xf) == 0) {// dont sleep always + msleep(1); + } } while (flag != expected_val); return ME_OK; } @@ -797,15 +799,39 @@ int mtcr_pciconf_set_addr_space(mfile* mf, u_int16_t space) return ME_OK; } -// adrianc: no support for auto-increment feature atm -int mtcr_pciconf_send_pci_cmd_int(mfile *mf, int space, unsigned int offset, u_int32_t* data, int rw) +int mtcr_pciconf_rw(mfile *mf, unsigned int offset, u_int32_t* data, int rw) { int rc = ME_OK; u_int32_t address = offset; - if ((offset >> 24) & 0xc) { + //last 2 bits must be zero as we only allow 30 bits addresses + if (EXTRACT(address, 30, 2)) { return ME_BAD_PARAMS; } + + address = MERGE(address,(rw ? 1 : 0), PCI_FLAG_BIT_OFFS, 1); + if (rw == WRITE_OP) { + // write data + WRITE4_PCI(mf, *data, CAP9_ADDR + PCI_DATA_OFFSET, "write value", return ME_PCI_WRITE_ERROR); + // write address + WRITE4_PCI(mf, address, CAP9_ADDR + PCI_ADDR_OFFSET, "write offset", return ME_PCI_WRITE_ERROR); + // wait on flag + rc = mtcr_pciconf_wait_on_flag(mf, 0); + } else { + // write address + WRITE4_PCI(mf, address, CAP9_ADDR + PCI_ADDR_OFFSET, "write offset", return ME_PCI_WRITE_ERROR); + // wait on flag + rc = mtcr_pciconf_wait_on_flag(mf, 1); + // read data + READ4_PCI(mf, data, CAP9_ADDR + PCI_DATA_OFFSET, "read value", return ME_PCI_READ_ERROR); + } + return rc; +} + +int mtcr_pciconf_send_pci_cmd_int(mfile *mf, int space, unsigned int offset, u_int32_t* data, int rw) +{ + int rc = ME_OK; + // take semaphore rc = mtcr_pciconf_cap9_sem(mf, 1); if (rc) { @@ -817,24 +843,11 @@ int mtcr_pciconf_send_pci_cmd_int(mfile *mf, int space, unsigned int offset, u_i if (rc) { goto cleanup; } - address = MERGE(address,(rw ? 1 : 0), PCI_FLAG_BIT_OFFS, 1); - if (rw) { - // write data - WRITE4_PCI(mf, *data, CAP9_ADDR + PCI_DATA_OFFSET, "write value", rc = ME_PCI_WRITE_ERROR; goto cleanup); - // write address - WRITE4_PCI(mf, address, CAP9_ADDR + PCI_ADDR_OFFSET, "write offset", rc = ME_PCI_WRITE_ERROR; goto cleanup); - // wait on flag - mtcr_pciconf_wait_on_flag(mf, 0); - } else { - // write address - WRITE4_PCI(mf, address, CAP9_ADDR + PCI_ADDR_OFFSET, "write offset", rc = ME_PCI_WRITE_ERROR; goto cleanup); - // wait on flag - mtcr_pciconf_wait_on_flag(mf, 1); - // read data - READ4_PCI(mf, data, CAP9_ADDR + PCI_DATA_OFFSET, "read value", rc = ME_PCI_READ_ERROR; goto cleanup); - } + // read/write the data + rc = mtcr_pciconf_rw(mf, offset, data, rw); cleanup: + // clear semaphore mtcr_pciconf_cap9_sem(mf, 0); return rc; } @@ -859,6 +872,48 @@ int mtcr_pciconf_mwrite4(mfile *mf, unsigned int offset, u_int32_t value) return 4; } +static int block_op_pciconf(mfile *mf, unsigned int offset, u_int32_t* data, int length, int rw) +{ + int i; + int rc = ME_OK; + int wrote_or_read = length; + if (length % 4) { + return -1; + } + // lock semaphore and set address space + rc = mtcr_pciconf_cap9_sem(mf, 1); + if (rc) { + return -1; + } + // set address space + rc = mtcr_pciconf_set_addr_space(mf, mf->address_domain); + if (rc) { + wrote_or_read = -1; + goto cleanup; + } + + for (i = 0; i < length ; i += 4) { + if (mtcr_pciconf_rw(mf, offset + i, &(data[(i >> 2)]), rw)) { + wrote_or_read = i; + goto cleanup; + } + } +cleanup: + mtcr_pciconf_cap9_sem(mf, 0); + return wrote_or_read; +} + +static int +mread4_block_pciconf(mfile *mf, unsigned int offset, u_int32_t* data, int length) +{ + return block_op_pciconf(mf, offset, data, length, READ_OP); +} + +static int +mwrite4_block_pciconf(mfile *mf, unsigned int offset, u_int32_t* data, int length) +{ + return block_op_pciconf(mf, offset, data, length, WRITE_OP); +} int mtcr_pciconf_mread4_old(mfile *mf, unsigned int offset, u_int32_t *value) { @@ -968,12 +1023,14 @@ int mtcr_pciconf_open(mfile *mf, const char *name) mf->address_domain = CR_SPACE_DOMAIN; mf->mread4 = mtcr_pciconf_mread4; mf->mwrite4 = mtcr_pciconf_mwrite4; + mf->mread4_block = mread4_block_pciconf; + mf->mwrite4_block = mwrite4_block_pciconf; } else { mf->mread4 = mtcr_pciconf_mread4_old; mf->mwrite4 = mtcr_pciconf_mwrite4_old; + mf->mread4_block = mread_chunk_as_multi_mread4; + mf->mwrite4_block = mwrite_chunk_as_multi_mwrite4; } - mf->mread4_block = mread_chunk_as_multi_mread4; - mf->mwrite4_block = mwrite_chunk_as_multi_mwrite4; mf->mclose = mtcr_pciconf_mclose; /* Kernels before 2.6.12 carry the high bit in each byte @@ -1162,12 +1219,12 @@ name_parsed: int mread4_block (mfile *mf, unsigned int offset, u_int32_t* data, int byte_len) { - return mread_chunk_as_multi_mread4(mf, offset, data, byte_len); + return mf->mread4_block(mf, offset, data, byte_len); } int mwrite4_block (mfile *mf, unsigned int offset, u_int32_t* data, int byte_len) { - return mwrite_chunk_as_multi_mwrite4(mf, offset, data, byte_len); + return mf->mwrite4_block(mf, offset, data, byte_len); } int msw_reset(mfile *mf) @@ -1352,12 +1409,8 @@ void mdevices_info_destroy(dev_info* dev_info, int len) free(dev_info); } -mfile *mopen(const char *name) -{ - return mopen_adv(name, MTCR_ACCESS_AUTO); -} -mfile *mopen_adv(const char *name, mtcr_access_method_t access_method) +mfile *mopen(const char *name) { mfile *mf; off_t offset; @@ -1401,25 +1454,6 @@ mfile *mopen_adv(const char *name, mtcr_access_method_t access_method) goto open_failed; } } - // update access according to access method - // if access_method != MTCR_ACCESS_AUTO then we only allow to open conf as memory and via versa - if (access_method == MTCR_ACCESS_AUTO || access_method == access) { - ; - } else if (access_method == MTCR_ACCESS_INBAND) { - char inband_dev[16] = {0}; - if (get_inband_dev_from_pci(inband_dev, mf->dev_name)) { - goto open_failed; - } - free(mf->dev_name); - mf->dev_name = strdup(name); - if (!mf->dev_name) { - goto open_failed; - } - force = 1; - access = access_method; - } else { - access = access_method; - } if (force) { switch (access) { @@ -1779,6 +1813,7 @@ int maccess_reg(mfile *mf, u_int32_t w_size_reg, int *reg_status) { + int rc; if (mf == NULL || reg_data == NULL || reg_status == NULL || reg_size <= 0) { return ME_BAD_PARAMS; @@ -1806,7 +1841,7 @@ int maccess_reg(mfile *mf, return ME_REG_ACCESS_NOT_SUPPORTED; } #endif - + //printf("-D- reg_id:0x%x, reg_size 0x%x, reg_status=0x%x, rc=0x%x\n", reg_id, reg_size, (unsigned int)*reg_status, (unsigned int)rc); if (rc ) { return rc; } else if (*reg_status) { @@ -1916,7 +1951,6 @@ static int mreg_send_raw(mfile *mf, u_int16_t reg_id, maccess_reg_method_t metho //put the reg itself into the buffer memcpy(buffer + OP_TLV_SIZE + REG_TLV_HEADER_LEN, reg_data, reg_size); cmdif_size += reg_size; - #ifdef _ENABLE_DEBUG_ fprintf(stdout, "-I-Tlv's of Data Sent:\n"); fprintf(stdout, "\tOperation Tlv\n"); diff --git a/mtcr_ul/mtcr_ul_icmd_cif.c b/mtcr_ul/mtcr_ul_icmd_cif.c index 61d2d7f..a9b93e2 100644 --- a/mtcr_ul/mtcr_ul_icmd_cif.c +++ b/mtcr_ul/mtcr_ul_icmd_cif.c @@ -45,7 +45,7 @@ #define VCR_CTRL_ADDR 0x0 #define VCR_SEMAPHORE62 0x0 // semaphore Domain -#define VCR_CMD_ADDR 0x1000000 // mailbox addr +#define VCR_CMD_ADDR 0x100000 // mailbox addr #define VCR_CMD_SIZE_ADDR 0x1000 // mailbox size /* @@ -139,7 +139,7 @@ mf->address_domain = AD_CR_SPACE;\ }while(0) - +//#define _DEBUG_MODE #ifdef _DEBUG_MODE #define DBG_PRINTF(...) fprintf(stderr, __VA_ARGS__) #else @@ -232,7 +232,7 @@ static int go(mfile *mf) { */ static int set_opcode(mfile *mf, u_int16_t opcode) { u_int32_t reg; - + DBG_PRINTF("-D- in set_opcode\n"); MREAD4_ICMD(mf, mf->icmd.ctrl_addr, ®, return ME_ICMD_STATUS_CR_FAIL); reg = MERGE(reg, opcode, OPCODE_BITOFF, OPCODE_BITLEN); MWRITE4_ICMD(mf, mf->icmd.ctrl_addr, reg, return ME_ICMD_STATUS_CR_FAIL); @@ -268,7 +268,7 @@ static int translate_status(int status) { } static int get_status(mfile *mf) { u_int32_t reg; - + DBG_PRINTF("-D- int get_status()"); MREAD4_ICMD(mf, mf->icmd.ctrl_addr, ®, return ME_ICMD_STATUS_CR_FAIL); return translate_status(EXTRACT(reg, STATUS_BITOFF, STATUS_BITLEN)); } @@ -278,6 +278,7 @@ static int get_status(mfile *mf) { */ static int icmd_is_cmd_ifc_ready(mfile *mf) { u_int32_t reg; + DBG_PRINTF("-D- in icmd_is_cmd_ifc_ready()\n"); if (MREAD4(mf, mf->icmd.static_cfg_not_done_addr, ®)) return ME_ICMD_STATUS_CR_FAIL; u_int32_t bit_val = EXTRACT(reg, mf->icmd.static_cfg_not_done_offs, 1); /* adrianc: for SWITCHIB the polarity of this bit is opposite than CONNECTIB/CONNECTX4 @@ -387,6 +388,7 @@ int icmd_send_command_int(mfile *mf, if (!skip_write) { + DBG_PRINTF("-D- Writing command to mailbox"); MWRITE_BUF_ICMD(mf, mf->icmd.cmd_addr, data, write_data_size, ret=ME_ICMD_STATUS_CR_FAIL; goto cleanup;); } @@ -397,7 +399,7 @@ int icmd_send_command_int(mfile *mf, if ((ret = get_status(mf))) { goto cleanup; } - + DBG_PRINTF("-D- Reading command from mailbox"); MREAD_BUF_ICMD(mf, mf->icmd.cmd_addr, data, read_data_size, ret=ME_ICMD_STATUS_CR_FAIL; goto cleanup;); ret = ME_OK; @@ -419,6 +421,11 @@ static int icmd_init_vcr(mfile* mf) GET_ADDR(mf,STAT_CFG_NOT_DONE_ADDR_CIB, STAT_CFG_NOT_DONE_ADDR_CX4, STAT_CFG_NOT_DONE_ADDR_SW_IB, mf->icmd.static_cfg_not_done_addr); GET_ADDR(mf, STAT_CFG_NOT_DONE_BITOFF_CIB, STAT_CFG_NOT_DONE_BITOFF_CIB, STAT_CFG_NOT_DONE_BITOFF_SW_IB, mf->icmd.static_cfg_not_done_offs); mf->icmd.icmd_opened = 1; + DBG_PRINTF("-D- iCMD command addr: 0x%x\n", mf->icmd.cmd_addr); + DBG_PRINTF("-D- iCMD ctrl addr: 0x%x\n", mf->icmd.ctrl_addr); + DBG_PRINTF("-D- iCMD semaphore addr(semaphore space): 0x%x\n", mf->icmd.semaphore_addr); + DBG_PRINTF("-D- iCMD max mailbox size: 0x%x\n", mf->icmd.max_cmd_size); + DBG_PRINTF("-D- iCMD stat_cfg_not_done addr: 0x%x:%d\n", mf->icmd.static_cfg_not_done_addr, mf->icmd.static_cfg_not_done_offs); return ME_OK; }