From 559e01f7a3694eb3a8e3535f3ef7ea6a50ff5001 Mon Sep 17 00:00:00 2001 From: leonidk Date: Thu, 30 Mar 2006 19:51:23 +0000 Subject: [PATCH] [MTHCA] added support for burning (for flint et al) git-svn-id: svn://openib.tc.cornell.edu/gen1@265 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/hw/mthca/kernel/SOURCES | 3 +- trunk/hw/mthca/kernel/hca_data.h | 1 + trunk/hw/mthca/kernel/hca_driver.c | 210 ++++++++++++++++++++++++++++- trunk/hw/mthca/kernel/hca_verbs.c | 11 +- trunk/hw/mthca/kernel/ib_verbs.h | 3 + trunk/inc/mthca/mthca_vc.h | 58 ++++++++ trunk/tools/mtcr/user/SOURCES | 1 + trunk/tools/mtcr/user/mtcr.c | 33 +---- 8 files changed, 279 insertions(+), 41 deletions(-) create mode 100644 trunk/inc/mthca/mthca_vc.h diff --git a/trunk/hw/mthca/kernel/SOURCES b/trunk/hw/mthca/kernel/SOURCES index 66f4f52e..22452cf2 100644 --- a/trunk/hw/mthca/kernel/SOURCES +++ b/trunk/hw/mthca/kernel/SOURCES @@ -59,8 +59,9 @@ SOURCES= \ INCLUDES=\ ..; \ $(TRUNK)\inc; \ - $(TRUNK)\inc\kernel; \ $(TRUNK)\inc\complib; \ + $(TRUNK)\inc\mthca; \ + $(TRUNK)\inc\kernel; \ $(TRUNK)\inc\kernel\complib; \ C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN diff --git a/trunk/hw/mthca/kernel/hca_data.h b/trunk/hw/mthca/kernel/hca_data.h index ca0ba9e6..5af25247 100644 --- a/trunk/hw/mthca/kernel/hca_data.h +++ b/trunk/hw/mthca/kernel/hca_data.h @@ -336,6 +336,7 @@ fw_access_ctrl( IN ib_ci_op_t* const p_ci_op, IN OUT ci_umv_buf_t *p_umv_buf OPTIONAL); +void unmap_crspace_for_all( struct ib_ucontext *p_context ); void cq_comp_handler(struct ib_cq *cq, void *context); diff --git a/trunk/hw/mthca/kernel/hca_driver.c b/trunk/hw/mthca/kernel/hca_driver.c index 13e62390..49db4bce 100644 --- a/trunk/hw/mthca/kernel/hca_driver.c +++ b/trunk/hw/mthca/kernel/hca_driver.c @@ -51,6 +51,7 @@ //#include "MdCard.h" #pragma warning( pop ) #include +#include "mthca_vc.h" /* from \inc\platform\evntrace.h #define TRACE_LEVEL_NONE 0 // Tracing is not on @@ -434,6 +435,194 @@ End: return status; } +typedef struct _mthca_map_space { + struct list_head list; + PMDL p_mdl; + PVOID va; + PEPROCESS p_pcs; +} mthca_map_space; + +static NTSTATUS +__map_crspace( + IN struct ib_ucontext * p_context, + IN mlnx_hob_t * p_hob, + IN PVOID p_buf, + IN ULONG buf_size + ) +{ + NTSTATUS status; + PMDL p_mdl; + PVOID ua, ka; + ULONG sz; + mthca_map_space *p_map; + hca_dev_ext_t *p_ext = EXT_FROM_HOB(p_hob); + map_crspace *p_res = (map_crspace *)p_buf; + + HCA_ENTER( HCA_DBG_PNP ); + + // sanity checks + if ( buf_size < sizeof *p_res ) { + status = STATUS_INVALID_PARAMETER; + goto err_invalid_params; + } + + // allocate a structure + p_map = (mthca_map_space *)kmalloc(sizeof *p_map, GFP_KERNEL); + if (p_map == NULL) { + status = STATUS_INSUFFICIENT_RESOURCES; + goto err_no_memory; + } + + // support several sim clients + down( &p_context->mutex ); + + // map memory + sz =(ULONG)p_ext->bar[HCA_BAR_TYPE_HCR].size; + if (!p_ext->bar[HCA_BAR_TYPE_HCR].virt) { + PHYSICAL_ADDRESS pa; + pa.QuadPart = p_ext->bar[HCA_BAR_TYPE_HCR].phys; + ka = MmMapIoSpace( pa, sz, MmNonCached ); + if ( ka == NULL) { + HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_SHIM, + ("No kernel mapping of CR space.\n") ); + status = STATUS_UNSUCCESSFUL; + goto err_map_to_kernel; + } + p_ext->bar[HCA_BAR_TYPE_HCR].virt = ka; + } + ka = p_ext->bar[HCA_BAR_TYPE_HCR].virt; + + // prepare for mapping to user space + p_mdl = IoAllocateMdl( ka, sz, FALSE,FALSE,NULL); + if (p_mdl == NULL) { + HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_SHIM, + ("IoAllocateMdl failed.\n") ); + status = STATUS_INSUFFICIENT_RESOURCES; + goto err_alloc_mdl; + } + + // fill MDL + MmBuildMdlForNonPagedPool(p_mdl); + + // map the buffer into user space + ua = MmMapLockedPagesSpecifyCache( p_mdl, UserMode, MmNonCached, + NULL, FALSE, NormalPagePriority ); + if (ua == NULL) { + HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_SHIM, + ("MmMapLockedPagesSpecifyCache failed.\n") ); + status = STATUS_UNSUCCESSFUL; + goto err_map_to_user; + } + + // fill the results + p_res->va = ua; + p_res->size = sz; + + // resource tracking + p_map->p_mdl = p_mdl; + p_map->va = ua; + p_map->p_pcs = PsGetCurrentProcess(); + list_add_tail(&p_map->list, &p_context->map_list); + + up( &p_context->mutex ); + status = STATUS_SUCCESS; + goto out; + +err_map_to_user: + IoFreeMdl( p_mdl ); +err_alloc_mdl: +err_map_to_kernel: + up( &p_context->mutex ); + kfree( p_map ); +err_no_memory: +err_invalid_params: +out: + HCA_EXIT( HCA_DBG_PNP ); + return status; +} + + +static NTSTATUS +__unmap_crspace( + IN struct ib_ucontext * p_context, + IN PVOID p_buf, + IN ULONG buf_size + ) +{ + NTSTATUS status; + unmap_crspace *parm = (unmap_crspace *)p_buf; + mthca_map_space *p_map, *p_tmp; + int found = FALSE; + PEPROCESS p_pcs = PsGetCurrentProcess(); + + HCA_ENTER( HCA_DBG_PNP ); + + // sanity checks + if ( buf_size < sizeof *parm ) { + status = STATUS_INVALID_PARAMETER; + goto err_invalid_params; + } + + // support several sim clients + down( &p_context->mutex ); + + // look for the mapping info + list_for_each_entry_safe(p_map, p_tmp, &p_context->map_list, list, + mthca_map_space, mthca_map_space) { + if (p_map->va == parm->va && p_map->p_pcs == p_pcs) { + found = TRUE; + break; + } + } + + if (!found) { + HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_SHIM, + ("Not found internal info for unmappipng.%p for PID %d.\n" , + parm->va, (int)(INT_PTR)PsGetCurrentProcessId())); + status = STATUS_UNSUCCESSFUL; + goto err_not_found; + } + + // release the resources + list_del( &p_map->list ); + MmUnmapLockedPages(p_map->va, p_map->p_mdl); + IoFreeMdl( p_map->p_mdl ); + kfree( p_map ); + + up( &p_context->mutex ); + status = STATUS_SUCCESS; + goto out; + +err_not_found: + up( &p_context->mutex ); +err_invalid_params: +out: + HCA_EXIT( HCA_DBG_PNP ); + return status; +} + +void +unmap_crspace_for_all( struct ib_ucontext *p_context ) +{ + mthca_map_space *p_map, *p_tmp; + + HCA_ENTER( HCA_DBG_PNP ); + + down( &p_context->mutex ); + + list_for_each_entry_safe(p_map, p_tmp, &p_context->map_list, list, + mthca_map_space, mthca_map_space) { + list_del( &p_map->list ); + MmUnmapLockedPages(p_map->va, p_map->p_mdl); + IoFreeMdl( p_map->p_mdl ); + kfree( p_map ); + } + + up( &p_context->mutex ); + + HCA_EXIT( HCA_DBG_PNP ); +} + ib_api_status_t fw_access_ctrl( IN const ib_ca_handle_t h_ca, @@ -451,6 +640,7 @@ fw_access_ctrl( ULONG POINTER_ALIGNMENT length; ib_ci_op_t *p_ci; mlnx_hob_t *p_hob; + struct ib_ucontext * p_context; UNREFERENCED_PARAMETER(handle_array); UNREFERENCED_PARAMETER(num_handles); @@ -458,17 +648,19 @@ fw_access_ctrl( status = STATUS_SUCCESS; - if( p_umv_buf ) + if( p_umv_buf ) { + p_context = (struct ib_ucontext *)h_ca; p_hob = HOB_FROM_IBDEV( ((struct ib_ucontext*)h_ca)->device ); - else + } + else { + p_context = NULL; p_hob = (mlnx_hob_t *)(const void *)h_ca; + } p_dev_obj = (DEVICE_OBJECT *)EXT_FROM_HOB(p_hob)->cl_ext.p_self_do; p_ci = p_ci_op; - if ( !p_ci ) - return STATUS_INVALID_DEVICE_REQUEST; - if ( !p_ci->buf_size ) + if ( !p_ci || !p_ci->buf_size ) return STATUS_INVALID_DEVICE_REQUEST; length = p_ci->buf_size; @@ -477,6 +669,14 @@ fw_access_ctrl( switch ( p_ci->command ) { + case FW_MAP_CRSPACE: + status = __map_crspace(p_context, p_hob, p_data, length); + break; + + case FW_UNMAP_CRSPACE: + status = __unmap_crspace(p_context, p_data, length); + break; + case FW_READ: // read data from flash if ( if_ready ) status = fw_flash_read_data(&BusInterface, p_data, offset, length); diff --git a/trunk/hw/mthca/kernel/hca_verbs.c b/trunk/hw/mthca/kernel/hca_verbs.c index ed458f88..58dca783 100644 --- a/trunk/hw/mthca/kernel/hca_verbs.c +++ b/trunk/hw/mthca/kernel/hca_verbs.c @@ -44,7 +44,6 @@ #define PTR_ALIGN(size) (((size) + sizeof(void*) - 1) & ~(sizeof(void*) - 1)) - /* Matches definition in IbAccess for MaxSMPsWatermark */ uint32_t g_sqp_max_avs = ((4096/sizeof(ib_mad_t))*32*5); @@ -423,6 +422,10 @@ mlnx_um_open( uresp_p->dev_id = (uint16_t)ext_p->hcaConfig.DeviceID; done: + // some more inits + INIT_LIST_HEAD(&context_p->map_list); + KeInitializeMutex( &context_p->mutex, 0 ); + // return the result if (ph_um_ca) *ph_um_ca = (ib_ca_handle_t)context_p; @@ -445,12 +448,14 @@ mlnx_um_close( IN ib_ca_handle_t h_ca, IN ib_ca_handle_t h_um_ca ) { + struct ib_ucontext *p_ucontext = (struct ib_ucontext *)h_um_ca; UNREFERENCED_PARAMETER(h_ca); - if( !((struct ib_ucontext*)h_um_ca)->pd ) + unmap_crspace_for_all(p_ucontext); + if( !p_ucontext->pd ) cl_free( h_um_ca ); else - ibv_um_close((struct ib_ucontext *)h_um_ca); + ibv_um_close(p_ucontext); return; } diff --git a/trunk/hw/mthca/kernel/ib_verbs.h b/trunk/hw/mthca/kernel/ib_verbs.h index 58e73aaf..3f2fdd78 100644 --- a/trunk/hw/mthca/kernel/ib_verbs.h +++ b/trunk/hw/mthca/kernel/ib_verbs.h @@ -531,6 +531,9 @@ struct ib_ucontext { struct ib_pd *pd; atomic_t usecnt; /* count all resources */ ULONG is_removing; + // for CR space mapping + KMUTEX mutex; + struct list_head map_list; }; struct ib_uobject { diff --git a/trunk/inc/mthca/mthca_vc.h b/trunk/inc/mthca/mthca_vc.h new file mode 100644 index 00000000..45c001ec --- /dev/null +++ b/trunk/inc/mthca/mthca_vc.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2005 SilverStorm Technologies. All rights reserved. + * Copyright (c) 2004-2005 Mellanox Technologies, Inc. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: hca_verbs.c 148 2005-07-12 07:48:46Z sleybo $ + */ + +#ifndef MTHCA_VC_H +#define MTHCA_VC_H + +typedef +struct _map_crspace { + void * va; /* address of CRSPACE, mapped to user space */ + unsigned long size; /* size of CRSPACE, mapped to user space */ +} map_crspace; + +typedef +struct _unmap_crspace { + void * va; /* address of CRSPACE, mapped to user space */ +} unmap_crspace; + + +/* Definitions for hca_driver commands*/ +#define FW_READ 0x00 +#define FW_WRITE 0x01 +#define FW_READ_CMD 0x08 +#define FW_WRITE_CMD 0x09 +#define FW_MAP_CRSPACE 0x0A +#define FW_UNMAP_CRSPACE 0x0B +#define FW_OPEN_IF 0xe7 +#define FW_CLOSE_IF 0x7e + +#endif diff --git a/trunk/tools/mtcr/user/SOURCES b/trunk/tools/mtcr/user/SOURCES index d0111cfa..1a2e6e54 100644 --- a/trunk/tools/mtcr/user/SOURCES +++ b/trunk/tools/mtcr/user/SOURCES @@ -23,6 +23,7 @@ TARGETPATH=$(WINIBHOME)\bin\user\obj$(BUILD_ALT_DIR) INCLUDES= $(WINIBHOME)\inc; \ $(WINIBHOME)\inc\user; \ $(WINIBHOME)\inc\iba; \ + $(WINIBHOME)\inc\mthca; \ .\usb; diff --git a/trunk/tools/mtcr/user/mtcr.c b/trunk/tools/mtcr/user/mtcr.c index 1105e0c9..d01463f5 100644 --- a/trunk/tools/mtcr/user/mtcr.c +++ b/trunk/tools/mtcr/user/mtcr.c @@ -10,6 +10,7 @@ #include "usb.h" #include "mtcr.h" #include "mtcr_i2c.h" +#include "mthca_vc.h" //----------------------------------------------------- // NEW FEATURES @@ -31,37 +32,6 @@ ULONG g_DebugLevel = DEBUG_LEVEL_LOW; //----------------------------------------------------- - - - -// Copied from : #include -/* Definitions intended to become shared with UM. Later... */ - -typedef -struct _map_crspace { - PVOID va; /* address of CRSPACE, mapped to user space */ - PVOID ctx; /* opaque operation context; to be used in FW_UNMAP_CRSPACE */ - ULONG size; /* size of CRSPACE, mapped to user space */ -} map_crspace; - -typedef -struct _unmap_crspace { - PVOID va; /* address of CRSPACE, mapped to user space */ - PVOID ctx; /* operation context, received in FW_MAP_CRSPACE */ -} unmap_crspace; - - -/* Definitions for hca_driver commands*/ -#define FW_READ 0x00 -#define FW_WRITE 0x01 -#define FW_READ_CMD 0x08 -#define FW_WRITE_CMD 0x09 -#define FW_MAP_CRSPACE 0x0A -#define FW_UNMAP_CRSPACE 0x0B -#define FW_OPEN_IF 0xe7 -#define FW_CLOSE_IF 0x7e - - #define MAX_HCA_NUM 16 @@ -735,7 +705,6 @@ MTCR_API int mclose(mfile *mf) unmap_crspace unmap; unmap.va = mfi->cr_map.va; - unmap.ctx = mfi->cr_map.ctx; if (ibal_access(mfi->h_ca, 0x0, &unmap, sizeof(unmap), FW_UNMAP_CRSPACE ) != IB_SUCCESS) { DPRINT1(("Unmap crspace failed")); } -- 2.41.0