rem in %IDIR%\r
rem\r
rem BuildRelease option\r
-rem option == all | allnoforce | compile | compilenoforce | makebin | msi | wix | clean {OPENIB_REV}\r
+rem option == all | allnoforce | compile | compilenoforce | makebin | msi | wix | clean | msi-label | msi-del | msi-dir {OPENIB_REV}\r
rem\r
\r
rem This script is an 'example' of a one-command entire IB stack build to\r
if "%1" == "msi" goto OK\r
if "%1" == "wix" goto OK\r
if "%1" == "clean" goto OK\r
+if "%1" == "msi-label" goto OK\r
+if "%1" == "msi-del" goto OK\r
+if "%1" == "msi-dir" goto OK\r
\r
echo Unknown arg '%1' ?\r
\r
:usage\r
\r
-echo "usage: BuildRelease all | allnoforce | compile | compilenoforce | makebin | msi | wix | clean {OPENIB_REV value}"\r
-echo where:\r
+echo "usage: BuildRelease command {OPENIB_REV value}"\r
+echo where 'command' may be:\r
echo all - force recompile, install binaries to WIX tree, sign drivers and\r
echo build installers.\r
\r
echo create installers (.msi files) in IDIR.\r
echo wix - build .msi installers, assumes (drivers signed) .cat files exist\r
echo clean - remove build artifacts for a clean build: .obj, .sys, ...\r
+echo msi-label {OPENIB_REV}\r
+echo rename WOF_os*_arch*.msi to WOF_os*_arch*_svn#.msi \r
+echo Uniquely identify installers just created.\r
+echo If OPENIB_REV arg used, then rename WOF_os*_arch*argVal.msi\r
+echo otherwise, use SVN# from path.\r
+echo msi-del - del %windir%\temp\WOF_os*_arch*.msi\r
+echo msi-dir - del %windir%\temp\WOF_os*_arch*.msi\r
echo :\r
echo {OPENIB_REV}\r
echo optional, if set then OPENIB_REV is assigned this value.\r
echo Missing WDK @ %_DDK%\r
exit /B 1\r
)\r
-echo %0 - Building with WDK @ %_DDK%\r
\r
rem Platform SDK path - watchout for missing LoadPerf.h (installsp.c)\r
set _PSDK=%SystemDrive%\progra~1\mi2578~1\r
\r
if not "%2" == "" (\r
set SVN=%2\r
+ set LBL=%2\r
) else (\r
rem WARNING - SVN value depends on the path '\openIB-windows-svn\XXX\gen1',\r
rem where SVN is set to XXX\r
rem Note - OPENIB_REV is assigned SVN in a child script.\r
\r
set SVN=%CD:~22,4%\r
+ set LBL=_svn.!SVN!\r
)\r
\r
rem assumes %CD% == '...\gen1\trunk'\r
rem \r
set IDIR=%SystemRoot%\temp\r
\r
+if "%1" == "msi-label" (\r
+ pushd %IDIR%\r
+ if exist WOF_wxp_x86.msi (\r
+ if exist WOF_wxp_x86%LBL%.msi del /F/Q WOF_wxp_x86%LBL%.msi\r
+ rename WOF_wxp_x86.msi WOF_wxp_x86%LBL%.msi\r
+ )\r
+\r
+ if exist WOF_wnet_x86.msi (\r
+ if exist WOF_wnet_x86%LBL%.msi del /F/Q WOF_wnet_x86%LBL%.msi\r
+ rename WOF_wnet_x86.msi WOF_wnet_x86%LBL%.msi\r
+ )\r
+ if exist WOF_wnet_x64.msi (\r
+ if exist WOF_wnet_x64%LBL%.msi del /F/Q WOF_wnet_x64%LBL%.msi\r
+ rename WOF_wnet_x64.msi WOF_wnet_x64%LBL%.msi\r
+ )\r
+ if exist WOF_wnet_ia64.msi (\r
+ if exist WOF_wnet_ia64%LBL%.msi del /F/Q WOF_wnet_ia64%LBL%.msi\r
+ rename WOF_wnet_ia64.msi WOF_wnet_ia64%LBL%.msi\r
+ )\r
+\r
+ if exist WOF_wlh_x86.msi (\r
+ if exist WOF_wlh_x86%LBL%.msi del /F/Q WOF_wlh_x86%LBL%.msi\r
+ rename WOF_wlh_x86.msi WOF_wlh_x86%LBL%.msi\r
+ )\r
+ if exist WOF_wlh_x64.msi (\r
+ if exist WOF_wlh_x64%LBL%.msi del /F/Q WOF_wlh_x64%LBL%.msi\r
+ rename WOF_wlh_x64.msi WOF_wlh_x64%LBL%.msi\r
+ )\r
+ if exist WOF_wlh_ia64.msi (\r
+ if exist WOF_wlh_ia64%LBL%.msi del /F/Q WOF_wlh_ia64%LBL%.msi\r
+ rename WOF_wlh_ia64.msi WOF_wlh_ia64%LBL%.msi\r
+ )\r
+ dir WOF_*%LBL%.msi\r
+ popd\r
+ exit /B 0\r
+)\r
+\r
+if "%1" == "msi-del" (\r
+ echo Deleting WOF_{wxp,wnet,wlh}_{x86,x64,ia64}%LBL%.msi\r
+ pushd %IDIR%\r
+ if exist WOF_wxp_x86%LBL%.msi del /F/P WOF_wxp_x86%LBL%.msi\r
+\r
+ if exist WOF_wnet_x86%LBL%.msi del /F/P WOF_wnet_x86%LBL%.msi\r
+ if exist WOF_wnet_x64%LBL%.msi del /F/P WOF_wnet_x64%LBL%.msi\r
+ if exist WOF_wnet_ia64%LBL%.msi del /F/P WOF_wnet_ia64%LBL%.msi\r
+\r
+ if exist WOF_wlh_x86%LBL%.msi del /F/P WOF_wlh_x86%LBL%.msi\r
+ if exist WOF_wlh_x64%LBL%.msi del /F/P WOF_wlh_x64%LBL%.msi\r
+ if exist WOF_wlh_ia64%LBL%.msi del /F/P WOF_wlh_ia64%LBL%.msi\r
+\r
+ dir /N/OD WOF_*.msi\r
+ popd\r
+ exit /B 0\r
+)\r
+\r
+if "%1" == "msi-dir" (\r
+ pushd %IDIR%\r
+ dir /N/OD WOF_*.msi\r
+ popd\r
+ exit /B 0\r
+)\r
+\r
+echo %0 - Building with WDK @ %_DDK%\r
echo Building for OPENIB_REV %SVN%, installer files @ %IDIR%\r
echo Drivers signed using Certificate '%CERTNAME%'\r
\r
mkdir %RBIN_WXP%\r
\r
if EXIST "%BSE%\etc\makebin.bat" (\r
- echo etc\makebin.bat %BSE% %RBIN_WLH% wlh\r
call %BSE%\etc\makebin.bat %BSE% %RBIN_WLH% wlh\r
if ERRORLEVEL 1 (\r
echo %0: Err in makebin.bat %BSE% %RBIN_WLH% wlh\r
exit /B 1\r
)\r
- echo etc\makebin.bat %BSE% %RBIN_WNET% wnet\r
call %BSE%\etc\makebin.bat %BSE% %RBIN_WNET% wnet\r
if ERRORLEVEL 1 (\r
echo %0: Err in makebin.bat %BSE% %RBIN_WNET% wnet\r
exit /B 1\r
)\r
- echo etc\makebin.bat %BSE% %RBIN_WXP% wxp\r
call %BSE%\etc\makebin.bat %BSE% %RBIN_WXP% wxp\r
if ERRORLEVEL 1 (\r
echo %0: Err in makebin.bat %BSE% %RBIN_WXP% wxp\r
--- /dev/null
+[CatalogHeader]\r
+Name=ib_bus.cat\r
+PublicVersion=0x0000001\r
+EncodingType=0x00010001\r
+CATATTR1=0x10010001:OSAttr:2:6.0\r
+[CatalogFiles]\r
+<hash>ib_bus.inf=ib_bus.inf\r
+<hash>ibbus.sys=ibbus.sys\r
+<hash>ibiou.sys=ibiou.sys\r
+<hash>ibal.dll=ibal.dll\r
+<hash>complib.dll=complib.dll\r
+<hash>ibald.dll=ibald.dll\r
+<hash>complibd.dll=complibd.dll\r
\r
rem usage: makebin src dst [wlh,wnet,wxp]\r
rem\r
-rem src - ...\gen1\trunk\r
-rem dst - full path where \bin is created.\r
-rem OSE - (Operating System Environment) which windows version\r
-\r
-echo %1 %2 %3\r
+rem src - OpenIB src path ...\gen1\trunk\r
+rem dst - full path tp where binaries are copied, 'bin\' created here.\r
+rem OSE - (Operating System Environment) which windows version {wxp,wlh,wnet}\r
+rem representing {XP, server 2008 & server 2003}\r
+\r
+rem makebin is designed to take an openIB build tree path and produce a folder\r
+rem tree of binaries suitable for input to a WIX builder which procduces\r
+rem an OS .msi installer.\r
+rem Building a WinOF release is done is 3 phases, makebin is the 2nd phase.\r
+rem makebin is commonly run from trunk\buildrelease.bat although it can be\r
+rem run standalone.\r
+\r
+echo Starting makebin\r
+echo Src %1\r
+echo Dst %2\r
+echo OS %3\r
\r
if "%1"=="" goto usage\r
if "%2"=="" goto usage\r
set bin_dir=%1\bin\kernel\objfre_%OSE%_amd64\amd64\r
set dest_dir=%2\HCA\amd64\\r
\r
-if "%DBG%" == "TRUE" echo DBG: AMD64 free drivers to %dest_dir%\r
+if "%DBG%" == "TRUE" echo DBG: AMD64 free drivers\r
\r
-set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys \r
+set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys mlx4_hca.inf mlx4_bus.inf \r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %dest_dir% /yq 1> nul\r
if ERRORLEVEL 1 (\r
set bin_dir=%1\bin\kernel\objfre_%OSE%_ia64\ia64\r
set dest_dir=%2\HCA\ia64\\r
\r
-if "%DBG%" == "TRUE" echo DBG: ia64 free drivers to %dest_dir%\r
+if "%DBG%" == "TRUE" echo DBG: ia64 free drivers\r
\r
set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf\r
for %%i in ( %F% ) do (\r
set bin_dir=%1\bin\kernel\objfre_%OSE%_x86\i386\r
set dest_dir=%2\HCA\x86\\r
\r
-set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys\r
+set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys mlx4_hca.inf mlx4_bus.inf \r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul\r
if ERRORLEVEL 1 (\r
)\r
)\r
\r
-echo xcopy %bin_dir%\*.exe %2\tools\amd64\debug\ /yq\r
+echo xcopy *.exe tools\amd64\debug\\r
xcopy %bin_dir%\*.exe %2\tools\amd64\debug\ /yq 1>nul\r
\r
set F=dapld.dll dapld.pdb datd.dll datd.lib datd.pdb\r
)\r
)\r
\r
-echo xcopy %bin_dir%\*.exe %2\tools\ia64\debug\ /yq \r
+echo xcopy *.exe tools\ia64\debug\ \r
xcopy %bin_dir%\*.exe %2\tools\ia64\debug\ /yq 1>nul\r
\r
set F=dapld.dll dapld.pdb datd.dll datd.lib datd.pdb\r
)\r
)\r
\r
-echo xcopy %bin_dir%\*.exe %2\tools\x86\debug\ /yq\r
+if "%DBG%" == "TRUE" echo DBG: copy debug *.exe to tools\x86\debug\ \r
xcopy %bin_dir%\*.exe %2\tools\x86\debug\ /yq 1>nul\r
\r
set F=dapld.dll dapld.pdb datd.dll datd.lib datd.pdb\r
set bin_dir=%1\bin\user\objfre_%OSE%_amd64\amd64\r
set dest_dir=%2\HCA\amd64\\r
\r
-if "%DBG%" == "TRUE" echo DBG: amd64 Free dlls to %dest_dir%\r
+if "%DBG%" == "TRUE" echo DBG: copy amd64 Free dlls\r
\r
for %%i in (ibal.dll complib.dll mthcau.dll IbInstaller.dll mlx4u.dll) do (\r
xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul\r
\r
xcopy %bin_dir%\ibwsd.dll %2\net\amd64\ /yq\r
xcopy %bin_dir%\installsp.exe %2\net\amd64\ /yq \r
+xcopy %bin_dir%\installsp.exe %2\tools\amd64\release\ /yq\r
+xcopy %bin_dir%\ndinstall.exe %2\net\amd64\ /yq \r
+xcopy %bin_dir%\ndinstall.exe %2\tools\amd64\release\ /yq\r
+xcopy %bin_dir%\ibndprov.dll %2\net\amd64\ /yq\r
\r
-echo xcopy %bin_dir%\*.exe %2\tools\amd64\release\ /y \r
+echo xcopy *.exe tools\amd64\release\ \r
xcopy %bin_dir%\*.exe %2\tools\amd64\release\ /yq 1>nul\r
\r
set F=dapl.dll dapl.pdb dat.dll dat.lib dat.pdb\r
set bin_dir=%1\bin\user\objfre_%OSE%_ia64\ia64\r
set dest_dir=%2\HCA\ia64\\r
\r
-if "%DBG%" == "TRUE" echo DBG: IA64 Free dlls to %dest_dir%\r
+if "%DBG%" == "TRUE" echo DBG: copy IA64 Free dlls\r
\r
for %%i in (ibal.dll complib.dll mthcau.dll IbInstaller.dll) do (\r
xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul\r
\r
xcopy %bin_dir%\ibwsd.dll %2\net\ia64\ /yq\r
xcopy %bin_dir%\installsp.exe %2\net\ia64\ /yq\r
+xcopy %bin_dir%\ndinstall.exe %2\net\ia64\ /yq\r
+xcopy %bin_dir%\ibndprov.dll %2\net\ia64\ /yq\r
\r
-echo xcopy %bin_dir%\*.exe %2\tools\ia64\release\ /y q\r
+echo xcopy *.exe tools\ia64\release\ \r
xcopy %bin_dir%\*.exe %2\tools\ia64\release\ /yq 1>nul\r
\r
set F=dapl.dll dapl.pdb dat.dll dat.lib dat.pdb \r
set bin_dir=%1\bin\user\objfre_%OSE%_x86\i386\r
set dest_dir=%2\HCA\x86\\r
\r
-if "%DBG%" == "TRUE" echo DBG: x86 Free dlls to %dest_dir%\r
+if "%DBG%" == "TRUE" echo DBG: copy x86 Free dlls\r
\r
for %%i in (ibal.dll complib.dll mthcau.dll IbInstaller.dll mlx4u.dll) do (\r
xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul\r
)\r
)\r
\r
-echo xcopy %bin_dir%\*.exe %2\tools\x86\release\ /yq\r
+echo xcopy *.exe tools\x86\release\\r
xcopy %bin_dir%\*.exe %2\tools\x86\release\ /yq 1>nul\r
\r
set F=dapl.dll dapl.pdb dat.dll dat.lib dat.pdb\r
\r
rem free x86 items\r
\r
-if "%DBG%" == "TRUE" echo DBG: x86 Free WSD to %dest_dir%\net\x86\r
+if "%DBG%" == "TRUE" echo DBG: copy x86 Free WSD\r
\r
copy %bin_dir%\ibwsd.dll %2\net\x86\ /y\r
copy %bin_dir%\installsp.exe %2\net\x86\ /y\r
-copy %bin_dir%\installsp.exe %2\tools\x86\release /y\r
+copy %bin_dir%\installsp.exe %2\tools\x86\release\ /y\r
+copy %bin_dir%\ndinstall.exe %2\net\x86\ /y\r
+copy %bin_dir%\ndinstall.exe %2\tools\x86\release\ /y\r
+copy %bin_dir%\ibndprov.dll %2\net\x86\ /y\r
\r
rem free x86 DLLs --> WOW64 DLLs\r
\r
if "%DBG%" == "TRUE" echo DBG: x86 Free dlls to WOW64\r
\r
+copy %bin_dir%\ibndprov.dll %2\net\amd64\ibndprov32.dll /y\r
+copy %bin_dir%\ibndprov.dll %2\net\ia64\ibndprov32.dll /y\r
copy /B %bin_dir%\ibwsd.dll %2\net\amd64\ibwsd32.dll /y\r
copy /B %bin_dir%\ibwsd.dll %2\net\ia64\ibwsd32.dll /y\r
copy /B %bin_dir%\ibal.dll %2\HCA\amd64\ibal32.dll /y\r
\r
if "%DBG%" == "TRUE" echo DBG: cpy %OSE% x86 INF files\r
\r
-xcopy %1\hw\mthca\kernel\mthca.cdf %2\HCA\x86\ /yq\r
-xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.inf %2\HCA\x86\ /yq\r
-xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.cdf %2\HCA\x86\ /yq\r
-xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.inf %2\HCA\x86\ /yq\r
-xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.cdf %2\HCA\x86\ /yq\r
-\r
-xcopy %1\core\bus\kernel\ib_bus.cdf %2\HCA\x86\ /yq\r
+copy /A/Y %1\hw\mthca\kernel\mthca32.cdf %2\HCA\x86\mthca.cdf\r
+copy /A/Y %1\hw\mlx4\kernel\hca\mlx4_hca32.cdf %2\HCA\x86\mlx4_hca.cdf\r
+copy /A/Y %1\hw\mlx4\kernel\bus\drv\mlx4_bus32.cdf %2\HCA\x86\mlx4_bus.cdf\r
+copy /A/Y %1\core\bus\kernel\ib_bus32.cdf %2\HCA\x86\ib_bus.cdf\r
\r
rem No WSD support for XP32\r
if /I "%OSE%" == "wxp" (\r
copy /A /Y %1\ulp\ipoib\kernel\netipoib-xp32.inf %2\net\x86\netipoib.inf\r
- copy /A /Y %1\ulp\ipoib\kernel\ipoib-xp32.cdf %2\net\x86\ipoib.cdf\r
+ copy /A /Y %1\ulp\ipoib\kernel\ipoib32-xp.cdf %2\net\x86\ipoib.cdf\r
) else (\r
- xcopy %1\ulp\ipoib\kernel\ipoib.cdf %2\net\x86\ /yq\r
+ copy /A/Y %1\ulp\ipoib\kernel\ipoib32.cdf %2\net\x86\ipoib.cdf\r
xcopy %1\tools\wsdinstall\user\installsp.exe.manifest %2\net\x86\ /yq\r
)\r
xcopy %1\ulp\qlgcvnic\kernel\netvnic.cdf %2\net\x86\ /yq\r
xcopy %1\ulp\srp\kernel\ib_srp.cdf %2\storage\x86\ /yq\r
\r
-\r
if /I "%OSE%" == "wxp" goto cpy_bin_files\r
\r
if "%DBG%" == "TRUE" echo DBG: copy 64bit INF files\r
\r
xcopy %1\hw\mthca\kernel\mthca.cdf %2\HCA\amd64 /yq\r
\r
-xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.inf %2\HCA\amd64\ /yq\r
xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.cdf %2\HCA\amd64\ /yq\r
xcopy %1\hw\mlx4\kernel\hca\mlx4_hca32.cdf %2\HCA\amd64\ /yq\r
-xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.inf %2\HCA\amd64\ /yq\r
xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.cdf %2\HCA\amd64\ /yq\r
xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus32.cdf %2\HCA\amd64\ /yq\r
\r
\r
rem bin\bin used to generate a web based symbol store in build-ofa-dist.bat.\r
\r
-echo 'Copy bin\obj{chk,fre}_%3_{x86,amd64,ia64} to %2\bin'\r
+echo 'Copy bin\obj{chk,fre}_%3_{x86,amd64,ia64} to bin'\r
rem xcopy %1\bin %2\bin\ /eyq\r
rem instead of copying the entire bin\ folder, pick items of OS interest.\r
\r
if NOT exist %2\Misc (\r
mkdir %2\Misc\r
)\r
-copy /Y %1\Docs\Manual.htm %2\Misc\Manual.htm /A\r
-copy /Y %1\tests\cmtest\user\cmtest_main.c %2\Misc\cmtest.c /A\r
+copy /Y/A %1\Docs\Manual.htm %2\Misc\Manual.htm\r
+copy /Y/A %1\tests\cmtest\user\cmtest_main.c %2\Misc\cmtest.c\r
\r
goto end\r
\r
echo %1\bin\user\objchk_%OSE%_x86\i386 missing \r
\r
:end\r
+echo .\r
+echo Finished OS %3\r
+echo .\r
+\r
endlocal\r
\r
TARGETLIBS= \\r
$(DDK_LIB_PATH)\ntstrsafe.lib \\r
- $(TARGETPATH)\*\complib.lib\r
- \r
+ $(TARGETPATH)\*\complib.lib \r
\r
!IFDEF ENABLE_EVENT_TRACING\r
\r
unsigned long flags;
int ret = 0;
+ if (mlx4_is_barred(device->dma_device))
+ return -EFAULT;
+
if (port_num < start_port(device) || port_num > end_port(device))
return -EINVAL;
int p, i;
int ret = -ENOENT;
+ if (mlx4_is_barred(device->dma_device))
+ return -EFAULT;
+
*port_num = (u8)-1;
if (index)
*index = (u16)-1;
unsigned long flags;
int ret = 0;
+ if (mlx4_is_barred(device->dma_device))
+ return -EFAULT;
+
if (port_num < start_port(device) || port_num > end_port(device))
return -EINVAL;
int i;
int ret = -ENOENT;
+ if (mlx4_is_barred(device->dma_device))
+ return -EFAULT;
+
if (port_num < start_port(device) || port_num > end_port(device))
return -EINVAL;
}
INIT_IB_EVENT_HANDLER(&device->cache.event_handler,
- device, ib_cache_event, NULL);
+ device, ib_cache_event, NULL, NULL, 0);
if (ib_register_event_handler(&device->cache.event_handler))
goto err_cache;
\r
#endif\r
\r
+static \r
NTSTATUS\r
__create_child(\r
__in WDFDEVICE Device,\r
return status;\r
}\r
\r
+static \r
NTSTATUS\r
__do_static_enumeration(\r
IN WDFDEVICE Device\r
NTSTATUS status = STATUS_SUCCESS;\r
int i;\r
int number_of_ib_ports;\r
-\r
- // TODO:Need to add an event log in the case of errors\r
+ PFDO_DEVICE_DATA p_fdo = FdoGetData(Device);\r
+ struct mlx4_dev *mdev = p_fdo->pci_dev.dev;\r
\r
MLX4_ENTER(MLX4_DBG_DRV);\r
\r
+ if ( p_fdo->children_created )\r
+ goto end;\r
+ \r
// eventually we'll have all information about children in Registry\r
// DriverEntry will read it into a Global storage and\r
// this routine will create all the children on base on this info\r
if(number_of_ib_ports > 0) {\r
status = __create_child(Device, BUS_HARDWARE_IDS, BUS_HARDWARE_DESCRIPTION, 0 );\r
if (!NT_SUCCESS(status)) {\r
- MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (ib)failed with 0x%x\n", status));\r
+ MLX4_PRINT_EV(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (ib)failed with 0x%x\n", status));\r
}\r
}\r
\r
if(mlx4_is_eth_port(i)) {\r
status = __create_child(Device, ETH_HARDWARE_IDS, ETH_HARDWARE_DESCRIPTION, i+1 );\r
if (!NT_SUCCESS(status)) {\r
- MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (eth) failed with 0x%x\n", status));\r
+ MLX4_PRINT_EV(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (eth) failed with 0x%x\n", status));\r
}\r
}\r
}\r
\r
+ p_fdo->children_created = TRUE;\r
+ \r
+end:\r
MLX4_EXIT( MLX4_DBG_DRV );\r
return status;\r
}\r
\r
+static \r
+NTSTATUS\r
+__start_card( \r
+ IN PFDO_DEVICE_DATA p_fdo \r
+ )\r
+{\r
+#ifndef USE_WDM_INTERRUPTS\r
+ int i;\r
+#endif\r
+ int err;\r
+ NTSTATUS status = STATUS_SUCCESS;\r
+ struct pci_dev *pdev = &p_fdo->pci_dev;\r
+\r
+ MLX4_ENTER(MLX4_DBG_DRV);\r
+\r
+ if ( p_fdo->card_started )\r
+ goto err; \r
+ \r
+ // enable the card\r
+ status = pci_hca_enable( &pdev->bus_pci_ifc, &pdev->pci_cfg_space );\r
+ if( !NT_SUCCESS( status ) ) \r
+ goto err;\r
+\r
+ //\r
+ // init the card\r
+ //\r
+\r
+#ifndef USE_WDM_INTERRUPTS\r
+ // enable interrupts for start up\r
+ for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) \r
+ WdfInterruptEnable(p_fdo->interrupt[i].WdfInterrupt);\r
+#endif \r
+\r
+ // NET library\r
+ err = mlx4_init_one( &p_fdo->pci_dev );\r
+ if (err) {\r
+ status = errno_to_ntstatus(err);\r
+ goto err;\r
+ }\r
+\r
+#ifndef USE_WDM_INTERRUPTS\r
+ //\r
+ // complete filling interrupt context (for more efficiency)\r
+ //\r
+ for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) {\r
+ struct mlx4_priv *priv = mlx4_priv( p_fdo->pci_dev.dev );\r
+ PINTERRUPT_DATA p_isr_ctx = WdfObjectGetTypedContext( \r
+ p_fdo->interrupt[i].WdfInterrupt, INTERRUPT_DATA );\r
+\r
+ p_isr_ctx->eq = &priv->eq_table.eq[i];\r
+ }\r
+#endif\r
+\r
+ //\r
+ // prepare MLX4 IB interface\r
+ //\r
+\r
+ // fill the header\r
+ p_fdo->bus_ib_ifc.i.Size = sizeof(MLX4_BUS_IB_INTERFACE);\r
+ p_fdo->bus_ib_ifc.i.Version = MLX4_BUS_IB_INTERFACE_VERSION;\r
+ // Let the framework handle reference counting.\r
+ p_fdo->bus_ib_ifc.i.InterfaceReference = WdfDeviceInterfaceReferenceNoOp;\r
+ p_fdo->bus_ib_ifc.i.InterfaceDereference = WdfDeviceInterfaceDereferenceNoOp;\r
+\r
+ p_fdo->bus_ib_ifc.pdev = &p_fdo->pci_dev;\r
+ p_fdo->bus_ib_ifc.p_ibdev = p_fdo->pci_dev.ib_dev;\r
+ p_fdo->bus_ib_ifc.pmlx4_dev = to_mdev(p_fdo->pci_dev.ib_dev)->dev;\r
+ p_fdo->bus_ib_ifc.is_livefish = mlx4_is_livefish(p_fdo->pci_dev.dev);\r
+\r
+ p_fdo->card_started = TRUE;\r
+\r
+err:\r
+ MLX4_EXIT( MLX4_DBG_DRV );\r
+ return status;\r
+}\r
+\r
+static\r
+void\r
+__stop_card(\r
+ IN PFDO_DEVICE_DATA p_fdo\r
+ )\r
+{\r
+ if ( p_fdo->card_started ) {\r
+ p_fdo->card_started = FALSE;\r
+ mlx4_remove_one( &p_fdo->pci_dev );\r
+ }\r
+}\r
+\r
+\r
NTSTATUS\r
EvtDeviceD0Entry(\r
IN WDFDEVICE Device,\r
)\r
{\r
NTSTATUS status = STATUS_SUCCESS;\r
- \r
- UNUSED_PARAM(Device);\r
- UNUSED_PARAM(PreviousState);\r
+ PFDO_DEVICE_DATA p_fdo = FdoGetData(Device);\r
+ struct pci_dev *pdev = &p_fdo->pci_dev;\r
+ struct mlx4_dev *mdev;\r
\r
MLX4_ENTER(MLX4_DBG_DRV);\r
\r
MLX4_PRINT(TRACE_LEVEL_INFORMATION, MLX4_DBG_DRV, ("PreviousState 0x%x\n", PreviousState));\r
\r
+ // start card (needed after Hibernetion)\r
+ if (PreviousState > WdfPowerDeviceD0)\r
+ __start_card( p_fdo );\r
+ mdev = pdev->dev;\r
+\r
+ // create child device\r
status = __do_static_enumeration(Device);\r
- if (!NT_SUCCESS(status)) {\r
- MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__do_static_enumeration failed with 0x%x\n", status));\r
- }\r
+ if (!NT_SUCCESS(status)) {\r
+ MLX4_PRINT_EV(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__do_static_enumeration failed with 0x%x\n", status));\r
+ goto err;\r
+ }\r
\r
- {\r
- PFDO_DEVICE_DATA p_fdo = FdoGetData(Device);\r
- struct pci_dev *pdev = &p_fdo->pci_dev;\r
- struct mlx4_dev *mdev = pdev->dev;\r
-\r
- MLX4_PRINT_EV(TRACE_LEVEL_INFORMATION ,MLX4_DBG_DRV ,\r
- ("Ven %x Dev %d Fw %d.%d.%d Drv %s (%s), BD %s\n", \r
- (unsigned)pdev->ven_id, (unsigned)pdev->dev_id,\r
- (int) (mdev->caps.fw_ver >> 32),\r
- (int) (mdev->caps.fw_ver >> 16) & 0xffff, \r
- (int) (mdev->caps.fw_ver & 0xffff),\r
- DRV_VERSION, DRV_RELDATE, \r
- mlx4_is_livefish(mdev) ? "Y" : "N"\r
- ));\r
- }\r
+ // Log Success Message\r
+ MLX4_PRINT_EV(TRACE_LEVEL_INFORMATION ,MLX4_DBG_DRV ,\r
+ ("Ven %x Dev %d Fw %d.%d.%d, IsBurnDevice %s\n", \r
+ (unsigned)pdev->ven_id, (unsigned)pdev->dev_id,\r
+ (int) (mdev->caps.fw_ver >> 32),\r
+ (int) (mdev->caps.fw_ver >> 16) & 0xffff, \r
+ (int) (mdev->caps.fw_ver & 0xffff),\r
+ mlx4_is_livefish(mdev) ? "Y" : "N"\r
+ ));\r
\r
+err:\r
MLX4_EXIT( MLX4_DBG_DRV );\r
- return STATUS_SUCCESS;\r
+ return status;\r
}\r
\r
NTSTATUS\r
IN WDF_POWER_DEVICE_STATE TargetState\r
)\r
{\r
- UNUSED_PARAM(Device);\r
- UNUSED_PARAM(TargetState);\r
+ PFDO_DEVICE_DATA p_fdo = FdoGetData(Device);\r
+ \r
MLX4_ENTER(MLX4_DBG_DRV);\r
MLX4_PRINT(TRACE_LEVEL_INFORMATION, MLX4_DBG_DRV, ("TargetState 0x%x\n", TargetState));\r
+\r
+ if (TargetState > WdfPowerDeviceD0)\r
+ __stop_card( p_fdo );\r
+\r
MLX4_EXIT( MLX4_DBG_DRV );\r
return STATUS_SUCCESS;\r
}\r
return status;\r
}\r
\r
-\r
NTSTATUS\r
EvtPrepareHardware(\r
IN WDFDEVICE Device,\r
IN WDFCMRESLIST ResourcesTranslated\r
)\r
{\r
-#ifndef USE_WDM_INTERRUPTS\r
- int i;\r
-#endif\r
- int err;\r
NTSTATUS status;\r
PFDO_DEVICE_DATA p_fdo = FdoGetData(Device);\r
- struct pci_dev *pdev = &p_fdo->pci_dev;\r
\r
MLX4_ENTER(MLX4_DBG_DRV);\r
\r
goto err;\r
}\r
\r
- // enable the card\r
- status = pci_hca_enable( &pdev->bus_pci_ifc, &pdev->pci_cfg_space );\r
- if( !NT_SUCCESS( status ) ) \r
- goto err;\r
-\r
- //\r
- // init the card\r
- //\r
-\r
-#ifndef USE_WDM_INTERRUPTS\r
- // enable interrupts for start up\r
- for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) \r
- WdfInterruptEnable(p_fdo->interrupt[i].WdfInterrupt);\r
-#endif \r
-\r
- // NET library\r
- err = mlx4_init_one( &p_fdo->pci_dev );\r
- if (err) {\r
- status = errno_to_ntstatus(err);\r
- goto err;\r
- }\r
-\r
-#ifndef USE_WDM_INTERRUPTS\r
- //\r
- // complete filling interrupt context (for more efficiency)\r
- //\r
- for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) {\r
- struct mlx4_priv *priv = mlx4_priv( p_fdo->pci_dev.dev );\r
- PINTERRUPT_DATA p_isr_ctx = WdfObjectGetTypedContext( \r
- p_fdo->interrupt[i].WdfInterrupt, INTERRUPT_DATA );\r
-\r
- p_isr_ctx->eq = &priv->eq_table.eq[i];\r
- }\r
-#endif\r
-\r
- //\r
- // prepare MLX4 IB interface\r
- //\r
+ // start the card\r
+ status = __start_card( p_fdo );\r
+ \r
+err:\r
+ MLX4_EXIT( MLX4_DBG_DRV );\r
+ return status;\r
+}\r
\r
- // fill the header\r
- p_fdo->bus_ib_ifc.i.Size = sizeof(MLX4_BUS_IB_INTERFACE);\r
- p_fdo->bus_ib_ifc.i.Version = MLX4_BUS_IB_INTERFACE_VERSION;\r
- // Let the framework handle reference counting.\r
- p_fdo->bus_ib_ifc.i.InterfaceReference = WdfDeviceInterfaceReferenceNoOp;\r
- p_fdo->bus_ib_ifc.i.InterfaceDereference = WdfDeviceInterfaceDereferenceNoOp;\r
+void fix_bus_ifc(struct pci_dev *pdev)\r
+{\r
+ PFDO_DEVICE_DATA p_fdo;\r
\r
- p_fdo->bus_ib_ifc.pdev = &p_fdo->pci_dev;\r
+ p_fdo = CONTAINING_RECORD(pdev, FDO_DEVICE_DATA, pci_dev);\r
p_fdo->bus_ib_ifc.p_ibdev = p_fdo->pci_dev.ib_dev;\r
p_fdo->bus_ib_ifc.pmlx4_dev = to_mdev(p_fdo->pci_dev.ib_dev)->dev;\r
p_fdo->bus_ib_ifc.is_livefish = mlx4_is_livefish(p_fdo->pci_dev.dev);\r
-\r
- status = STATUS_SUCCESS;\r
- \r
-err:\r
- MLX4_EXIT( MLX4_DBG_DRV );\r
- return status;\r
}\r
\r
NTSTATUS\r
\r
MLX4_ENTER(MLX4_DBG_DRV);\r
\r
- mlx4_remove_one( &p_fdo->pci_dev );\r
+ __stop_card( p_fdo );\r
__put_resources( p_fdo );\r
\r
MLX4_EXIT( MLX4_DBG_DRV );\r
// "Ports L2 type (ib/eth/auto, entry per port, comma seperated, default ib for all)"\r
DECLARE_CONST_UNICODE_STRING(PortType, L"PortType");\r
\r
- \r
+\r
ULONG value;\r
WDFKEY hKey = NULL;\r
NTSTATUS status = STATUS_SUCCESS;\r
else\r
g.mod_interrupt_from_first = 1;\r
\r
+\r
uvalue.Buffer = uvalue_data;\r
uvalue.MaximumLength = MAX_UVALUE;\r
uvalue.Length = 0;\r
WDFWAITLOCK ChildLock;\r
WDFDEVICE FdoDevice;\r
struct pci_dev pci_dev;\r
+ int card_started;\r
int pci_bus_ifc_taken;\r
WDFDMAENABLER dma_enabler;\r
int dma_adapter_taken;\r
res_interrupt_t interrupt[MLX4_MAX_INTERRUPTS];\r
MLX4_BUS_IB_INTERFACE bus_ib_ifc;\r
+ int children_created;\r
// Data for the Ethernet device\r
-#ifdef _WIN64 \r
- UCHAR pad[0x8];\r
-#endif\r
-\r
struct VipBusIfc mtnic_Ifc;\r
\r
} FDO_DEVICE_DATA, *PFDO_DEVICE_DATA;\r
\r
-#ifdef _WIN64 \r
-C_ASSERT((FIELD_OFFSET(FDO_DEVICE_DATA, mtnic_Ifc) % 16) == 0);\r
-#endif\r
-\r
WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(FDO_DEVICE_DATA, FdoGetData)\r
\r
//\r
%MT25448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6368\r
%MT26418.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6732\r
%MT26428.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_673c\r
+%MT26448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6750\r
%MT00401.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_0191\r
\r
[MLX4BUS.DeviceSection.ntamd64]\r
%MT26418.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6732\r
%MT25448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6368\r
%MT26428.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_673c\r
+%MT26448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6750\r
%MT00401.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_0191\r
\r
[MLX4BUS.DeviceSection.ntia64]\r
%MT25448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6368\r
%MT26418.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6732\r
%MT26428.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_673c\r
+%MT26448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6750\r
%MT00401.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_0191\r
\r
-[MLX4BUS.DDInstall.ntx86.hw]\r
-AddReg = MLX4BUS.HwReg\r
-\r
-[MLX4BUS.DDInstall.ntamd64.hw]\r
-AddReg = MLX4BUS.HwReg\r
-\r
-[MLX4BUS.DDInstall.ntia64.hw]\r
-AddReg = MLX4BUS.HwReg\r
-\r
-[MLX4BUS.HwReg]\r
-HKR,"Interrupt Management", 0x00000010\r
-HKR,"Interrupt Management\MessageSignaledInterruptProperties",0x00000010\r
-\r
-; MSI/MSI-X support\r
-HKR,"Interrupt Management\MessageSignaledInterruptProperties",MSISupported,0x00010001,0\r
-HKR,"Interrupt Management\MessageSignaledInterruptProperties",MessageNumberLimit,0x00010001,8\r
-HKR,"Interrupt Management\Affinity Policy",0x00000010\r
-\r
-; AssignmentSetOverride - processors KAFFINITY mask \r
-HKR,"Interrupt Management\Affinity Policy",AssignmentSetOverride,0x00000001,0x0\r
-\r
-; IrqPolicyMachineDefault (0) - use default policy for the computer\r
-; IrqPolicyAllCloseProcessors (1) - connect interrupts to all processors of the near NUMA node\r
-; IrqPolicyOneCloseProcessor (2) - connect interrupts to one processor\r
-; IrqPolicyAllProcessorsInMachine (3) - connect interrupts to all processors in the machine\r
-; IrqPolicySpecifiedProcessors (4) - connects interrupts according to AssignmentSetOverride\r
-HKR,"Interrupt Management\Affinity Policy",DevicePolicy,0x00010001,0x0\r
-\r
-; IrqArbPriorityUndefined (0) - no interrupt priority policy. \r
-; IrqArbPriorityLow (1) - device can tolerate low IRQL\r
-; IrqArbPriorityNormal (2) - device expects normal interrupt latencies\r
-; IrqArbPriorityHigh (3) - device requires the lowest possible interrupt latency\r
-HKR,"Interrupt Management\Affinity Policy",DevicePriority,0x00010001,0x0\r
-\r
[MLX4BUS.DDInstall.ntx86]\r
CopyFiles = MLX4BUS.CopyFiles\r
\r
MT25448.DeviceDesc="ConnectX (MT25448) - Mellanox ConnectX Condor Ethernet Adapter"\r
MT26418.DeviceDesc="ConnectX (MT26418) - Mellanox ConnectX DDR_G2 Channel Adapter"\r
MT26428.DeviceDesc="ConnectX (MT26428) - Mellanox ConnectX QDR_G2 Channel Adapter"\r
+MT26448.DeviceDesc="ConnectX (MT26448) - Mellanox ConnectX Condor_G2 Ethernet Adapter"\r
MT00401.DeviceDesc="ConnectX (MT00401) - Mellanox ConnectX Channel Adapter in Burning Mode"\r
DiskId = "Mellanox Mlx4 Bus installation disk"\r
SPSVCINST_NULL = 0x0\r
\r
#define MAX_ID_LEN 80\r
\r
+NTSTATUS\r
+EvtDeviceProcessQueryInterfaceRequest(\r
+ IN WDFDEVICE Device,\r
+ IN LPGUID InterfaceType,\r
+ IN OUT PINTERFACE ExposedInterface,\r
+ IN OUT PVOID ExposedInterfaceSpecificData\r
+ )\r
+{\r
+ PPDO_DEVICE_DATA p_pdo = PdoGetData(Device);\r
+ PFDO_DEVICE_DATA p_fdo = p_pdo->p_fdo;\r
+ PMLX4_BUS_IB_INTERFACE p_ib_ifc = (PMLX4_BUS_IB_INTERFACE)ExposedInterface;\r
+\r
+ UNUSED_PARAM(InterfaceType);\r
+ UNUSED_PARAM(ExposedInterfaceSpecificData);\r
+\r
+ p_ib_ifc->p_ibdev = p_fdo->bus_ib_ifc.p_ibdev;\r
+ p_ib_ifc->pmlx4_dev = p_fdo->bus_ib_ifc.pmlx4_dev;\r
+ p_ib_ifc->is_livefish = p_fdo->bus_ib_ifc.is_livefish;\r
+\r
+ return STATUS_SUCCESS;\r
+}\r
+\r
NTSTATUS\r
create_pdo(\r
__in WDFDEVICE Device,\r
\r
p_fdo->bus_ib_ifc.port_id = (u8) SerialNo;\r
p_fdo->bus_ib_ifc.pVipBusIfc = &p_fdo->mtnic_Ifc;\r
- p_fdo->bus_ib_ifc.pVipBusIfc->ulAllocatePortObjSize = MAX_PORT_SIZE;\r
- p_fdo->bus_ib_ifc.register_interface = mlx4_register_interface;\r
- p_fdo->bus_ib_ifc.unregister_interface = mlx4_unregister_interface;\r
p_fdo->bus_ib_ifc.mlx4_interface.mlx4_pd_alloc = mlx4_pd_alloc;\r
p_fdo->bus_ib_ifc.mlx4_interface.mlx4_pd_free = mlx4_pd_free;\r
p_fdo->bus_ib_ifc.mlx4_interface.mlx4_uar_alloc = mlx4_uar_alloc;\r
p_fdo->bus_ib_ifc.mlx4_interface.mlx4_CLOSE_PORT = mlx4_CLOSE_PORT;\r
p_fdo->bus_ib_ifc.mlx4_interface.mlx4_add_eq = mlx4_add_eq;\r
p_fdo->bus_ib_ifc.mlx4_interface.mlx4_remove_eq = mlx4_remove_eq;\r
- p_fdo->bus_ib_ifc.mlx4_interface.mlx4_register_ev_cb = ib_register_event_handler;\r
- p_fdo->bus_ib_ifc.mlx4_interface.mlx4_unregister_ev_cb = ib_unregister_event_handler;\r
+ p_fdo->bus_ib_ifc.mlx4_interface.mlx4_register_ev_cb = mlx4_reset_cb_register;\r
+ p_fdo->bus_ib_ifc.mlx4_interface.mlx4_unregister_ev_cb = mlx4_reset_cb_unregister;\r
+ p_fdo->bus_ib_ifc.mlx4_interface.mlx4_reset_request = mlx4_reset_request;\r
+ p_fdo->bus_ib_ifc.mlx4_interface.mlx4_reset_execute = mlx4_reset_execute;\r
\r
//\r
// Create a custom interface so that other drivers can\r
\r
WDF_QUERY_INTERFACE_CONFIG_INIT( &p_pdo->qiMlx4Bus,\r
(PINTERFACE) &p_fdo->bus_ib_ifc,\r
- &MLX4_BUS_IB_INTERFACE_GUID, NULL);\r
+ &MLX4_BUS_IB_INTERFACE_GUID, EvtDeviceProcessQueryInterfaceRequest);\r
\r
status = WdfDeviceAddQueryInterface( hChild, &p_pdo->qiMlx4Bus );\r
if (!NT_SUCCESS(status))\r
(PINTERFACE) &p_fdo->pci_dev.bus_pci_ifc,\r
&GUID_BUS_INTERFACE_STANDARD, NULL);\r
\r
+ // TODO: Soft Reset - how tobar getting interface during RESET_IN_PROGRESS\r
+ // maybe - using EvtDeviceProcessQueryInterfaceRequest\r
status = WdfDeviceAddQueryInterface( hChild, &p_pdo->qiPciBus );\r
if (!NT_SUCCESS(status))\r
goto Cleanup;\r
struct mlx4_dev *dev = to_mdev(pd->device)->dev;
struct mlx4_ib_ah *ah;
+ if (mlx4_is_barred(pd->device->dma_device))
+ return ERR_PTR(-EFAULT);
+
ah = kmalloc(sizeof *ah, GFP_ATOMIC);
if (!ah)
return ERR_PTR(-ENOMEM);
{
struct mlx4_ib_ah *ah = to_mah(ibah);
+ if (mlx4_is_barred(ibah->device->dma_device))
+ return -EFAULT;
+
memset(ah_attr, 0, sizeof *ah_attr);
ah_attr->dlid = be16_to_cpu(ah->av.dlid);
ah_attr->sl = (u8)(be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28);
struct mlx4_av *av = &to_mah(ibah)->av;
struct mlx4_dev *dev = to_mdev(ibah->pd->device)->dev;
+ if (mlx4_is_barred(dev))
+ return -EFAULT;
+
// taken from mthca_create_av
av->port_pd = cpu_to_be32(to_mpd(ibah->pd)->pdn | (ah_attr->port_num << 24));
av->g_slid = ah_attr->src_path_bits;
-/*
- * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <mlx4_debug.h>
-#include "mlx4_ib.h"
-#include "cq.h"
-#include "qp.h"
-#include "user.h"
-
-static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
-{
- struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
- ibcq->comp_handler(ibcq->cq_context);
-}
-
-static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)
-{
- ib_event_rec_t event;
- struct ib_cq *ibcq;
-
- if (type != MLX4_EVENT_TYPE_CQ_ERROR) {
- printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
- "on CQ %06x\n", type, cq->cqn);
- return;
- }
-
- ibcq = &to_mibcq(cq)->ibcq;
- event.type = IB_EVENT_CQ_ERR;
- event.context = ibcq->cq_context;
- event.vendor_specific = type;
- ibcq->event_handler(&event);
-}
-
-static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)
-{
- int offset = n * sizeof (struct mlx4_cqe);
-
- if (buf->buf.nbufs == 1)
- return buf->buf.u.direct.buf + offset;
- else
- return buf->buf.u.page_list[offset >> PAGE_SHIFT].buf +
- (offset & (PAGE_SIZE - 1));
-}
-
-static void *get_cqe(struct mlx4_ib_cq *cq, int n)
-{
- return get_cqe_from_buf(&cq->buf, n);
-}
-
-static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)
-{
- struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);
-
- return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
- !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;
-}
-
-static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)
-{
- return get_sw_cqe(cq, cq->mcq.cons_index);
-}
-
-int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
-{
- struct mlx4_ib_cq *mcq = to_mcq(cq);
- struct mlx4_ib_dev *dev = to_mdev(cq->device);
- struct mlx4_cq_context *context;
- int err;
-
- context = kzalloc(sizeof *context, GFP_KERNEL);
- if (!context)
- return -ENOMEM;
-
- context->cq_period = cpu_to_be16(cq_period);
- context->cq_max_count = cpu_to_be16(cq_count);
- err = mlx4_cq_modify(dev->dev, &mcq->mcq, context, 1);
-
- kfree(context);
- return err;
-}
-
-struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,
- struct ib_ucontext *context,
- struct ib_udata *udata)
-{
- struct mlx4_ib_dev *dev = to_mdev(ibdev);
- struct mlx4_ib_cq *cq;
- struct mlx4_uar *uar;
- int buf_size;
- int err;
-
- UNUSED_PARAM(vector);
-
- if (entries < 1 || entries > dev->dev->caps.max_cqes)
- return ERR_PTR(-EINVAL);
-
- cq = kzalloc(sizeof *cq, GFP_KERNEL);
- if (!cq)
- return ERR_PTR(-ENOMEM);
-
- entries = roundup_pow_of_two(entries + 1);
- cq->ibcq.cqe = entries - 1;
- buf_size = entries * sizeof (struct mlx4_cqe);
- spin_lock_init(&cq->lock);
-
- if (context) {
- struct mlx4_ib_create_cq ucmd;
-
- if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
- err = -EFAULT;
- goto err_cq;
- }
-
- cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,
- IB_ACCESS_LOCAL_WRITE, FALSE);
- if (IS_ERR(cq->umem)) {
- err = PTR_ERR(cq->umem);
- goto err_cq;
- }
-
- err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem),
- ilog2(cq->umem->page_size), &cq->buf.mtt);
- if (err)
- goto err_buf;
-
- err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);
- if (err)
- goto err_mtt;
-
- err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
- &cq->db);
- if (err)
- goto err_mtt;
-
- // add mapping to user's arm_sn variable
- // we have no way pass the completion event to provider library
- // so we'll increment user's arm_sn in kernel
- err = ib_umem_map( ucmd.arm_sn_addr, sizeof(int),
- IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn );
- if (err)
- goto err_dbmap;
-
- uar = &to_mucontext(context)->uar;
- } else {
- err = mlx4_ib_db_alloc(dev, &cq->db, 1);
- if (err)
- goto err_cq;
-
- cq->mcq.set_ci_db = cq->db.db;
- cq->mcq.arm_db = cq->db.db + 1;
- *cq->mcq.set_ci_db = 0;
- *cq->mcq.arm_db = 0;
-
- if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) {
- err = -ENOMEM;
- goto err_db;
- }
-
- err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift,
- &cq->buf.mtt);
- if (err)
- goto err_buf;
-
- err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf);
- if (err)
- goto err_mtt;
-
- cq->mcq.p_u_arm_sn = NULL;
- uar = &dev->priv_uar;
- }
-
- err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,
- cq->db.dma.da, &cq->mcq, 0, 0);
- if (err)
- goto err_dbmap;
-
- cq->mcq.comp = mlx4_ib_cq_comp;
- cq->mcq.event = mlx4_ib_cq_event;
-
- if (context)
- if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
- err = -EFAULT;
- goto err_dbmap;
- }
-
- return &cq->ibcq;
-
-err_dbmap:
- ib_umem_unmap( cq->mcq.mdl, cq->mcq.p_u_arm_sn );
- if (context)
- mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);
-
-err_mtt:
- mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
-
-err_buf:
- if (context)
- ib_umem_release(cq->umem);
- else
- mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe),
- &cq->buf.buf);
-
-err_db:
- if (!context)
- mlx4_ib_db_free(dev, &cq->db);
-
-err_cq:
- kfree(cq);
-
- return ERR_PTR(err);
-}
-
-int mlx4_ib_destroy_cq(struct ib_cq *cq)
-{
- struct mlx4_ib_dev *dev = to_mdev(cq->device);
- struct mlx4_ib_cq *mcq = to_mcq(cq);
-
- mlx4_cq_free(dev->dev, &mcq->mcq);
- mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);
-
- if (cq->p_uctx) {
- ib_umem_unmap( mcq->mcq.mdl, mcq->mcq.p_u_arm_sn );
- mlx4_ib_db_unmap_user(to_mucontext(cq->p_uctx), &mcq->db);
- ib_umem_release(mcq->umem);
- } else {
- mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe),
- &mcq->buf.buf);
- mlx4_ib_db_free(dev, &mcq->db);
- }
-
- kfree(mcq);
-
- return 0;
-}
-
-static void dump_cqe(void *cqe)
-{
- __be32 *buf = cqe;
-
- printk(KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
- be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),
- be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),
- be32_to_cpu(buf[6]), be32_to_cpu(buf[7]));
-}
-
-static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,
- ib_wc_t *wc)
-{
- if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {
- printk(KERN_DEBUG "local QP operation err "
- "(QPN %06x, WQE index %x, vendor syndrome %02x, "
- "opcode = %02x)\n",
- be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),
- cqe->vendor_err_syndrome,
- cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
- dump_cqe(cqe);
- }
-
- switch (cqe->syndrome) {
- case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
- wc->status = IB_WCS_LOCAL_LEN_ERR;
- break;
- case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
- wc->status = IB_WCS_LOCAL_OP_ERR;
- break;
- case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
- wc->status = IB_WCS_LOCAL_PROTECTION_ERR;
- break;
- case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
- wc->status = IB_WCS_WR_FLUSHED_ERR;
- break;
- case MLX4_CQE_SYNDROME_MW_BIND_ERR:
- wc->status = IB_WCS_MEM_WINDOW_BIND_ERR;
- break;
- case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
- wc->status = IB_WCS_BAD_RESP_ERR;
- break;
- case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
- wc->status = IB_WCS_LOCAL_ACCESS_ERR;
- break;
- case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
- wc->status = IB_WCS_REM_INVALID_REQ_ERR;
- break;
- case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
- wc->status = IB_WCS_REM_ACCESS_ERR;
- break;
- case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
- wc->status = IB_WCS_REM_OP_ERR;
- break;
- case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
- wc->status = IB_WCS_TIMEOUT_RETRY_ERR;
- break;
- case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
- wc->status = IB_WCS_RNR_RETRY_ERR;
- break;
- case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
- wc->status = IB_WCS_REM_ABORT_ERR;
- break;
- default:
- wc->status = IB_WC_GENERAL_ERR;
- break;
- }
-
- wc->vendor_specific = cqe->vendor_err_syndrome;
-}
-
-static uint8_t mlx4_ib_ipoib_csum_ok(__be32 status, __be16 checksum) {
-
- #define CSUM_VALID_NUM 0xffff
- uint8_t res = 0;
-
- // Verify that IP_OK bit is set and the packet is pure IPv4 packet
- if ((status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 |
- MLX4_CQE_IPOIB_STATUS_IPV4 |
- MLX4_CQE_IPOIB_STATUS_IPV4OPT |
- MLX4_CQE_IPOIB_STATUS_IPV6 |
- MLX4_CQE_IPOIB_STATUS_IPOK)) ==
- cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 |
- MLX4_CQE_IPOIB_STATUS_IPOK))
- {
- // IP checksum calculated by MLX4 matched the checksum in the receive packet's
- res |= MLX4_NdisPacketIpChecksumSucceeded;
- if (checksum == CSUM_VALID_NUM) {
- // TCP or UDP checksum calculated by MLX4 matched the checksum in the receive packet's
- res |= (MLX4_NdisPacketUdpChecksumSucceeded |
- MLX4_NdisPacketTcpChecksumSucceeded );
- ASSERT( status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_TCP | MLX4_CQE_IPOIB_STATUS_UDP));
- }
- }
- return res;
-}
-
-static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
- struct mlx4_ib_qp **cur_qp,
- ib_wc_t *wc)
-{
- struct mlx4_cqe *cqe;
- struct mlx4_qp *mqp;
- struct mlx4_ib_wq *wq;
- struct mlx4_ib_srq *srq;
- int is_send;
- int is_error;
- u16 wqe_ctr;
-
- cqe = next_cqe_sw(cq);
- if (!cqe)
- return -EAGAIN;
-
- ++cq->mcq.cons_index;
-
- /*
- * Make sure we read CQ entry contents after we've checked the
- * ownership bit.
- */
- rmb();
-
- is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
- is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
- MLX4_CQE_OPCODE_ERROR;
-
- if (!*cur_qp || (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (u32)(*cur_qp)->mqp.qpn) {
- /*
- * We do not have to take the QP table lock here,
- * because CQs will be locked while QPs are removed
- * from the table.
- */
-#if 1
- // radix_tree_insert in current implementation seems like
- // can cause radix_tree_lookup to miss an existing QP
- // so we call qp_lookup under the spinlock
- mqp = mlx4_qp_lookup_locked( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));
-#else
- mqp = __mlx4_qp_lookup( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));
-#endif
-
- if (unlikely(!mqp)) {
- printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n",
- cq->mcq.cqn, be32_to_cpu(cqe->my_qpn) & 0xffffff);
- return -EINVAL;
- }
-
- *cur_qp = to_mibqp(mqp);
- }
-
- if (is_send) {
- wq = &(*cur_qp)->sq;
- wqe_ctr = be16_to_cpu(cqe->wqe_index);
- wq->tail += (u16) (wqe_ctr - (u16) wq->tail);
- wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
- ++wq->tail;
- } else if ((*cur_qp)->ibqp.srq) {
- srq = to_msrq((*cur_qp)->ibqp.srq);
- wqe_ctr = be16_to_cpu(cqe->wqe_index);
- wc->wr_id = srq->wrid[wqe_ctr];
- mlx4_ib_free_srq_wqe(srq, wqe_ctr);
- } else {
- wq = &(*cur_qp)->rq;
- wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
- ++wq->tail;
- }
-
- if (is_send) {
- wc->recv.ud.recv_opt = 0;
- switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
- case MLX4_OPCODE_RDMA_WRITE_IMM:
- wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;
- case MLX4_OPCODE_RDMA_WRITE:
- wc->wc_type = IB_WC_RDMA_WRITE;
- break;
- case MLX4_OPCODE_SEND_IMM:
- wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;
- case MLX4_OPCODE_SEND:
- wc->wc_type = IB_WC_SEND;
- break;
- case MLX4_OPCODE_RDMA_READ:
- wc->wc_type = IB_WC_RDMA_READ;
- wc->length = be32_to_cpu(cqe->byte_cnt);
- break;
- case MLX4_OPCODE_ATOMIC_CS:
- wc->wc_type = IB_WC_COMPARE_SWAP;
- wc->length = 8;
- break;
- case MLX4_OPCODE_ATOMIC_FA:
- wc->wc_type = IB_WC_FETCH_ADD;
- wc->length = 8;
- break;
- case MLX4_OPCODE_BIND_MW:
- wc->wc_type = IB_WC_MW_BIND;
- break;
- default:
- wc->wc_type = IB_WC_SEND;
- break;
- }
- } else {
- wc->length = be32_to_cpu(cqe->byte_cnt);
-
- switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
- case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
- wc->wc_type = IB_WC_RECV_RDMA_WRITE;
- wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;
- wc->recv.ud.immediate_data = cqe->immed_rss_invalid;
- break;
- case MLX4_RECV_OPCODE_SEND:
- wc->wc_type = IB_WC_RECV;
- wc->recv.ud.recv_opt = 0;
- break;
- case MLX4_RECV_OPCODE_SEND_IMM:
- wc->wc_type = IB_WC_RECV;
- wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;
- wc->recv.ud.immediate_data = cqe->immed_rss_invalid;
- break;
- default:
- wc->recv.ud.recv_opt = 0;
- wc->wc_type = IB_WC_RECV;
- break;
- }
-
- wc->recv.ud.remote_lid = cqe->rlid;
- wc->recv.ud.remote_sl = cqe->sl >> 4;
- wc->recv.ud.remote_qp = cqe->g_mlpath_rqpn & 0xffffff00;
- wc->recv.ud.path_bits = (u8)(cqe->g_mlpath_rqpn & 0x7f);
- wc->recv.ud.recv_opt |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0;
- wc->recv.ud.pkey_index = (u16)(be32_to_cpu(cqe->immed_rss_invalid) & 0x7f);
- wc->csum_ok = mlx4_ib_ipoib_csum_ok(cqe->ipoib_status,cqe->checksum);
- }
- if (!is_send && cqe->rlid == 0){
- MLX4_PRINT(TRACE_LEVEL_INFORMATION,MLX4_DBG_CQ,("found rlid == 0 \n "));
- wc->recv.ud.recv_opt |= IB_RECV_OPT_FORWARD;
- }
-
- if (unlikely(is_error))
- mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
- else
- wc->status = IB_WCS_SUCCESS;
-
- return 0;
-}
-
-int mlx4_ib_poll_cq(
- IN struct ib_cq *ibcq,
- IN OUT ib_wc_t** const pp_free_wclist,
- OUT ib_wc_t** const pp_done_wclist )
-{
- struct mlx4_ib_cq *cq = to_mcq(ibcq);
- struct mlx4_ib_qp *cur_qp = NULL;
- unsigned long flags;
- int err = 0;
- int npolled = 0;
- ib_wc_t *wc_p, **next_pp;
-
- spin_lock_irqsave(&cq->lock, &flags);
-
- // loop through CQ
- next_pp = pp_done_wclist;
- wc_p = *pp_free_wclist;
- while( wc_p ) {
- // poll one CQE
- err = mlx4_ib_poll_one(cq, &cur_qp, wc_p);
- if (err)
- break;
-
- // prepare for the next loop
- *next_pp = wc_p;
- next_pp = &wc_p->p_next;
- wc_p = wc_p->p_next;
- ++npolled;
- }
-
- // prepare the results
- *pp_free_wclist = wc_p; /* Set the head of the free list. */
- *next_pp = NULL; /* Clear the tail of the done list. */
-
- // update consumer index
- if (npolled)
- mlx4_cq_set_ci(&cq->mcq);
-
- spin_unlock_irqrestore(&cq->lock, flags);
- return (err == 0 || err == -EAGAIN)? npolled : err;
-}
-
-int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
-{
- mlx4_cq_arm(&to_mcq(ibcq)->mcq,
- (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
- MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,
- to_mdev(ibcq->device)->uar_map,
- MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock));
-
- return 0;
-}
-
-void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
-{
- u32 prod_index;
- int nfreed = 0;
- struct mlx4_cqe *cqe, *dest;
- u8 owner_bit;
-
- /*
- * First we need to find the current producer index, so we
- * know where to start cleaning from. It doesn't matter if HW
- * adds new entries after this loop -- the QP we're worried
- * about is already in RESET, so the new entries won't come
- * from our QP and therefore don't need to be checked.
- */
- for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
- if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
- break;
-
- /*
- * Now sweep backwards through the CQ, removing CQ entries
- * that match our QP by copying older entries on top of them.
- */
- while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
- cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
- if ((be32_to_cpu(cqe->my_qpn) & 0xffffff) == qpn) {
- if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
- mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));
- ++nfreed;
- } else if (nfreed) {
- dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
- owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
- memcpy(dest, cqe, sizeof *cqe);
- dest->owner_sr_opcode = owner_bit |
- (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
- }
- }
-
- if (nfreed) {
- cq->mcq.cons_index += nfreed;
- /*
- * Make sure update of buffer contents is done before
- * updating consumer index.
- */
- wmb();
- mlx4_cq_set_ci(&cq->mcq);
- }
-}
-
-void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
-{
- spin_lock_irq(&cq->lock);
- __mlx4_ib_cq_clean(cq, qpn, srq);
- spin_unlock_irq(&cq->lock);
-}
+/*\r
+ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses. You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+#include <mlx4_debug.h>\r
+#include "mlx4_ib.h"\r
+#include "cq.h"\r
+#include "qp.h"\r
+#include "user.h"\r
+\r
+static void mlx4_ib_cq_comp(struct mlx4_cq *cq)\r
+{\r
+ struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;\r
+ ibcq->comp_handler(ibcq->cq_context);\r
+}\r
+\r
+static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)\r
+{\r
+ ib_event_rec_t event;\r
+ struct ib_cq *ibcq;\r
+\r
+ if (type != MLX4_EVENT_TYPE_CQ_ERROR) {\r
+ printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "\r
+ "on CQ %06x\n", type, cq->cqn);\r
+ return;\r
+ }\r
+\r
+ ibcq = &to_mibcq(cq)->ibcq;\r
+ event.type = IB_EVENT_CQ_ERR;\r
+ event.context = ibcq->cq_context;\r
+ event.vendor_specific = type;\r
+ ibcq->event_handler(&event);\r
+}\r
+\r
+static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)\r
+{\r
+ int offset = n * sizeof (struct mlx4_cqe);\r
+\r
+ if (buf->buf.nbufs == 1)\r
+ return buf->buf.u.direct.buf + offset;\r
+ else\r
+ return buf->buf.u.page_list[offset >> PAGE_SHIFT].buf +\r
+ (offset & (PAGE_SIZE - 1));\r
+}\r
+\r
+static void *get_cqe(struct mlx4_ib_cq *cq, int n)\r
+{\r
+ return get_cqe_from_buf(&cq->buf, n);\r
+}\r
+\r
+static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)\r
+{\r
+ struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);\r
+\r
+ return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^\r
+ !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;\r
+}\r
+\r
+static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)\r
+{\r
+ return get_sw_cqe(cq, cq->mcq.cons_index);\r
+}\r
+\r
+int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)\r
+{\r
+ struct mlx4_ib_cq *mcq = to_mcq(cq);\r
+ struct mlx4_ib_dev *dev = to_mdev(cq->device);\r
+ struct mlx4_cq_context *context;\r
+ int err;\r
+\r
+ if (mlx4_is_barred(dev->dev))\r
+ return -EFAULT;\r
+\r
+ context = kzalloc(sizeof *context, GFP_KERNEL);\r
+ if (!context)\r
+ return -ENOMEM;\r
+\r
+ context->cq_period = cpu_to_be16(cq_period);\r
+ context->cq_max_count = cpu_to_be16(cq_count);\r
+ err = mlx4_cq_modify(dev->dev, &mcq->mcq, context, 1);\r
+\r
+ kfree(context);\r
+ return err;\r
+}\r
+\r
+struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,\r
+ struct ib_ucontext *context,\r
+ struct ib_udata *udata)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(ibdev);\r
+ struct mlx4_ib_cq *cq;\r
+ struct mlx4_uar *uar;\r
+ int buf_size;\r
+ int err;\r
+\r
+ UNUSED_PARAM(vector);\r
+\r
+ if (mlx4_is_barred(ibdev->dma_device))\r
+ return ERR_PTR(-EFAULT);\r
+\r
+ if (entries < 1 || entries > dev->dev->caps.max_cqes)\r
+ return ERR_PTR(-EINVAL);\r
+\r
+ cq = kzalloc(sizeof *cq, GFP_KERNEL);\r
+ if (!cq)\r
+ return ERR_PTR(-ENOMEM);\r
+\r
+ entries = roundup_pow_of_two(entries + 1);\r
+ cq->ibcq.cqe = entries - 1;\r
+ buf_size = entries * sizeof (struct mlx4_cqe);\r
+ spin_lock_init(&cq->lock);\r
+\r
+ if (context) {\r
+ struct mlx4_ib_create_cq ucmd;\r
+\r
+ if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {\r
+ err = -EFAULT;\r
+ goto err_cq;\r
+ }\r
+\r
+ cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,\r
+ IB_ACCESS_LOCAL_WRITE, FALSE);\r
+ if (IS_ERR(cq->umem)) {\r
+ err = PTR_ERR(cq->umem);\r
+ goto err_cq;\r
+ }\r
+\r
+ err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem),\r
+ ilog2(cq->umem->page_size), &cq->buf.mtt);\r
+ if (err)\r
+ goto err_buf;\r
+\r
+ err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,\r
+ &cq->db);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ // add mapping to user's arm_sn variable\r
+ // we have no way pass the completion event to provider library\r
+ // so we'll increment user's arm_sn in kernel\r
+ err = ib_umem_map( ucmd.arm_sn_addr, sizeof(int), \r
+ IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn );\r
+ if (err)\r
+ goto err_dbmap;\r
+\r
+ uar = &to_mucontext(context)->uar;\r
+ } else {\r
+ err = mlx4_ib_db_alloc(dev, &cq->db, 1);\r
+ if (err)\r
+ goto err_cq;\r
+\r
+ cq->mcq.set_ci_db = cq->db.db;\r
+ cq->mcq.arm_db = cq->db.db + 1;\r
+ *cq->mcq.set_ci_db = 0;\r
+ *cq->mcq.arm_db = 0;\r
+\r
+ if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) {\r
+ err = -ENOMEM;\r
+ goto err_db;\r
+ }\r
+\r
+ err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift,\r
+ &cq->buf.mtt);\r
+ if (err)\r
+ goto err_buf;\r
+\r
+ err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ cq->mcq.p_u_arm_sn = NULL;\r
+ uar = &dev->priv_uar;\r
+ }\r
+\r
+ err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,\r
+ cq->db.dma.da, &cq->mcq, 0, 0);\r
+ if (err)\r
+ goto err_dbmap;\r
+\r
+ cq->mcq.comp = mlx4_ib_cq_comp;\r
+ cq->mcq.event = mlx4_ib_cq_event;\r
+\r
+ if (context) \r
+ if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {\r
+ err = -EFAULT;\r
+ goto err_dbmap;\r
+ }\r
+\r
+ return &cq->ibcq;\r
+\r
+err_dbmap:\r
+ ib_umem_unmap( cq->mcq.mdl, cq->mcq.p_u_arm_sn );\r
+ if (context)\r
+ mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);\r
+\r
+err_mtt:\r
+ mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);\r
+\r
+err_buf:\r
+ if (context)\r
+ ib_umem_release(cq->umem);\r
+ else\r
+ mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe),\r
+ &cq->buf.buf);\r
+\r
+err_db:\r
+ if (!context)\r
+ mlx4_ib_db_free(dev, &cq->db);\r
+\r
+err_cq:\r
+ kfree(cq);\r
+\r
+ return ERR_PTR(err);\r
+}\r
+\r
+int mlx4_ib_destroy_cq(struct ib_cq *cq)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(cq->device);\r
+ struct mlx4_ib_cq *mcq = to_mcq(cq);\r
+\r
+ mlx4_cq_free(dev->dev, &mcq->mcq);\r
+ mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);\r
+\r
+ if (cq->p_uctx) {\r
+ ib_umem_unmap( mcq->mcq.mdl, mcq->mcq.p_u_arm_sn );\r
+ mlx4_ib_db_unmap_user(to_mucontext(cq->p_uctx), &mcq->db);\r
+ ib_umem_release(mcq->umem);\r
+ } else {\r
+ mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe),\r
+ &mcq->buf.buf);\r
+ mlx4_ib_db_free(dev, &mcq->db);\r
+ }\r
+\r
+ kfree(mcq);\r
+\r
+ return 0;\r
+}\r
+\r
+static void dump_cqe(void *cqe)\r
+{\r
+ __be32 *buf = cqe;\r
+\r
+ printk(KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",\r
+ be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),\r
+ be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),\r
+ be32_to_cpu(buf[6]), be32_to_cpu(buf[7]));\r
+}\r
+\r
+static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,\r
+ ib_wc_t *wc)\r
+{\r
+ if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {\r
+ printk(KERN_DEBUG "local QP operation err "\r
+ "(QPN %06x, WQE index %x, vendor syndrome %02x, "\r
+ "opcode = %02x)\n",\r
+ be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),\r
+ cqe->vendor_err_syndrome,\r
+ cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
+ dump_cqe(cqe);\r
+ }\r
+\r
+ switch (cqe->syndrome) {\r
+ case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:\r
+ wc->status = IB_WCS_LOCAL_LEN_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:\r
+ wc->status = IB_WCS_LOCAL_OP_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:\r
+ wc->status = IB_WCS_LOCAL_PROTECTION_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:\r
+ wc->status = IB_WCS_WR_FLUSHED_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_MW_BIND_ERR:\r
+ wc->status = IB_WCS_MEM_WINDOW_BIND_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_BAD_RESP_ERR:\r
+ wc->status = IB_WCS_BAD_RESP_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:\r
+ wc->status = IB_WCS_LOCAL_ACCESS_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:\r
+ wc->status = IB_WCS_REM_INVALID_REQ_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:\r
+ wc->status = IB_WCS_REM_ACCESS_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:\r
+ wc->status = IB_WCS_REM_OP_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:\r
+ wc->status = IB_WCS_TIMEOUT_RETRY_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:\r
+ wc->status = IB_WCS_RNR_RETRY_ERR;\r
+ break;\r
+ case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:\r
+ wc->status = IB_WCS_REM_ABORT_ERR;\r
+ break;\r
+ default:\r
+ wc->status = IB_WC_GENERAL_ERR;\r
+ break;\r
+ }\r
+\r
+ wc->vendor_specific = cqe->vendor_err_syndrome;\r
+}\r
+\r
+static uint8_t mlx4_ib_ipoib_csum_ok(__be32 status, __be16 checksum) {\r
+ \r
+ #define CSUM_VALID_NUM 0xffff\r
+ uint8_t res = 0;\r
+\r
+ // Verify that IP_OK bit is set and the packet is pure IPv4 packet\r
+ if ((status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 |\r
+ MLX4_CQE_IPOIB_STATUS_IPV4 |\r
+ MLX4_CQE_IPOIB_STATUS_IPV4OPT |\r
+ MLX4_CQE_IPOIB_STATUS_IPV6 |\r
+ MLX4_CQE_IPOIB_STATUS_IPOK)) ==\r
+ cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 |\r
+ MLX4_CQE_IPOIB_STATUS_IPOK))\r
+ {\r
+ // IP checksum calculated by MLX4 matched the checksum in the receive packet's \r
+ res |= MLX4_NdisPacketIpChecksumSucceeded;\r
+ if (checksum == CSUM_VALID_NUM) {\r
+ // TCP or UDP checksum calculated by MLX4 matched the checksum in the receive packet's \r
+ res |= (MLX4_NdisPacketUdpChecksumSucceeded |\r
+ MLX4_NdisPacketTcpChecksumSucceeded );\r
+ ASSERT( status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_TCP | MLX4_CQE_IPOIB_STATUS_UDP));\r
+ }\r
+ }\r
+ return res;\r
+}\r
+\r
+static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,\r
+ struct mlx4_ib_qp **cur_qp,\r
+ ib_wc_t *wc)\r
+{\r
+ struct mlx4_cqe *cqe;\r
+ struct mlx4_qp *mqp;\r
+ struct mlx4_ib_wq *wq;\r
+ struct mlx4_ib_srq *srq;\r
+ int is_send;\r
+ int is_error;\r
+ u16 wqe_ctr;\r
+\r
+ cqe = next_cqe_sw(cq);\r
+ if (!cqe)\r
+ return -EAGAIN;\r
+\r
+ ++cq->mcq.cons_index;\r
+\r
+ /*\r
+ * Make sure we read CQ entry contents after we've checked the\r
+ * ownership bit.\r
+ */\r
+ rmb();\r
+\r
+ is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;\r
+ is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==\r
+ MLX4_CQE_OPCODE_ERROR;\r
+\r
+ if (!*cur_qp || (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (u32)(*cur_qp)->mqp.qpn) {\r
+ /*\r
+ * We do not have to take the QP table lock here,\r
+ * because CQs will be locked while QPs are removed\r
+ * from the table.\r
+ */\r
+#if 1\r
+ // radix_tree_insert in current implementation seems like\r
+ // can cause radix_tree_lookup to miss an existing QP\r
+ // so we call qp_lookup under the spinlock\r
+ mqp = mlx4_qp_lookup_locked( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));\r
+#else\r
+ mqp = __mlx4_qp_lookup( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));\r
+#endif\r
+\r
+ if (unlikely(!mqp)) {\r
+ printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n",\r
+ cq->mcq.cqn, be32_to_cpu(cqe->my_qpn) & 0xffffff);\r
+ return -EINVAL;\r
+ }\r
+\r
+ *cur_qp = to_mibqp(mqp);\r
+ }\r
+\r
+ if (is_send) {\r
+ wq = &(*cur_qp)->sq;\r
+ wqe_ctr = be16_to_cpu(cqe->wqe_index);\r
+ wq->tail += (u16) (wqe_ctr - (u16) wq->tail);\r
+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
+ ++wq->tail;\r
+ } else if ((*cur_qp)->ibqp.srq) {\r
+ srq = to_msrq((*cur_qp)->ibqp.srq);\r
+ wqe_ctr = be16_to_cpu(cqe->wqe_index);\r
+ wc->wr_id = srq->wrid[wqe_ctr];\r
+ mlx4_ib_free_srq_wqe(srq, wqe_ctr);\r
+ } else {\r
+ wq = &(*cur_qp)->rq;\r
+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
+ ++wq->tail;\r
+ }\r
+\r
+ if (is_send) {\r
+ wc->recv.ud.recv_opt = 0;\r
+ switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
+ case MLX4_OPCODE_RDMA_WRITE_IMM:\r
+ wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
+ case MLX4_OPCODE_RDMA_WRITE:\r
+ wc->wc_type = IB_WC_RDMA_WRITE;\r
+ break;\r
+ case MLX4_OPCODE_SEND_IMM:\r
+ wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
+ case MLX4_OPCODE_SEND:\r
+ wc->wc_type = IB_WC_SEND;\r
+ break;\r
+ case MLX4_OPCODE_RDMA_READ:\r
+ wc->wc_type = IB_WC_RDMA_READ;\r
+ wc->length = be32_to_cpu(cqe->byte_cnt);\r
+ break;\r
+ case MLX4_OPCODE_ATOMIC_CS:\r
+ wc->wc_type = IB_WC_COMPARE_SWAP;\r
+ wc->length = 8;\r
+ break;\r
+ case MLX4_OPCODE_ATOMIC_FA:\r
+ wc->wc_type = IB_WC_FETCH_ADD;\r
+ wc->length = 8;\r
+ break;\r
+ case MLX4_OPCODE_BIND_MW:\r
+ wc->wc_type = IB_WC_MW_BIND;\r
+ break;\r
+ default:\r
+ wc->wc_type = IB_WC_SEND;\r
+ break;\r
+ }\r
+ } else {\r
+ wc->length = be32_to_cpu(cqe->byte_cnt);\r
+\r
+ switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
+ case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:\r
+ wc->wc_type = IB_WC_RECV_RDMA_WRITE;\r
+ wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;\r
+ wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
+ break;\r
+ case MLX4_RECV_OPCODE_SEND:\r
+ wc->wc_type = IB_WC_RECV;\r
+ wc->recv.ud.recv_opt = 0;\r
+ break;\r
+ case MLX4_RECV_OPCODE_SEND_IMM:\r
+ wc->wc_type = IB_WC_RECV;\r
+ wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;\r
+ wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
+ break;\r
+ default:\r
+ wc->recv.ud.recv_opt = 0;\r
+ wc->wc_type = IB_WC_RECV;\r
+ break;\r
+ }\r
+\r
+ wc->recv.ud.remote_lid = cqe->rlid;\r
+ wc->recv.ud.remote_sl = cqe->sl >> 4;\r
+ wc->recv.ud.remote_qp = cqe->g_mlpath_rqpn & 0xffffff00;\r
+ wc->recv.ud.path_bits = (u8)(cqe->g_mlpath_rqpn & 0x7f);\r
+ wc->recv.ud.recv_opt |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0;\r
+ wc->recv.ud.pkey_index = (u16)(be32_to_cpu(cqe->immed_rss_invalid) & 0x7f);\r
+ wc->csum_ok = mlx4_ib_ipoib_csum_ok(cqe->ipoib_status,cqe->checksum);\r
+ }\r
+ if (!is_send && cqe->rlid == 0){\r
+ MLX4_PRINT(TRACE_LEVEL_INFORMATION,MLX4_DBG_CQ,("found rlid == 0 \n "));\r
+ wc->recv.ud.recv_opt |= IB_RECV_OPT_FORWARD;\r
+ }\r
+\r
+ if (unlikely(is_error))\r
+ mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);\r
+ else\r
+ wc->status = IB_WCS_SUCCESS;\r
+\r
+ return 0;\r
+}\r
+\r
+int mlx4_ib_poll_cq(\r
+ IN struct ib_cq *ibcq, \r
+ IN OUT ib_wc_t** const pp_free_wclist,\r
+ OUT ib_wc_t** const pp_done_wclist )\r
+{\r
+ struct mlx4_ib_cq *cq = to_mcq(ibcq);\r
+ struct mlx4_ib_qp *cur_qp = NULL;\r
+ unsigned long flags;\r
+ int err = 0;\r
+ int npolled = 0;\r
+ ib_wc_t *wc_p, **next_pp;\r
+\r
+ spin_lock_irqsave(&cq->lock, &flags);\r
+\r
+ // loop through CQ\r
+ next_pp = pp_done_wclist;\r
+ wc_p = *pp_free_wclist;\r
+ while( wc_p ) {\r
+ // poll one CQE\r
+ err = mlx4_ib_poll_one(cq, &cur_qp, wc_p);\r
+ if (err)\r
+ break;\r
+\r
+ // prepare for the next loop\r
+ *next_pp = wc_p;\r
+ next_pp = &wc_p->p_next;\r
+ wc_p = wc_p->p_next;\r
+ ++npolled;\r
+ }\r
+\r
+ // prepare the results\r
+ *pp_free_wclist = wc_p; /* Set the head of the free list. */\r
+ *next_pp = NULL; /* Clear the tail of the done list. */\r
+\r
+ // update consumer index\r
+ if (npolled)\r
+ mlx4_cq_set_ci(&cq->mcq);\r
+\r
+ spin_unlock_irqrestore(&cq->lock, flags);\r
+ return (err == 0 || err == -EAGAIN)? npolled : err;\r
+}\r
+\r
+int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)\r
+{\r
+ if (!mlx4_is_barred(ibcq->device->dma_device))\r
+ mlx4_cq_arm(&to_mcq(ibcq)->mcq,\r
+ (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?\r
+ MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,\r
+ to_mdev(ibcq->device)->uar_map,\r
+ MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock));\r
+\r
+ return 0;\r
+}\r
+\r
+void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)\r
+{\r
+ u32 prod_index;\r
+ int nfreed = 0;\r
+ struct mlx4_cqe *cqe, *dest;\r
+ u8 owner_bit;\r
+\r
+ /*\r
+ * First we need to find the current producer index, so we\r
+ * know where to start cleaning from. It doesn't matter if HW\r
+ * adds new entries after this loop -- the QP we're worried\r
+ * about is already in RESET, so the new entries won't come\r
+ * from our QP and therefore don't need to be checked.\r
+ */\r
+ for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)\r
+ if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)\r
+ break;\r
+\r
+ /*\r
+ * Now sweep backwards through the CQ, removing CQ entries\r
+ * that match our QP by copying older entries on top of them.\r
+ */\r
+ while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {\r
+ cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);\r
+ if ((be32_to_cpu(cqe->my_qpn) & 0xffffff) == qpn) {\r
+ if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))\r
+ mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));\r
+ ++nfreed;\r
+ } else if (nfreed) {\r
+ dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);\r
+ owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;\r
+ memcpy(dest, cqe, sizeof *cqe);\r
+ dest->owner_sr_opcode = owner_bit |\r
+ (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
+ }\r
+ }\r
+\r
+ if (nfreed) {\r
+ cq->mcq.cons_index += nfreed;\r
+ /*\r
+ * Make sure update of buffer contents is done before\r
+ * updating consumer index.\r
+ */\r
+ wmb();\r
+ mlx4_cq_set_ci(&cq->mcq);\r
+ }\r
+}\r
+\r
+void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)\r
+{\r
+ spin_lock_irq(&cq->lock);\r
+ __mlx4_ib_cq_clean(cq, qpn, srq);\r
+ spin_unlock_irq(&cq->lock);\r
+}\r
u16 slid;
int err;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
slid = in_wc ? be16_to_cpu(in_wc->recv.ud.remote_lid) : be16_to_cpu(XIB_LID_PERMISSIVE);
if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) {
struct ib_smp *out_mad = NULL;
int err = -ENOMEM;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
if (!in_mad || !out_mad)
struct ib_smp *out_mad = NULL;
int err = -ENOMEM;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
if (!in_mad || !out_mad)
__be64 subnet_prefix;
int err = -ENOMEM;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
if (!in_mad || !out_mad)
struct ib_smp *out_mad = NULL;
int err = -ENOMEM;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
if (!in_mad || !out_mad)
static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,
struct ib_device_modify *props)
{
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
return -EOPNOTSUPP;
u32 cap_mask;
int err;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return -EFAULT;
+
mutex_lock(&to_mdev(ibdev)->cap_mask_mutex);
err = mlx4_ib_query_port(ibdev, port, &attr);
struct mlx4_ib_alloc_ucontext_resp resp;
int err;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return ERR_PTR(-EFAULT);
+
resp.qp_tab_size = dev->dev->caps.num_qps;
resp.bf_reg_size = (__u16)dev->dev->caps.bf_reg_size;
resp.bf_regs_per_page = (__u16)dev->dev->caps.bf_regs_per_page;
struct mlx4_ib_pd *pd;
int err;
+ if (mlx4_is_barred(ibdev->dma_device))
+ return ERR_PTR(-EFAULT);
+
pd = kmalloc(sizeof *pd, GFP_KERNEL);
if (!pd)
return ERR_PTR(-ENOMEM);
static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
UNUSED_PARAM(lid);
+ if (mlx4_is_barred(ibqp->device->dma_device))
+ return -EFAULT;
return mlx4_multicast_attach(to_mdev(ibqp->device)->dev,
&to_mqp(ibqp)->mqp, gid->raw);
}
static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
UNUSED_PARAM(lid);
+ if (mlx4_is_barred(ibqp->device->dma_device))
+ return -EFAULT;
return mlx4_multicast_detach(to_mdev(ibqp->device)->dev,
&to_mqp(ibqp)->mqp, gid->raw);
}
mlx4_uar_free(dev, &ibdev->priv_uar);
mlx4_pd_free(dev, ibdev->priv_pdn);
dealloc_dev:
- ib_dealloc_device(&ibdev->ib_dev);
mlx4_dbg(ibdev->dev, "MLX4_BUS: IB interface is REMOVED ! \n");
+ ibdev->ib_dev.reg_state = IB_DEV_UNINITIALIZED;
+ ib_dealloc_device(&ibdev->ib_dev);
}
static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
struct mlx4_ib_mr *mr;
int err;
+ if (mlx4_is_barred(pd->device->dma_device))
+ return ERR_PTR(-EFAULT);
+
mr = kmalloc(sizeof *mr, GFP_KERNEL);
if (!mr)
return ERR_PTR(-ENOMEM);
int n;
UNUSED_PARAM(udata);
+
+ if (mlx4_is_barred(pd->device->dma_device))
+ return ERR_PTR(-EFAULT);
mr = kmalloc(sizeof *mr, GFP_KERNEL);
if (!mr)
struct mlx4_ib_fmr *fmr;
int err = -ENOMEM;
+ if (mlx4_is_barred(pd->device->dma_device))
+ return ERR_PTR(-EFAULT);
+
fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
if (!fmr)
return ERR_PTR(-ENOMEM);
struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);
+ if (mlx4_is_barred(ifmr->ibfmr.device->dma_device))
+ return -EFAULT;
+
return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,
&ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
}
int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
{
struct ib_fmr *ibfmr;
- int err;
+ int err = 0;
struct mlx4_dev *mdev = NULL;
list_for_each_entry(ibfmr, fmr_list, list, struct ib_fmr) {
*/
wmb();
- err = mlx4_SYNC_TPT(mdev);
+ if (!mlx4_is_barred(mdev))
+ err = mlx4_SYNC_TPT(mdev);
if (err)
printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when "
"unmapping FMRs\n", err);
-/*
- * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "mlx4_ib.h"
-#include "ib_cache.h"
-#include "ib_pack.h"
-#include "qp.h"
-#include "user.h"
-
-enum {
- MLX4_IB_ACK_REQ_FREQ = 8,
-};
-
-enum {
- MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
- MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f
-};
-
-enum {
- /*
- * Largest possible UD header: send with GRH and immediate data.
- */
- MLX4_IB_UD_HEADER_SIZE = 72
-};
-
-struct mlx4_ib_sqp {
- struct mlx4_ib_qp qp;
- int pkey_index;
- u32 qkey;
- u32 send_psn;
- struct ib_ud_header ud_header;
- u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
-};
-
-enum {
- MLX4_IB_MIN_SQ_STRIDE = 6
-};
-
-static const __be32 mlx4_ib_opcode[] = {
- __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), /* [IB_WR_RDMA_WRITE] */
- __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), /* [IB_WR_RDMA_WRITE_WITH_IMM] */
- __constant_cpu_to_be32(MLX4_OPCODE_SEND), /* [IB_WR_SEND] */
- __constant_cpu_to_be32(MLX4_OPCODE_SEND_IMM), /* [IB_WR_SEND_WITH_IMM] */
- __constant_cpu_to_be32(MLX4_OPCODE_RDMA_READ), /* [IB_WR_RDMA_READ] */
- __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), /* [IB_WR_ATOMIC_CMP_AND_SWP] */
- __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), /* [IB_WR_ATOMIC_FETCH_AND_ADD]*/
-};
-
-static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
-{
- return container_of(mqp, struct mlx4_ib_sqp, qp);
-}
-
-static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
-{
- return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
- qp->mqp.qpn <= dev->dev->caps.sqp_start + 3;
-}
-
-static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
-{
- return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
- qp->mqp.qpn <= dev->dev->caps.sqp_start + 1;
-}
-
-static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
-{
- if (qp->buf.nbufs == 1)
- return qp->buf.u.direct.buf + offset;
- else
- return qp->buf.u.page_list[offset >> PAGE_SHIFT].buf +
- (offset & (PAGE_SIZE - 1));
-}
-
-static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
-{
- return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
-}
-
-static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
-{
- return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
-}
-
-/*
- * Stamp a SQ WQE so that it is invalid if prefetched by marking the
- * first four bytes of every 64 byte chunk with 0xffffffff, except for
- * the very first chunk of the WQE.
- */
-static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n)
-{
- u32 *wqe = get_send_wqe(qp, n);
- int i;
-
- for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16)
- wqe[i] = 0xffffffff;
-}
-
-static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
-{
- ib_event_rec_t event;
- struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
-
- if (type == MLX4_EVENT_TYPE_PATH_MIG)
- to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
-
- switch (type) {
- case MLX4_EVENT_TYPE_PATH_MIG:
- event.type = IB_EVENT_PATH_MIG;
- break;
- case MLX4_EVENT_TYPE_COMM_EST:
- event.type = IB_EVENT_COMM_EST;
- break;
- case MLX4_EVENT_TYPE_SQ_DRAINED:
- event.type = IB_EVENT_SQ_DRAINED;
- break;
- case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
- event.type = IB_EVENT_QP_LAST_WQE_REACHED;
- break;
- case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
- event.type = IB_EVENT_QP_FATAL;
- break;
- case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
- event.type = IB_EVENT_PATH_MIG_ERR;
- break;
- case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
- event.type = IB_EVENT_QP_REQ_ERR;
- break;
- case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
- event.type = IB_EVENT_QP_ACCESS_ERR;
- break;
- default:
- printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
- "on QP %06x\n", type, qp->qpn);
- return;
- }
-
- event.context = ibqp->qp_context;
- ibqp->event_handler(&event);
-}
-
-static int send_wqe_overhead(enum ib_qp_type type)
-{
- /*
- * UD WQEs must have a datagram segment.
- * RC and UC WQEs might have a remote address segment.
- * MLX WQEs need two extra inline data segments (for the UD
- * header and space for the ICRC).
- */
- switch (type) {
- case IB_QPT_UD:
- return sizeof (struct mlx4_wqe_ctrl_seg) +
- sizeof (struct mlx4_wqe_datagram_seg);
- case IB_QPT_UC:
- return sizeof (struct mlx4_wqe_ctrl_seg) +
- sizeof (struct mlx4_wqe_raddr_seg);
- case IB_QPT_RC:
- return sizeof (struct mlx4_wqe_ctrl_seg) +
- sizeof (struct mlx4_wqe_atomic_seg) +
- sizeof (struct mlx4_wqe_raddr_seg);
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- return sizeof (struct mlx4_wqe_ctrl_seg) +
- ALIGN(MLX4_IB_UD_HEADER_SIZE +
- DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
- MLX4_INLINE_ALIGN) *
- sizeof (struct mlx4_wqe_inline_seg),
- sizeof (struct mlx4_wqe_data_seg)) +
- ALIGN(4 +
- sizeof (struct mlx4_wqe_inline_seg),
- sizeof (struct mlx4_wqe_data_seg));
- default:
- return sizeof (struct mlx4_wqe_ctrl_seg);
- }
-}
-
-static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
- int is_user, int has_srq, struct mlx4_ib_qp *qp)
-{
- /* Sanity check RQ size before proceeding */
- if ((int)cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
- (int)cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
- return -EINVAL;
-
- if (has_srq) {
- /* QPs attached to an SRQ should have no RQ */
- if (cap->max_recv_wr)
- return -EINVAL;
-
- qp->rq.wqe_cnt = qp->rq.max_gs = 0;
- } else {
- /* HW requires >= 1 RQ entry with >= 1 gather entry */
- if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
- return -EINVAL;
-
- qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
- qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
- qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
- }
-
- /* leave userspace return values as they were, so as not to break ABI */
- if (is_user) {
- cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
- cap->max_recv_sge = qp->rq.max_gs;
- } else {
- cap->max_recv_wr = qp->rq.max_post =
- min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
- cap->max_recv_sge = min(qp->rq.max_gs,
- min(dev->dev->caps.max_sq_sg,
- dev->dev->caps.max_rq_sg));
- }
- /* We don't support inline sends for kernel QPs (yet) */
-
- return 0;
-}
-
-static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
- enum ib_qp_type type, struct mlx4_ib_qp *qp)
-{
- /* Sanity check SQ size before proceeding */
- if ((int)cap->max_send_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
- (int)cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
- (int)cap->max_inline_data + send_wqe_overhead(type) +
- (int)sizeof(struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
- return -EINVAL;
-
- /*
- * For MLX transport we need 2 extra S/G entries:
- * one for the header and one for the checksum at the end
- */
- if ((type == IB_QPT_SMI || type == IB_QPT_GSI) &&
- (int)cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
- return -EINVAL;
-
- qp->sq.wqe_shift = ilog2(roundup_pow_of_two(max(cap->max_send_sge *
- sizeof (struct mlx4_wqe_data_seg),
- cap->max_inline_data +
- sizeof (struct mlx4_wqe_inline_seg)) +
- send_wqe_overhead(type)));
- qp->sq.wqe_shift = max(MLX4_IB_SQ_MIN_WQE_SHIFT, qp->sq.wqe_shift);
- qp->sq.max_gs = ((1 << qp->sq.wqe_shift) - send_wqe_overhead(type)) /
- sizeof (struct mlx4_wqe_data_seg);
-
- /*
- * We need to leave 2 KB + 1 WQE of headroom in the SQ to
- * allow HW to prefetch.
- */
- qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);
- qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr + qp->sq_spare_wqes);
-
- qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
- (qp->sq.wqe_cnt << qp->sq.wqe_shift);
- if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
- qp->rq.offset = 0;
- qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
- } else {
- qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
- qp->sq.offset = 0;
- }
-
- cap->max_send_wr = qp->sq.max_post =
- min(qp->sq.wqe_cnt - qp->sq_spare_wqes,
- dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE);
- cap->max_send_sge = min(qp->sq.max_gs,
- min(dev->dev->caps.max_sq_sg,
- dev->dev->caps.max_rq_sg));
- /* We don't support inline sends for kernel QPs (yet) */
- cap->max_inline_data = 0;
-
- return 0;
-}
-
-static int set_user_sq_size(struct mlx4_ib_dev *dev,
- struct mlx4_ib_qp *qp,
- struct mlx4_ib_create_qp *ucmd)
-{
- /* Sanity check SQ size before proceeding */
- if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
- ucmd->log_sq_stride >
- ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
- ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
- return -EINVAL;
-
- qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
- qp->sq.wqe_shift = ucmd->log_sq_stride;
-
- qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
- (qp->sq.wqe_cnt << qp->sq.wqe_shift);
-
- return 0;
-}
-
-static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata, u32 sqpn, struct mlx4_ib_qp *qp)
-{
- int err;
-
- mutex_init(&qp->mutex);
- spin_lock_init(&qp->sq.lock);
- spin_lock_init(&qp->rq.lock);
-
- qp->state = XIB_QPS_RESET;
- qp->atomic_rd_en = 0;
- qp->resp_depth = 0;
-
- qp->rq.head = 0;
- qp->rq.tail = 0;
- qp->sq.head = 0;
- qp->sq.tail = 0;
-
- err = set_rq_size(dev, &init_attr->cap, !!pd->p_uctx, !!init_attr->srq, qp);
- if (err)
- goto err;
-
- if (pd->p_uctx) {
- struct mlx4_ib_create_qp ucmd;
-
- if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
- err = -EFAULT;
- goto err;
- }
-
- qp->sq_no_prefetch = ucmd.sq_no_prefetch;
-
- err = set_user_sq_size(dev, qp, &ucmd);
- if (err)
- goto err;
-
- qp->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr,
- qp->buf_size, 0, FALSE);
- if (IS_ERR(qp->umem)) {
- err = PTR_ERR(qp->umem);
- goto err;
- }
-
- err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),
- ilog2(qp->umem->page_size), &qp->mtt);
- if (err)
- goto err_buf;
-
- err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
- if (err)
- goto err_mtt;
-
- if (!init_attr->srq) {
- err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx),
- ucmd.db_addr, &qp->db);
- if (err)
- goto err_mtt;
- }
- } else {
- qp->sq_no_prefetch = 0;
-
- err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp);
- if (err)
- goto err;
-
- if (!init_attr->srq) {
- err = mlx4_ib_db_alloc(dev, &qp->db, 0);
- if (err)
- goto err;
-
- *qp->db.db = 0;
- }
-
- if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) {
- err = -ENOMEM;
- goto err_db;
- }
-
- err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
- &qp->mtt);
- if (err)
- goto err_buf;
-
- err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
- if (err)
- goto err_mtt;
-
- qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL);
- qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL);
-
- if (!qp->sq.wrid || !qp->rq.wrid) {
- err = -ENOMEM;
- goto err_wrid;
- }
- }
-
- if (!sqpn)
- err = mlx4_qp_reserve_range(dev->dev, 1, 1, &sqpn);
- if (err)
- goto err_wrid;
-
- err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp);
- if (err)
- goto err_wrid;
-
- /*
- * Hardware wants QPN written in big-endian order (after
- * shifting) for send doorbell. Precompute this value to save
- * a little bit when posting sends.
- */
- qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
-
- if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
- qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
- else
- qp->sq_signal_bits = 0;
-
- qp->mqp.event = mlx4_ib_qp_event;
-
- return 0;
-
-err_wrid:
- if (pd->p_uctx) {
- if (!init_attr->srq)
- mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx),
- &qp->db);
- } else {
- kfree(qp->sq.wrid);
- kfree(qp->rq.wrid);
- }
-
-err_mtt:
- mlx4_mtt_cleanup(dev->dev, &qp->mtt);
-
-err_buf:
- if (pd->p_uctx)
- ib_umem_release(qp->umem);
- else
- mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
-
-err_db:
- if (!pd->p_uctx && !init_attr->srq)
- mlx4_ib_db_free(dev, &qp->db);
-
-err:
- return err;
-}
-
-static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
-{
- switch (state) {
- case XIB_QPS_RESET: return MLX4_QP_STATE_RST;
- case XIB_QPS_INIT: return MLX4_QP_STATE_INIT;
- case XIB_QPS_RTR: return MLX4_QP_STATE_RTR;
- case XIB_QPS_RTS: return MLX4_QP_STATE_RTS;
- case XIB_QPS_SQD: return MLX4_QP_STATE_SQD;
- case XIB_QPS_SQE: return MLX4_QP_STATE_SQER;
- case XIB_QPS_ERR: return MLX4_QP_STATE_ERR;
- default: return -1;
- }
-}
-
-static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
-{
- if (send_cq == recv_cq)
- spin_lock_irq(&send_cq->lock);
- else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
- spin_lock_irq(&send_cq->lock);
- spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
- } else {
- spin_lock_irq(&recv_cq->lock);
- spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
- }
-}
-
-static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
-{
- if (send_cq == recv_cq)
- spin_unlock_irq(&send_cq->lock);
- else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
- spin_unlock(&recv_cq->lock);
- spin_unlock_irq(&send_cq->lock);
- } else {
- spin_unlock(&send_cq->lock);
- spin_unlock_irq(&recv_cq->lock);
- }
-}
-
-static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
- int is_user)
-{
- struct mlx4_ib_cq *send_cq, *recv_cq;
-
- if (qp->state != XIB_QPS_RESET)
- if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
- MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
- printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n",
- qp->mqp.qpn);
-
- send_cq = to_mcq(qp->ibqp.send_cq);
- recv_cq = to_mcq(qp->ibqp.recv_cq);
-
- mlx4_ib_lock_cqs(send_cq, recv_cq);
-
- if (!is_user) {
- __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
- qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
- if (send_cq != recv_cq)
- __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
- }
-
- mlx4_qp_remove(dev->dev, &qp->mqp);
-
- mlx4_ib_unlock_cqs(send_cq, recv_cq);
-
- mlx4_qp_free(dev->dev, &qp->mqp);
- mlx4_mtt_cleanup(dev->dev, &qp->mtt);
-
- if (is_user) {
- if (!qp->ibqp.srq)
- mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.p_uctx),
- &qp->db);
- ib_umem_release(qp->umem);
- } else {
- kfree(qp->sq.wrid);
- kfree(qp->rq.wrid);
- mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
- if (!qp->ibqp.srq)
- mlx4_ib_db_free(dev, &qp->db);
- }
-}
-
-struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata)
-{
- struct mlx4_ib_dev *dev = to_mdev(pd->device);
- struct mlx4_ib_sqp *sqp;
- struct mlx4_ib_qp *qp;
- int err;
-
- switch (init_attr->qp_type) {
- case IB_QPT_RC:
- case IB_QPT_UC:
- case IB_QPT_UD:
- {
- qp = kzalloc(sizeof *qp, GFP_KERNEL);
- if (!qp)
- return ERR_PTR(-ENOMEM);
-
- err = create_qp_common(dev, pd, init_attr, udata, 0, qp);
- if (err) {
- kfree(qp);
- return ERR_PTR(err);
- }
-
- qp->ibqp.qp_num = qp->mqp.qpn;
-
- break;
- }
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- {
- /* Userspace is not allowed to create special QPs: */
- if (pd->p_uctx)
- return ERR_PTR(-EINVAL);
-
- sqp = kzalloc(sizeof *sqp, GFP_KERNEL);
- if (!sqp)
- return ERR_PTR(-ENOMEM);
-
- qp = &sqp->qp;
-
- err = create_qp_common(dev, pd, init_attr, udata,
- dev->dev->caps.sqp_start +
- (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) +
- init_attr->port_num - 1,
- qp);
- if (err) {
- kfree(sqp);
- return ERR_PTR(err);
- }
-
- qp->port = init_attr->port_num;
- qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
-
- break;
- }
- default:
- /* Don't support raw QPs */
- return ERR_PTR(-EINVAL);
- }
-
- return &qp->ibqp;
-}
-
-int mlx4_ib_destroy_qp(struct ib_qp *qp)
-{
- struct mlx4_ib_dev *dev = to_mdev(qp->device);
- struct mlx4_ib_qp *mqp = to_mqp(qp);
-
- if (is_qp0(dev, mqp))
- mlx4_CLOSE_PORT(dev->dev, mqp->port);
-
- destroy_qp_common(dev, mqp, !!qp->pd->p_uctx);
-
- if (is_sqp(dev, mqp))
- kfree(to_msqp(mqp));
- else
- kfree(mqp);
-
- return 0;
-}
-
-static int to_mlx4_st(enum ib_qp_type type)
-{
- switch (type) {
- case IB_QPT_RC: return MLX4_QP_ST_RC;
- case IB_QPT_UC: return MLX4_QP_ST_UC;
- case IB_QPT_UD: return MLX4_QP_ST_UD;
- case IB_QPT_SMI:
- case IB_QPT_GSI: return MLX4_QP_ST_MLX;
- default: return -1;
- }
-}
-
-static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
- int attr_mask)
-{
- u8 dest_rd_atomic;
- u32 access_flags;
- u32 hw_access_flags = 0;
-
- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
- dest_rd_atomic = attr->max_dest_rd_atomic;
- else
- dest_rd_atomic = qp->resp_depth;
-
- if (attr_mask & IB_QP_ACCESS_FLAGS)
- access_flags = attr->qp_access_flags;
- else
- access_flags = qp->atomic_rd_en;
-
- if (!dest_rd_atomic)
- access_flags &= IB_ACCESS_REMOTE_WRITE;
-
- if (access_flags & IB_ACCESS_REMOTE_READ)
- hw_access_flags |= MLX4_QP_BIT_RRE;
- if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
- hw_access_flags |= MLX4_QP_BIT_RAE;
- if (access_flags & IB_ACCESS_REMOTE_WRITE)
- hw_access_flags |= MLX4_QP_BIT_RWE;
-
- return cpu_to_be32(hw_access_flags);
-}
-
-static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
- int attr_mask)
-{
- if (attr_mask & IB_QP_PKEY_INDEX)
- sqp->pkey_index = attr->pkey_index;
- if (attr_mask & IB_QP_QKEY)
- sqp->qkey = attr->qkey;
- if (attr_mask & IB_QP_SQ_PSN)
- sqp->send_psn = attr->sq_psn;
-}
-
-static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
-{
- path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
-}
-
-static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
- struct mlx4_qp_path *path, u8 port)
-{
- path->grh_mylmc = ah->src_path_bits & 0x7f;
- path->rlid = cpu_to_be16(ah->dlid);
- if (ah->static_rate) {
- path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;
- while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
- !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
- --path->static_rate;
- } else
- path->static_rate = 0;
- path->counter_index = 0xff;
-
- if (ah->ah_flags & IB_AH_GRH) {
- if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
- printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
- ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
- return -1;
- }
-
- path->grh_mylmc |= 1 << 7;
- path->mgid_index = ah->grh.sgid_index;
- path->hop_limit = ah->grh.hop_limit;
- path->tclass_flowlabel =
- cpu_to_be32((ah->grh.traffic_class << 20) |
- (ah->grh.flow_label));
- memcpy(path->rgid, ah->grh.dgid.raw, 16);
- }
-
- path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
- ((port - 1) << 6) | ((ah->sl & 0xf) << 2);
-
- return 0;
-}
-
-static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
- const struct ib_qp_attr *attr, int attr_mask,
- enum ib_qp_state cur_state, enum ib_qp_state new_state)
-{
- struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
- struct mlx4_ib_qp *qp = to_mqp(ibqp);
- struct mlx4_qp_context *context;
- enum mlx4_qp_optpar optpar = 0;
- int sqd_event;
- int err = -EINVAL;
-
- context = kzalloc(sizeof *context, GFP_KERNEL);
- if (!context)
- return -ENOMEM;
-
- context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
- (to_mlx4_st(ibqp->qp_type) << 16));
- context->flags |= cpu_to_be32(1 << 8); /* DE? */
-
- if (!(attr_mask & IB_QP_PATH_MIG_STATE))
- context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
- else {
- optpar |= MLX4_QP_OPTPAR_PM_STATE;
- switch (attr->path_mig_state) {
- case IB_MIG_MIGRATED:
- context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
- break;
- case IB_MIG_REARM:
- context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
- break;
- case IB_MIG_ARMED:
- context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
- break;
- }
- }
-
- if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
- ibqp->qp_type == IB_QPT_UD)
- context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
- else if (attr_mask & IB_QP_PATH_MTU) {
- if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
- printk(KERN_ERR "path MTU (%u) is invalid\n",
- attr->path_mtu);
- goto out;
- }
- context->mtu_msgmax = (u8)((attr->path_mtu << 5) |
- ilog2(dev->dev->caps.max_msg_sz));
- }
-
- if (qp->rq.wqe_cnt)
- context->rq_size_stride = (u8)(ilog2(qp->rq.wqe_cnt) << 3);
- context->rq_size_stride |= qp->rq.wqe_shift - 4;
-
- if (qp->sq.wqe_cnt)
- context->sq_size_stride = (u8)(ilog2(qp->sq.wqe_cnt) << 3);
- context->sq_size_stride |= qp->sq.wqe_shift - 4;
-
- if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT)
- context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
-
- if (qp->ibqp.p_uctx)
- context->usr_page = cpu_to_be32(to_mucontext(ibqp->p_uctx)->uar.index);
- else
- context->usr_page = cpu_to_be32(dev->priv_uar.index);
-
- if (attr_mask & IB_QP_DEST_QPN)
- context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
-
- if (attr_mask & IB_QP_PORT) {
- if (cur_state == XIB_QPS_SQD && new_state == XIB_QPS_SQD &&
- !(attr_mask & IB_QP_AV)) {
- mlx4_set_sched(&context->pri_path, attr->port_num);
- optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
- }
- }
-
- if (attr_mask & IB_QP_PKEY_INDEX) {
- context->pri_path.pkey_index = (u8)attr->pkey_index;
- optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
- }
-
- if (attr_mask & IB_QP_AV) {
- if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
- attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
- goto out;
-
- optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
- MLX4_QP_OPTPAR_SCHED_QUEUE);
- }
-
- if (attr_mask & IB_QP_TIMEOUT) {
- context->pri_path.ackto = attr->timeout << 3;
- optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
- }
-
- if (attr_mask & IB_QP_ALT_PATH) {
- if (attr->alt_port_num == 0 ||
- attr->alt_port_num > dev->dev->caps.num_ports)
- goto out;
-
- if (attr->alt_pkey_index >=
- dev->dev->caps.pkey_table_len[attr->alt_port_num])
- goto out;
-
- if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
- attr->alt_port_num))
- goto out;
-
- context->alt_path.pkey_index = (u8)attr->alt_pkey_index;
- context->alt_path.ackto = attr->alt_timeout << 3;
- optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
- }
-
- context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn);
- context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
-
- if (attr_mask & IB_QP_RNR_RETRY) {
- context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
- optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
- }
-
- if (attr_mask & IB_QP_RETRY_CNT) {
- context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
- optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
- }
-
- if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
- if (attr->max_rd_atomic)
- context->params1 |=
- cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
- optpar |= MLX4_QP_OPTPAR_SRA_MAX;
- }
-
- if (attr_mask & IB_QP_SQ_PSN)
- context->next_send_psn = cpu_to_be32(attr->sq_psn);
-
- context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn);
-
- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
- if (attr->max_dest_rd_atomic)
- context->params2 |=
- cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
- optpar |= MLX4_QP_OPTPAR_RRA_MAX;
- }
-
- if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
- context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
- optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
- }
-
- if (ibqp->srq)
- context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
-
- if (attr_mask & IB_QP_MIN_RNR_TIMER) {
- context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
- optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
- }
- if (attr_mask & IB_QP_RQ_PSN)
- context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
-
- context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn);
-
- if (attr_mask & IB_QP_QKEY) {
- context->qkey = cpu_to_be32(attr->qkey);
- optpar |= MLX4_QP_OPTPAR_Q_KEY;
- }
-
- if (ibqp->srq)
- context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
-
- if (!ibqp->srq && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT)
- context->db_rec_addr = cpu_to_be64(qp->db.dma.da);
-
- if (cur_state == XIB_QPS_INIT &&
- new_state == XIB_QPS_RTR &&
- (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
- ibqp->qp_type == IB_QPT_UD)) {
- context->pri_path.sched_queue = (qp->port - 1) << 6;
- if (is_qp0(dev, qp))
- context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
- else
- context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
- }
-
- if (cur_state == XIB_QPS_RTS && new_state == XIB_QPS_SQD &&
- attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
- sqd_event = 1;
- else
- sqd_event = 0;
-
- /*
- * Before passing a kernel QP to the HW, make sure that the
- * ownership bits of the send queue are set and the SQ
- * headroom is stamped so that the hardware doesn't start
- * processing stale work requests.
- */
- if (!ibqp->p_uctx && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) {
- struct mlx4_wqe_ctrl_seg *ctrl;
- int i;
-
- for (i = 0; i < qp->sq.wqe_cnt; ++i) {
- ctrl = get_send_wqe(qp, i);
- ctrl->owner_opcode = cpu_to_be32(1 << 31);
-
- stamp_send_wqe(qp, i);
- }
- }
-
- err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
- to_mlx4_state(new_state), context, optpar,
- sqd_event, &qp->mqp);
- if (err)
- goto out;
-
- qp->state = new_state;
-
- if (attr_mask & IB_QP_ACCESS_FLAGS)
- qp->atomic_rd_en = (u8)attr->qp_access_flags;
- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
- qp->resp_depth = attr->max_dest_rd_atomic;
- if (attr_mask & IB_QP_PORT)
- qp->port = attr->port_num;
- if (attr_mask & IB_QP_ALT_PATH)
- qp->alt_port = attr->alt_port_num;
-
- if (is_sqp(dev, qp))
- store_sqp_attrs(to_msqp(qp), attr, attr_mask);
-
- /*
- * If we moved QP0 to RTR, bring the IB link up; if we moved
- * QP0 to RESET or ERROR, bring the link back down.
- */
- if (is_qp0(dev, qp)) {
- if (cur_state != XIB_QPS_RTR && new_state == XIB_QPS_RTR)
- if (mlx4_INIT_PORT(dev->dev, qp->port))
- printk(KERN_WARNING "INIT_PORT failed for port %d\n",
- qp->port);
-
- if (cur_state != XIB_QPS_RESET && cur_state != XIB_QPS_ERR &&
- (new_state == XIB_QPS_RESET || new_state == XIB_QPS_ERR))
- mlx4_CLOSE_PORT(dev->dev, qp->port);
- }
-
- /*
- * If we moved a kernel QP to RESET, clean up all old CQ
- * entries and reinitialize the QP.
- */
- if (new_state == XIB_QPS_RESET && !ibqp->p_uctx) {
- mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn,
- ibqp->srq ? to_msrq(ibqp->srq): NULL);
- if (ibqp->send_cq != ibqp->recv_cq)
- mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL);
-
- qp->rq.head = 0;
- qp->rq.tail = 0;
- qp->sq.head = 0;
- qp->sq.tail = 0;
- if (!ibqp->srq)
- *qp->db.db = 0;
- }
-
-out:
- kfree(context);
- return err;
-}
-
-static struct ib_qp_attr mlx4_ib_qp_attr;
-static int mlx4_ib_qp_attr_mask_table[IB_QPT_UD + 1];
-
-void mlx4_ib_qp_init()
-{
- memset( &mlx4_ib_qp_attr, 0, sizeof(mlx4_ib_qp_attr) );
- mlx4_ib_qp_attr.port_num = 1;
-
- memset( &mlx4_ib_qp_attr_mask_table, 0, sizeof(mlx4_ib_qp_attr_mask_table) );
- mlx4_ib_qp_attr_mask_table[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
- IB_QP_PORT |
- IB_QP_QKEY);
- mlx4_ib_qp_attr_mask_table[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
- IB_QP_PORT |
- IB_QP_ACCESS_FLAGS);
- mlx4_ib_qp_attr_mask_table[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
- IB_QP_PORT |
- IB_QP_ACCESS_FLAGS);
- mlx4_ib_qp_attr_mask_table[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
- IB_QP_QKEY);
- mlx4_ib_qp_attr_mask_table[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
- IB_QP_QKEY);
-}
-
-int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
- int attr_mask, struct ib_udata *udata)
-{
- struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
- struct mlx4_ib_qp *qp = to_mqp(ibqp);
- enum ib_qp_state cur_state, new_state;
- int err = -EINVAL;
-
- UNUSED_PARAM(udata);
-
- mutex_lock(&qp->mutex);
-
- cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
- new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
-
- if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
- goto out;
-
- if ((attr_mask & IB_QP_PORT) &&
- (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {
- goto out;
- }
-
- if (attr_mask & IB_QP_PKEY_INDEX) {
- int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
- if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
- goto out;
- }
-
- if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
- attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
- goto out;
- }
-
- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
- attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
- goto out;
- }
-
- if (cur_state == new_state && cur_state == XIB_QPS_RESET) {
- err = 0;
- goto out;
- }
-
- if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_ERR) {
- err = __mlx4_ib_modify_qp(ibqp, &mlx4_ib_qp_attr,
- mlx4_ib_qp_attr_mask_table[ibqp->qp_type],
- XIB_QPS_RESET, XIB_QPS_INIT);
- if (err)
- goto out;
- cur_state = XIB_QPS_INIT;
- }
-
- err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
-
-out:
- mutex_unlock(&qp->mutex);
- return err;
-}
-
-static enum ib_wr_opcode to_wr_opcode(struct _ib_send_wr *wr)
-{
-
- enum ib_wr_opcode opcode = -1; //= wr->wr_type;
-
- switch (wr->wr_type) {
- case WR_SEND:
- opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_SEND_WITH_IMM : IB_WR_SEND;
- break;
- case WR_RDMA_WRITE:
- opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_RDMA_WRITE_WITH_IMM : IB_WR_RDMA_WRITE;
- break;
- case WR_RDMA_READ: opcode = IB_WR_RDMA_READ; break;
- case WR_COMPARE_SWAP: opcode = IB_WR_ATOMIC_CMP_AND_SWP; break;
- case WR_FETCH_ADD: opcode = IB_WR_ATOMIC_FETCH_AND_ADD; break;
- }
- return opcode;
-}
-
-static int build_mlx_header(struct mlx4_ib_sqp *sqp, ib_send_wr_t *wr,
- void *wqe)
-{
- enum ib_wr_opcode opcode = to_wr_opcode(wr);
- struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev;
- struct mlx4_wqe_mlx_seg *mlx = wqe;
- struct mlx4_wqe_inline_seg *inl = (void*)((u8*)wqe + sizeof *mlx);
- struct mlx4_ib_ah *ah = to_mah((struct ib_ah *)wr->dgrm.ud.h_av);
- __be16 pkey;
- int send_size;
- int header_size;
- int spc;
- u32 i;
-
- send_size = 0;
- for (i = 0; i < wr->num_ds; ++i)
- send_size += wr->ds_array[i].length;
-
- ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->ud_header);
-
- sqp->ud_header.lrh.service_level =
- (u8)(be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28);
- sqp->ud_header.lrh.destination_lid = ah->av.dlid;
- sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.g_slid & 0x7f);
- if (mlx4_ib_ah_grh_present(ah)) {
- sqp->ud_header.grh.traffic_class =
- (u8)((be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20) & 0xff);
- sqp->ud_header.grh.flow_label =
- ah->av.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
- sqp->ud_header.grh.hop_limit = ah->av.hop_limit;
- ib_get_cached_gid(ib_dev, (u8)(be32_to_cpu(ah->av.port_pd) >> 24),
- ah->av.gid_index, &sqp->ud_header.grh.source_gid);
- memcpy(sqp->ud_header.grh.destination_gid.raw,
- ah->av.dgid, 16);
- }
-
- mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
- mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
- (sqp->ud_header.lrh.destination_lid ==
- XIB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
- (sqp->ud_header.lrh.service_level << 8));
- mlx->rlid = sqp->ud_header.lrh.destination_lid;
-
- switch (opcode) {
- case IB_WR_SEND:
- sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
- sqp->ud_header.immediate_present = 0;
- break;
- case IB_WR_SEND_WITH_IMM:
- sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
- sqp->ud_header.immediate_present = 1;
- sqp->ud_header.immediate_data = wr->immediate_data;
- break;
- default:
- return -EINVAL;
- }
-
- sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;
- if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
- sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
- sqp->ud_header.bth.solicited_event = (u8)(!!(wr->send_opt & IB_SEND_OPT_SOLICITED));
- if (!sqp->qp.ibqp.qp_num)
- ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
- else
- ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->dgrm.ud.pkey_index, &pkey);
- sqp->ud_header.bth.pkey = pkey;
- sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp;
- sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
- sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ?
- cpu_to_be32(sqp->qkey) : wr->dgrm.ud.remote_qkey;
- sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
-
- header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
-
-#if 0
- {
- printk(KERN_ERR "built UD header of size %d:\n", header_size);
- for (i = 0; i < header_size / 4; ++i) {
- if (i % 8 == 0)
- printk(" [%02x] ", i * 4);
- printk(" %08x",
- be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
- if ((i + 1) % 8 == 0)
- printk("\n");
- }
- printk("\n");
- }
-#endif
-
- /*
- * Inline data segments may not cross a 64 byte boundary. If
- * our UD header is bigger than the space available up to the
- * next 64 byte boundary in the WQE, use two inline data
- * segments to hold the UD header.
- */
- spc = MLX4_INLINE_ALIGN -
- ((u32)(ULONG_PTR)(inl + 1) & (MLX4_INLINE_ALIGN - 1));
- if (header_size <= spc) {
- inl->byte_count = cpu_to_be32(1 << 31 | header_size);
- memcpy(inl + 1, sqp->header_buf, header_size);
- i = 1;
- } else {
- inl->byte_count = cpu_to_be32(1 << 31 | spc);
- memcpy(inl + 1, sqp->header_buf, spc);
-
- inl = (void*)((u8*)(inl + 1) + spc);
- memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
- /*
- * Need a barrier here to make sure all the data is
- * visible before the byte_count field is set.
- * Otherwise the HCA prefetcher could grab the 64-byte
- * chunk with this inline segment and get a valid (!=
- * 0xffffffff) byte count but stale data, and end up
- * generating a packet with bad headers.
- *
- * The first inline segment's byte_count field doesn't
- * need a barrier, because it comes after a
- * control/MLX segment and therefore is at an offset
- * of 16 mod 64.
- */
- wmb();
- inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
- i = 2;
- }
-
- return ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
-}
-
-static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
-{
- unsigned cur;
- struct mlx4_ib_cq *cq;
-
- cur = wq->head - wq->tail;
- if (likely((int)cur + nreq < wq->max_post))
- return 0;
-
- cq = to_mcq(ib_cq);
- spin_lock(&cq->lock);
- cur = wq->head - wq->tail;
- spin_unlock(&cq->lock);
-
- return (int)cur + nreq >= wq->max_post;
-}
-
-static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
- u64 remote_addr, __be32 rkey)
-{
- rseg->raddr = cpu_to_be64(remote_addr);
- rseg->rkey = rkey;
- rseg->reserved = 0;
-}
-
-static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, ib_send_wr_t *wr)
-{
- if (wr->wr_type == WR_COMPARE_SWAP) {
- aseg->swap_add = wr->remote_ops.atomic2;
- aseg->compare = wr->remote_ops.atomic1;
- } else {
- aseg->swap_add = wr->remote_ops.atomic1;
- aseg->compare = 0;
- }
-
-}
-
-static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
- ib_send_wr_t *wr)
-{
- memcpy(dseg->av, &to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, sizeof (struct mlx4_av));
- dseg->dqpn = wr->dgrm.ud.remote_qp;
- dseg->qkey = wr->dgrm.ud.remote_qkey;
-}
-
-static void set_mlx_icrc_seg(void *dseg)
-{
- u32 *t = dseg;
- struct mlx4_wqe_inline_seg *iseg = dseg;
-
- t[1] = 0;
-
- /*
- * Need a barrier here before writing the byte_count field to
- * make sure that all the data is visible before the
- * byte_count field is set. Otherwise, if the segment begins
- * a new cacheline, the HCA prefetcher could grab the 64-byte
- * chunk and get a valid (!= * 0xffffffff) byte count but
- * stale data, and end up sending the wrong data.
- */
- wmb();
-
- iseg->byte_count = cpu_to_be32((1 << 31) | 4);
-}
-
-static void set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg)
-{
- dseg->lkey = cpu_to_be32(sg->lkey);
- dseg->addr = cpu_to_be64(sg->vaddr);
-
- /*
- * Need a barrier here before writing the byte_count field to
- * make sure that all the data is visible before the
- * byte_count field is set. Otherwise, if the segment begins
- * a new cacheline, the HCA prefetcher could grab the 64-byte
- * chunk and get a valid (!= * 0xffffffff) byte count but
- * stale data, and end up sending the wrong data.
- */
- wmb();
-
- dseg->byte_count = cpu_to_be32(sg->length);
-}
-
-static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg)
-{
- dseg->byte_count = cpu_to_be32(sg->length);
- dseg->lkey = cpu_to_be32(sg->lkey);
- dseg->addr = cpu_to_be64(sg->vaddr);
-}
-
-int mlx4_ib_post_send(struct ib_qp *ibqp, ib_send_wr_t *wr,
- ib_send_wr_t **bad_wr)
-{
- enum ib_wr_opcode opcode;
- struct mlx4_ib_qp *qp = to_mqp(ibqp);
- u8 *wqe;
- struct mlx4_wqe_ctrl_seg *ctrl;
- struct mlx4_wqe_data_seg *dseg;
- unsigned long flags;
- int nreq;
- int err = 0;
- int ind;
- int size;
- int i;
-
- spin_lock_irqsave(&qp->sq.lock, &flags);
-
- ind = qp->sq.head;
-
- for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
- if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
- err = -ENOMEM;
- if (bad_wr)
- *bad_wr = wr;
- goto out;
- }
-
- if (unlikely(wr->num_ds > (u32)qp->sq.max_gs)) {
- err = -EINVAL;
- if (bad_wr)
- *bad_wr = wr;
- goto out;
- }
-
- wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
- ctrl = (void*)wqe;
- qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
- opcode = to_wr_opcode(wr);
-
- ctrl->srcrb_flags =
- (wr->send_opt & IB_SEND_OPT_SIGNALED ?
- cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
- (wr->send_opt & IB_SEND_OPT_SOLICITED ?
- cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
- (wr->send_opt & IB_SEND_OPT_TX_IP_CSUM ?
- cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM) : 0) |
- (wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM ?
- cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
- qp->sq_signal_bits;
-
- if (opcode == IB_WR_SEND_WITH_IMM ||
- opcode == IB_WR_RDMA_WRITE_WITH_IMM)
- ctrl->imm = wr->immediate_data;
- else
- ctrl->imm = 0;
-
- wqe += sizeof *ctrl;
- size = sizeof *ctrl / 16;
-
- switch (ibqp->qp_type) {
- case IB_QPT_RC:
- case IB_QPT_UC:
- switch (opcode) {
- case IB_WR_ATOMIC_CMP_AND_SWP:
- case IB_WR_ATOMIC_FETCH_AND_ADD:
- set_raddr_seg((void*)wqe, wr->remote_ops.vaddr,
- wr->remote_ops.rkey);
- wqe += sizeof (struct mlx4_wqe_raddr_seg);
-
- set_atomic_seg((void*)wqe, wr);
- wqe += sizeof (struct mlx4_wqe_atomic_seg);
-
- size += (sizeof (struct mlx4_wqe_raddr_seg) +
- sizeof (struct mlx4_wqe_atomic_seg)) / 16;
-
- break;
-
- case IB_WR_RDMA_READ:
- case IB_WR_RDMA_WRITE:
- case IB_WR_RDMA_WRITE_WITH_IMM:
- set_raddr_seg((void*)wqe, wr->remote_ops.vaddr,
- wr->remote_ops.rkey);
- wqe += sizeof (struct mlx4_wqe_raddr_seg);
- size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
- break;
-
- default:
- /* No extra segments required for sends */
- break;
- }
- break;
-
- case IB_QPT_UD:
- set_datagram_seg((void*)wqe, wr);
- wqe += sizeof (struct mlx4_wqe_datagram_seg);
- size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
- break;
-
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- err = build_mlx_header(to_msqp(qp), wr, ctrl);
- if (err < 0) {
- if (bad_wr)
- *bad_wr = wr;
- goto out;
- }
- wqe += err;
- size += err / 16;
-
- err = 0;
- break;
-
- default:
- break;
- }
-
- /*
- * Write data segments in reverse order, so as to
- * overwrite cacheline stamp last within each
- * cacheline. This avoids issues with WQE
- * prefetching.
- */
-
- dseg = (void*)wqe;
- dseg += wr->num_ds - 1;
- size += wr->num_ds * (sizeof (struct mlx4_wqe_data_seg) / 16);
-
- /* Add one more inline data segment for ICRC for MLX sends */
- if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI ||
- qp->ibqp.qp_type == IB_QPT_GSI)) {
- set_mlx_icrc_seg(dseg + 1);
- size += sizeof (struct mlx4_wqe_data_seg) / 16;
- }
-
- for (i = wr->num_ds - 1; i >= 0; --i, --dseg)
- set_data_seg(dseg, wr->ds_array + i);
-
- ctrl->fence_size = (u8)((wr->send_opt & IB_SEND_OPT_FENCE ?
- MLX4_WQE_CTRL_FENCE : 0) | size);
-
- /*
- * Make sure descriptor is fully written before
- * setting ownership bit (because HW can start
- * executing as soon as we do).
- */
- wmb();
-
- if (opcode < 0 || opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
- err = -EINVAL;
- goto out;
- }
-
- ctrl->owner_opcode = mlx4_ib_opcode[opcode] |
- (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
-
- /*
- * We can improve latency by not stamping the last
- * send queue WQE until after ringing the doorbell, so
- * only stamp here if there are still more WQEs to post.
- */
- if (wr->p_next)
- stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) &
- (qp->sq.wqe_cnt - 1));
-
- ++ind;
- }
-
-out:
- if (likely(nreq)) {
- qp->sq.head += nreq;
-
- /*
- * Make sure that descriptors are written before
- * doorbell record.
- */
- wmb();
-
- writel(qp->doorbell_qpn,
- (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
-
-#if 0
- if (qp->mqp.qpn == 0x41)
- DbgPrint( "[MLX4_BUS] mlx4_ib_post_send : qtype %d, qpn %#x, nreq %d, sq.head %#x, wqe_ix %d, db %p \n",
- ibqp->qp_type, qp->mqp.qpn, nreq, qp->sq.head, ind,
- (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL );
-#endif
- /*
- * Make sure doorbells don't leak out of SQ spinlock
- * and reach the HCA out of order.
- */
- mmiowb();
-
- stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) &
- (qp->sq.wqe_cnt - 1));
- }
-
- spin_unlock_irqrestore(&qp->sq.lock, flags);
-
- return err;
-}
-
-int mlx4_ib_post_recv(struct ib_qp *ibqp, ib_recv_wr_t *wr,
- ib_recv_wr_t **bad_wr)
-{
- struct mlx4_ib_qp *qp = to_mqp(ibqp);
- struct mlx4_wqe_data_seg *scat;
- unsigned long flags;
- int err = 0;
- int nreq;
- int ind;
- int i;
-
- spin_lock_irqsave(&qp->rq.lock, &flags);
-
- ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
-
- for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
- if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.send_cq)) {
- err = -ENOMEM;
- if (bad_wr)
- *bad_wr = wr;
- goto out;
- }
-
- if (unlikely(wr->num_ds > (u32)qp->rq.max_gs)) {
- err = -EINVAL;
- if (bad_wr)
- *bad_wr = wr;
- goto out;
- }
-
- scat = get_recv_wqe(qp, ind);
-
- for (i = 0; i < (int)wr->num_ds; ++i)
- __set_data_seg(scat + i, wr->ds_array + i);
-
- if (i < qp->rq.max_gs) {
- scat[i].byte_count = 0;
- scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
- scat[i].addr = 0;
- }
-
- qp->rq.wrid[ind] = wr->wr_id;
-
- ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
- }
-
-out:
- if (likely(nreq)) {
- qp->rq.head += nreq;
-
- /*
- * Make sure that descriptors are written before
- * doorbell record.
- */
- wmb();
-
- *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
-
-#if 0
- if (qp->mqp.qpn == 0x41)
- DbgPrint( "[MLX4_BUS] mlx4_ib_post_recv : qtype %d, qpn %#x, nreq %d, rq.head %#x, wqe_ix %d, db_obj %p, db %p \n",
- ibqp->qp_type, qp->mqp.qpn, nreq, qp->rq.head, ind, &qp->db, qp->db.db );
-#endif
- }
-
- spin_unlock_irqrestore(&qp->rq.lock, flags);
-
- return err;
-}
-
-static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
-{
- switch (mlx4_state) {
- case MLX4_QP_STATE_RST: return XIB_QPS_RESET;
- case MLX4_QP_STATE_INIT: return XIB_QPS_INIT;
- case MLX4_QP_STATE_RTR: return XIB_QPS_RTR;
- case MLX4_QP_STATE_RTS: return XIB_QPS_RTS;
- case MLX4_QP_STATE_SQ_DRAINING:
- case MLX4_QP_STATE_SQD: return XIB_QPS_SQD;
- case MLX4_QP_STATE_SQER: return XIB_QPS_SQE;
- case MLX4_QP_STATE_ERR: return XIB_QPS_ERR;
- default: return -1;
- }
-}
-
-static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
-{
- switch (mlx4_mig_state) {
- case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;
- case MLX4_QP_PM_REARM: return IB_MIG_REARM;
- case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
- default: return -1;
- }
-}
-
-static int to_ib_qp_access_flags(int mlx4_flags)
-{
- int ib_flags = 0;
-
- if (mlx4_flags & MLX4_QP_BIT_RRE)
- ib_flags |= IB_ACCESS_REMOTE_READ;
- if (mlx4_flags & MLX4_QP_BIT_RWE)
- ib_flags |= IB_ACCESS_REMOTE_WRITE;
- if (mlx4_flags & MLX4_QP_BIT_RAE)
- ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
-
- return ib_flags;
-}
-
-static void to_ib_ah_attr(struct mlx4_dev *dev, struct ib_ah_attr *ib_ah_attr,
- struct mlx4_qp_path *path)
-{
- memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
- ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1;
-
- if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)
- return;
-
- ib_ah_attr->dlid = be16_to_cpu(path->rlid);
- ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;
- ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;
- ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0;
- ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
- if (ib_ah_attr->ah_flags) {
- ib_ah_attr->grh.sgid_index = path->mgid_index;
- ib_ah_attr->grh.hop_limit = path->hop_limit;
- ib_ah_attr->grh.traffic_class =
- (u8)((be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff);
- ib_ah_attr->grh.flow_label =
- be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
- memcpy(ib_ah_attr->grh.dgid.raw,
- path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
- }
-}
-
-int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
- struct ib_qp_init_attr *qp_init_attr)
-{
- struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
- struct mlx4_ib_qp *qp = to_mqp(ibqp);
- struct mlx4_qp_context context;
- int mlx4_state;
- int err;
-
- UNUSED_PARAM(qp_attr_mask);
-
- if (qp->state == XIB_QPS_RESET) {
- qp_attr->qp_state = XIB_QPS_RESET;
- goto done;
- }
-
- err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
- if (err)
- return -EINVAL;
-
- mlx4_state = be32_to_cpu(context.flags) >> 28;
-
- qp_attr->qp_state = to_ib_qp_state(mlx4_state);
- qp_attr->path_mtu = context.mtu_msgmax >> 5;
- qp_attr->path_mig_state =
- to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
- qp_attr->qkey = be32_to_cpu(context.qkey);
- qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
- qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;
- qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;
- qp_attr->qp_access_flags =
- to_ib_qp_access_flags(be32_to_cpu(context.params2));
-
- if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
- to_ib_ah_attr(dev->dev, &qp_attr->ah_attr, &context.pri_path);
- to_ib_ah_attr(dev->dev, &qp_attr->alt_ah_attr, &context.alt_path);
- qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
- qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;
- }
-
- qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
- if (qp_attr->qp_state == XIB_QPS_INIT)
- qp_attr->port_num = qp->port;
- else
- qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
-
- /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
- qp_attr->sq_draining = (u8)(mlx4_state == MLX4_QP_STATE_SQ_DRAINING);
-
- qp_attr->max_rd_atomic = (u8)(1 << ((be32_to_cpu(context.params1) >> 21) & 0x7));
-
- qp_attr->max_dest_rd_atomic =
- (u8)(1 << ((be32_to_cpu(context.params2) >> 21) & 0x7));
- qp_attr->min_rnr_timer =
- (u8)((be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f);
- qp_attr->timeout = context.pri_path.ackto >> 3;
- qp_attr->retry_cnt = (u8)((be32_to_cpu(context.params1) >> 16) & 0x7);
- qp_attr->rnr_retry = (u8)((be32_to_cpu(context.params1) >> 13) & 0x7);
- qp_attr->alt_timeout = context.alt_path.ackto >> 3;
-
-done:
- qp_attr->cur_qp_state = qp_attr->qp_state;
- qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
- qp_attr->cap.max_recv_sge = qp->rq.max_gs;
-
- if (!ibqp->p_uctx) {
- qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;
- qp_attr->cap.max_send_sge = qp->sq.max_gs;
- } else {
- qp_attr->cap.max_send_wr = 0;
- qp_attr->cap.max_send_sge = 0;
- }
-
- /*
- * We don't support inline sends for kernel QPs (yet), and we
- * don't know what userspace's value should be.
- */
- qp_attr->cap.max_inline_data = 0;
-
- qp_init_attr->cap = qp_attr->cap;
-
- return 0;
-}
-
+/*\r
+ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses. You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include "mlx4_ib.h"\r
+#include "ib_cache.h"\r
+#include "ib_pack.h"\r
+#include "qp.h"\r
+#include "user.h"\r
+\r
+enum {\r
+ MLX4_IB_ACK_REQ_FREQ = 8,\r
+};\r
+\r
+enum {\r
+ MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,\r
+ MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f\r
+};\r
+\r
+enum {\r
+ /*\r
+ * Largest possible UD header: send with GRH and immediate data.\r
+ */\r
+ MLX4_IB_UD_HEADER_SIZE = 72\r
+};\r
+\r
+struct mlx4_ib_sqp {\r
+ struct mlx4_ib_qp qp;\r
+ int pkey_index;\r
+ u32 qkey;\r
+ u32 send_psn;\r
+ struct ib_ud_header ud_header;\r
+ u8 header_buf[MLX4_IB_UD_HEADER_SIZE];\r
+};\r
+\r
+enum {\r
+ MLX4_IB_MIN_SQ_STRIDE = 6\r
+};\r
+\r
+static const __be32 mlx4_ib_opcode[] = {\r
+ __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), /* [IB_WR_RDMA_WRITE] */\r
+ __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), /* [IB_WR_RDMA_WRITE_WITH_IMM] */\r
+ __constant_cpu_to_be32(MLX4_OPCODE_SEND), /* [IB_WR_SEND] */\r
+ __constant_cpu_to_be32(MLX4_OPCODE_SEND_IMM), /* [IB_WR_SEND_WITH_IMM] */\r
+ __constant_cpu_to_be32(MLX4_OPCODE_RDMA_READ), /* [IB_WR_RDMA_READ] */\r
+ __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), /* [IB_WR_ATOMIC_CMP_AND_SWP] */\r
+ __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), /* [IB_WR_ATOMIC_FETCH_AND_ADD]*/\r
+};\r
+\r
+static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)\r
+{\r
+ return container_of(mqp, struct mlx4_ib_sqp, qp);\r
+}\r
+\r
+static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)\r
+{\r
+ return qp->mqp.qpn >= dev->dev->caps.sqp_start &&\r
+ qp->mqp.qpn <= dev->dev->caps.sqp_start + 3;\r
+}\r
+\r
+static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)\r
+{\r
+ return qp->mqp.qpn >= dev->dev->caps.sqp_start &&\r
+ qp->mqp.qpn <= dev->dev->caps.sqp_start + 1;\r
+}\r
+\r
+static void *get_wqe(struct mlx4_ib_qp *qp, int offset)\r
+{\r
+ if (qp->buf.nbufs == 1)\r
+ return qp->buf.u.direct.buf + offset;\r
+ else\r
+ return qp->buf.u.page_list[offset >> PAGE_SHIFT].buf +\r
+ (offset & (PAGE_SIZE - 1));\r
+}\r
+\r
+static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)\r
+{\r
+ return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));\r
+}\r
+\r
+static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)\r
+{\r
+ return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));\r
+}\r
+\r
+/*\r
+ * Stamp a SQ WQE so that it is invalid if prefetched by marking the\r
+ * first four bytes of every 64 byte chunk with 0xffffffff, except for\r
+ * the very first chunk of the WQE.\r
+ */\r
+static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n)\r
+{\r
+ u32 *wqe = get_send_wqe(qp, n);\r
+ int i;\r
+\r
+ for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16)\r
+ wqe[i] = 0xffffffff;\r
+}\r
+\r
+static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)\r
+{\r
+ ib_event_rec_t event;\r
+ struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;\r
+\r
+ if (type == MLX4_EVENT_TYPE_PATH_MIG)\r
+ to_mibqp(qp)->port = to_mibqp(qp)->alt_port;\r
+\r
+ switch (type) {\r
+ case MLX4_EVENT_TYPE_PATH_MIG:\r
+ event.type = IB_EVENT_PATH_MIG;\r
+ break;\r
+ case MLX4_EVENT_TYPE_COMM_EST:\r
+ event.type = IB_EVENT_COMM_EST;\r
+ break;\r
+ case MLX4_EVENT_TYPE_SQ_DRAINED:\r
+ event.type = IB_EVENT_SQ_DRAINED;\r
+ break;\r
+ case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:\r
+ event.type = IB_EVENT_QP_LAST_WQE_REACHED;\r
+ break;\r
+ case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:\r
+ event.type = IB_EVENT_QP_FATAL;\r
+ break;\r
+ case MLX4_EVENT_TYPE_PATH_MIG_FAILED:\r
+ event.type = IB_EVENT_PATH_MIG_ERR;\r
+ break;\r
+ case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:\r
+ event.type = IB_EVENT_QP_REQ_ERR;\r
+ break;\r
+ case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:\r
+ event.type = IB_EVENT_QP_ACCESS_ERR;\r
+ break;\r
+ default:\r
+ printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "\r
+ "on QP %06x\n", type, qp->qpn);\r
+ return;\r
+ }\r
+\r
+ event.context = ibqp->qp_context;\r
+ ibqp->event_handler(&event);\r
+}\r
+\r
+static int send_wqe_overhead(enum ib_qp_type type)\r
+{\r
+ /*\r
+ * UD WQEs must have a datagram segment.\r
+ * RC and UC WQEs might have a remote address segment.\r
+ * MLX WQEs need two extra inline data segments (for the UD\r
+ * header and space for the ICRC).\r
+ */\r
+ switch (type) {\r
+ case IB_QPT_UD:\r
+ return sizeof (struct mlx4_wqe_ctrl_seg) +\r
+ sizeof (struct mlx4_wqe_datagram_seg);\r
+ case IB_QPT_UC:\r
+ return sizeof (struct mlx4_wqe_ctrl_seg) +\r
+ sizeof (struct mlx4_wqe_raddr_seg);\r
+ case IB_QPT_RC:\r
+ return sizeof (struct mlx4_wqe_ctrl_seg) +\r
+ sizeof (struct mlx4_wqe_atomic_seg) +\r
+ sizeof (struct mlx4_wqe_raddr_seg);\r
+ case IB_QPT_SMI:\r
+ case IB_QPT_GSI:\r
+ return sizeof (struct mlx4_wqe_ctrl_seg) +\r
+ ALIGN(MLX4_IB_UD_HEADER_SIZE +\r
+ DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,\r
+ MLX4_INLINE_ALIGN) *\r
+ sizeof (struct mlx4_wqe_inline_seg),\r
+ sizeof (struct mlx4_wqe_data_seg)) +\r
+ ALIGN(4 +\r
+ sizeof (struct mlx4_wqe_inline_seg),\r
+ sizeof (struct mlx4_wqe_data_seg));\r
+ default:\r
+ return sizeof (struct mlx4_wqe_ctrl_seg);\r
+ }\r
+}\r
+\r
+static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,\r
+ int is_user, int has_srq, struct mlx4_ib_qp *qp)\r
+{\r
+ /* Sanity check RQ size before proceeding */\r
+ if ((int)cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||\r
+ (int)cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))\r
+ return -EINVAL;\r
+\r
+ if (has_srq) {\r
+ /* QPs attached to an SRQ should have no RQ */\r
+ if (cap->max_recv_wr)\r
+ return -EINVAL;\r
+\r
+ qp->rq.wqe_cnt = qp->rq.max_gs = 0;\r
+ } else {\r
+ /* HW requires >= 1 RQ entry with >= 1 gather entry */\r
+ if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))\r
+ return -EINVAL;\r
+\r
+ qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));\r
+ qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));\r
+ qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));\r
+ }\r
+\r
+ /* leave userspace return values as they were, so as not to break ABI */\r
+ if (is_user) {\r
+ cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;\r
+ cap->max_recv_sge = qp->rq.max_gs;\r
+ } else {\r
+ cap->max_recv_wr = qp->rq.max_post =\r
+ min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);\r
+ cap->max_recv_sge = min(qp->rq.max_gs,\r
+ min(dev->dev->caps.max_sq_sg,\r
+ dev->dev->caps.max_rq_sg));\r
+ }\r
+ /* We don't support inline sends for kernel QPs (yet) */\r
+\r
+ return 0;\r
+}\r
+\r
+static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,\r
+ enum ib_qp_type type, struct mlx4_ib_qp *qp)\r
+{\r
+ /* Sanity check SQ size before proceeding */\r
+ if ((int)cap->max_send_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||\r
+ (int)cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||\r
+ (int)cap->max_inline_data + send_wqe_overhead(type) +\r
+ (int)sizeof(struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)\r
+ return -EINVAL;\r
+\r
+ /*\r
+ * For MLX transport we need 2 extra S/G entries:\r
+ * one for the header and one for the checksum at the end\r
+ */\r
+ if ((type == IB_QPT_SMI || type == IB_QPT_GSI) &&\r
+ (int)cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)\r
+ return -EINVAL;\r
+\r
+ qp->sq.wqe_shift = ilog2(roundup_pow_of_two(max(cap->max_send_sge *\r
+ sizeof (struct mlx4_wqe_data_seg),\r
+ cap->max_inline_data +\r
+ sizeof (struct mlx4_wqe_inline_seg)) +\r
+ send_wqe_overhead(type)));\r
+ qp->sq.wqe_shift = max(MLX4_IB_SQ_MIN_WQE_SHIFT, qp->sq.wqe_shift);\r
+ qp->sq.max_gs = ((1 << qp->sq.wqe_shift) - send_wqe_overhead(type)) /\r
+ sizeof (struct mlx4_wqe_data_seg);\r
+\r
+ /*\r
+ * We need to leave 2 KB + 1 WQE of headroom in the SQ to\r
+ * allow HW to prefetch.\r
+ */\r
+ qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);\r
+ qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr + qp->sq_spare_wqes);\r
+\r
+ qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +\r
+ (qp->sq.wqe_cnt << qp->sq.wqe_shift);\r
+ if (qp->rq.wqe_shift > qp->sq.wqe_shift) {\r
+ qp->rq.offset = 0;\r
+ qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;\r
+ } else {\r
+ qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;\r
+ qp->sq.offset = 0;\r
+ }\r
+\r
+ cap->max_send_wr = qp->sq.max_post =\r
+ min(qp->sq.wqe_cnt - qp->sq_spare_wqes,\r
+ dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE);\r
+ cap->max_send_sge = min(qp->sq.max_gs,\r
+ min(dev->dev->caps.max_sq_sg,\r
+ dev->dev->caps.max_rq_sg));\r
+ /* We don't support inline sends for kernel QPs (yet) */\r
+ cap->max_inline_data = 0;\r
+\r
+ return 0;\r
+}\r
+\r
+static int set_user_sq_size(struct mlx4_ib_dev *dev,\r
+ struct mlx4_ib_qp *qp,\r
+ struct mlx4_ib_create_qp *ucmd)\r
+{\r
+ /* Sanity check SQ size before proceeding */\r
+ if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||\r
+ ucmd->log_sq_stride >\r
+ ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||\r
+ ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)\r
+ return -EINVAL;\r
+\r
+ qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;\r
+ qp->sq.wqe_shift = ucmd->log_sq_stride;\r
+\r
+ qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +\r
+ (qp->sq.wqe_cnt << qp->sq.wqe_shift);\r
+\r
+ return 0;\r
+}\r
+\r
+static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,\r
+ struct ib_qp_init_attr *init_attr,\r
+ struct ib_udata *udata, u32 sqpn, struct mlx4_ib_qp *qp)\r
+{\r
+ int err;\r
+\r
+ mutex_init(&qp->mutex);\r
+ spin_lock_init(&qp->sq.lock);\r
+ spin_lock_init(&qp->rq.lock);\r
+\r
+ qp->state = XIB_QPS_RESET;\r
+ qp->atomic_rd_en = 0;\r
+ qp->resp_depth = 0;\r
+\r
+ qp->rq.head = 0;\r
+ qp->rq.tail = 0;\r
+ qp->sq.head = 0;\r
+ qp->sq.tail = 0;\r
+\r
+ err = set_rq_size(dev, &init_attr->cap, !!pd->p_uctx, !!init_attr->srq, qp);\r
+ if (err)\r
+ goto err;\r
+\r
+ if (pd->p_uctx) {\r
+ struct mlx4_ib_create_qp ucmd;\r
+\r
+ if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {\r
+ err = -EFAULT;\r
+ goto err;\r
+ }\r
+\r
+ qp->sq_no_prefetch = ucmd.sq_no_prefetch;\r
+\r
+ err = set_user_sq_size(dev, qp, &ucmd);\r
+ if (err)\r
+ goto err;\r
+\r
+ qp->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr,\r
+ qp->buf_size, 0, FALSE);\r
+ if (IS_ERR(qp->umem)) {\r
+ err = PTR_ERR(qp->umem);\r
+ goto err;\r
+ }\r
+\r
+ err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),\r
+ ilog2(qp->umem->page_size), &qp->mtt);\r
+ if (err)\r
+ goto err_buf;\r
+\r
+ err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ if (!init_attr->srq) {\r
+ err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx),\r
+ ucmd.db_addr, &qp->db);\r
+ if (err)\r
+ goto err_mtt;\r
+ }\r
+ } else {\r
+ qp->sq_no_prefetch = 0;\r
+\r
+ err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp);\r
+ if (err)\r
+ goto err;\r
+\r
+ if (!init_attr->srq) {\r
+ err = mlx4_ib_db_alloc(dev, &qp->db, 0);\r
+ if (err)\r
+ goto err;\r
+\r
+ *qp->db.db = 0;\r
+ }\r
+\r
+ if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) {\r
+ err = -ENOMEM;\r
+ goto err_db;\r
+ }\r
+\r
+ err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,\r
+ &qp->mtt);\r
+ if (err)\r
+ goto err_buf;\r
+\r
+ err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL);\r
+ qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL);\r
+\r
+ if (!qp->sq.wrid || !qp->rq.wrid) {\r
+ err = -ENOMEM;\r
+ goto err_wrid;\r
+ }\r
+ }\r
+\r
+ if (!sqpn)\r
+ err = mlx4_qp_reserve_range(dev->dev, 1, 1, &sqpn);\r
+ if (err)\r
+ goto err_wrid;\r
+\r
+ err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp);\r
+ if (err)\r
+ goto err_wrid;\r
+\r
+ /*\r
+ * Hardware wants QPN written in big-endian order (after\r
+ * shifting) for send doorbell. Precompute this value to save\r
+ * a little bit when posting sends.\r
+ */\r
+ qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);\r
+\r
+ if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)\r
+ qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);\r
+ else\r
+ qp->sq_signal_bits = 0;\r
+\r
+ qp->mqp.event = mlx4_ib_qp_event;\r
+\r
+ return 0;\r
+\r
+err_wrid:\r
+ if (pd->p_uctx) {\r
+ if (!init_attr->srq)\r
+ mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx),\r
+ &qp->db);\r
+ } else {\r
+ kfree(qp->sq.wrid);\r
+ kfree(qp->rq.wrid);\r
+ }\r
+\r
+err_mtt:\r
+ mlx4_mtt_cleanup(dev->dev, &qp->mtt);\r
+\r
+err_buf:\r
+ if (pd->p_uctx)\r
+ ib_umem_release(qp->umem);\r
+ else\r
+ mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);\r
+\r
+err_db:\r
+ if (!pd->p_uctx && !init_attr->srq)\r
+ mlx4_ib_db_free(dev, &qp->db);\r
+\r
+err:\r
+ return err;\r
+}\r
+\r
+static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)\r
+{\r
+ switch (state) {\r
+ case XIB_QPS_RESET: return MLX4_QP_STATE_RST;\r
+ case XIB_QPS_INIT: return MLX4_QP_STATE_INIT;\r
+ case XIB_QPS_RTR: return MLX4_QP_STATE_RTR;\r
+ case XIB_QPS_RTS: return MLX4_QP_STATE_RTS;\r
+ case XIB_QPS_SQD: return MLX4_QP_STATE_SQD;\r
+ case XIB_QPS_SQE: return MLX4_QP_STATE_SQER;\r
+ case XIB_QPS_ERR: return MLX4_QP_STATE_ERR;\r
+ default: return -1;\r
+ }\r
+}\r
+\r
+static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)\r
+{\r
+ if (send_cq == recv_cq)\r
+ spin_lock_irq(&send_cq->lock);\r
+ else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {\r
+ spin_lock_irq(&send_cq->lock);\r
+ spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);\r
+ } else {\r
+ spin_lock_irq(&recv_cq->lock);\r
+ spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);\r
+ }\r
+}\r
+\r
+static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)\r
+{\r
+ if (send_cq == recv_cq)\r
+ spin_unlock_irq(&send_cq->lock);\r
+ else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {\r
+ spin_unlock(&recv_cq->lock);\r
+ spin_unlock_irq(&send_cq->lock);\r
+ } else {\r
+ spin_unlock(&send_cq->lock);\r
+ spin_unlock_irq(&recv_cq->lock);\r
+ }\r
+}\r
+\r
+static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,\r
+ int is_user)\r
+{\r
+ struct mlx4_ib_cq *send_cq, *recv_cq;\r
+\r
+ if (qp->state != XIB_QPS_RESET)\r
+ if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),\r
+ MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))\r
+ printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n",\r
+ qp->mqp.qpn);\r
+\r
+ send_cq = to_mcq(qp->ibqp.send_cq);\r
+ recv_cq = to_mcq(qp->ibqp.recv_cq);\r
+\r
+ mlx4_ib_lock_cqs(send_cq, recv_cq);\r
+\r
+ if (!is_user) {\r
+ __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,\r
+ qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);\r
+ if (send_cq != recv_cq)\r
+ __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);\r
+ }\r
+\r
+ mlx4_qp_remove(dev->dev, &qp->mqp);\r
+\r
+ mlx4_ib_unlock_cqs(send_cq, recv_cq);\r
+\r
+ mlx4_qp_free(dev->dev, &qp->mqp);\r
+ mlx4_mtt_cleanup(dev->dev, &qp->mtt);\r
+\r
+ if (is_user) {\r
+ if (!qp->ibqp.srq)\r
+ mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.p_uctx),\r
+ &qp->db);\r
+ ib_umem_release(qp->umem);\r
+ } else {\r
+ kfree(qp->sq.wrid);\r
+ kfree(qp->rq.wrid);\r
+ mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);\r
+ if (!qp->ibqp.srq)\r
+ mlx4_ib_db_free(dev, &qp->db);\r
+ }\r
+}\r
+\r
+struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,\r
+ struct ib_qp_init_attr *init_attr,\r
+ struct ib_udata *udata)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(pd->device);\r
+ struct mlx4_ib_sqp *sqp;\r
+ struct mlx4_ib_qp *qp;\r
+ int err;\r
+\r
+ if (mlx4_is_barred(pd->device->dma_device))\r
+ return ERR_PTR(-EFAULT);\r
+\r
+ switch (init_attr->qp_type) {\r
+ case IB_QPT_RC:\r
+ case IB_QPT_UC:\r
+ case IB_QPT_UD:\r
+ {\r
+ qp = kzalloc(sizeof *qp, GFP_KERNEL);\r
+ if (!qp)\r
+ return ERR_PTR(-ENOMEM);\r
+\r
+ err = create_qp_common(dev, pd, init_attr, udata, 0, qp);\r
+ if (err) {\r
+ kfree(qp);\r
+ return ERR_PTR(err);\r
+ }\r
+\r
+ qp->ibqp.qp_num = qp->mqp.qpn;\r
+\r
+ break;\r
+ }\r
+ case IB_QPT_SMI:\r
+ case IB_QPT_GSI:\r
+ {\r
+ /* Userspace is not allowed to create special QPs: */\r
+ if (pd->p_uctx)\r
+ return ERR_PTR(-EINVAL);\r
+\r
+ sqp = kzalloc(sizeof *sqp, GFP_KERNEL);\r
+ if (!sqp)\r
+ return ERR_PTR(-ENOMEM);\r
+\r
+ qp = &sqp->qp;\r
+\r
+ err = create_qp_common(dev, pd, init_attr, udata,\r
+ dev->dev->caps.sqp_start +\r
+ (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) +\r
+ init_attr->port_num - 1,\r
+ qp);\r
+ if (err) {\r
+ kfree(sqp);\r
+ return ERR_PTR(err);\r
+ }\r
+\r
+ qp->port = init_attr->port_num;\r
+ qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;\r
+\r
+ break;\r
+ }\r
+ default:\r
+ /* Don't support raw QPs */\r
+ return ERR_PTR(-EINVAL);\r
+ }\r
+\r
+ return &qp->ibqp;\r
+}\r
+\r
+int mlx4_ib_destroy_qp(struct ib_qp *qp)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(qp->device);\r
+ struct mlx4_ib_qp *mqp = to_mqp(qp);\r
+\r
+ if (!mlx4_is_barred(dev->dev) && is_qp0(dev, mqp))\r
+ mlx4_CLOSE_PORT(dev->dev, mqp->port);\r
+\r
+ destroy_qp_common(dev, mqp, !!qp->pd->p_uctx);\r
+\r
+ if (is_sqp(dev, mqp))\r
+ kfree(to_msqp(mqp));\r
+ else\r
+ kfree(mqp);\r
+\r
+ return 0;\r
+}\r
+\r
+static int to_mlx4_st(enum ib_qp_type type)\r
+{\r
+ switch (type) {\r
+ case IB_QPT_RC: return MLX4_QP_ST_RC;\r
+ case IB_QPT_UC: return MLX4_QP_ST_UC;\r
+ case IB_QPT_UD: return MLX4_QP_ST_UD;\r
+ case IB_QPT_SMI:\r
+ case IB_QPT_GSI: return MLX4_QP_ST_MLX;\r
+ default: return -1;\r
+ }\r
+}\r
+\r
+static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,\r
+ int attr_mask)\r
+{\r
+ u8 dest_rd_atomic;\r
+ u32 access_flags;\r
+ u32 hw_access_flags = 0;\r
+\r
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)\r
+ dest_rd_atomic = attr->max_dest_rd_atomic;\r
+ else\r
+ dest_rd_atomic = qp->resp_depth;\r
+\r
+ if (attr_mask & IB_QP_ACCESS_FLAGS)\r
+ access_flags = attr->qp_access_flags;\r
+ else\r
+ access_flags = qp->atomic_rd_en;\r
+\r
+ if (!dest_rd_atomic)\r
+ access_flags &= IB_ACCESS_REMOTE_WRITE;\r
+\r
+ if (access_flags & IB_ACCESS_REMOTE_READ)\r
+ hw_access_flags |= MLX4_QP_BIT_RRE;\r
+ if (access_flags & IB_ACCESS_REMOTE_ATOMIC)\r
+ hw_access_flags |= MLX4_QP_BIT_RAE;\r
+ if (access_flags & IB_ACCESS_REMOTE_WRITE)\r
+ hw_access_flags |= MLX4_QP_BIT_RWE;\r
+\r
+ return cpu_to_be32(hw_access_flags);\r
+}\r
+\r
+static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,\r
+ int attr_mask)\r
+{\r
+ if (attr_mask & IB_QP_PKEY_INDEX)\r
+ sqp->pkey_index = attr->pkey_index;\r
+ if (attr_mask & IB_QP_QKEY)\r
+ sqp->qkey = attr->qkey;\r
+ if (attr_mask & IB_QP_SQ_PSN)\r
+ sqp->send_psn = attr->sq_psn;\r
+}\r
+\r
+static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)\r
+{\r
+ path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);\r
+}\r
+\r
+static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,\r
+ struct mlx4_qp_path *path, u8 port)\r
+{\r
+ path->grh_mylmc = ah->src_path_bits & 0x7f;\r
+ path->rlid = cpu_to_be16(ah->dlid);\r
+ if (ah->static_rate) {\r
+ path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;\r
+ while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&\r
+ !(1 << path->static_rate & dev->dev->caps.stat_rate_support))\r
+ --path->static_rate;\r
+ } else\r
+ path->static_rate = 0;\r
+ path->counter_index = 0xff;\r
+\r
+ if (ah->ah_flags & IB_AH_GRH) {\r
+ if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {\r
+ printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",\r
+ ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);\r
+ return -1;\r
+ }\r
+\r
+ path->grh_mylmc |= 1 << 7;\r
+ path->mgid_index = ah->grh.sgid_index;\r
+ path->hop_limit = ah->grh.hop_limit;\r
+ path->tclass_flowlabel =\r
+ cpu_to_be32((ah->grh.traffic_class << 20) |\r
+ (ah->grh.flow_label));\r
+ memcpy(path->rgid, ah->grh.dgid.raw, 16);\r
+ }\r
+\r
+ path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |\r
+ ((port - 1) << 6) | ((ah->sl & 0xf) << 2);\r
+\r
+ return 0;\r
+}\r
+\r
+static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,\r
+ const struct ib_qp_attr *attr, int attr_mask,\r
+ enum ib_qp_state cur_state, enum ib_qp_state new_state)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(ibqp->device);\r
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);\r
+ struct mlx4_qp_context *context;\r
+ enum mlx4_qp_optpar optpar = 0;\r
+ int sqd_event;\r
+ int err = -EINVAL;\r
+\r
+ context = kzalloc(sizeof *context, GFP_KERNEL);\r
+ if (!context)\r
+ return -ENOMEM;\r
+\r
+ context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |\r
+ (to_mlx4_st(ibqp->qp_type) << 16));\r
+ context->flags |= cpu_to_be32(1 << 8); /* DE? */\r
+\r
+ if (!(attr_mask & IB_QP_PATH_MIG_STATE))\r
+ context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);\r
+ else {\r
+ optpar |= MLX4_QP_OPTPAR_PM_STATE;\r
+ switch (attr->path_mig_state) {\r
+ case IB_MIG_MIGRATED:\r
+ context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);\r
+ break;\r
+ case IB_MIG_REARM:\r
+ context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);\r
+ break;\r
+ case IB_MIG_ARMED:\r
+ context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);\r
+ break;\r
+ }\r
+ }\r
+\r
+ if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||\r
+ ibqp->qp_type == IB_QPT_UD)\r
+ context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;\r
+ else if (attr_mask & IB_QP_PATH_MTU) {\r
+ if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {\r
+ printk(KERN_ERR "path MTU (%u) is invalid\n",\r
+ attr->path_mtu);\r
+ goto out;\r
+ }\r
+ context->mtu_msgmax = (u8)((attr->path_mtu << 5) |\r
+ ilog2(dev->dev->caps.max_msg_sz));\r
+ }\r
+\r
+ if (qp->rq.wqe_cnt)\r
+ context->rq_size_stride = (u8)(ilog2(qp->rq.wqe_cnt) << 3);\r
+ context->rq_size_stride |= qp->rq.wqe_shift - 4;\r
+\r
+ if (qp->sq.wqe_cnt)\r
+ context->sq_size_stride = (u8)(ilog2(qp->sq.wqe_cnt) << 3);\r
+ context->sq_size_stride |= qp->sq.wqe_shift - 4;\r
+\r
+ if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT)\r
+ context->sq_size_stride |= !!qp->sq_no_prefetch << 7;\r
+\r
+ if (qp->ibqp.p_uctx)\r
+ context->usr_page = cpu_to_be32(to_mucontext(ibqp->p_uctx)->uar.index);\r
+ else\r
+ context->usr_page = cpu_to_be32(dev->priv_uar.index);\r
+\r
+ if (attr_mask & IB_QP_DEST_QPN)\r
+ context->remote_qpn = cpu_to_be32(attr->dest_qp_num);\r
+\r
+ if (attr_mask & IB_QP_PORT) {\r
+ if (cur_state == XIB_QPS_SQD && new_state == XIB_QPS_SQD &&\r
+ !(attr_mask & IB_QP_AV)) {\r
+ mlx4_set_sched(&context->pri_path, attr->port_num);\r
+ optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;\r
+ }\r
+ }\r
+\r
+ if (attr_mask & IB_QP_PKEY_INDEX) {\r
+ context->pri_path.pkey_index = (u8)attr->pkey_index;\r
+ optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_AV) {\r
+ if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,\r
+ attr_mask & IB_QP_PORT ? attr->port_num : qp->port))\r
+ goto out;\r
+\r
+ optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |\r
+ MLX4_QP_OPTPAR_SCHED_QUEUE);\r
+ }\r
+\r
+ if (attr_mask & IB_QP_TIMEOUT) {\r
+ context->pri_path.ackto = attr->timeout << 3;\r
+ optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_ALT_PATH) {\r
+ if (attr->alt_port_num == 0 ||\r
+ attr->alt_port_num > dev->dev->caps.num_ports)\r
+ goto out;\r
+\r
+ if (attr->alt_pkey_index >=\r
+ dev->dev->caps.pkey_table_len[attr->alt_port_num])\r
+ goto out;\r
+\r
+ if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,\r
+ attr->alt_port_num))\r
+ goto out;\r
+\r
+ context->alt_path.pkey_index = (u8)attr->alt_pkey_index;\r
+ context->alt_path.ackto = attr->alt_timeout << 3;\r
+ optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;\r
+ }\r
+\r
+ context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn);\r
+ context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);\r
+\r
+ if (attr_mask & IB_QP_RNR_RETRY) {\r
+ context->params1 |= cpu_to_be32(attr->rnr_retry << 13);\r
+ optpar |= MLX4_QP_OPTPAR_RNR_RETRY;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_RETRY_CNT) {\r
+ context->params1 |= cpu_to_be32(attr->retry_cnt << 16);\r
+ optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {\r
+ if (attr->max_rd_atomic)\r
+ context->params1 |=\r
+ cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);\r
+ optpar |= MLX4_QP_OPTPAR_SRA_MAX;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_SQ_PSN)\r
+ context->next_send_psn = cpu_to_be32(attr->sq_psn);\r
+\r
+ context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn);\r
+\r
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {\r
+ if (attr->max_dest_rd_atomic)\r
+ context->params2 |=\r
+ cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);\r
+ optpar |= MLX4_QP_OPTPAR_RRA_MAX;\r
+ }\r
+\r
+ if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {\r
+ context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);\r
+ optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;\r
+ }\r
+\r
+ if (ibqp->srq)\r
+ context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);\r
+\r
+ if (attr_mask & IB_QP_MIN_RNR_TIMER) {\r
+ context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);\r
+ optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;\r
+ }\r
+ if (attr_mask & IB_QP_RQ_PSN)\r
+ context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);\r
+\r
+ context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn);\r
+\r
+ if (attr_mask & IB_QP_QKEY) {\r
+ context->qkey = cpu_to_be32(attr->qkey);\r
+ optpar |= MLX4_QP_OPTPAR_Q_KEY;\r
+ }\r
+\r
+ if (ibqp->srq)\r
+ context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);\r
+\r
+ if (!ibqp->srq && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT)\r
+ context->db_rec_addr = cpu_to_be64(qp->db.dma.da);\r
+\r
+ if (cur_state == XIB_QPS_INIT &&\r
+ new_state == XIB_QPS_RTR &&\r
+ (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||\r
+ ibqp->qp_type == IB_QPT_UD)) {\r
+ context->pri_path.sched_queue = (qp->port - 1) << 6;\r
+ if (is_qp0(dev, qp))\r
+ context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;\r
+ else\r
+ context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;\r
+ }\r
+\r
+ if (cur_state == XIB_QPS_RTS && new_state == XIB_QPS_SQD &&\r
+ attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)\r
+ sqd_event = 1;\r
+ else\r
+ sqd_event = 0;\r
+\r
+ /*\r
+ * Before passing a kernel QP to the HW, make sure that the\r
+ * ownership bits of the send queue are set and the SQ\r
+ * headroom is stamped so that the hardware doesn't start\r
+ * processing stale work requests.\r
+ */\r
+ if (!ibqp->p_uctx && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) {\r
+ struct mlx4_wqe_ctrl_seg *ctrl;\r
+ int i;\r
+\r
+ for (i = 0; i < qp->sq.wqe_cnt; ++i) {\r
+ ctrl = get_send_wqe(qp, i);\r
+ ctrl->owner_opcode = cpu_to_be32(1 << 31);\r
+\r
+ stamp_send_wqe(qp, i);\r
+ }\r
+ }\r
+\r
+ err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),\r
+ to_mlx4_state(new_state), context, optpar,\r
+ sqd_event, &qp->mqp);\r
+ if (err)\r
+ goto out;\r
+\r
+ qp->state = new_state;\r
+\r
+ if (attr_mask & IB_QP_ACCESS_FLAGS)\r
+ qp->atomic_rd_en = (u8)attr->qp_access_flags;\r
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)\r
+ qp->resp_depth = attr->max_dest_rd_atomic;\r
+ if (attr_mask & IB_QP_PORT)\r
+ qp->port = attr->port_num;\r
+ if (attr_mask & IB_QP_ALT_PATH)\r
+ qp->alt_port = attr->alt_port_num;\r
+\r
+ if (is_sqp(dev, qp))\r
+ store_sqp_attrs(to_msqp(qp), attr, attr_mask);\r
+\r
+ /*\r
+ * If we moved QP0 to RTR, bring the IB link up; if we moved\r
+ * QP0 to RESET or ERROR, bring the link back down.\r
+ */\r
+ if (is_qp0(dev, qp)) {\r
+ if (cur_state != XIB_QPS_RTR && new_state == XIB_QPS_RTR)\r
+ if (mlx4_INIT_PORT(dev->dev, qp->port))\r
+ printk(KERN_WARNING "INIT_PORT failed for port %d\n",\r
+ qp->port);\r
+\r
+ if (cur_state != XIB_QPS_RESET && cur_state != XIB_QPS_ERR &&\r
+ (new_state == XIB_QPS_RESET || new_state == XIB_QPS_ERR))\r
+ mlx4_CLOSE_PORT(dev->dev, qp->port);\r
+ }\r
+\r
+ /*\r
+ * If we moved a kernel QP to RESET, clean up all old CQ\r
+ * entries and reinitialize the QP.\r
+ */\r
+ if (new_state == XIB_QPS_RESET && !ibqp->p_uctx) {\r
+ mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn,\r
+ ibqp->srq ? to_msrq(ibqp->srq): NULL);\r
+ if (ibqp->send_cq != ibqp->recv_cq)\r
+ mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL);\r
+\r
+ qp->rq.head = 0;\r
+ qp->rq.tail = 0;\r
+ qp->sq.head = 0;\r
+ qp->sq.tail = 0;\r
+ if (!ibqp->srq)\r
+ *qp->db.db = 0;\r
+ }\r
+\r
+out:\r
+ kfree(context);\r
+ return err;\r
+}\r
+\r
+static struct ib_qp_attr mlx4_ib_qp_attr;\r
+static int mlx4_ib_qp_attr_mask_table[IB_QPT_UD + 1];\r
+\r
+void mlx4_ib_qp_init()\r
+{\r
+ memset( &mlx4_ib_qp_attr, 0, sizeof(mlx4_ib_qp_attr) );\r
+ mlx4_ib_qp_attr.port_num = 1;\r
+\r
+ memset( &mlx4_ib_qp_attr_mask_table, 0, sizeof(mlx4_ib_qp_attr_mask_table) );\r
+ mlx4_ib_qp_attr_mask_table[IB_QPT_UD] = (IB_QP_PKEY_INDEX |\r
+ IB_QP_PORT |\r
+ IB_QP_QKEY);\r
+ mlx4_ib_qp_attr_mask_table[IB_QPT_UC] = (IB_QP_PKEY_INDEX |\r
+ IB_QP_PORT |\r
+ IB_QP_ACCESS_FLAGS);\r
+ mlx4_ib_qp_attr_mask_table[IB_QPT_RC] = (IB_QP_PKEY_INDEX |\r
+ IB_QP_PORT |\r
+ IB_QP_ACCESS_FLAGS);\r
+ mlx4_ib_qp_attr_mask_table[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |\r
+ IB_QP_QKEY);\r
+ mlx4_ib_qp_attr_mask_table[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |\r
+ IB_QP_QKEY);\r
+}\r
+\r
+int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,\r
+ int attr_mask, struct ib_udata *udata)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(ibqp->device);\r
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);\r
+ enum ib_qp_state cur_state, new_state;\r
+ int err = -EINVAL;\r
+\r
+ UNUSED_PARAM(udata);\r
+ \r
+ if (mlx4_is_barred(dev->dev))\r
+ return -EFAULT; \r
+\r
+ mutex_lock(&qp->mutex);\r
+\r
+ cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;\r
+ new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;\r
+\r
+ if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))\r
+ goto out;\r
+\r
+ if ((attr_mask & IB_QP_PORT) &&\r
+ (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {\r
+ goto out;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_PKEY_INDEX) {\r
+ int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;\r
+ if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])\r
+ goto out;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&\r
+ attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {\r
+ goto out;\r
+ }\r
+\r
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&\r
+ attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {\r
+ goto out;\r
+ }\r
+\r
+ if (cur_state == new_state && cur_state == XIB_QPS_RESET) {\r
+ err = 0;\r
+ goto out;\r
+ }\r
+\r
+ if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_ERR) {\r
+ err = __mlx4_ib_modify_qp(ibqp, &mlx4_ib_qp_attr,\r
+ mlx4_ib_qp_attr_mask_table[ibqp->qp_type],\r
+ XIB_QPS_RESET, XIB_QPS_INIT);\r
+ if (err)\r
+ goto out;\r
+ cur_state = XIB_QPS_INIT;\r
+ }\r
+\r
+ err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);\r
+\r
+out:\r
+ mutex_unlock(&qp->mutex);\r
+ return err;\r
+}\r
+\r
+static enum ib_wr_opcode to_wr_opcode(struct _ib_send_wr *wr)\r
+{\r
+\r
+ enum ib_wr_opcode opcode = -1; //= wr->wr_type;\r
+\r
+ switch (wr->wr_type) {\r
+ case WR_SEND: \r
+ opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_SEND_WITH_IMM : IB_WR_SEND;\r
+ break;\r
+ case WR_RDMA_WRITE: \r
+ opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_RDMA_WRITE_WITH_IMM : IB_WR_RDMA_WRITE;\r
+ break;\r
+ case WR_RDMA_READ: opcode = IB_WR_RDMA_READ; break;\r
+ case WR_COMPARE_SWAP: opcode = IB_WR_ATOMIC_CMP_AND_SWP; break;\r
+ case WR_FETCH_ADD: opcode = IB_WR_ATOMIC_FETCH_AND_ADD; break;\r
+ }\r
+ return opcode;\r
+}\r
+\r
+static int build_mlx_header(struct mlx4_ib_sqp *sqp, ib_send_wr_t *wr,\r
+ void *wqe)\r
+{\r
+ enum ib_wr_opcode opcode = to_wr_opcode(wr);\r
+ struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev;\r
+ struct mlx4_wqe_mlx_seg *mlx = wqe;\r
+ struct mlx4_wqe_inline_seg *inl = (void*)((u8*)wqe + sizeof *mlx);\r
+ struct mlx4_ib_ah *ah = to_mah((struct ib_ah *)wr->dgrm.ud.h_av);\r
+ __be16 pkey;\r
+ int send_size;\r
+ int header_size;\r
+ int spc;\r
+ u32 i;\r
+\r
+ send_size = 0;\r
+ for (i = 0; i < wr->num_ds; ++i)\r
+ send_size += wr->ds_array[i].length;\r
+\r
+ ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->ud_header);\r
+\r
+ sqp->ud_header.lrh.service_level =\r
+ (u8)(be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28);\r
+ sqp->ud_header.lrh.destination_lid = ah->av.dlid;\r
+ sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.g_slid & 0x7f);\r
+ if (mlx4_ib_ah_grh_present(ah)) {\r
+ sqp->ud_header.grh.traffic_class =\r
+ (u8)((be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20) & 0xff);\r
+ sqp->ud_header.grh.flow_label =\r
+ ah->av.sl_tclass_flowlabel & cpu_to_be32(0xfffff);\r
+ sqp->ud_header.grh.hop_limit = ah->av.hop_limit;\r
+ ib_get_cached_gid(ib_dev, (u8)(be32_to_cpu(ah->av.port_pd) >> 24),\r
+ ah->av.gid_index, &sqp->ud_header.grh.source_gid);\r
+ memcpy(sqp->ud_header.grh.destination_gid.raw,\r
+ ah->av.dgid, 16);\r
+ }\r
+\r
+ mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);\r
+ mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |\r
+ (sqp->ud_header.lrh.destination_lid ==\r
+ XIB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |\r
+ (sqp->ud_header.lrh.service_level << 8));\r
+ mlx->rlid = sqp->ud_header.lrh.destination_lid;\r
+\r
+ switch (opcode) {\r
+ case IB_WR_SEND:\r
+ sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;\r
+ sqp->ud_header.immediate_present = 0;\r
+ break;\r
+ case IB_WR_SEND_WITH_IMM:\r
+ sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;\r
+ sqp->ud_header.immediate_present = 1;\r
+ sqp->ud_header.immediate_data = wr->immediate_data;\r
+ break;\r
+ default:\r
+ return -EINVAL;\r
+ }\r
+\r
+ sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;\r
+ if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)\r
+ sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;\r
+ sqp->ud_header.bth.solicited_event = (u8)(!!(wr->send_opt & IB_SEND_OPT_SOLICITED));\r
+ if (!sqp->qp.ibqp.qp_num)\r
+ ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);\r
+ else\r
+ ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->dgrm.ud.pkey_index, &pkey);\r
+ sqp->ud_header.bth.pkey = pkey;\r
+ sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp;\r
+ sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));\r
+ sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ?\r
+ cpu_to_be32(sqp->qkey) : wr->dgrm.ud.remote_qkey;\r
+ sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);\r
+\r
+ header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);\r
+\r
+#if 0\r
+ {\r
+ printk(KERN_ERR "built UD header of size %d:\n", header_size);\r
+ for (i = 0; i < header_size / 4; ++i) {\r
+ if (i % 8 == 0)\r
+ printk(" [%02x] ", i * 4);\r
+ printk(" %08x",\r
+ be32_to_cpu(((__be32 *) sqp->header_buf)[i]));\r
+ if ((i + 1) % 8 == 0)\r
+ printk("\n");\r
+ }\r
+ printk("\n");\r
+ }\r
+#endif\r
+\r
+ /*\r
+ * Inline data segments may not cross a 64 byte boundary. If\r
+ * our UD header is bigger than the space available up to the\r
+ * next 64 byte boundary in the WQE, use two inline data\r
+ * segments to hold the UD header.\r
+ */\r
+ spc = MLX4_INLINE_ALIGN -\r
+ ((u32)(ULONG_PTR)(inl + 1) & (MLX4_INLINE_ALIGN - 1));\r
+ if (header_size <= spc) {\r
+ inl->byte_count = cpu_to_be32(1 << 31 | header_size);\r
+ memcpy(inl + 1, sqp->header_buf, header_size);\r
+ i = 1;\r
+ } else {\r
+ inl->byte_count = cpu_to_be32(1 << 31 | spc);\r
+ memcpy(inl + 1, sqp->header_buf, spc);\r
+\r
+ inl = (void*)((u8*)(inl + 1) + spc);\r
+ memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);\r
+ /*\r
+ * Need a barrier here to make sure all the data is\r
+ * visible before the byte_count field is set.\r
+ * Otherwise the HCA prefetcher could grab the 64-byte\r
+ * chunk with this inline segment and get a valid (!=\r
+ * 0xffffffff) byte count but stale data, and end up\r
+ * generating a packet with bad headers.\r
+ *\r
+ * The first inline segment's byte_count field doesn't\r
+ * need a barrier, because it comes after a\r
+ * control/MLX segment and therefore is at an offset\r
+ * of 16 mod 64.\r
+ */\r
+ wmb();\r
+ inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));\r
+ i = 2;\r
+ }\r
+\r
+ return ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);\r
+}\r
+\r
+static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)\r
+{\r
+ unsigned cur;\r
+ struct mlx4_ib_cq *cq;\r
+\r
+ cur = wq->head - wq->tail;\r
+ if (likely((int)cur + nreq < wq->max_post))\r
+ return 0;\r
+\r
+ cq = to_mcq(ib_cq);\r
+ spin_lock(&cq->lock);\r
+ cur = wq->head - wq->tail;\r
+ spin_unlock(&cq->lock);\r
+\r
+ return (int)cur + nreq >= wq->max_post;\r
+}\r
+\r
+static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,\r
+ u64 remote_addr, __be32 rkey)\r
+{\r
+ rseg->raddr = cpu_to_be64(remote_addr);\r
+ rseg->rkey = rkey;\r
+ rseg->reserved = 0;\r
+}\r
+\r
+static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, ib_send_wr_t *wr)\r
+{\r
+ if (wr->wr_type == WR_COMPARE_SWAP) {\r
+ aseg->swap_add = wr->remote_ops.atomic2;\r
+ aseg->compare = wr->remote_ops.atomic1;\r
+ } else {\r
+ aseg->swap_add = wr->remote_ops.atomic1;\r
+ aseg->compare = 0;\r
+ }\r
+\r
+}\r
+\r
+static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,\r
+ ib_send_wr_t *wr)\r
+{\r
+ memcpy(dseg->av, &to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, sizeof (struct mlx4_av));\r
+ dseg->dqpn = wr->dgrm.ud.remote_qp;\r
+ dseg->qkey = wr->dgrm.ud.remote_qkey;\r
+}\r
+\r
+static void set_mlx_icrc_seg(void *dseg)\r
+{\r
+ u32 *t = dseg;\r
+ struct mlx4_wqe_inline_seg *iseg = dseg;\r
+\r
+ t[1] = 0;\r
+\r
+ /*\r
+ * Need a barrier here before writing the byte_count field to\r
+ * make sure that all the data is visible before the\r
+ * byte_count field is set. Otherwise, if the segment begins\r
+ * a new cacheline, the HCA prefetcher could grab the 64-byte\r
+ * chunk and get a valid (!= * 0xffffffff) byte count but\r
+ * stale data, and end up sending the wrong data.\r
+ */\r
+ wmb();\r
+\r
+ iseg->byte_count = cpu_to_be32((1 << 31) | 4);\r
+}\r
+\r
+static void set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg)\r
+{\r
+ dseg->lkey = cpu_to_be32(sg->lkey);\r
+ dseg->addr = cpu_to_be64(sg->vaddr);\r
+\r
+ /*\r
+ * Need a barrier here before writing the byte_count field to\r
+ * make sure that all the data is visible before the\r
+ * byte_count field is set. Otherwise, if the segment begins\r
+ * a new cacheline, the HCA prefetcher could grab the 64-byte\r
+ * chunk and get a valid (!= * 0xffffffff) byte count but\r
+ * stale data, and end up sending the wrong data.\r
+ */\r
+ wmb();\r
+\r
+ dseg->byte_count = cpu_to_be32(sg->length);\r
+}\r
+\r
+static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg)\r
+{\r
+ dseg->byte_count = cpu_to_be32(sg->length);\r
+ dseg->lkey = cpu_to_be32(sg->lkey);\r
+ dseg->addr = cpu_to_be64(sg->vaddr);\r
+}\r
+\r
+int mlx4_ib_post_send(struct ib_qp *ibqp, ib_send_wr_t *wr,\r
+ ib_send_wr_t **bad_wr)\r
+{\r
+ enum ib_wr_opcode opcode;\r
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);\r
+ u8 *wqe;\r
+ struct mlx4_wqe_ctrl_seg *ctrl;\r
+ struct mlx4_wqe_data_seg *dseg;\r
+ unsigned long flags;\r
+ int nreq;\r
+ int err = 0;\r
+ int ind;\r
+ int size;\r
+ int i;\r
+\r
+ if (mlx4_is_barred(ibqp->device->dma_device))\r
+ return -EFAULT;\r
+\r
+ spin_lock_irqsave(&qp->sq.lock, &flags);\r
+\r
+ ind = qp->sq.head;\r
+\r
+ for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
+ if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {\r
+ err = -ENOMEM;\r
+ if (bad_wr)\r
+ *bad_wr = wr;\r
+ goto out;\r
+ }\r
+\r
+ if (unlikely(wr->num_ds > (u32)qp->sq.max_gs)) {\r
+ err = -EINVAL;\r
+ if (bad_wr)\r
+ *bad_wr = wr;\r
+ goto out;\r
+ }\r
+\r
+ wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));\r
+ ctrl = (void*)wqe;\r
+ qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;\r
+ opcode = to_wr_opcode(wr);\r
+\r
+ ctrl->srcrb_flags =\r
+ (wr->send_opt & IB_SEND_OPT_SIGNALED ?\r
+ cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |\r
+ (wr->send_opt & IB_SEND_OPT_SOLICITED ?\r
+ cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |\r
+ (wr->send_opt & IB_SEND_OPT_TX_IP_CSUM ?\r
+ cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM) : 0) |\r
+ (wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM ?\r
+ cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |\r
+ qp->sq_signal_bits;\r
+\r
+ if (opcode == IB_WR_SEND_WITH_IMM ||\r
+ opcode == IB_WR_RDMA_WRITE_WITH_IMM)\r
+ ctrl->imm = wr->immediate_data;\r
+ else\r
+ ctrl->imm = 0;\r
+\r
+ wqe += sizeof *ctrl;\r
+ size = sizeof *ctrl / 16;\r
+\r
+ switch (ibqp->qp_type) {\r
+ case IB_QPT_RC:\r
+ case IB_QPT_UC:\r
+ switch (opcode) {\r
+ case IB_WR_ATOMIC_CMP_AND_SWP:\r
+ case IB_WR_ATOMIC_FETCH_AND_ADD:\r
+ set_raddr_seg((void*)wqe, wr->remote_ops.vaddr,\r
+ wr->remote_ops.rkey);\r
+ wqe += sizeof (struct mlx4_wqe_raddr_seg);\r
+\r
+ set_atomic_seg((void*)wqe, wr);\r
+ wqe += sizeof (struct mlx4_wqe_atomic_seg);\r
+\r
+ size += (sizeof (struct mlx4_wqe_raddr_seg) +\r
+ sizeof (struct mlx4_wqe_atomic_seg)) / 16;\r
+\r
+ break;\r
+\r
+ case IB_WR_RDMA_READ:\r
+ case IB_WR_RDMA_WRITE:\r
+ case IB_WR_RDMA_WRITE_WITH_IMM:\r
+ set_raddr_seg((void*)wqe, wr->remote_ops.vaddr,\r
+ wr->remote_ops.rkey);\r
+ wqe += sizeof (struct mlx4_wqe_raddr_seg);\r
+ size += sizeof (struct mlx4_wqe_raddr_seg) / 16;\r
+ break;\r
+\r
+ default:\r
+ /* No extra segments required for sends */\r
+ break;\r
+ }\r
+ break;\r
+\r
+ case IB_QPT_UD:\r
+ set_datagram_seg((void*)wqe, wr);\r
+ wqe += sizeof (struct mlx4_wqe_datagram_seg);\r
+ size += sizeof (struct mlx4_wqe_datagram_seg) / 16;\r
+ break;\r
+\r
+ case IB_QPT_SMI:\r
+ case IB_QPT_GSI:\r
+ err = build_mlx_header(to_msqp(qp), wr, ctrl);\r
+ if (err < 0) {\r
+ if (bad_wr)\r
+ *bad_wr = wr;\r
+ goto out;\r
+ }\r
+ wqe += err;\r
+ size += err / 16;\r
+\r
+ err = 0;\r
+ break;\r
+\r
+ default:\r
+ break;\r
+ }\r
+\r
+ /*\r
+ * Write data segments in reverse order, so as to\r
+ * overwrite cacheline stamp last within each\r
+ * cacheline. This avoids issues with WQE\r
+ * prefetching.\r
+ */\r
+\r
+ dseg = (void*)wqe;\r
+ dseg += wr->num_ds - 1;\r
+ size += wr->num_ds * (sizeof (struct mlx4_wqe_data_seg) / 16);\r
+\r
+ /* Add one more inline data segment for ICRC for MLX sends */\r
+ if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI ||\r
+ qp->ibqp.qp_type == IB_QPT_GSI)) {\r
+ set_mlx_icrc_seg(dseg + 1);\r
+ size += sizeof (struct mlx4_wqe_data_seg) / 16;\r
+ }\r
+\r
+ for (i = wr->num_ds - 1; i >= 0; --i, --dseg)\r
+ set_data_seg(dseg, wr->ds_array + i);\r
+\r
+ ctrl->fence_size = (u8)((wr->send_opt & IB_SEND_OPT_FENCE ?\r
+ MLX4_WQE_CTRL_FENCE : 0) | size);\r
+\r
+ /*\r
+ * Make sure descriptor is fully written before\r
+ * setting ownership bit (because HW can start\r
+ * executing as soon as we do).\r
+ */\r
+ wmb();\r
+\r
+ if (opcode < 0 || opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {\r
+ err = -EINVAL;\r
+ goto out;\r
+ }\r
+\r
+ ctrl->owner_opcode = mlx4_ib_opcode[opcode] |\r
+ (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);\r
+\r
+ /*\r
+ * We can improve latency by not stamping the last\r
+ * send queue WQE until after ringing the doorbell, so\r
+ * only stamp here if there are still more WQEs to post.\r
+ */\r
+ if (wr->p_next)\r
+ stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) &\r
+ (qp->sq.wqe_cnt - 1));\r
+\r
+ ++ind;\r
+ }\r
+\r
+out:\r
+ if (likely(nreq)) {\r
+ qp->sq.head += nreq;\r
+\r
+ /*\r
+ * Make sure that descriptors are written before\r
+ * doorbell record.\r
+ */\r
+ wmb();\r
+\r
+ writel(qp->doorbell_qpn,\r
+ (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);\r
+\r
+#if 0\r
+ if (qp->mqp.qpn == 0x41)\r
+ DbgPrint( "[MLX4_BUS] mlx4_ib_post_send : qtype %d, qpn %#x, nreq %d, sq.head %#x, wqe_ix %d, db %p \n", \r
+ ibqp->qp_type, qp->mqp.qpn, nreq, qp->sq.head, ind, \r
+ (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL );\r
+#endif \r
+ /*\r
+ * Make sure doorbells don't leak out of SQ spinlock\r
+ * and reach the HCA out of order.\r
+ */\r
+ mmiowb();\r
+\r
+ stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) &\r
+ (qp->sq.wqe_cnt - 1));\r
+ }\r
+\r
+ spin_unlock_irqrestore(&qp->sq.lock, flags);\r
+\r
+ return err;\r
+}\r
+\r
+int mlx4_ib_post_recv(struct ib_qp *ibqp, ib_recv_wr_t *wr,\r
+ ib_recv_wr_t **bad_wr)\r
+{\r
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);\r
+ struct mlx4_wqe_data_seg *scat;\r
+ unsigned long flags;\r
+ int err = 0;\r
+ int nreq;\r
+ int ind;\r
+ int i;\r
+\r
+ if (mlx4_is_barred(ibqp->device->dma_device))\r
+ return -EFAULT;\r
+\r
+ spin_lock_irqsave(&qp->rq.lock, &flags);\r
+\r
+ ind = qp->rq.head & (qp->rq.wqe_cnt - 1);\r
+\r
+ for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
+ if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.send_cq)) {\r
+ err = -ENOMEM;\r
+ if (bad_wr)\r
+ *bad_wr = wr;\r
+ goto out;\r
+ }\r
+\r
+ if (unlikely(wr->num_ds > (u32)qp->rq.max_gs)) {\r
+ err = -EINVAL;\r
+ if (bad_wr)\r
+ *bad_wr = wr;\r
+ goto out;\r
+ }\r
+\r
+ scat = get_recv_wqe(qp, ind);\r
+\r
+ for (i = 0; i < (int)wr->num_ds; ++i)\r
+ __set_data_seg(scat + i, wr->ds_array + i);\r
+\r
+ if (i < qp->rq.max_gs) {\r
+ scat[i].byte_count = 0;\r
+ scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);\r
+ scat[i].addr = 0;\r
+ }\r
+\r
+ qp->rq.wrid[ind] = wr->wr_id;\r
+\r
+ ind = (ind + 1) & (qp->rq.wqe_cnt - 1);\r
+ }\r
+\r
+out:\r
+ if (likely(nreq)) {\r
+ qp->rq.head += nreq;\r
+\r
+ /*\r
+ * Make sure that descriptors are written before\r
+ * doorbell record.\r
+ */\r
+ wmb();\r
+\r
+ *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);\r
+\r
+#if 0\r
+ if (qp->mqp.qpn == 0x41)\r
+ DbgPrint( "[MLX4_BUS] mlx4_ib_post_recv : qtype %d, qpn %#x, nreq %d, rq.head %#x, wqe_ix %d, db_obj %p, db %p \n", \r
+ ibqp->qp_type, qp->mqp.qpn, nreq, qp->rq.head, ind, &qp->db, qp->db.db );\r
+#endif \r
+ }\r
+\r
+ spin_unlock_irqrestore(&qp->rq.lock, flags);\r
+\r
+ return err;\r
+}\r
+\r
+static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)\r
+{\r
+ switch (mlx4_state) {\r
+ case MLX4_QP_STATE_RST: return XIB_QPS_RESET;\r
+ case MLX4_QP_STATE_INIT: return XIB_QPS_INIT;\r
+ case MLX4_QP_STATE_RTR: return XIB_QPS_RTR;\r
+ case MLX4_QP_STATE_RTS: return XIB_QPS_RTS;\r
+ case MLX4_QP_STATE_SQ_DRAINING:\r
+ case MLX4_QP_STATE_SQD: return XIB_QPS_SQD;\r
+ case MLX4_QP_STATE_SQER: return XIB_QPS_SQE;\r
+ case MLX4_QP_STATE_ERR: return XIB_QPS_ERR;\r
+ default: return -1;\r
+ }\r
+}\r
+\r
+static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)\r
+{\r
+ switch (mlx4_mig_state) {\r
+ case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;\r
+ case MLX4_QP_PM_REARM: return IB_MIG_REARM;\r
+ case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;\r
+ default: return -1;\r
+ }\r
+}\r
+\r
+static int to_ib_qp_access_flags(int mlx4_flags)\r
+{\r
+ int ib_flags = 0;\r
+\r
+ if (mlx4_flags & MLX4_QP_BIT_RRE)\r
+ ib_flags |= IB_ACCESS_REMOTE_READ;\r
+ if (mlx4_flags & MLX4_QP_BIT_RWE)\r
+ ib_flags |= IB_ACCESS_REMOTE_WRITE;\r
+ if (mlx4_flags & MLX4_QP_BIT_RAE)\r
+ ib_flags |= IB_ACCESS_REMOTE_ATOMIC;\r
+\r
+ return ib_flags;\r
+}\r
+\r
+static void to_ib_ah_attr(struct mlx4_dev *dev, struct ib_ah_attr *ib_ah_attr,\r
+ struct mlx4_qp_path *path)\r
+{\r
+ memset(ib_ah_attr, 0, sizeof *ib_ah_attr);\r
+ ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1;\r
+\r
+ if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)\r
+ return;\r
+\r
+ ib_ah_attr->dlid = be16_to_cpu(path->rlid);\r
+ ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;\r
+ ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;\r
+ ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0;\r
+ ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;\r
+ if (ib_ah_attr->ah_flags) {\r
+ ib_ah_attr->grh.sgid_index = path->mgid_index;\r
+ ib_ah_attr->grh.hop_limit = path->hop_limit;\r
+ ib_ah_attr->grh.traffic_class =\r
+ (u8)((be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff);\r
+ ib_ah_attr->grh.flow_label =\r
+ be32_to_cpu(path->tclass_flowlabel) & 0xfffff;\r
+ memcpy(ib_ah_attr->grh.dgid.raw,\r
+ path->rgid, sizeof ib_ah_attr->grh.dgid.raw);\r
+ }\r
+}\r
+\r
+int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,\r
+ struct ib_qp_init_attr *qp_init_attr)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(ibqp->device);\r
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);\r
+ struct mlx4_qp_context context;\r
+ int mlx4_state;\r
+ int err;\r
+\r
+ UNUSED_PARAM(qp_attr_mask);\r
+\r
+ if (mlx4_is_barred(dev->dev))\r
+ return -EFAULT;\r
+ \r
+ if (qp->state == XIB_QPS_RESET) {\r
+ qp_attr->qp_state = XIB_QPS_RESET;\r
+ goto done;\r
+ }\r
+\r
+ err = mlx4_qp_query(dev->dev, &qp->mqp, &context);\r
+ if (err)\r
+ return -EINVAL;\r
+\r
+ mlx4_state = be32_to_cpu(context.flags) >> 28;\r
+\r
+ qp_attr->qp_state = to_ib_qp_state(mlx4_state);\r
+ qp_attr->path_mtu = context.mtu_msgmax >> 5;\r
+ qp_attr->path_mig_state =\r
+ to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);\r
+ qp_attr->qkey = be32_to_cpu(context.qkey);\r
+ qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;\r
+ qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;\r
+ qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;\r
+ qp_attr->qp_access_flags =\r
+ to_ib_qp_access_flags(be32_to_cpu(context.params2));\r
+\r
+ if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {\r
+ to_ib_ah_attr(dev->dev, &qp_attr->ah_attr, &context.pri_path);\r
+ to_ib_ah_attr(dev->dev, &qp_attr->alt_ah_attr, &context.alt_path);\r
+ qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;\r
+ qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;\r
+ }\r
+\r
+ qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;\r
+ if (qp_attr->qp_state == XIB_QPS_INIT)\r
+ qp_attr->port_num = qp->port;\r
+ else\r
+ qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;\r
+\r
+ /* qp_attr->en_sqd_async_notify is only applicable in modify qp */\r
+ qp_attr->sq_draining = (u8)(mlx4_state == MLX4_QP_STATE_SQ_DRAINING);\r
+\r
+ qp_attr->max_rd_atomic = (u8)(1 << ((be32_to_cpu(context.params1) >> 21) & 0x7));\r
+\r
+ qp_attr->max_dest_rd_atomic =\r
+ (u8)(1 << ((be32_to_cpu(context.params2) >> 21) & 0x7));\r
+ qp_attr->min_rnr_timer =\r
+ (u8)((be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f);\r
+ qp_attr->timeout = context.pri_path.ackto >> 3;\r
+ qp_attr->retry_cnt = (u8)((be32_to_cpu(context.params1) >> 16) & 0x7);\r
+ qp_attr->rnr_retry = (u8)((be32_to_cpu(context.params1) >> 13) & 0x7);\r
+ qp_attr->alt_timeout = context.alt_path.ackto >> 3;\r
+\r
+done:\r
+ qp_attr->cur_qp_state = qp_attr->qp_state;\r
+ qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;\r
+ qp_attr->cap.max_recv_sge = qp->rq.max_gs;\r
+\r
+ if (!ibqp->p_uctx) {\r
+ qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;\r
+ qp_attr->cap.max_send_sge = qp->sq.max_gs;\r
+ } else {\r
+ qp_attr->cap.max_send_wr = 0;\r
+ qp_attr->cap.max_send_sge = 0;\r
+ }\r
+\r
+ /*\r
+ * We don't support inline sends for kernel QPs (yet), and we\r
+ * don't know what userspace's value should be.\r
+ */\r
+ qp_attr->cap.max_inline_data = 0;\r
+\r
+ qp_init_attr->cap = qp_attr->cap;\r
+\r
+ return 0;\r
+}\r
+\r
-/*
- * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "mlx4_ib.h"
-#include "qp.h"
-#include "srq.h"
-#include "user.h"
-
-static void *get_wqe(struct mlx4_ib_srq *srq, int n)
-{
- int offset = n << srq->msrq.wqe_shift;
-
- if (srq->buf.nbufs == 1)
- return srq->buf.u.direct.buf + offset;
- else
- return srq->buf.u.page_list[offset >> PAGE_SHIFT].buf +
- (offset & (PAGE_SIZE - 1));
-}
-
-static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
-{
- ib_event_rec_t event;
- struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
-
- switch (type) {
- case MLX4_EVENT_TYPE_SRQ_LIMIT:
- event.type = IB_EVENT_SRQ_LIMIT_REACHED;
- break;
- case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
- event.type = IB_EVENT_SRQ_ERR;
- break;
- default:
- printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
- "on SRQ %06x\n", type, srq->srqn);
- return;
- }
-
- event.context = ibsrq->srq_context;
- ibsrq->event_handler(&event);
-}
-
-struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
- struct ib_srq_init_attr *init_attr,
- struct ib_udata *udata)
-{
- struct mlx4_ib_dev *dev = to_mdev(pd->device);
- struct mlx4_ib_srq *srq;
- struct mlx4_wqe_srq_next_seg *next;
- int desc_size;
- int buf_size;
- int err;
- int i;
- u32 cqn = 0;
- u16 xrcd = 0;
-
- /* Sanity check SRQ size before proceeding */
- if ((int)init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes ||
- (int)init_attr->attr.max_sge > dev->dev->caps.max_srq_sge)
- return ERR_PTR(-EINVAL);
-
- srq = kzalloc(sizeof *srq, GFP_KERNEL);
- if (!srq)
- return ERR_PTR(-ENOMEM);
-
- mutex_init(&srq->mutex);
- spin_lock_init(&srq->lock);
- srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1);
- srq->msrq.max_gs = init_attr->attr.max_sge;
-
- desc_size = max(32UL,
- roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) +
- srq->msrq.max_gs *
- sizeof (struct mlx4_wqe_data_seg)));
- srq->msrq.wqe_shift = ilog2(desc_size);
-
- buf_size = srq->msrq.max * desc_size;
-
- if (pd->p_uctx) {
- struct mlx4_ib_create_srq ucmd;
-
- if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
- err = -EFAULT;
- goto err_srq;
- }
-
- srq->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr,
- buf_size, 0, FALSE);
- if (IS_ERR(srq->umem)) {
- err = PTR_ERR(srq->umem);
- goto err_srq;
- }
-
- err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem),
- ilog2(srq->umem->page_size), &srq->mtt);
- if (err)
- goto err_buf;
-
- err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem);
- if (err)
- goto err_mtt;
-
- err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx),
- ucmd.db_addr, &srq->db);
- if (err)
- goto err_mtt;
- } else {
- err = mlx4_ib_db_alloc(dev, &srq->db, 0);
- if (err)
- goto err_srq;
-
- *srq->db.db = 0;
-
- if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) {
- err = -ENOMEM;
- goto err_db;
- }
-
- srq->head = 0;
- srq->tail = srq->msrq.max - 1;
- srq->wqe_ctr = 0;
-
- for (i = 0; i < srq->msrq.max; ++i) {
- next = get_wqe(srq, i);
- next->next_wqe_index =
- cpu_to_be16((i + 1) & (srq->msrq.max - 1));
- }
-
- err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift,
- &srq->mtt);
- if (err)
- goto err_buf;
-
- err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf);
- if (err)
- goto err_mtt;
-
- srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL);
- if (!srq->wrid) {
- err = -ENOMEM;
- goto err_mtt;
- }
- }
- err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcd, &srq->mtt,
- srq->db.dma.da, &srq->msrq);
- if (err)
- goto err_wrid;
-
- srq->msrq.event = mlx4_ib_srq_event;
-
- if (pd->p_uctx)
- if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
- err = -EFAULT;
- goto err_wrid;
- }
-
- init_attr->attr.max_wr = srq->msrq.max - 1;
-
- return &srq->ibsrq;
-
-err_wrid:
- if (pd->p_uctx)
- mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx), &srq->db);
- else
- kfree(srq->wrid);
-
-err_mtt:
- mlx4_mtt_cleanup(dev->dev, &srq->mtt);
-
-err_buf:
- if (pd->p_uctx)
- ib_umem_release(srq->umem);
- else
- mlx4_buf_free(dev->dev, buf_size, &srq->buf);
-
-err_db:
- if (!pd->p_uctx)
- mlx4_ib_db_free(dev, &srq->db);
-
-err_srq:
- kfree(srq);
-
- return ERR_PTR(err);
-}
-
-int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
- enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
-{
- struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);
- struct mlx4_ib_srq *srq = to_msrq(ibsrq);
- int ret;
-
- UNUSED_PARAM(udata);
-
- /* We don't support resizing SRQs (yet?) */
- if (attr_mask & XIB_SRQ_MAX_WR)
- return -ENOSYS;
-
- if (attr_mask & XIB_SRQ_LIMIT) {
- if ((int)attr->srq_limit >= srq->msrq.max)
- return -ERANGE;
-
- mutex_lock(&srq->mutex);
- ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit);
- mutex_unlock(&srq->mutex);
-
- if (ret)
- return ret;
- }
-
- return 0;
-}
-
-int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
-{
- struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);
- struct mlx4_ib_srq *srq = to_msrq(ibsrq);
- int ret;
- int limit_watermark;
-
- ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark);
- if (ret)
- return ret;
-
- srq_attr->srq_limit = limit_watermark;
- srq_attr->max_wr = srq->msrq.max - 1;
- srq_attr->max_sge = srq->msrq.max_gs;
-
- return 0;
-}
-
-int mlx4_ib_destroy_srq(struct ib_srq *srq)
-{
- struct mlx4_ib_dev *dev = to_mdev(srq->device);
- struct mlx4_ib_srq *msrq = to_msrq(srq);
-
- mlx4_srq_invalidate(dev->dev, &msrq->msrq);
- mlx4_srq_remove(dev->dev, &msrq->msrq);
-
- mlx4_srq_free(dev->dev, &msrq->msrq);
- mlx4_mtt_cleanup(dev->dev, &msrq->mtt);
-
- if (srq->p_uctx) {
- mlx4_ib_db_unmap_user(to_mucontext(srq->p_uctx), &msrq->db);
- ib_umem_release(msrq->umem);
- } else {
- kfree(msrq->wrid);
- mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,
- &msrq->buf);
- mlx4_ib_db_free(dev, &msrq->db);
- }
-
- kfree(msrq);
-
- return 0;
-}
-
-void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)
-{
- struct mlx4_wqe_srq_next_seg *next;
-
- /* always called with interrupts disabled. */
- spin_lock(&srq->lock);
-
- next = get_wqe(srq, srq->tail);
- next->next_wqe_index = cpu_to_be16(wqe_index);
- srq->tail = wqe_index;
-
- spin_unlock(&srq->lock);
-}
-
-int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, ib_recv_wr_t *wr,
- ib_recv_wr_t **bad_wr)
-{
- struct mlx4_ib_srq *srq = to_msrq(ibsrq);
- struct mlx4_wqe_srq_next_seg *next;
- struct mlx4_wqe_data_seg *scat;
- unsigned long flags;
- int err = 0;
- int nreq;
- int i;
-
- spin_lock_irqsave(&srq->lock, &flags);
-
- for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
- if (unlikely(wr->num_ds > (u32)srq->msrq.max_gs)) {
- err = -EINVAL;
- *bad_wr = wr;
- break;
- }
-
- if (unlikely(srq->head == srq->tail)) {
- err = -ENOMEM;
- *bad_wr = wr;
- break;
- }
-
- srq->wrid[srq->head] = wr->wr_id;
-
- next = get_wqe(srq, srq->head);
- srq->head = be16_to_cpu(next->next_wqe_index);
- scat = (struct mlx4_wqe_data_seg *) (next + 1);
-
- for (i = 0; i < (int)wr->num_ds; ++i) {
- scat[i].byte_count = cpu_to_be32(wr->ds_array[i].length);
- scat[i].lkey = cpu_to_be32(wr->ds_array[i].lkey);
- scat[i].addr = cpu_to_be64(wr->ds_array[i].vaddr);
- }
-
- if (i < srq->msrq.max_gs) {
- scat[i].byte_count = 0;
- scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
- scat[i].addr = 0;
- }
- }
-
- if (likely(nreq)) {
- srq->wqe_ctr = (u16)(srq->wqe_ctr + nreq);
-
- /*
- * Make sure that descriptors are written before
- * doorbell record.
- */
- wmb();
-
- *srq->db.db = cpu_to_be32(srq->wqe_ctr);
- }
-
- spin_unlock_irqrestore(&srq->lock, flags);
-
- return err;
-}
+/*\r
+ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses. You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include "mlx4_ib.h"\r
+#include "qp.h"\r
+#include "srq.h"\r
+#include "user.h"\r
+\r
+static void *get_wqe(struct mlx4_ib_srq *srq, int n)\r
+{\r
+ int offset = n << srq->msrq.wqe_shift;\r
+\r
+ if (srq->buf.nbufs == 1)\r
+ return srq->buf.u.direct.buf + offset;\r
+ else\r
+ return srq->buf.u.page_list[offset >> PAGE_SHIFT].buf +\r
+ (offset & (PAGE_SIZE - 1));\r
+}\r
+\r
+static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)\r
+{\r
+ ib_event_rec_t event;\r
+ struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;\r
+\r
+ switch (type) {\r
+ case MLX4_EVENT_TYPE_SRQ_LIMIT:\r
+ event.type = IB_EVENT_SRQ_LIMIT_REACHED;\r
+ break;\r
+ case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:\r
+ event.type = IB_EVENT_SRQ_ERR;\r
+ break;\r
+ default:\r
+ printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "\r
+ "on SRQ %06x\n", type, srq->srqn);\r
+ return;\r
+ }\r
+\r
+ event.context = ibsrq->srq_context;\r
+ ibsrq->event_handler(&event);\r
+}\r
+\r
+struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,\r
+ struct ib_srq_init_attr *init_attr,\r
+ struct ib_udata *udata)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(pd->device);\r
+ struct mlx4_ib_srq *srq;\r
+ struct mlx4_wqe_srq_next_seg *next;\r
+ int desc_size;\r
+ int buf_size;\r
+ int err;\r
+ int i;\r
+ u32 cqn = 0;\r
+ u16 xrcd = 0;\r
+\r
+ if (mlx4_is_barred(pd->device->dma_device))\r
+ return ERR_PTR(-EFAULT);\r
+\r
+ /* Sanity check SRQ size before proceeding */\r
+ if ((int)init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes ||\r
+ (int)init_attr->attr.max_sge > dev->dev->caps.max_srq_sge)\r
+ return ERR_PTR(-EINVAL);\r
+\r
+ srq = kzalloc(sizeof *srq, GFP_KERNEL);\r
+ if (!srq)\r
+ return ERR_PTR(-ENOMEM);\r
+\r
+ mutex_init(&srq->mutex);\r
+ spin_lock_init(&srq->lock);\r
+ srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1);\r
+ srq->msrq.max_gs = init_attr->attr.max_sge;\r
+\r
+ desc_size = max(32UL,\r
+ roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) +\r
+ srq->msrq.max_gs *\r
+ sizeof (struct mlx4_wqe_data_seg)));\r
+ srq->msrq.wqe_shift = ilog2(desc_size);\r
+\r
+ buf_size = srq->msrq.max * desc_size;\r
+\r
+ if (pd->p_uctx) {\r
+ struct mlx4_ib_create_srq ucmd;\r
+\r
+ if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {\r
+ err = -EFAULT;\r
+ goto err_srq;\r
+ }\r
+\r
+ srq->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr,\r
+ buf_size, 0, FALSE);\r
+ if (IS_ERR(srq->umem)) {\r
+ err = PTR_ERR(srq->umem);\r
+ goto err_srq;\r
+ }\r
+\r
+ err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem),\r
+ ilog2(srq->umem->page_size), &srq->mtt);\r
+ if (err)\r
+ goto err_buf;\r
+\r
+ err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx),\r
+ ucmd.db_addr, &srq->db);\r
+ if (err)\r
+ goto err_mtt;\r
+ } else {\r
+ err = mlx4_ib_db_alloc(dev, &srq->db, 0);\r
+ if (err)\r
+ goto err_srq;\r
+\r
+ *srq->db.db = 0;\r
+\r
+ if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) {\r
+ err = -ENOMEM;\r
+ goto err_db;\r
+ }\r
+\r
+ srq->head = 0;\r
+ srq->tail = srq->msrq.max - 1;\r
+ srq->wqe_ctr = 0;\r
+\r
+ for (i = 0; i < srq->msrq.max; ++i) {\r
+ next = get_wqe(srq, i);\r
+ next->next_wqe_index =\r
+ cpu_to_be16((i + 1) & (srq->msrq.max - 1));\r
+ }\r
+\r
+ err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift,\r
+ &srq->mtt);\r
+ if (err)\r
+ goto err_buf;\r
+\r
+ err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf);\r
+ if (err)\r
+ goto err_mtt;\r
+\r
+ srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL);\r
+ if (!srq->wrid) {\r
+ err = -ENOMEM;\r
+ goto err_mtt;\r
+ }\r
+ }\r
+ err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcd, &srq->mtt,\r
+ srq->db.dma.da, &srq->msrq);\r
+ if (err)\r
+ goto err_wrid;\r
+\r
+ srq->msrq.event = mlx4_ib_srq_event;\r
+\r
+ if (pd->p_uctx)\r
+ if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {\r
+ err = -EFAULT;\r
+ goto err_wrid;\r
+ }\r
+\r
+ init_attr->attr.max_wr = srq->msrq.max - 1;\r
+\r
+ return &srq->ibsrq;\r
+\r
+err_wrid:\r
+ if (pd->p_uctx)\r
+ mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx), &srq->db);\r
+ else\r
+ kfree(srq->wrid);\r
+\r
+err_mtt:\r
+ mlx4_mtt_cleanup(dev->dev, &srq->mtt);\r
+\r
+err_buf:\r
+ if (pd->p_uctx)\r
+ ib_umem_release(srq->umem);\r
+ else\r
+ mlx4_buf_free(dev->dev, buf_size, &srq->buf);\r
+\r
+err_db:\r
+ if (!pd->p_uctx)\r
+ mlx4_ib_db_free(dev, &srq->db);\r
+\r
+err_srq:\r
+ kfree(srq);\r
+\r
+ return ERR_PTR(err);\r
+}\r
+\r
+int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,\r
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);\r
+ struct mlx4_ib_srq *srq = to_msrq(ibsrq);\r
+ int ret;\r
+\r
+ UNUSED_PARAM(udata);\r
+\r
+ if (mlx4_is_barred(ibsrq->device->dma_device))\r
+ return -EFAULT;\r
+ \r
+ /* We don't support resizing SRQs (yet?) */\r
+ if (attr_mask & XIB_SRQ_MAX_WR)\r
+ return -ENOSYS;\r
+\r
+ if (attr_mask & XIB_SRQ_LIMIT) {\r
+ if ((int)attr->srq_limit >= srq->msrq.max)\r
+ return -ERANGE;\r
+\r
+ mutex_lock(&srq->mutex);\r
+ ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit);\r
+ mutex_unlock(&srq->mutex);\r
+\r
+ if (ret)\r
+ return ret;\r
+ }\r
+\r
+ return 0;\r
+}\r
+\r
+int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);\r
+ struct mlx4_ib_srq *srq = to_msrq(ibsrq);\r
+ int ret;\r
+ int limit_watermark;\r
+\r
+ if (mlx4_is_barred(ibsrq->device->dma_device))\r
+ return -EFAULT;\r
+\r
+ ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark);\r
+ if (ret)\r
+ return ret;\r
+\r
+ srq_attr->srq_limit = limit_watermark;\r
+ srq_attr->max_wr = srq->msrq.max - 1;\r
+ srq_attr->max_sge = srq->msrq.max_gs;\r
+\r
+ return 0;\r
+}\r
+\r
+int mlx4_ib_destroy_srq(struct ib_srq *srq)\r
+{\r
+ struct mlx4_ib_dev *dev = to_mdev(srq->device);\r
+ struct mlx4_ib_srq *msrq = to_msrq(srq);\r
+\r
+ if (!mlx4_is_barred(dev->dev))\r
+ mlx4_srq_invalidate(dev->dev, &msrq->msrq);\r
+ mlx4_srq_remove(dev->dev, &msrq->msrq);\r
+\r
+ mlx4_srq_free(dev->dev, &msrq->msrq);\r
+ mlx4_mtt_cleanup(dev->dev, &msrq->mtt);\r
+\r
+ if (srq->p_uctx) {\r
+ mlx4_ib_db_unmap_user(to_mucontext(srq->p_uctx), &msrq->db);\r
+ ib_umem_release(msrq->umem);\r
+ } else {\r
+ kfree(msrq->wrid);\r
+ mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,\r
+ &msrq->buf);\r
+ mlx4_ib_db_free(dev, &msrq->db);\r
+ }\r
+\r
+ kfree(msrq);\r
+\r
+ return 0;\r
+}\r
+\r
+void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)\r
+{\r
+ struct mlx4_wqe_srq_next_seg *next;\r
+\r
+ /* always called with interrupts disabled. */\r
+ spin_lock(&srq->lock);\r
+\r
+ next = get_wqe(srq, srq->tail);\r
+ next->next_wqe_index = cpu_to_be16(wqe_index);\r
+ srq->tail = wqe_index;\r
+\r
+ spin_unlock(&srq->lock);\r
+}\r
+\r
+int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, ib_recv_wr_t *wr,\r
+ ib_recv_wr_t **bad_wr)\r
+{\r
+ struct mlx4_ib_srq *srq = to_msrq(ibsrq);\r
+ struct mlx4_wqe_srq_next_seg *next;\r
+ struct mlx4_wqe_data_seg *scat;\r
+ unsigned long flags;\r
+ int err = 0;\r
+ int nreq;\r
+ int i;\r
+\r
+ if (mlx4_is_barred(ibsrq->device->dma_device))\r
+ return -EFAULT;\r
+\r
+ spin_lock_irqsave(&srq->lock, &flags);\r
+\r
+ for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
+ if (unlikely(wr->num_ds > (u32)srq->msrq.max_gs)) {\r
+ err = -EINVAL;\r
+ *bad_wr = wr;\r
+ break;\r
+ }\r
+\r
+ if (unlikely(srq->head == srq->tail)) {\r
+ err = -ENOMEM;\r
+ *bad_wr = wr;\r
+ break;\r
+ }\r
+\r
+ srq->wrid[srq->head] = wr->wr_id;\r
+\r
+ next = get_wqe(srq, srq->head);\r
+ srq->head = be16_to_cpu(next->next_wqe_index);\r
+ scat = (struct mlx4_wqe_data_seg *) (next + 1);\r
+\r
+ for (i = 0; i < (int)wr->num_ds; ++i) {\r
+ scat[i].byte_count = cpu_to_be32(wr->ds_array[i].length);\r
+ scat[i].lkey = cpu_to_be32(wr->ds_array[i].lkey);\r
+ scat[i].addr = cpu_to_be64(wr->ds_array[i].vaddr);\r
+ }\r
+\r
+ if (i < srq->msrq.max_gs) {\r
+ scat[i].byte_count = 0;\r
+ scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);\r
+ scat[i].addr = 0;\r
+ }\r
+ }\r
+\r
+ if (likely(nreq)) {\r
+ srq->wqe_ctr = (u16)(srq->wqe_ctr + nreq);\r
+\r
+ /*\r
+ * Make sure that descriptors are written before\r
+ * doorbell record.\r
+ */\r
+ wmb();\r
+\r
+ *srq->db.db = cpu_to_be32(srq->wqe_ctr);\r
+ }\r
+\r
+ spin_unlock_irqrestore(&srq->lock, flags);\r
+\r
+ return err;\r
+}\r
typedef int (*MLX4_REGISTER_EVENT_HANDLER) (struct ib_event_handler *event_handler);
typedef int (*MLX4_UNREGISTER_EVENT_HANDLER)(struct ib_event_handler *event_handler);
+typedef int (*MLX4_RESET_REQUEST) (struct ib_event_handler *event_handler);
+typedef int (*MLX4_RESET_EXECUTE) (struct ib_event_handler *event_handler);
struct mlx4_interface_ex {
MLX4_PD_ALLOC mlx4_pd_alloc;
MLX4_REGISTER_EVENT_HANDLER mlx4_register_ev_cb;
MLX4_UNREGISTER_EVENT_HANDLER mlx4_unregister_ev_cb;
+ MLX4_RESET_REQUEST mlx4_reset_request;
+ MLX4_RESET_EXECUTE mlx4_reset_execute;
};
struct mlx4_dev * pmlx4_dev;
struct mlx4_interface_ex mlx4_interface;
int is_livefish;
- MLX4_REGISTER_INTERFACE register_interface;
- MLX4_UNREGISTER_INTERFACE unregister_interface;
u8 port_id;
struct VipBusIfc *pVipBusIfc;
enum {
MLX4_FLAG_MSI_X = 1 << 0,
MLX4_FLAG_OLD_PORT_CMDS = 1 << 1,
- MLX4_FLAG_LIVEFISH = 1 << 10
+ MLX4_FLAG_LIVEFISH = 1 << 10,
+ MLX4_FLAG_RESET_CLIENT = 1 << 11,
+ MLX4_FLAG_RESET_DRIVER = 1 << 12,
+ MLX4_FLAG_RESET_STARTED = 1 << 13
};
enum {
IB_EVENT_LID_CHANGE = IB_AE_UNKNOWN + 1,\r
IB_EVENT_PKEY_CHANGE,\r
IB_EVENT_SM_CHANGE,\r
- IB_EVENT_CLIENT_REREGISTER\r
+ IB_EVENT_CLIENT_REREGISTER,\r
+ IB_EVENT_RESET_DRIVER, // device will be reset upon fatal error\r
+ IB_EVENT_RESET_CLIENT, // device will be upon client request\r
+ IB_EVENT_RESET_END // device has been reset \r
};\r
\r
struct ib_event {\r
} element;\r
enum ib_event_type event;\r
struct ib_event_ex x;\r
- };\r
+};\r
+\r
+enum ib_event_handler_flags {\r
+ IB_IVH_RESET_CB = (1 << 0),\r
+ IB_IVH_NOTIFIED = (1 << 1),\r
+ IB_IVH_RESET_READY = (1 << 2)\r
+};\r
+\r
\r
struct ib_event_handler {\r
struct ib_device *device;\r
void (*handler)(struct ib_event_handler *, struct ib_event *);\r
- void * ctx;\r
struct list_head list;\r
+ void * ctx;\r
+ void * rsrv_ptr;\r
+ u32 flags;\r
};\r
\r
-#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler, _ctx) \\r
+#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler, _ctx, _rptr, _flags) \\r
{ \\r
(_ptr)->device = _device; \\r
(_ptr)->handler = _handler; \\r
+ INIT_LIST_HEAD(&(_ptr)->list); \\r
(_ptr)->ctx = _ctx; \\r
- INIT_LIST_HEAD(&(_ptr)->list); \\r
+ (_ptr)->rsrv_ptr = _rptr; \\r
+ (_ptr)->flags = _flags; \\r
}\r
\r
struct ib_global_route {\r
{
int err;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
err = mlx4_db_alloc(dev, &wqres->db, 1);
if (err)
return err;
// "Reset device on internal errors if non-zero (default 1)")
int g_internal_err_reset = 1;
+static void dispatch_event(struct ib_device *ibdev, enum ib_event_type type)
+{
+ unsigned long flags;
+ struct ib_event event;
+ struct ib_event_handler *handler;
+
+ event.device = ibdev;
+ event.event = type;
+
+ spin_lock_irqsave(&ibdev->event_handler_lock, &flags);
+
+ list_for_each_entry(handler, &ibdev->event_handler_list, list, struct ib_event_handler)
+ {
+ // notify only those, that are not notified
+ if ( handler->flags & IB_IVH_RESET_CB )
+ if ( !(handler->flags & IB_IVH_NOTIFIED) ) {
+ handler->flags |= IB_IVH_NOTIFIED;
+ handler->handler(handler, &event);
+ }
+ }
+
+ spin_unlock_irqrestore(&ibdev->event_handler_lock, flags);
+}
+
+/**
+ * get_event_handlers - return list of handlers of the device
+ * @device:device
+ * @tlist:list
+ *
+ * get_event_handlers() remove all the device event handlers and put them in 'tlist'
+ */
+static void get_event_handlers(struct ib_device *device, struct list_head *tlist)
+{
+ unsigned long flags;
+ struct ib_event_handler *handler, *thandler;
+
+ spin_lock_irqsave(&device->event_handler_lock, &flags);
+
+ list_for_each_entry_safe(handler, thandler, &device->event_handler_list,
+ list, struct ib_event_handler, struct ib_event_handler)
+ {
+ // take out only reset callbacks
+ if ( handler->flags & IB_IVH_RESET_CB ) {
+ list_del( &handler->list );
+ list_add_tail( &handler->list, tlist );
+ }
+ }
+
+ spin_unlock_irqrestore(&device->event_handler_lock, flags);
+}
+
+
static void dump_err_buf(struct mlx4_dev *dev)
{
struct mlx4_priv *priv = mlx4_priv(dev);
mlx4_dispatch_event(dev, MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR, 0, 0);
+ // bar the device
+ dev->flags |= MLX4_FLAG_RESET_DRIVER;
+
+ // notify the clients
+ dispatch_event(dev->pdev->ib_dev, IB_EVENT_RESET_DRIVER);
+
if (g_internal_err_reset) {
PIO_WORKITEM catas_work = IoAllocateWorkItem( dev->pdev->p_self_do );
struct mlx4_priv *priv = mlx4_priv(dev);
spin_lock_irq(&catas_lock);
+ if (priv->catas_err.stop) {
+ spin_unlock_irq(&catas_lock);
+ return;
+ }
priv->catas_err.stop = 1;
spin_unlock_irq(&catas_lock);
spin_unlock_irq(&catas_lock);
}
+static int wait4reset(struct ib_event_handler *event_handler)
+{
+ int n_not_ready = 0;
+ unsigned long flags;
+ struct ib_event_handler *handler;
+ struct ib_device *ibdev = event_handler->device;
+
+ spin_lock_irqsave(&ibdev->event_handler_lock, &flags);
+
+ // mark this handler (=client) reset-ready
+ event_handler->flags |= IB_IVH_RESET_READY;
+
+ // check the number of still not ready client
+
+ list_for_each_entry(handler, &ibdev->event_handler_list, list, struct ib_event_handler)
+ if ( handler->flags & IB_IVH_RESET_CB )
+ if ( !(handler->flags & IB_IVH_RESET_READY) )
+ ++n_not_ready;
+
+ spin_unlock_irqrestore(&ibdev->event_handler_lock, flags);
+
+ return n_not_ready;
+}
+
+int mlx4_reset_execute( struct ib_event_handler *event_handler )
+{
+ int err;
+ struct ib_event event;
+ struct list_head tlist;
+ struct ib_event_handler *handler, *thandler;
+ struct ib_device *ibdev = event_handler->device;
+ struct pci_dev *pdev = ibdev->dma_device->pdev;
+
+ // mark client as "ready for reset" and check whether we can do reset
+ if (wait4reset(event_handler))
+ return 0;
+
+ // fully bar the device
+ ibdev->dma_device->flags |= MLX4_FLAG_RESET_STARTED;
+
+ // get old handler list
+ INIT_LIST_HEAD(&tlist);
+ get_event_handlers(ibdev, &tlist);
+
+ // restart the device
+ err = mlx4_restart_one(pdev);
+
+ // recreate interfaces
+ fix_bus_ifc(pdev);
+
+ // notify the clients
+ event.event = IB_EVENT_RESET_END;
+ list_for_each_entry_safe(handler, thandler, &tlist,
+ list, struct ib_event_handler, struct ib_event_handler)
+ {
+ // because 'handler' will be re-registered during the next call
+ list_del( &handler->list );
+ handler->handler(handler, &event);
+ }
+
+ return err;
+}
+
+static void
+card_reset_wi(
+ IN DEVICE_OBJECT* p_dev_obj,
+ IN struct ib_event_handler * event_handler )
+{
+ NTSTATUS status;
+ struct ib_device *ibdev = event_handler->device;
+ struct mlx4_dev *dev = ibdev->dma_device;
+
+ UNUSED_PARAM(p_dev_obj);
+ IoFreeWorkItem( event_handler->rsrv_ptr );
+
+ // reset the card
+ mlx4_stop_catas_poll( dev );
+ status = mlx4_reset( dev );
+ if ( !NT_SUCCESS( status ) )
+ mlx4_err( dev, "Failed to reset HCA, aborting. (status %#x)\n", status );
+
+ // notify the clients
+ dispatch_event(ibdev, IB_EVENT_RESET_CLIENT);
+}
+
+int mlx4_reset_request( struct ib_event_handler *event_handler )
+{
+ struct ib_device *ibdev = event_handler->device;
+ struct mlx4_dev *dev = ibdev->dma_device;
+
+ // set device to RESET_PENDING mode
+ if (!mlx4_is_barred(dev)) {
+ PIO_WORKITEM reset_work;
+
+ // bar the device
+ dev->flags |= MLX4_FLAG_RESET_CLIENT;
+
+ // delay reset to a system thread
+ // to allow for end of operations that are in progress
+ reset_work = IoAllocateWorkItem( dev->pdev->p_self_do );
+ if (!reset_work)
+ return -EFAULT;
+ event_handler->rsrv_ptr = reset_work;
+ IoQueueWorkItem( reset_work, card_reset_wi, DelayedWorkQueue, event_handler );
+ }
+
+ return 0;
+}
+
+int mlx4_reset_cb_register( struct ib_event_handler *event_handler )
+{
+ if (mlx4_is_in_reset(event_handler->device->dma_device))
+ return -EBUSY;
+
+ return ib_register_event_handler(event_handler);
+}
+
+int mlx4_reset_cb_unregister( struct ib_event_handler *event_handler )
+{
+ return ib_unregister_event_handler(event_handler);
+}
+
{
struct mlx4_cmd_mailbox *mailbox;
+ if ( mlx4_is_barred(dev) )
+ return ERR_PTR(-EFAULT);
+
mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL);
if (!mailbox)
return ERR_PTR(-ENOMEM);
int imlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm,
u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout)
{
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
return __mlx4_cmd(dev, in_param, out_param, out_is_imm, in_modifier,
op_modifier, op, timeout);
}
#define COLLAPSED_SHIFT 18
#define ENTRIES_SHIFT 24
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
if (cq->cqn == -1)
return -ENOMEM;
struct mlx4_cmd_mailbox *mailbox;
int err;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cq_table *cq_table = &priv->cq_table;
- int err;
+ int err = 0;
- err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
+ if (!mlx4_is_barred(dev))
+ err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
if (err)
mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
if (atomic_dec_and_test(&cq->refcount))
complete(&cq->free);
- wait_for_completion(&cq->free);
+ if (!mlx4_is_barred(dev))
+ wait_for_completion(&cq->free);
mlx4_table_put(dev, &cq_table->table, cq->cqn);
mlx4_bitmap_free(&cq_table->bitmap, cq->cqn);
UNREFERENCED_PARAMETER(intr);
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
for (i = MLX4_NUM_EQ; i < MLX4_NUM_EQ + MLX4_MAX_EXTRA_EQS ; i++) {
if(priv->eq_table.eq[MLX4_NUM_EQ].isr == NULL) {
new_eq = i;
u32 flags;
u16 field;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
#define INIT_PORT_IN_SIZE 256
#define INIT_PORT_FLAGS_OFFSET 0x00
int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port)
{
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000);
}
EXPORT_SYMBOL_GPL(mlx4_CLOSE_PORT);
HCA(MELLANOX, QDR, HERMON),
HCA(MELLANOX, DDR_G2, HERMON),
HCA(MELLANOX, QDR_G2, HERMON),
- HCA(MELLANOX, ETH_CONDOR, HERMON),
+ HCA(MELLANOX, ETH_CONDOR, HERMON),
+ HCA(MELLANOX, ETH_CONDOR_G2, HERMON),
HCA(MELLANOX, BD, LIVEFISH),
};
#define MLX4_PCI_TABLE_SIZE (sizeof(mlx4_pci_table)/sizeof(struct pci_device_id))
struct mlx4_priv *priv;
struct mlx4_dev *dev;
int err;
+ NTSTATUS status;
#ifdef FORCE_LIVEFISH
if (pdev)
* attempt a firmware command, since a boot ROM may have left
* the HCA in an undefined state.
*/
- err = mlx4_reset(dev);
- if (err) {
- mlx4_err(dev, "Failed to reset HCA, aborting.\n");
+ status = mlx4_reset(dev);
+ if ( !NT_SUCCESS( status ) ) {
+ mlx4_err(dev, "Failed to reset HCA, aborting.(status %#x)\n", status);
+ err = -EFAULT;
goto err_free_dev;
}
u8* p_eq_num, struct mlx4_eq ** p_eq);
void mlx4_remove_eq(struct mlx4_dev *dev, u8 eq_num);
+
+int mlx4_reset_execute( struct ib_event_handler *event_handler );
+
+int mlx4_reset_request( struct ib_event_handler *event_handler );
+
+int mlx4_reset_cb_register( struct ib_event_handler *event_handler );
+
+int mlx4_reset_cb_unregister( struct ib_event_handler *event_handler );
+
+void fix_bus_ifc(struct pci_dev *pdev);
+
+
+
#endif /* MLX4_H */
u32 index;
int err;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
index = mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap);
if (index == -1)
return -ENOMEM;
struct mlx4_priv *priv = mlx4_priv(dev);
int err;
- if (mr->enabled) {
+ if (!mlx4_is_barred(dev) && mr->enabled) {
err = mlx4_HW2SW_MPT(dev, NULL,
key_to_hw_index(mr->key) &
(dev->caps.num_mpts - 1));
struct mlx4_mpt_entry *mpt_entry;
int err;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
err = mlx4_table_get(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key));
if (err)
return err;
{
struct mlx4_priv *priv = mlx4_priv(dev);
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
*pdn = mlx4_bitmap_alloc(&priv->pd_bitmap);
if (*pdn == -1)
return -ENOMEM;
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar)
{
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap);
if (uar->index == -1)
return -ENOMEM;
-/*
- * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include "mlx4.h"
-#include "cmd.h"
-
-
-void mlx4_init_mac_table(struct mlx4_dev *dev, u8 port)
-{
- struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table;
- int i;
-
- sema_init(&table->mac_sem, 1);
- for (i = 0; i < MLX4_MAX_MAC_NUM; i++) {
- table->entries[i] = 0;
- table->refs[i] = 0;
- }
- table->max = 1 << dev->caps.log_num_macs;
- table->total = 0;
-}
-
-void mlx4_init_vlan_table(struct mlx4_dev *dev, u8 port)
-{
- struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table;
- int i;
-
- sema_init(&table->vlan_sem, 1);
- for (i = 0; i < MLX4_MAX_MAC_NUM; i++) {
- table->entries[i] = 0;
- table->refs[i] = 0;
- }
- table->max = 1 << dev->caps.log_num_vlans;
- table->total = 0;
-}
-
-static int mlx4_SET_PORT_mac_table(struct mlx4_dev *dev, u8 port,
- __be64 *entries)
-{
- struct mlx4_cmd_mailbox *mailbox;
- u32 in_mod;
- int err;
-
- mailbox = mlx4_alloc_cmd_mailbox(dev);
- if (IS_ERR(mailbox))
- return PTR_ERR(mailbox);
-
- memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE);
-
- in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port;
- err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT,
- MLX4_CMD_TIME_CLASS_B);
-
- mlx4_free_cmd_mailbox(dev, mailbox);
- return err;
-}
-
-int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index)
-{
- struct mlx4_mac_table *table =
- &mlx4_priv(dev)->port[port - 1].mac_table;
- int i, err = 0;
- int free = -1;
- u64 valid = 1;
-
- mlx4_dbg(dev, "Registering mac : 0x%llx\n", mac);
- down(&table->mac_sem);
- for (i = 0; i < MLX4_MAX_MAC_NUM - 1; i++) {
- if (free < 0 && !table->refs[i]) {
- free = i;
- continue;
- }
-
- if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) {
- /* Mac already registered, increase refernce count */
- *index = i;
- ++table->refs[i];
- goto out;
- }
- }
- mlx4_dbg(dev, "Free mac index is %d\n", free);
-
- if (table->total == table->max) {
- /* No free mac entries */
- err = -ENOSPC;
- goto out;
- }
-
- /* Register new MAC */
- table->refs[free] = 1;
- table->entries[free] = cpu_to_be64(mac | valid << MLX4_MAC_VALID_SHIFT);
-
- err = mlx4_SET_PORT_mac_table(dev, port, table->entries);
- if (unlikely(err)) {
- mlx4_err(dev, "Failed adding mac: 0x%llx\n", mac);
- table->refs[free] = 0;
- table->entries[free] = 0;
- goto out;
- }
-
- *index = free;
- ++table->total;
-out:
- up(&table->mac_sem);
- return err;
-}
-EXPORT_SYMBOL_GPL(mlx4_register_mac);
-
-void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index)
-{
- struct mlx4_mac_table *table =
- &mlx4_priv(dev)->port[port - 1].mac_table;
-
- down(&table->mac_sem);
- if (!table->refs[index]) {
- mlx4_warn(dev, "No mac entry for index %d\n", index);
- goto out;
- }
- if (--table->refs[index]) {
- mlx4_warn(dev, "Have more references for index %d,"
- "no need to modify mac table\n", index);
- goto out;
- }
- table->entries[index] = 0;
- mlx4_SET_PORT_mac_table(dev, port, table->entries);
- --table->total;
-out:
- up(&table->mac_sem);
-}
-EXPORT_SYMBOL_GPL(mlx4_unregister_mac);
-
-static int mlx4_SET_PORT_vlan_table(struct mlx4_dev *dev, u8 port,
- __be32 *entries)
-{
- struct mlx4_cmd_mailbox *mailbox;
- u32 in_mod;
- int err;
-
- mailbox = mlx4_alloc_cmd_mailbox(dev);
- if (IS_ERR(mailbox))
- return PTR_ERR(mailbox);
-
- memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE);
- in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port;
- err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT,
- MLX4_CMD_TIME_CLASS_B);
-
- mlx4_free_cmd_mailbox(dev, mailbox);
-
- return err;
-}
-
-int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index)
-{
- struct mlx4_vlan_table *table =
- &mlx4_priv(dev)->port[port - 1].vlan_table;
- int i, err = 0;
- int free = -1;
-
- down(&table->vlan_sem);
- for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) {
- if (free < 0 && (table->refs[i] == 0)) {
- free = i;
- continue;
- }
-
- if (table->refs[i] &&
- (vlan == (MLX4_VLAN_MASK &
- be32_to_cpu(table->entries[i])))) {
- /* Vlan already registered, increase refernce count */
- *index = i;
- ++table->refs[i];
- goto out;
- }
- }
-
- if (table->total == table->max) {
- /* No free vlan entries */
- err = -ENOSPC;
- goto out;
- }
-
- /* Register new MAC */
- table->refs[free] = 1;
- table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID);
-
- err = mlx4_SET_PORT_vlan_table(dev, port, table->entries);
- if (unlikely(err)) {
- mlx4_warn(dev, "Failed adding vlan: %u\n", vlan);
- table->refs[free] = 0;
- table->entries[free] = 0;
- goto out;
- }
-
- *index = free;
- ++table->total;
-out:
- up(&table->vlan_sem);
- return err;
-}
-EXPORT_SYMBOL_GPL(mlx4_register_vlan);
-
-void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index)
-{
- struct mlx4_vlan_table *table =
- &mlx4_priv(dev)->port[port - 1].vlan_table;
-
- down(&table->vlan_sem);
- if (!table->refs[index]) {
- mlx4_warn(dev, "No vlan entry for index %d\n", index);
- goto out;
- }
- if (--table->refs[index]) {
- mlx4_dbg(dev, "Have more references for index %d,"
- "no need to modify vlan table\n", index);
- goto out;
- }
- table->entries[index] = 0;
- mlx4_SET_PORT_vlan_table(dev, port, table->entries);
- --table->total;
-out:
- up(&table->vlan_sem);
-}
-EXPORT_SYMBOL_GPL(mlx4_unregister_vlan);
-
-int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port,
- int reset_qkey_viols, u32 cap_mask)
-{
- struct mlx4_cmd_mailbox *mailbox;
- int err;
- u8 is_eth = (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) ? 1 : 0;
-
- mailbox = mlx4_alloc_cmd_mailbox(dev);
- if (IS_ERR(mailbox))
- return PTR_ERR(mailbox);
-
- memset(mailbox->buf, 0, 256);
- if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
+/*\r
+ * Copyright (c) 2007 Mellanox Technologies. All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses. You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ */\r
+\r
+#include "mlx4.h"\r
+#include "cmd.h"\r
+\r
+\r
+void mlx4_init_mac_table(struct mlx4_dev *dev, u8 port)\r
+{\r
+ struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table;\r
+ int i;\r
+\r
+ sema_init(&table->mac_sem, 1);\r
+ for (i = 0; i < MLX4_MAX_MAC_NUM; i++) {\r
+ table->entries[i] = 0;\r
+ table->refs[i] = 0;\r
+ }\r
+ table->max = 1 << dev->caps.log_num_macs;\r
+ table->total = 0;\r
+}\r
+\r
+void mlx4_init_vlan_table(struct mlx4_dev *dev, u8 port)\r
+{\r
+ struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table;\r
+ int i;\r
+\r
+ sema_init(&table->vlan_sem, 1);\r
+ for (i = 0; i < MLX4_MAX_MAC_NUM; i++) {\r
+ table->entries[i] = 0;\r
+ table->refs[i] = 0;\r
+ }\r
+ table->max = 1 << dev->caps.log_num_vlans;\r
+ table->total = 0;\r
+}\r
+\r
+static int mlx4_SET_PORT_mac_table(struct mlx4_dev *dev, u8 port,\r
+ __be64 *entries)\r
+{\r
+ struct mlx4_cmd_mailbox *mailbox;\r
+ u32 in_mod;\r
+ int err;\r
+\r
+ mailbox = mlx4_alloc_cmd_mailbox(dev);\r
+ if (IS_ERR(mailbox))\r
+ return PTR_ERR(mailbox);\r
+\r
+ memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE);\r
+\r
+ in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port;\r
+ err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT,\r
+ MLX4_CMD_TIME_CLASS_B);\r
+\r
+ mlx4_free_cmd_mailbox(dev, mailbox);\r
+ return err;\r
+}\r
+\r
+int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index)\r
+{\r
+ struct mlx4_mac_table *table =\r
+ &mlx4_priv(dev)->port[port - 1].mac_table;\r
+ int i, err = 0;\r
+ int free = -1;\r
+ u64 valid = 1;\r
+\r
+ if ( mlx4_is_barred(dev) )\r
+ return -EFAULT;\r
+\r
+ mlx4_dbg(dev, "Registering mac : 0x%llx\n", mac);\r
+ down(&table->mac_sem);\r
+ for (i = 0; i < MLX4_MAX_MAC_NUM - 1; i++) {\r
+ if (free < 0 && !table->refs[i]) {\r
+ free = i;\r
+ continue;\r
+ }\r
+\r
+ if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) {\r
+ /* Mac already registered, increase refernce count */\r
+ *index = i;\r
+ ++table->refs[i];\r
+ goto out;\r
+ }\r
+ }\r
+ mlx4_dbg(dev, "Free mac index is %d\n", free);\r
+\r
+ if (table->total == table->max) {\r
+ /* No free mac entries */\r
+ err = -ENOSPC;\r
+ goto out;\r
+ }\r
+\r
+ /* Register new MAC */\r
+ table->refs[free] = 1;\r
+ table->entries[free] = cpu_to_be64(mac | valid << MLX4_MAC_VALID_SHIFT);\r
+\r
+ err = mlx4_SET_PORT_mac_table(dev, port, table->entries);\r
+ if (unlikely(err)) {\r
+ mlx4_err(dev, "Failed adding mac: 0x%llx\n", mac);\r
+ table->refs[free] = 0;\r
+ table->entries[free] = 0;\r
+ goto out;\r
+ }\r
+\r
+ *index = free;\r
+ ++table->total;\r
+out:\r
+ up(&table->mac_sem);\r
+ return err;\r
+}\r
+EXPORT_SYMBOL_GPL(mlx4_register_mac);\r
+\r
+void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index)\r
+{\r
+ struct mlx4_mac_table *table =\r
+ &mlx4_priv(dev)->port[port - 1].mac_table;\r
+\r
+ down(&table->mac_sem);\r
+ if (!table->refs[index]) {\r
+ mlx4_warn(dev, "No mac entry for index %d\n", index);\r
+ goto out;\r
+ }\r
+ if (--table->refs[index]) {\r
+ mlx4_warn(dev, "Have more references for index %d,"\r
+ "no need to modify mac table\n", index);\r
+ goto out;\r
+ }\r
+ table->entries[index] = 0;\r
+ if ( !mlx4_is_barred(dev) )\r
+ mlx4_SET_PORT_mac_table(dev, port, table->entries);\r
+ --table->total;\r
+out:\r
+ up(&table->mac_sem);\r
+}\r
+EXPORT_SYMBOL_GPL(mlx4_unregister_mac);\r
+\r
+static int mlx4_SET_PORT_vlan_table(struct mlx4_dev *dev, u8 port,\r
+ __be32 *entries)\r
+{\r
+ struct mlx4_cmd_mailbox *mailbox;\r
+ u32 in_mod;\r
+ int err;\r
+\r
+ mailbox = mlx4_alloc_cmd_mailbox(dev);\r
+ if (IS_ERR(mailbox))\r
+ return PTR_ERR(mailbox);\r
+\r
+ memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE);\r
+ in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port;\r
+ err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT,\r
+ MLX4_CMD_TIME_CLASS_B);\r
+\r
+ mlx4_free_cmd_mailbox(dev, mailbox);\r
+\r
+ return err;\r
+}\r
+\r
+int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index)\r
+{\r
+ struct mlx4_vlan_table *table =\r
+ &mlx4_priv(dev)->port[port - 1].vlan_table;\r
+ int i, err = 0;\r
+ int free = -1;\r
+\r
+ down(&table->vlan_sem);\r
+ for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) {\r
+ if (free < 0 && (table->refs[i] == 0)) {\r
+ free = i;\r
+ continue;\r
+ }\r
+\r
+ if (table->refs[i] &&\r
+ (vlan == (MLX4_VLAN_MASK &\r
+ be32_to_cpu(table->entries[i])))) {\r
+ /* Vlan already registered, increase refernce count */\r
+ *index = i;\r
+ ++table->refs[i];\r
+ goto out;\r
+ }\r
+ }\r
+\r
+ if (table->total == table->max) {\r
+ /* No free vlan entries */\r
+ err = -ENOSPC;\r
+ goto out;\r
+ }\r
+\r
+ /* Register new MAC */\r
+ table->refs[free] = 1;\r
+ table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID);\r
+\r
+ err = mlx4_SET_PORT_vlan_table(dev, port, table->entries);\r
+ if (unlikely(err)) {\r
+ mlx4_warn(dev, "Failed adding vlan: %u\n", vlan);\r
+ table->refs[free] = 0;\r
+ table->entries[free] = 0;\r
+ goto out;\r
+ }\r
+\r
+ *index = free;\r
+ ++table->total;\r
+out:\r
+ up(&table->vlan_sem);\r
+ return err;\r
+}\r
+EXPORT_SYMBOL_GPL(mlx4_register_vlan);\r
+\r
+void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index)\r
+{\r
+ struct mlx4_vlan_table *table =\r
+ &mlx4_priv(dev)->port[port - 1].vlan_table;\r
+\r
+ down(&table->vlan_sem);\r
+ if (!table->refs[index]) {\r
+ mlx4_warn(dev, "No vlan entry for index %d\n", index);\r
+ goto out;\r
+ }\r
+ if (--table->refs[index]) {\r
+ mlx4_dbg(dev, "Have more references for index %d,"\r
+ "no need to modify vlan table\n", index);\r
+ goto out;\r
+ }\r
+ table->entries[index] = 0;\r
+ mlx4_SET_PORT_vlan_table(dev, port, table->entries);\r
+ --table->total;\r
+out:\r
+ up(&table->vlan_sem);\r
+}\r
+EXPORT_SYMBOL_GPL(mlx4_unregister_vlan);\r
+\r
+int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, \r
+ int reset_qkey_viols, u32 cap_mask)\r
+{\r
+ struct mlx4_cmd_mailbox *mailbox;\r
+ int err;\r
+ u8 is_eth = (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) ? 1 : 0;\r
+\r
+ mailbox = mlx4_alloc_cmd_mailbox(dev);\r
+ if (IS_ERR(mailbox))\r
+ return PTR_ERR(mailbox);\r
+\r
+ memset(mailbox->buf, 0, 256);\r
+ if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {\r
*(u8 *) mailbox->buf = (u8)(!!reset_qkey_viols << 6);
((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask);
} else {
((u8 *) mailbox->buf)[3] = (u8)!!reset_qkey_viols;
((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask);
}
-
- if (is_eth) {
- ((u8 *) mailbox->buf)[3] = 7;
- ((__be16 *) mailbox->buf)[3] =
- cpu_to_be16(dev->caps.eth_mtu_cap[port] +
- ETH_HLEN + ETH_FCS_LEN);
- ((__be16 *) mailbox->buf)[4] = cpu_to_be16(1 << 15);
- ((__be16 *) mailbox->buf)[6] = cpu_to_be16(1 << 15);
- }
- err = mlx4_cmd(dev, mailbox->dma.da, port, is_eth, MLX4_CMD_SET_PORT,
- MLX4_CMD_TIME_CLASS_B);
-
- mlx4_free_cmd_mailbox(dev, mailbox);
- return err;
-}
-
+ \r
+ if (is_eth) {\r
+ ((u8 *) mailbox->buf)[3] = 7;\r
+ ((__be16 *) mailbox->buf)[3] =\r
+ cpu_to_be16(dev->caps.eth_mtu_cap[port] +\r
+ ETH_HLEN + ETH_FCS_LEN);\r
+ ((__be16 *) mailbox->buf)[4] = cpu_to_be16(1 << 15);\r
+ ((__be16 *) mailbox->buf)[6] = cpu_to_be16(1 << 15);\r
+ }\r
+ err = mlx4_cmd(dev, mailbox->dma.da, port, is_eth, MLX4_CMD_SET_PORT,\r
+ MLX4_CMD_TIME_CLASS_B);\r
+\r
+ mlx4_free_cmd_mailbox(dev, mailbox);\r
+ return err;\r
+}\r
+\r
static u16 op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE];
static int op_inited = 0;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
if (!op_inited) {
op[MLX4_QP_STATE_RST][MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP;
op[MLX4_QP_STATE_RST][MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP;
struct mlx4_qp_table *qp_table = &priv->qp_table;
int err;
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
if (!qpn)
return -EINVAL;
if (atomic_dec_and_test(&qp->refcount))
complete(&qp->free);
- wait_for_completion(&qp->free);
+ if (!mlx4_is_barred(dev))
+ wait_for_completion(&qp->free);
mlx4_table_put(dev, &qp_table->cmpt_table, qp->qpn);
mlx4_table_put(dev, &qp_table->rdmarc_table, qp->qpn);
int err;
UNREFERENCED_PARAMETER(xrcd);
+ if ( mlx4_is_barred(dev) )
+ return -EFAULT;
+
srq->srqn = mlx4_bitmap_alloc(&srq_table->bitmap);
if (srq->srqn == -1)
return -ENOMEM;
{
int err;
+ if ( mlx4_is_barred(dev) )
+ return;
+
err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn);
if (err)
mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn);
if (atomic_dec_and_test(&srq->refcount))
complete(&srq->free);
- wait_for_completion(&srq->free);
+ if (!mlx4_is_barred(dev))
+ wait_for_completion(&srq->free);
mlx4_table_put(dev, &srq_table->table, srq->srqn);
mlx4_bitmap_free(&srq_table->bitmap, srq->srqn);
# WDK build only - transform .inx --> .inf adding date & version stamp.\r
# see .\makefile.inc\r
INF_NAME=$(TARGETNAME)\r
-INF_TARGET==..\..\..\..\bin\kernel\$(O)\$(INF_NAME).inf\r
+INF_TARGET=..\..\..\..\bin\kernel\$(O)\$(INF_NAME).inf\r
NTTARGETFILES=$(INF_TARGET)\r
!endif\r
\r
\r
KMDF_VERSION=1\r
\r
-C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUSE_WDM_FRAMEWORK\r
+C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUSE_WDM_FRAMEWORK -DUSE_WDM_INTERRUPTS\r
\r
TARGETLIBS= \\r
$(TARGETPATH)\*\complib.lib \\r
&p_ib_ah_attr->grh.traffic_class, &p_ib_ah_attr->grh.flow_label );\r
err = p_ib_dev->x.find_cached_gid((struct ib_device *)p_ib_dev, \r
(union ib_gid *)p_ib_av_attr->grh.src_gid.raw, &port_num, &gid_index);\r
- if (err) {\r
-\r
+ if (err) {\r\r
HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_SHIM ,\r
("ib_find_cached_gid failed %d (%#x). Using default: sgid_index = 0\n", err, err));\r
gid_index = 0;\r
err = p_ib_dev->x.get_cached_gid((struct ib_device *)p_ib_dev, \r
p_ib_ah_attr->port_num, p_ib_ah_attr->grh.sgid_index,\r
(union ib_gid*)p_ib_av_attr->grh.src_gid.raw );\r
- if (err) {\r
-\r
+ if (err) {\r\r
HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_SHIM ,\r
("ib_get_cached_gid failed %d (%#x). Using default: sgid_index = 0\n", err, err));\r
}\r
#include "precomp.h"\r
#include <initguid.h>\r
#include <wdmguid.h>\r
+#include <rdma\verbs.h>\r
\r
#if defined(EVENT_TRACING)\r
#ifdef offsetof\r
#define DRV_VERSION "1.0"\r
#define DRV_RELDATE "02/01/2008"\r
\r
+#define MLX_VERBS_MIN_VERSION 2\r
+#define MLX_VERBS_MAX_VERSION 2\r
+\r
GLOBALS g;\r
\r
/*\r
\r
#ifndef USE_WDM_FRAMEWORK\r
\r
+//\r
+// TODO: add support for Hibernate/Standby as in WDM version below\r
+//\r
+\r
#ifdef ALLOC_PRAGMA\r
#pragma alloc_text (INIT, DriverEntry)\r
#pragma alloc_text (PAGE, EvtDeviceAdd)\r
\r
/* get node GUID */\r
err = __get_dev_info( p_fdo, &p_fdo->hca.guid, &p_fdo->hca.hw_ver );\r
- if (err) {\r\r
+ if (err) {\r
+\r
HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_LOW ,\r
("can't get guid - ib_query_device() failed (%08X)\n", err ));\r
//TODO: no cleanup on error\r
IN IRP* const p_irp, \r
OUT cl_irp_action_t* const p_action );\r
\r
+static NTSTATUS\r
+hca_query_interface(\r
+ IN DEVICE_OBJECT* const p_dev_obj,\r
+ IN IRP* const p_irp, \r
+ OUT cl_irp_action_t* const p_action );\r
+\r
static NTSTATUS\r
hca_query_pnp_state(\r
IN DEVICE_OBJECT* const p_dev_obj,\r
#pragma alloc_text (PAGE, hca_cancel_remove)\r
#pragma alloc_text (PAGE, hca_surprise_remove)\r
#pragma alloc_text (PAGE, hca_query_capabilities)\r
+#pragma alloc_text (PAGE, hca_query_interface)\r
#pragma alloc_text (PAGE, hca_query_pnp_state)\r
#pragma alloc_text (PAGE, hca_query_bus_relations)\r
#pragma alloc_text (PAGE, hca_query_removal_relations)\r
\r
HCA_ENTER( HCA_DBG_PNP );\r
\r
- for( i = 0; i < HCA_BAR_TYPE_MAX; i++ ) {\r
- if (pdev->bar[i].virt) {\r
- MmUnmapIoSpace( pdev->bar[i].virt, pdev->bar[i].size );\r
- cl_memclr( &pdev->bar[i], sizeof(hca_bar_t) );\r
+ if ( pdev )\r
+ for( i = 0; i < HCA_BAR_TYPE_MAX; i++ ) {\r
+ if (pdev->bar[i].virt) {\r
+ MmUnmapIoSpace( pdev->bar[i].virt, pdev->bar[i].size );\r
+ cl_memclr( &pdev->bar[i], sizeof(hca_bar_t) );\r
+ }\r
}\r
- }\r
\r
HCA_EXIT( HCA_DBG_PNP );\r
}\r
p_fdo->bus_ib_ifc_taken = TRUE;\r
p_fdo->bus_ib_ifc.p_ibdev->x.p_fdo = p_fdo;\r
\r
+ InitializeListHead(&p_fdo->hca.event_list);\r
+ KeInitializeSpinLock(&p_fdo->hca.event_list_lock);\r
\r
/* get node GUID */\r
err = __get_dev_info( p_fdo, &p_fdo->hca.guid, &p_fdo->hca.hw_ver );\r
- if (err) {\r\r
+ if (err) {\r
+\r
HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_LOW ,\r
("can't get guid - ib_query_device() failed (%08X)\n", err ));\r
//TODO: no cleanup on error\r
}\r
\r
\r
+static VOID\r
+__hca_noop( VOID *context )\r
+{\r
+ UNREFERENCED_PARAMETER(context);\r
+}\r
+\r
+\r
+static NTSTATUS\r
+__query_ci_ifc(\r
+ IN DEVICE_OBJECT* const p_dev_obj,\r
+ IN IO_STACK_LOCATION* const p_io_stack )\r
+{\r
+ RDMA_INTERFACE_VERBS *p_ifc;\r
+ PFDO_DEVICE_DATA p_fdo;\r
+ ci_interface_t *p_hca_ifc;\r
+ NTSTATUS status;\r
+ UINT8 version;\r
+\r
+ HCA_ENTER( HCA_DBG_PNP );\r
+\r
+ version = VerbsVersionMajor(p_io_stack->Parameters.QueryInterface.Version);\r
+ if( version < MLX_VERBS_MIN_VERSION || version > MLX_VERBS_MAX_VERSION )\r
+ {\r
+ status = STATUS_NOT_SUPPORTED;\r
+ goto exit;\r
+ }\r
+\r
+ if( p_io_stack->Parameters.QueryInterface.Size < sizeof(RDMA_INTERFACE_VERBS) )\r
+ {\r
+ status = STATUS_BUFFER_TOO_SMALL;\r
+ goto exit;\r
+ }\r
+\r
+ p_fdo = (PFDO_DEVICE_DATA)p_dev_obj->DeviceExtension;\r
+ p_hca_ifc = __alloc_hca_ifc( p_fdo );\r
+ if( !p_hca_ifc )\r
+ {\r
+ status = STATUS_NO_MEMORY;\r
+ goto exit;\r
+ }\r
+\r
+ p_ifc = (RDMA_INTERFACE_VERBS *) p_io_stack->Parameters.QueryInterface.Interface;\r
+\r
+ p_ifc->InterfaceHeader.Size = sizeof(RDMA_INTERFACE_VERBS);\r
+ p_ifc->InterfaceHeader.Version = VerbsVersion(version, 0);\r
+ p_ifc->InterfaceHeader.Context = p_dev_obj;\r
+ p_ifc->InterfaceHeader.InterfaceReference = __hca_noop;\r
+ p_ifc->InterfaceHeader.InterfaceDereference = __hca_noop;\r
+ p_ifc->Verbs = *p_hca_ifc;\r
+ p_ifc->Verbs.p_hca_dev = &p_fdo->hca;\r
+\r
+ ExFreePool( p_hca_ifc );\r
+ status = STATUS_SUCCESS;\r
+\r
+exit:\r
+ HCA_EXIT( HCA_DBG_PNP );\r
+ return status;\r
+}\r
+\r
+\r
+static NTSTATUS\r
+hca_query_interface(\r
+ IN DEVICE_OBJECT* const p_dev_obj,\r
+ IN IRP* const p_irp, \r
+ OUT cl_irp_action_t* const p_action )\r
+{\r
+ NTSTATUS status;\r
+ IO_STACK_LOCATION *p_io_stack;\r
+\r
+ HCA_ENTER( HCA_DBG_PNP );\r
+\r
+#pragma warning( push, 3 )\r
+ PAGED_CODE();\r
+#pragma warning( pop )\r
+\r
+ p_io_stack = IoGetCurrentIrpStackLocation( p_irp );\r
+ \r
+ /* Compare requested GUID with our supported interface GUIDs. */\r
+ if( IsEqualGUID( p_io_stack->Parameters.QueryInterface.InterfaceType,\r
+ &GUID_RDMA_INTERFACE_VERBS ) )\r
+ {\r
+ status = __query_ci_ifc( p_dev_obj, p_io_stack );\r
+ *p_action = IrpComplete;\r
+ }\r
+ else\r
+ {\r
+ status = p_irp->IoStatus.Status;\r
+ *p_action = IrpSkip;\r
+ }\r
+\r
+ HCA_EXIT( HCA_DBG_PNP );\r
+ return status;\r
+}\r
+\r
+\r
static NTSTATUS\r
hca_query_pnp_state(\r
IN DEVICE_OBJECT* const p_dev_obj,\r
IN DEVICE_OBJECT* p_dev_obj,\r
IN void* context )\r
{\r
+ int err;\r
NTSTATUS status;\r
IO_STACK_LOCATION *pIoStack;\r
PFDO_DEVICE_DATA p_fdo;\r
HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_PO, \r
("***** Restart the HCA, IRQL %d\n", KeGetCurrentIrql()));\r
\r
+ /* get MLX4_BUS IB interface */\r
+ status = __get_ifc( p_dev_obj, &MLX4_BUS_IB_INTERFACE_GUID,\r
+ sizeof(MLX4_BUS_IB_INTERFACE), MLX4_BUS_IB_INTERFACE_VERSION, NULL, (PINTERFACE)&p_fdo->bus_ib_ifc);\r
+ if( !NT_SUCCESS( status ) ) {\r
+ HCA_PRINT(TRACE_LEVEL_ERROR, HCA_DBG_PNP, ("Getting MLX4 BUS interface failed: status=0x%x\n", status));\r
+ goto err_hca_reg;\r
+ }\r
+ p_fdo->bus_ib_ifc_taken = TRUE;\r
+ p_fdo->bus_ib_ifc.p_ibdev->x.p_fdo = p_fdo;\r
+\r
+ /* get node GUID */\r
+ err = __get_dev_info( p_fdo, &p_fdo->hca.guid, &p_fdo->hca.hw_ver );\r
+ if (err) {\r
+\r
+ HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_LOW ,\r
+ ("can't get guid - ib_query_device() failed (%08X)\n", err ));\r
+ //TODO: no cleanup on error\r
+ goto err_hca_reg;\r
+ }\r
+\r
if( p_fdo->p_al_dev ) {\r
status = __hca_register( p_dev_obj );\r
if( !NT_SUCCESS( status ) ) {\r
\r
{\r
__hca_deregister( p_fdo );\r
+\r
+ // release MLX4_BUS resources\r
+ if(p_fdo->bus_ib_ifc_taken) {\r
+ p_fdo->bus_ib_ifc_taken = FALSE;\r
+ __put_ifc( (PINTERFACE)&p_fdo->bus_ib_ifc );\r
+ }\r
}\r
\r
IoCopyCurrentIrpStackLocationToNext( p_irp );\r
vfptrHcaPnp.pfn_query_resources = cl_irp_ignore;\r
vfptrHcaPnp.pfn_query_res_req = cl_irp_ignore;\r
vfptrHcaPnp.pfn_query_bus_info = cl_irp_ignore;\r
- vfptrHcaPnp.pfn_query_interface = cl_irp_ignore;\r
+ vfptrHcaPnp.pfn_query_interface = hca_query_interface;\r
vfptrHcaPnp.pfn_read_config = cl_irp_ignore;\r
vfptrHcaPnp.pfn_write_config = cl_irp_ignore;\r
vfptrHcaPnp.pfn_eject = cl_irp_ignore;\r
USHORT dev_id;
DMA_ADAPTER * p_dma_adapter; /* HCA adapter object */
DEVICE_OBJECT * p_self_do; /* mlx4 FDO */
+ // mlx4_ib: various objects and info
+ struct ib_device * ib_dev;
// mlx4_net: various objects and info
struct mlx4_dev * dev;
volatile long dpc_lock;
PKINTERRUPT int_obj; /* HCA interrupt object */
KSPIN_LOCK isr_lock; /* lock for the ISR */
#endif
- // mlx4_ib: various objects and info
- struct ib_device * ib_dev;
};
/* DPC */
return dev->flags & MLX4_FLAG_LIVEFISH;
}
+static inline int mlx4_is_barred(struct mlx4_dev *dev)
+{
+ return dev->flags & (MLX4_FLAG_RESET_CLIENT | MLX4_FLAG_RESET_DRIVER);
+}
+
+static inline int mlx4_is_in_reset(struct mlx4_dev *dev)
+{
+ return dev->flags & MLX4_FLAG_RESET_STARTED;
+}
+
#endif
#define DEVID_HERMON_ETH_CONDOR 0x6368 /* 25448 */
#define DEVID_HERMON_DDR_G2 0x6732 /* 26418 */
#define DEVID_HERMON_QDR_G2 0x673c /* 26428 */
+#define DEVID_HERMON_ETH_CONDOR_G2 0x6750 /* 26448 */
/* livefish */
#define DEVID_HERMON_BD 0x0191 /* 401 */
\r
#define MTNIC_MAX_PORTS 2\r
\r
-#define MAX_PORT_SIZE 250000\r
#define MXE_INTERFACE_VERSION 2\r
\r
enum mtnic_state {\r
CARD_DISABLED\r
};\r
\r
+struct _MP_PORT;\r
+\r
typedef struct {\r
enum mtnic_state state;\r
- NDIS_WORK_ITEM PortStateWorkItem;\r
- NDIS_WORK_ITEM ResetWorkItem;\r
KEVENT ConfigChangeEvent; \r
- KTIMER HeardBeatTimer; \r
- LONG ResetCount;\r
\r
// Objects that are needed in order to work with the hw\r
\r
{\r
PVOID Context;\r
LONG NoOfConnectedPorts;\r
- ULONG ulAllocatePortObjSize;\r
+\r
NicData_t NicData;\r
-#ifdef _WIN64 \r
- UCHAR pad[0x8];\r
-#else\r
- UCHAR pad[0x8]; \r
-#endif \r
-#ifdef MTNIC\r
- C_ASSERT(MAX_PORT_SIZE >= sizeof(struct _MP_PORT));\r
- MP_PORT ports[MTNIC_MAX_PORTS];\r
-#else\r
- UCHAR ports[MAX_PORT_SIZE * MTNIC_MAX_PORTS];\r
-#endif \r
-};\r
\r
-C_ASSERT((FIELD_OFFSET(struct VipBusIfc, ports) % 16) == 0);\r
+ struct _MP_PORT *ports[MTNIC_MAX_PORTS];\r
+ \r
+};\r
\r
\r
\r
--- /dev/null
+[CatalogHeader]\r
+Name=ib_bus.cat\r
+PublicVersion=0x0000001\r
+EncodingType=0x00010001\r
+CATATTR1=0x10010001:OSAttr:2:6.0\r
+[CatalogFiles]\r
+<hash>ib_bus.inf=ib_bus.inf\r
+<hash>ibbus.sys=ibbus.sys\r
+<hash>ibiou.sys=ibiou.sys\r
+<hash>ibal.dll=ibal.dll\r
+<hash>complib.dll=complib.dll\r
+<hash>ibald.dll=ibald.dll\r
+<hash>complibd.dll=complibd.dll\r
struct ib_event *event)
{
struct ib_update_work *work;
- static int temp_skip = 10;
-
- if (temp_skip-- <= 0)
- return;
if (event->event == IB_EVENT_PORT_ERR ||
event->event == IB_EVENT_PORT_ACTIVE ||
srp \\r
wsd \\r
qlgcvnic \\r
- libibverbs\r
+ libibverbs \\r
+ nd\r
+++ /dev/null
-[CatalogHeader]\r
-Name=ipoib.cat\r
-PublicVersion=0x0000001\r
-EncodingType=0x00010001\r
-CATATTR1=0x10010001:OSAttr:2:6.0\r
-[CatalogFiles]\r
-<hash>netipoib.inf=netipoib.inf\r
-<hash>ipoib.sys=ipoib.sys\r
-\r
<hash>ipoib.sys=ipoib.sys\r
<hash>ibwsd.dll=ibwsd.dll\r
<hash>ibwsd32.dll=ibwsd32.dll\r
+<hash>ibndprov.dll=ibndprov.dll\r
+<hash>ibndprov32.dll=ibndprov32.dll\r
--- /dev/null
+[CatalogHeader]\r
+Name=ipoib.cat\r
+PublicVersion=0x0000001\r
+EncodingType=0x00010001\r
+CATATTR1=0x10010001:OSAttr:2:6.0\r
+[CatalogFiles]\r
+<hash>netipoib.inf=netipoib.inf\r
+<hash>ipoib.sys=ipoib.sys\r
+\r
--- /dev/null
+[CatalogHeader]\r
+Name=ipoib.cat\r
+PublicVersion=0x0000001\r
+EncodingType=0x00010001\r
+CATATTR1=0x10010001:OSAttr:2:6.0\r
+[CatalogFiles]\r
+<hash>netipoib.inf=netipoib.inf\r
+<hash>ipoib.sys=ipoib.sys\r
+<hash>ibwsd.dll=ibwsd.dll\r
+<hash>ibndprov.dll=ibndprov.dll\r
}\r
else\r
{\r
+ ip_stat_sel_t ip_stat;\r
p_desc->len =\r
len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
if( p_dst->h_mcast)\r
p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
{\r
p_desc->type = PKT_TYPE_BCAST;\r
+ ip_stat = IP_STAT_BCAST_BYTES;\r
}\r
else\r
{\r
p_desc->type = PKT_TYPE_MCAST;\r
+ ip_stat = IP_STAT_MCAST_BYTES;\r
}\r
}\r
else\r
{\r
p_desc->type = PKT_TYPE_UCAST;\r
+ ip_stat = IP_STAT_UCAST_BYTES;\r
+ \r
}\r
cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
+ ipoib_inc_recv_stat( p_port->p_adapter,ip_stat , len ); \r
}\r
}\r
\r
Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL\r
AddReg = IpoibAddReg\r
CopyFiles = IpoibCopyFiles\r
-CopyFiles = WsdCopyFiles\r
-CopyFiles = WOW64CopyFiles\r
+CopyFiles = WsdCopyFiles\r
+CopyFiles = NdCopyFiles\r
+CopyFiles = WOW64CopyFiles\r
\r
[Ipoib.DDInstall.ntia64]\r
Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL\r
AddReg = IpoibAddReg\r
CopyFiles = IpoibCopyFiles\r
-CopyFiles = WsdCopyFiles\r
-CopyFiles = WOW64CopyFiles\r
+CopyFiles = WsdCopyFiles\r
+CopyFiles = NdCopyFiles\r
+CopyFiles = WOW64CopyFiles\r
\r
[Ipoib.DDInstall.ntx86.Services]\r
AddService = ipoib, 2, IpoibService, IpoibEventLog\r
AddService = ipoib, 2, IpoibService, IpoibEventLog\r
\r
[IpoibAddReg]\r
+HKR, ,RDMACapable, %REG_DWORD%, 1\r
HKR, Ndi, Service, 0, "ipoib"\r
HKR, Ndi\Interfaces, UpperRange, 0, "ndis5"\r
HKR, Ndi\Interfaces, LowerRange, 0, "ethernet"\r
HKR, Ndi\Params\PayloadMtu, Min, 0, "60"\r
HKR, Ndi\Params\PayloadMtu, Max, 0, "4092"\r
\r
+HKR, Ndi\Params\MCLeaveRescan, ParamDesc, 0, "MC leave rescan (sec)"\r
+HKR, Ndi\Params\MCLeaveRescan, Type, 0, "dword"\r
+HKR, Ndi\Params\MCLeaveRescan, Default, 0, "260"\r
+HKR, Ndi\Params\MCLeaveRescan, Optional, 0, "0"\r
+HKR, Ndi\Params\MCLeaveRescan, Min, 0, "1"\r
+HKR, Ndi\Params\MCLeaveRescan, Max, 0, "3600"\r
[IpoibService]\r
DisplayName = %IpoibServiceDispName%\r
ServiceType = 1 ;%SERVICE_KERNEL_DRIVER%\r
[WsdCopyFiles]\r
ibwsd.dll,,,0x00000002\r
\r
+[NdCopyFiles]\r
+ibndprov.dll,,,0x00000002\r
+\r
[WOW64CopyFiles]\r
ibwsd.dll,ibwsd32.dll,,0x00000002\r
+ibndprov.dll,ibndprov32.dll,,0x00000002\r
\r
[SourceDisksNames.x86]\r
1 = %IcsDisk1%,,,""\r
ipoib.sys = 1\r
ibwsd.dll = 1\r
ibwsd32.dll = 1\r
+ibndprov.dll = 1\r
+ibndprov32.dll = 1\r
\r
[SourceDisksFiles.ia64]\r
ipoib.sys = 1\r
ibwsd.dll = 1\r
ibwsd32.dll = 1\r
+ibndprov.dll = 1\r
+ibndprov32.dll = 1\r
\r
[DestinationDirs]\r
IpoibCopyFiles = %DIRID_DRIVERS%\r
WsdCopyFiles = %DIRID_SYSTEM%\r
+NdCopyFiles = %DIRID_SYSTEM%\r
WOW64CopyFiles = %DIRID_SYSTEM_X86%\r
DefaultDestDir = %DIRID_SYSTEM%\r
\r
DIRID_SYSTEM = 11\r
DIRID_DRIVERS = 12\r
DIRID_SYSTEM_X86 = 16425\r
+REG_DWORD = 0x00010001\r
REG_DWORD_NO_CLOBBER = 0x00010003\r
Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL\r
AddReg = IpoibAddReg\r
CopyFiles = IpoibCopyFiles\r
-CopyFiles = WsdCopyFiles\r
+CopyFiles = WsdCopyFiles\r
+CopyFiles = NdCopyFiles\r
\r
[Ipoib.DDInstall.ntamd64]\r
Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL\r
AddReg = IpoibAddReg\r
CopyFiles = IpoibCopyFiles\r
-CopyFiles = WsdCopyFiles\r
-CopyFiles = WOW64CopyFiles\r
+CopyFiles = WsdCopyFiles\r
+CopyFiles = NdCopyFiles\r
+CopyFiles = WOW64CopyFiles\r
\r
[Ipoib.DDInstall.ntia64]\r
Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL\r
AddReg = IpoibAddReg\r
CopyFiles = IpoibCopyFiles\r
-CopyFiles = WsdCopyFiles\r
-CopyFiles = WOW64CopyFiles\r
+CopyFiles = WsdCopyFiles\r
+CopyFiles = NdCopyFiles\r
+CopyFiles = WOW64CopyFiles\r
\r
[Ipoib.DDInstall.ntx86.Services]\r
AddService = ipoib, 2, IpoibService, IpoibEventLog\r
[WsdCopyFiles]\r
ibwsd.dll,,,0x00000002\r
\r
+[NdCopyFiles]\r
+ibndprov.dll,,,0x00000002\r
+\r
[WOW64CopyFiles]\r
ibwsd.dll,ibwsd32.dll,,0x00000002\r
+ibndprov.dll,ibndprov32.dll,,0x00000002\r
\r
[SourceDisksNames.x86]\r
1 = %IcsDisk1%,,,""\r
[SourceDisksFiles.x86]\r
ipoib.sys = 1\r
ibwsd.dll = 1\r
+ibndprov.dll = 1\r
\r
[SourceDisksFiles.amd64]\r
ipoib.sys = 1\r
ibwsd.dll = 1\r
ibwsd32.dll = 1\r
+ibndprov.dll = 1\r
+ibndprov32.dll = 1\r
\r
[SourceDisksFiles.ia64]\r
ipoib.sys = 1\r
ibwsd.dll = 1\r
ibwsd32.dll = 1\r
+ibndprov.dll = 1\r
+ibndprov32.dll = 1\r
\r
[DestinationDirs]\r
IpoibCopyFiles = %DIRID_DRIVERS%\r
WsdCopyFiles = %DIRID_SYSTEM%\r
+NdCopyFiles = %DIRID_SYSTEM%\r
WOW64CopyFiles = %DIRID_SYSTEM_X86%\r
DefaultDestDir = %DIRID_SYSTEM%\r
\r
--- /dev/null
+DIRS=\\r
+ user \r
--- /dev/null
+TARGETNAME=fake\r
+TARGETPATH=..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE=PROGRAM\r
+UMTYPE=console\r
+USE_MSVCRT=1\r
+\r
+SOURCES=fake.c\r
+\r
+INCLUDES=..;..\..\..\inc;..\..\..\inc\kernel;\r
+\r
+ND_TARGET1=..\..\..\bin\user\$(O)\ibndprov.dll\r
+ND_TARGET2=..\..\..\bin\user\$(O)\ndinstall.exe\r
+\r
+NTTARGETFILES=$(ND_TARGET1) $(ND_TARGET2)\r
+\r
--- /dev/null
+\r
+int __cdecl main() { return 0; }\r
--- /dev/null
+#\r
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source\r
+# file to this component. This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+MINIMUM_NT_TARGET_VERSION=0x502\r
+\r
+!INCLUDE ..\..\..\inc\openib.def\r
--- /dev/null
+\r
+# files copied to the $(ND_TARGET*) folder \r
+# When ND sources hit svn, this hack will be removed.\r
+# recreate dir structure and nd binaries are obj*\ folders tend to get\r
+# removed; don't want to lose the ONLY copies of the ND binaries we have.\r
+#\r
+# ND on ia64 is not supported [8-07-08]. Building the ia64\ folders is a\r
+# blantant hack to keep the build happy; WIX installer does not use the\r
+# ia64 .exe or .dll. If someone knows how to force the build to skip for ia64,\r
+# please enlighten.\r
+\r
+.\$(O)\ibndprov.dll : \r
+ xcopy objfre_svr-03_amd64 objfre_wnet_amd64 /E/I/K/Y \r
+ xcopy objfre_svr-03_amd64 objchk_wnet_amd64 /E/I/K/Y \r
+ xcopy objfre_svr-03_x86 objfre_wnet_x86 /E/I/K/Y \r
+ xcopy objfre_svr-03_x86 objchk_wnet_x86 /E/I/K/Y \r
+ xcopy objfre_svr-08_amd64 objfre_wlh_amd64 /E/I/K/Y \r
+ xcopy objfre_svr-08_amd64 objchk_wlh_amd64 /E/I/K/Y \r
+ xcopy objfre_svr-08_x86 objfre_wlh_x86 /E/I/K/Y \r
+ xcopy objfre_svr-08_x86 objchk_wlh_x86 /E/I/K/Y \r
+ xcopy objfre_svr-08_ia64 objfre_wlh_ia64 /E/I/K/Y \r
+ xcopy objfre_svr-08_ia64 objchk_wlh_ia64 /E/I/K/Y \r
+ xcopy objfre_svr-08_ia64 objfre_wnet_ia64 /E/I/K/Y \r
+ xcopy objfre_svr-08_ia64 objchk_wnet_ia64 /E/I/K/Y \r
+\r
+$(ND_TARGET1) : .\$(O)\ibndprov.dll\r
+ copy /B/Y .\$(O)\$(@B).dll $@\r
+\r
+$(ND_TARGET2) : .\$(O)\ndinstall.exe\r
+ copy /B/Y .\$(O)\$(@B).exe $@\r
+\r
projects / ~shefty / rdma-win.git / commitdiff