From f229da0c1217c24567f3b850fa2c68c7ff69cb89 Mon Sep 17 00:00:00 2001 From: shefty Date: Mon, 11 Aug 2008 16:19:25 +0000 Subject: [PATCH] update to svn 1477 git-svn-id: svn://openib.tc.cornell.edu/gen1@1478 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- branches/winverbs/BuildRelease.bat | 85 +- .../winverbs/core/bus/kernel/ib_bus32.cdf | 13 + branches/winverbs/etc/makebin.bat | 90 +- .../winverbs/hw/mlx4/kernel/bus/core/SOURCES | 3 +- .../winverbs/hw/mlx4/kernel/bus/core/cache.c | 14 +- .../winverbs/hw/mlx4/kernel/bus/drv/drv.c | 232 +- .../winverbs/hw/mlx4/kernel/bus/drv/drv.h | 10 +- .../hw/mlx4/kernel/bus/drv/mlx4_bus.inx | 38 +- .../winverbs/hw/mlx4/kernel/bus/drv/pdo.c | 35 +- branches/winverbs/hw/mlx4/kernel/bus/ib/ah.c | 9 + branches/winverbs/hw/mlx4/kernel/bus/ib/cq.c | 1233 +++--- branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c | 3 + .../winverbs/hw/mlx4/kernel/bus/ib/main.c | 31 +- branches/winverbs/hw/mlx4/kernel/bus/ib/mr.c | 17 +- branches/winverbs/hw/mlx4/kernel/bus/ib/qp.c | 3489 +++++++++-------- branches/winverbs/hw/mlx4/kernel/bus/ib/srq.c | 733 ++-- .../hw/mlx4/kernel/bus/inc/bus_intf.h | 6 +- .../winverbs/hw/mlx4/kernel/bus/inc/device.h | 5 +- .../hw/mlx4/kernel/bus/inc/ib_verbs.h | 24 +- .../winverbs/hw/mlx4/kernel/bus/net/alloc.c | 3 + .../winverbs/hw/mlx4/kernel/bus/net/catas.c | 184 + .../winverbs/hw/mlx4/kernel/bus/net/cmd.c | 6 + branches/winverbs/hw/mlx4/kernel/bus/net/cq.c | 14 +- branches/winverbs/hw/mlx4/kernel/bus/net/eq.c | 3 + branches/winverbs/hw/mlx4/kernel/bus/net/fw.c | 6 + .../winverbs/hw/mlx4/kernel/bus/net/main.c | 11 +- .../winverbs/hw/mlx4/kernel/bus/net/mlx4.h | 13 + branches/winverbs/hw/mlx4/kernel/bus/net/mr.c | 8 +- branches/winverbs/hw/mlx4/kernel/bus/net/pd.c | 6 + .../winverbs/hw/mlx4/kernel/bus/net/port.c | 570 +-- branches/winverbs/hw/mlx4/kernel/bus/net/qp.c | 9 +- .../winverbs/hw/mlx4/kernel/bus/net/srq.c | 9 +- branches/winverbs/hw/mlx4/kernel/hca/SOURCES | 4 +- branches/winverbs/hw/mlx4/kernel/hca/data.c | 6 +- branches/winverbs/hw/mlx4/kernel/hca/drv.c | 158 +- branches/winverbs/hw/mlx4/kernel/inc/l2w.h | 14 +- .../winverbs/hw/mlx4/kernel/inc/l2w_pci.h | 1 + .../winverbs/hw/mlx4/kernel/inc/vip_dev.h | 25 +- .../winverbs/hw/mthca/kernel/ib_bus32.cdf | 13 + branches/winverbs/hw/mthca/kernel/mt_cache.c | 4 - branches/winverbs/ulp/dirs | 3 +- branches/winverbs/ulp/ipoib/kernel/ipoib.cdf | 2 + .../kernel/{ipoib-xp32.cdf => ipoib32-xp.cdf} | 0 .../winverbs/ulp/ipoib/kernel/ipoib32.cdf | 10 + .../winverbs/ulp/ipoib/kernel/ipoib_port.c | 6 + .../ulp/ipoib/kernel/netipoib-xp32.inf | 27 +- .../winverbs/ulp/ipoib/kernel/netipoib.inx | 23 +- branches/winverbs/ulp/nd/dirs | 2 + branches/winverbs/ulp/nd/user/SOURCES | 15 + branches/winverbs/ulp/nd/user/fake.c | 2 + branches/winverbs/ulp/nd/user/makefile | 8 + branches/winverbs/ulp/nd/user/makefile.inc | 31 + .../objfre_svr-03_amd64/amd64/ibndprov.dll | Bin 0 -> 43520 bytes .../objfre_svr-03_amd64/amd64/ndinstall.exe | Bin 0 -> 11776 bytes .../user/objfre_svr-03_x86/i386/ibndprov.dll | Bin 0 -> 37376 bytes .../user/objfre_svr-03_x86/i386/ndinstall.exe | Bin 0 -> 11264 bytes .../objfre_svr-08_amd64/amd64/ibndprov.dll | Bin 0 -> 44032 bytes .../objfre_svr-08_amd64/amd64/ndinstall.exe | Bin 0 -> 10752 bytes .../user/objfre_svr-08_ia64/ia64/ibndprov.dll | Bin 0 -> 43520 bytes .../objfre_svr-08_ia64/ia64/ndinstall.exe | Bin 0 -> 11776 bytes .../user/objfre_svr-08_x86/i386/ibndprov.dll | Bin 0 -> 37376 bytes .../user/objfre_svr-08_x86/i386/ndinstall.exe | Bin 0 -> 9216 bytes 62 files changed, 4023 insertions(+), 3243 deletions(-) create mode 100644 branches/winverbs/core/bus/kernel/ib_bus32.cdf create mode 100644 branches/winverbs/hw/mthca/kernel/ib_bus32.cdf rename branches/winverbs/ulp/ipoib/kernel/{ipoib-xp32.cdf => ipoib32-xp.cdf} (100%) create mode 100644 branches/winverbs/ulp/ipoib/kernel/ipoib32.cdf create mode 100644 branches/winverbs/ulp/nd/dirs create mode 100644 branches/winverbs/ulp/nd/user/SOURCES create mode 100644 branches/winverbs/ulp/nd/user/fake.c create mode 100644 branches/winverbs/ulp/nd/user/makefile create mode 100644 branches/winverbs/ulp/nd/user/makefile.inc create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-03_amd64/amd64/ibndprov.dll create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-03_amd64/amd64/ndinstall.exe create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-03_x86/i386/ibndprov.dll create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-03_x86/i386/ndinstall.exe create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-08_amd64/amd64/ibndprov.dll create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-08_amd64/amd64/ndinstall.exe create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-08_ia64/ia64/ibndprov.dll create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-08_ia64/ia64/ndinstall.exe create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-08_x86/i386/ibndprov.dll create mode 100644 branches/winverbs/ulp/nd/user/objfre_svr-08_x86/i386/ndinstall.exe diff --git a/branches/winverbs/BuildRelease.bat b/branches/winverbs/BuildRelease.bat index 798ee920..e9119626 100644 --- a/branches/winverbs/BuildRelease.bat +++ b/branches/winverbs/BuildRelease.bat @@ -7,7 +7,7 @@ rem Processor architecture specific WIX installers are constructed rem in %IDIR% rem rem BuildRelease option -rem option == all | allnoforce | compile | compilenoforce | makebin | msi | wix | clean {OPENIB_REV} +rem option == all | allnoforce | compile | compilenoforce | makebin | msi | wix | clean | msi-label | msi-del | msi-dir {OPENIB_REV} rem rem This script is an 'example' of a one-command entire IB stack build to @@ -29,13 +29,16 @@ if "%1" == "makebin" goto OK if "%1" == "msi" goto OK if "%1" == "wix" goto OK if "%1" == "clean" goto OK +if "%1" == "msi-label" goto OK +if "%1" == "msi-del" goto OK +if "%1" == "msi-dir" goto OK echo Unknown arg '%1' ? :usage -echo "usage: BuildRelease all | allnoforce | compile | compilenoforce | makebin | msi | wix | clean {OPENIB_REV value}" -echo where: +echo "usage: BuildRelease command {OPENIB_REV value}" +echo where 'command' may be: echo all - force recompile, install binaries to WIX tree, sign drivers and echo build installers. @@ -49,6 +52,13 @@ echo msi - assumes binaries are installed in WIX tree, signs drivers and echo create installers (.msi files) in IDIR. echo wix - build .msi installers, assumes (drivers signed) .cat files exist echo clean - remove build artifacts for a clean build: .obj, .sys, ... +echo msi-label {OPENIB_REV} +echo rename WOF_os*_arch*.msi to WOF_os*_arch*_svn#.msi +echo Uniquely identify installers just created. +echo If OPENIB_REV arg used, then rename WOF_os*_arch*argVal.msi +echo otherwise, use SVN# from path. +echo msi-del - del %windir%\temp\WOF_os*_arch*.msi +echo msi-dir - del %windir%\temp\WOF_os*_arch*.msi echo : echo {OPENIB_REV} echo optional, if set then OPENIB_REV is assigned this value. @@ -103,7 +113,6 @@ if NOT EXIST %_DDK% ( echo Missing WDK @ %_DDK% exit /B 1 ) -echo %0 - Building with WDK @ %_DDK% rem Platform SDK path - watchout for missing LoadPerf.h (installsp.c) set _PSDK=%SystemDrive%\progra~1\mi2578~1 @@ -112,6 +121,7 @@ rem setup value for OPENIB_REV assignment. if not "%2" == "" ( set SVN=%2 + set LBL=%2 ) else ( rem WARNING - SVN value depends on the path '\openIB-windows-svn\XXX\gen1', rem where SVN is set to XXX @@ -119,6 +129,7 @@ if not "%2" == "" ( rem Note - OPENIB_REV is assigned SVN in a child script. set SVN=%CD:~22,4% + set LBL=_svn.!SVN! ) rem assumes %CD% == '...\gen1\trunk' @@ -139,6 +150,69 @@ rem WIX Installer files (.msi) destination folder - set for local environment. rem set IDIR=%SystemRoot%\temp +if "%1" == "msi-label" ( + pushd %IDIR% + if exist WOF_wxp_x86.msi ( + if exist WOF_wxp_x86%LBL%.msi del /F/Q WOF_wxp_x86%LBL%.msi + rename WOF_wxp_x86.msi WOF_wxp_x86%LBL%.msi + ) + + if exist WOF_wnet_x86.msi ( + if exist WOF_wnet_x86%LBL%.msi del /F/Q WOF_wnet_x86%LBL%.msi + rename WOF_wnet_x86.msi WOF_wnet_x86%LBL%.msi + ) + if exist WOF_wnet_x64.msi ( + if exist WOF_wnet_x64%LBL%.msi del /F/Q WOF_wnet_x64%LBL%.msi + rename WOF_wnet_x64.msi WOF_wnet_x64%LBL%.msi + ) + if exist WOF_wnet_ia64.msi ( + if exist WOF_wnet_ia64%LBL%.msi del /F/Q WOF_wnet_ia64%LBL%.msi + rename WOF_wnet_ia64.msi WOF_wnet_ia64%LBL%.msi + ) + + if exist WOF_wlh_x86.msi ( + if exist WOF_wlh_x86%LBL%.msi del /F/Q WOF_wlh_x86%LBL%.msi + rename WOF_wlh_x86.msi WOF_wlh_x86%LBL%.msi + ) + if exist WOF_wlh_x64.msi ( + if exist WOF_wlh_x64%LBL%.msi del /F/Q WOF_wlh_x64%LBL%.msi + rename WOF_wlh_x64.msi WOF_wlh_x64%LBL%.msi + ) + if exist WOF_wlh_ia64.msi ( + if exist WOF_wlh_ia64%LBL%.msi del /F/Q WOF_wlh_ia64%LBL%.msi + rename WOF_wlh_ia64.msi WOF_wlh_ia64%LBL%.msi + ) + dir WOF_*%LBL%.msi + popd + exit /B 0 +) + +if "%1" == "msi-del" ( + echo Deleting WOF_{wxp,wnet,wlh}_{x86,x64,ia64}%LBL%.msi + pushd %IDIR% + if exist WOF_wxp_x86%LBL%.msi del /F/P WOF_wxp_x86%LBL%.msi + + if exist WOF_wnet_x86%LBL%.msi del /F/P WOF_wnet_x86%LBL%.msi + if exist WOF_wnet_x64%LBL%.msi del /F/P WOF_wnet_x64%LBL%.msi + if exist WOF_wnet_ia64%LBL%.msi del /F/P WOF_wnet_ia64%LBL%.msi + + if exist WOF_wlh_x86%LBL%.msi del /F/P WOF_wlh_x86%LBL%.msi + if exist WOF_wlh_x64%LBL%.msi del /F/P WOF_wlh_x64%LBL%.msi + if exist WOF_wlh_ia64%LBL%.msi del /F/P WOF_wlh_ia64%LBL%.msi + + dir /N/OD WOF_*.msi + popd + exit /B 0 +) + +if "%1" == "msi-dir" ( + pushd %IDIR% + dir /N/OD WOF_*.msi + popd + exit /B 0 +) + +echo %0 - Building with WDK @ %_DDK% echo Building for OPENIB_REV %SVN%, installer files @ %IDIR% echo Drivers signed using Certificate '%CERTNAME%' @@ -280,19 +354,16 @@ mkdir %RBIN_WNET% mkdir %RBIN_WXP% if EXIST "%BSE%\etc\makebin.bat" ( - echo etc\makebin.bat %BSE% %RBIN_WLH% wlh call %BSE%\etc\makebin.bat %BSE% %RBIN_WLH% wlh if ERRORLEVEL 1 ( echo %0: Err in makebin.bat %BSE% %RBIN_WLH% wlh exit /B 1 ) - echo etc\makebin.bat %BSE% %RBIN_WNET% wnet call %BSE%\etc\makebin.bat %BSE% %RBIN_WNET% wnet if ERRORLEVEL 1 ( echo %0: Err in makebin.bat %BSE% %RBIN_WNET% wnet exit /B 1 ) - echo etc\makebin.bat %BSE% %RBIN_WXP% wxp call %BSE%\etc\makebin.bat %BSE% %RBIN_WXP% wxp if ERRORLEVEL 1 ( echo %0: Err in makebin.bat %BSE% %RBIN_WXP% wxp diff --git a/branches/winverbs/core/bus/kernel/ib_bus32.cdf b/branches/winverbs/core/bus/kernel/ib_bus32.cdf new file mode 100644 index 00000000..de22a915 --- /dev/null +++ b/branches/winverbs/core/bus/kernel/ib_bus32.cdf @@ -0,0 +1,13 @@ +[CatalogHeader] +Name=ib_bus.cat +PublicVersion=0x0000001 +EncodingType=0x00010001 +CATATTR1=0x10010001:OSAttr:2:6.0 +[CatalogFiles] +ib_bus.inf=ib_bus.inf +ibbus.sys=ibbus.sys +ibiou.sys=ibiou.sys +ibal.dll=ibal.dll +complib.dll=complib.dll +ibald.dll=ibald.dll +complibd.dll=complibd.dll diff --git a/branches/winverbs/etc/makebin.bat b/branches/winverbs/etc/makebin.bat index ad96055f..8600ac0a 100644 --- a/branches/winverbs/etc/makebin.bat +++ b/branches/winverbs/etc/makebin.bat @@ -3,11 +3,22 @@ setlocal rem usage: makebin src dst [wlh,wnet,wxp] rem -rem src - ...\gen1\trunk -rem dst - full path where \bin is created. -rem OSE - (Operating System Environment) which windows version - -echo %1 %2 %3 +rem src - OpenIB src path ...\gen1\trunk +rem dst - full path tp where binaries are copied, 'bin\' created here. +rem OSE - (Operating System Environment) which windows version {wxp,wlh,wnet} +rem representing {XP, server 2008 & server 2003} + +rem makebin is designed to take an openIB build tree path and produce a folder +rem tree of binaries suitable for input to a WIX builder which procduces +rem an OS .msi installer. +rem Building a WinOF release is done is 3 phases, makebin is the 2nd phase. +rem makebin is commonly run from trunk\buildrelease.bat although it can be +rem run standalone. + +echo Starting makebin +echo Src %1 +echo Dst %2 +echo OS %3 if "%1"=="" goto usage if "%2"=="" goto usage @@ -56,9 +67,9 @@ rem Copy AMD64 drivers set bin_dir=%1\bin\kernel\objfre_%OSE%_amd64\amd64 set dest_dir=%2\HCA\amd64\ -if "%DBG%" == "TRUE" echo DBG: AMD64 free drivers to %dest_dir% +if "%DBG%" == "TRUE" echo DBG: AMD64 free drivers -set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys +set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys mlx4_hca.inf mlx4_bus.inf for %%i in ( %F% ) do ( xcopy %bin_dir%\%%i %dest_dir% /yq 1> nul if ERRORLEVEL 1 ( @@ -81,7 +92,7 @@ rem Copy IA64 drivers set bin_dir=%1\bin\kernel\objfre_%OSE%_ia64\ia64 set dest_dir=%2\HCA\ia64\ -if "%DBG%" == "TRUE" echo DBG: ia64 free drivers to %dest_dir% +if "%DBG%" == "TRUE" echo DBG: ia64 free drivers set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf for %%i in ( %F% ) do ( @@ -109,7 +120,7 @@ if "%DBG%" == "TRUE" echo DBG: x86 free drivers set bin_dir=%1\bin\kernel\objfre_%OSE%_x86\i386 set dest_dir=%2\HCA\x86\ -set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys +set F=ibbus.sys ib_bus.inf ibiou.sys mthca.sys mthca.inf mlx4_hca.sys mlx4_bus.sys mlx4_hca.inf mlx4_bus.inf for %%i in ( %F% ) do ( xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul if ERRORLEVEL 1 ( @@ -152,7 +163,7 @@ for %%i in (ibald.lib ibald.pdb complibd.lib complibd.pdb) do ( ) ) -echo xcopy %bin_dir%\*.exe %2\tools\amd64\debug\ /yq +echo xcopy *.exe tools\amd64\debug\ xcopy %bin_dir%\*.exe %2\tools\amd64\debug\ /yq 1>nul set F=dapld.dll dapld.pdb datd.dll datd.lib datd.pdb @@ -199,7 +210,7 @@ for %%i in (ibald.lib ibald.pdb complibd.lib complibd.pdb) do ( ) ) -echo xcopy %bin_dir%\*.exe %2\tools\ia64\debug\ /yq +echo xcopy *.exe tools\ia64\debug\ xcopy %bin_dir%\*.exe %2\tools\ia64\debug\ /yq 1>nul set F=dapld.dll dapld.pdb datd.dll datd.lib datd.pdb @@ -248,7 +259,7 @@ for %%i in (ibald.lib ibald.pdb complibd.lib complibd.pdb) do ( ) ) -echo xcopy %bin_dir%\*.exe %2\tools\x86\debug\ /yq +if "%DBG%" == "TRUE" echo DBG: copy debug *.exe to tools\x86\debug\ xcopy %bin_dir%\*.exe %2\tools\x86\debug\ /yq 1>nul set F=dapld.dll dapld.pdb datd.dll datd.lib datd.pdb @@ -315,7 +326,7 @@ rem Copy Free x64 dll set bin_dir=%1\bin\user\objfre_%OSE%_amd64\amd64 set dest_dir=%2\HCA\amd64\ -if "%DBG%" == "TRUE" echo DBG: amd64 Free dlls to %dest_dir% +if "%DBG%" == "TRUE" echo DBG: copy amd64 Free dlls for %%i in (ibal.dll complib.dll mthcau.dll IbInstaller.dll mlx4u.dll) do ( xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul @@ -335,8 +346,12 @@ for %%i in (ibal.lib ibal.pdb complib.lib complib.pdb) do ( xcopy %bin_dir%\ibwsd.dll %2\net\amd64\ /yq xcopy %bin_dir%\installsp.exe %2\net\amd64\ /yq +xcopy %bin_dir%\installsp.exe %2\tools\amd64\release\ /yq +xcopy %bin_dir%\ndinstall.exe %2\net\amd64\ /yq +xcopy %bin_dir%\ndinstall.exe %2\tools\amd64\release\ /yq +xcopy %bin_dir%\ibndprov.dll %2\net\amd64\ /yq -echo xcopy %bin_dir%\*.exe %2\tools\amd64\release\ /y +echo xcopy *.exe tools\amd64\release\ xcopy %bin_dir%\*.exe %2\tools\amd64\release\ /yq 1>nul set F=dapl.dll dapl.pdb dat.dll dat.lib dat.pdb @@ -365,7 +380,7 @@ rem Copy Free IA64 dlls set bin_dir=%1\bin\user\objfre_%OSE%_ia64\ia64 set dest_dir=%2\HCA\ia64\ -if "%DBG%" == "TRUE" echo DBG: IA64 Free dlls to %dest_dir% +if "%DBG%" == "TRUE" echo DBG: copy IA64 Free dlls for %%i in (ibal.dll complib.dll mthcau.dll IbInstaller.dll) do ( xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul @@ -385,8 +400,10 @@ for %%i in (ibal.lib ibal.pdb complib.lib complib.pdb) do ( xcopy %bin_dir%\ibwsd.dll %2\net\ia64\ /yq xcopy %bin_dir%\installsp.exe %2\net\ia64\ /yq +xcopy %bin_dir%\ndinstall.exe %2\net\ia64\ /yq +xcopy %bin_dir%\ibndprov.dll %2\net\ia64\ /yq -echo xcopy %bin_dir%\*.exe %2\tools\ia64\release\ /y q +echo xcopy *.exe tools\ia64\release\ xcopy %bin_dir%\*.exe %2\tools\ia64\release\ /yq 1>nul set F=dapl.dll dapl.pdb dat.dll dat.lib dat.pdb @@ -417,7 +434,7 @@ rem Copy Free x86 drivers set bin_dir=%1\bin\user\objfre_%OSE%_x86\i386 set dest_dir=%2\HCA\x86\ -if "%DBG%" == "TRUE" echo DBG: x86 Free dlls to %dest_dir% +if "%DBG%" == "TRUE" echo DBG: copy x86 Free dlls for %%i in (ibal.dll complib.dll mthcau.dll IbInstaller.dll mlx4u.dll) do ( xcopy %bin_dir%\%%i %dest_dir% /yq 1>nul @@ -435,7 +452,7 @@ for %%i in (ibal.lib ibal.pdb complib.lib complib.pdb) do ( ) ) -echo xcopy %bin_dir%\*.exe %2\tools\x86\release\ /yq +echo xcopy *.exe tools\x86\release\ xcopy %bin_dir%\*.exe %2\tools\x86\release\ /yq 1>nul set F=dapl.dll dapl.pdb dat.dll dat.lib dat.pdb @@ -463,16 +480,21 @@ if /I "%OSE%" == "wxp" goto inf_files rem free x86 items -if "%DBG%" == "TRUE" echo DBG: x86 Free WSD to %dest_dir%\net\x86 +if "%DBG%" == "TRUE" echo DBG: copy x86 Free WSD copy %bin_dir%\ibwsd.dll %2\net\x86\ /y copy %bin_dir%\installsp.exe %2\net\x86\ /y -copy %bin_dir%\installsp.exe %2\tools\x86\release /y +copy %bin_dir%\installsp.exe %2\tools\x86\release\ /y +copy %bin_dir%\ndinstall.exe %2\net\x86\ /y +copy %bin_dir%\ndinstall.exe %2\tools\x86\release\ /y +copy %bin_dir%\ibndprov.dll %2\net\x86\ /y rem free x86 DLLs --> WOW64 DLLs if "%DBG%" == "TRUE" echo DBG: x86 Free dlls to WOW64 +copy %bin_dir%\ibndprov.dll %2\net\amd64\ibndprov32.dll /y +copy %bin_dir%\ibndprov.dll %2\net\ia64\ibndprov32.dll /y copy /B %bin_dir%\ibwsd.dll %2\net\amd64\ibwsd32.dll /y copy /B %bin_dir%\ibwsd.dll %2\net\ia64\ibwsd32.dll /y copy /B %bin_dir%\ibal.dll %2\HCA\amd64\ibal32.dll /y @@ -512,36 +534,30 @@ rem .inf files now resident alongside .sys files if "%DBG%" == "TRUE" echo DBG: cpy %OSE% x86 INF files -xcopy %1\hw\mthca\kernel\mthca.cdf %2\HCA\x86\ /yq -xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.inf %2\HCA\x86\ /yq -xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.cdf %2\HCA\x86\ /yq -xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.inf %2\HCA\x86\ /yq -xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.cdf %2\HCA\x86\ /yq - -xcopy %1\core\bus\kernel\ib_bus.cdf %2\HCA\x86\ /yq +copy /A/Y %1\hw\mthca\kernel\mthca32.cdf %2\HCA\x86\mthca.cdf +copy /A/Y %1\hw\mlx4\kernel\hca\mlx4_hca32.cdf %2\HCA\x86\mlx4_hca.cdf +copy /A/Y %1\hw\mlx4\kernel\bus\drv\mlx4_bus32.cdf %2\HCA\x86\mlx4_bus.cdf +copy /A/Y %1\core\bus\kernel\ib_bus32.cdf %2\HCA\x86\ib_bus.cdf rem No WSD support for XP32 if /I "%OSE%" == "wxp" ( copy /A /Y %1\ulp\ipoib\kernel\netipoib-xp32.inf %2\net\x86\netipoib.inf - copy /A /Y %1\ulp\ipoib\kernel\ipoib-xp32.cdf %2\net\x86\ipoib.cdf + copy /A /Y %1\ulp\ipoib\kernel\ipoib32-xp.cdf %2\net\x86\ipoib.cdf ) else ( - xcopy %1\ulp\ipoib\kernel\ipoib.cdf %2\net\x86\ /yq + copy /A/Y %1\ulp\ipoib\kernel\ipoib32.cdf %2\net\x86\ipoib.cdf xcopy %1\tools\wsdinstall\user\installsp.exe.manifest %2\net\x86\ /yq ) xcopy %1\ulp\qlgcvnic\kernel\netvnic.cdf %2\net\x86\ /yq xcopy %1\ulp\srp\kernel\ib_srp.cdf %2\storage\x86\ /yq - if /I "%OSE%" == "wxp" goto cpy_bin_files if "%DBG%" == "TRUE" echo DBG: copy 64bit INF files xcopy %1\hw\mthca\kernel\mthca.cdf %2\HCA\amd64 /yq -xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.inf %2\HCA\amd64\ /yq xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.cdf %2\HCA\amd64\ /yq xcopy %1\hw\mlx4\kernel\hca\mlx4_hca32.cdf %2\HCA\amd64\ /yq -xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.inf %2\HCA\amd64\ /yq xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.cdf %2\HCA\amd64\ /yq xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus32.cdf %2\HCA\amd64\ /yq @@ -572,7 +588,7 @@ xcopy %1\ulp\srp\kernel\ib_srp.cdf %2\storage\ia64\ /yq rem bin\bin used to generate a web based symbol store in build-ofa-dist.bat. -echo 'Copy bin\obj{chk,fre}_%3_{x86,amd64,ia64} to %2\bin' +echo 'Copy bin\obj{chk,fre}_%3_{x86,amd64,ia64} to bin' rem xcopy %1\bin %2\bin\ /eyq rem instead of copying the entire bin\ folder, pick items of OS interest. @@ -671,8 +687,8 @@ if "%DBG%" == "TRUE" echo DBG: Docs and SDK files if NOT exist %2\Misc ( mkdir %2\Misc ) -copy /Y %1\Docs\Manual.htm %2\Misc\Manual.htm /A -copy /Y %1\tests\cmtest\user\cmtest_main.c %2\Misc\cmtest.c /A +copy /Y/A %1\Docs\Manual.htm %2\Misc\Manual.htm +copy /Y/A %1\tests\cmtest\user\cmtest_main.c %2\Misc\cmtest.c goto end @@ -711,4 +727,8 @@ goto end echo %1\bin\user\objchk_%OSE%_x86\i386 missing :end +echo . +echo Finished OS %3 +echo . + endlocal diff --git a/branches/winverbs/hw/mlx4/kernel/bus/core/SOURCES b/branches/winverbs/hw/mlx4/kernel/bus/core/SOURCES index d138a00a..2f921072 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/core/SOURCES +++ b/branches/winverbs/hw/mlx4/kernel/bus/core/SOURCES @@ -35,8 +35,7 @@ C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUS TARGETLIBS= \ $(DDK_LIB_PATH)\ntstrsafe.lib \ - $(TARGETPATH)\*\complib.lib - + $(TARGETPATH)\*\complib.lib !IFDEF ENABLE_EVENT_TRACING diff --git a/branches/winverbs/hw/mlx4/kernel/bus/core/cache.c b/branches/winverbs/hw/mlx4/kernel/bus/core/cache.c index a313236f..5b819358 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/core/cache.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/core/cache.c @@ -83,6 +83,9 @@ int ib_get_cached_gid(struct ib_device *device, unsigned long flags; int ret = 0; + if (mlx4_is_barred(device->dma_device)) + return -EFAULT; + if (port_num < start_port(device) || port_num > end_port(device)) return -EINVAL; @@ -111,6 +114,9 @@ int ib_find_cached_gid(struct ib_device *device, int p, i; int ret = -ENOENT; + if (mlx4_is_barred(device->dma_device)) + return -EFAULT; + *port_num = (u8)-1; if (index) *index = (u16)-1; @@ -145,6 +151,9 @@ int ib_get_cached_pkey(struct ib_device *device, unsigned long flags; int ret = 0; + if (mlx4_is_barred(device->dma_device)) + return -EFAULT; + if (port_num < start_port(device) || port_num > end_port(device)) return -EINVAL; @@ -173,6 +182,9 @@ int ib_find_cached_pkey(struct ib_device *device, int i; int ret = -ENOENT; + if (mlx4_is_barred(device->dma_device)) + return -EFAULT; + if (port_num < start_port(device) || port_num > end_port(device)) return -EINVAL; @@ -382,7 +394,7 @@ static void ib_cache_setup_one(struct ib_device *device) } INIT_IB_EVENT_HANDLER(&device->cache.event_handler, - device, ib_cache_event, NULL); + device, ib_cache_event, NULL, NULL, 0); if (ib_register_event_handler(&device->cache.event_handler)) goto err_cache; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.c b/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.c index 5e8c80c8..35bc591b 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.c @@ -101,6 +101,7 @@ EvtInterruptIsr( #endif +static NTSTATUS __create_child( __in WDFDEVICE Device, @@ -200,6 +201,7 @@ Routine Description: return status; } +static NTSTATUS __do_static_enumeration( IN WDFDEVICE Device @@ -223,11 +225,14 @@ Routine Description: NTSTATUS status = STATUS_SUCCESS; int i; int number_of_ib_ports; - - // TODO:Need to add an event log in the case of errors + PFDO_DEVICE_DATA p_fdo = FdoGetData(Device); + struct mlx4_dev *mdev = p_fdo->pci_dev.dev; MLX4_ENTER(MLX4_DBG_DRV); + if ( p_fdo->children_created ) + goto end; + // eventually we'll have all information about children in Registry // DriverEntry will read it into a Global storage and // this routine will create all the children on base on this info @@ -237,7 +242,7 @@ Routine Description: if(number_of_ib_ports > 0) { status = __create_child(Device, BUS_HARDWARE_IDS, BUS_HARDWARE_DESCRIPTION, 0 ); if (!NT_SUCCESS(status)) { - MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (ib)failed with 0x%x\n", status)); + MLX4_PRINT_EV(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (ib)failed with 0x%x\n", status)); } } @@ -246,15 +251,107 @@ Routine Description: if(mlx4_is_eth_port(i)) { status = __create_child(Device, ETH_HARDWARE_IDS, ETH_HARDWARE_DESCRIPTION, i+1 ); if (!NT_SUCCESS(status)) { - MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (eth) failed with 0x%x\n", status)); + MLX4_PRINT_EV(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__create_child (eth) failed with 0x%x\n", status)); } } } + p_fdo->children_created = TRUE; + +end: MLX4_EXIT( MLX4_DBG_DRV ); return status; } +static +NTSTATUS +__start_card( + IN PFDO_DEVICE_DATA p_fdo + ) +{ +#ifndef USE_WDM_INTERRUPTS + int i; +#endif + int err; + NTSTATUS status = STATUS_SUCCESS; + struct pci_dev *pdev = &p_fdo->pci_dev; + + MLX4_ENTER(MLX4_DBG_DRV); + + if ( p_fdo->card_started ) + goto err; + + // enable the card + status = pci_hca_enable( &pdev->bus_pci_ifc, &pdev->pci_cfg_space ); + if( !NT_SUCCESS( status ) ) + goto err; + + // + // init the card + // + +#ifndef USE_WDM_INTERRUPTS + // enable interrupts for start up + for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) + WdfInterruptEnable(p_fdo->interrupt[i].WdfInterrupt); +#endif + + // NET library + err = mlx4_init_one( &p_fdo->pci_dev ); + if (err) { + status = errno_to_ntstatus(err); + goto err; + } + +#ifndef USE_WDM_INTERRUPTS + // + // complete filling interrupt context (for more efficiency) + // + for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) { + struct mlx4_priv *priv = mlx4_priv( p_fdo->pci_dev.dev ); + PINTERRUPT_DATA p_isr_ctx = WdfObjectGetTypedContext( + p_fdo->interrupt[i].WdfInterrupt, INTERRUPT_DATA ); + + p_isr_ctx->eq = &priv->eq_table.eq[i]; + } +#endif + + // + // prepare MLX4 IB interface + // + + // fill the header + p_fdo->bus_ib_ifc.i.Size = sizeof(MLX4_BUS_IB_INTERFACE); + p_fdo->bus_ib_ifc.i.Version = MLX4_BUS_IB_INTERFACE_VERSION; + // Let the framework handle reference counting. + p_fdo->bus_ib_ifc.i.InterfaceReference = WdfDeviceInterfaceReferenceNoOp; + p_fdo->bus_ib_ifc.i.InterfaceDereference = WdfDeviceInterfaceDereferenceNoOp; + + p_fdo->bus_ib_ifc.pdev = &p_fdo->pci_dev; + p_fdo->bus_ib_ifc.p_ibdev = p_fdo->pci_dev.ib_dev; + p_fdo->bus_ib_ifc.pmlx4_dev = to_mdev(p_fdo->pci_dev.ib_dev)->dev; + p_fdo->bus_ib_ifc.is_livefish = mlx4_is_livefish(p_fdo->pci_dev.dev); + + p_fdo->card_started = TRUE; + +err: + MLX4_EXIT( MLX4_DBG_DRV ); + return status; +} + +static +void +__stop_card( + IN PFDO_DEVICE_DATA p_fdo + ) +{ + if ( p_fdo->card_started ) { + p_fdo->card_started = FALSE; + mlx4_remove_one( &p_fdo->pci_dev ); + } +} + + NTSTATUS EvtDeviceD0Entry( IN WDFDEVICE Device, @@ -262,37 +359,39 @@ EvtDeviceD0Entry( ) { NTSTATUS status = STATUS_SUCCESS; - - UNUSED_PARAM(Device); - UNUSED_PARAM(PreviousState); + PFDO_DEVICE_DATA p_fdo = FdoGetData(Device); + struct pci_dev *pdev = &p_fdo->pci_dev; + struct mlx4_dev *mdev; MLX4_ENTER(MLX4_DBG_DRV); MLX4_PRINT(TRACE_LEVEL_INFORMATION, MLX4_DBG_DRV, ("PreviousState 0x%x\n", PreviousState)); + // start card (needed after Hibernetion) + if (PreviousState > WdfPowerDeviceD0) + __start_card( p_fdo ); + mdev = pdev->dev; + + // create child device status = __do_static_enumeration(Device); - if (!NT_SUCCESS(status)) { - MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__do_static_enumeration failed with 0x%x\n", status)); - } + if (!NT_SUCCESS(status)) { + MLX4_PRINT_EV(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, ("__do_static_enumeration failed with 0x%x\n", status)); + goto err; + } - { - PFDO_DEVICE_DATA p_fdo = FdoGetData(Device); - struct pci_dev *pdev = &p_fdo->pci_dev; - struct mlx4_dev *mdev = pdev->dev; - - MLX4_PRINT_EV(TRACE_LEVEL_INFORMATION ,MLX4_DBG_DRV , - ("Ven %x Dev %d Fw %d.%d.%d Drv %s (%s), BD %s\n", - (unsigned)pdev->ven_id, (unsigned)pdev->dev_id, - (int) (mdev->caps.fw_ver >> 32), - (int) (mdev->caps.fw_ver >> 16) & 0xffff, - (int) (mdev->caps.fw_ver & 0xffff), - DRV_VERSION, DRV_RELDATE, - mlx4_is_livefish(mdev) ? "Y" : "N" - )); - } + // Log Success Message + MLX4_PRINT_EV(TRACE_LEVEL_INFORMATION ,MLX4_DBG_DRV , + ("Ven %x Dev %d Fw %d.%d.%d, IsBurnDevice %s\n", + (unsigned)pdev->ven_id, (unsigned)pdev->dev_id, + (int) (mdev->caps.fw_ver >> 32), + (int) (mdev->caps.fw_ver >> 16) & 0xffff, + (int) (mdev->caps.fw_ver & 0xffff), + mlx4_is_livefish(mdev) ? "Y" : "N" + )); +err: MLX4_EXIT( MLX4_DBG_DRV ); - return STATUS_SUCCESS; + return status; } NTSTATUS @@ -301,10 +400,14 @@ EvtDeviceD0Exit( IN WDF_POWER_DEVICE_STATE TargetState ) { - UNUSED_PARAM(Device); - UNUSED_PARAM(TargetState); + PFDO_DEVICE_DATA p_fdo = FdoGetData(Device); + MLX4_ENTER(MLX4_DBG_DRV); MLX4_PRINT(TRACE_LEVEL_INFORMATION, MLX4_DBG_DRV, ("TargetState 0x%x\n", TargetState)); + + if (TargetState > WdfPowerDeviceD0) + __stop_card( p_fdo ); + MLX4_EXIT( MLX4_DBG_DRV ); return STATUS_SUCCESS; } @@ -516,7 +619,6 @@ err: return status; } - NTSTATUS EvtPrepareHardware( IN WDFDEVICE Device, @@ -524,13 +626,8 @@ EvtPrepareHardware( IN WDFCMRESLIST ResourcesTranslated ) { -#ifndef USE_WDM_INTERRUPTS - int i; -#endif - int err; NTSTATUS status; PFDO_DEVICE_DATA p_fdo = FdoGetData(Device); - struct pci_dev *pdev = &p_fdo->pci_dev; MLX4_ENTER(MLX4_DBG_DRV); @@ -541,62 +638,22 @@ EvtPrepareHardware( goto err; } - // enable the card - status = pci_hca_enable( &pdev->bus_pci_ifc, &pdev->pci_cfg_space ); - if( !NT_SUCCESS( status ) ) - goto err; - - // - // init the card - // - -#ifndef USE_WDM_INTERRUPTS - // enable interrupts for start up - for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) - WdfInterruptEnable(p_fdo->interrupt[i].WdfInterrupt); -#endif - - // NET library - err = mlx4_init_one( &p_fdo->pci_dev ); - if (err) { - status = errno_to_ntstatus(err); - goto err; - } - -#ifndef USE_WDM_INTERRUPTS - // - // complete filling interrupt context (for more efficiency) - // - for ( i = 0; i < MLX4_MAX_INTERRUPTS; ++i ) { - struct mlx4_priv *priv = mlx4_priv( p_fdo->pci_dev.dev ); - PINTERRUPT_DATA p_isr_ctx = WdfObjectGetTypedContext( - p_fdo->interrupt[i].WdfInterrupt, INTERRUPT_DATA ); - - p_isr_ctx->eq = &priv->eq_table.eq[i]; - } -#endif - - // - // prepare MLX4 IB interface - // + // start the card + status = __start_card( p_fdo ); + +err: + MLX4_EXIT( MLX4_DBG_DRV ); + return status; +} - // fill the header - p_fdo->bus_ib_ifc.i.Size = sizeof(MLX4_BUS_IB_INTERFACE); - p_fdo->bus_ib_ifc.i.Version = MLX4_BUS_IB_INTERFACE_VERSION; - // Let the framework handle reference counting. - p_fdo->bus_ib_ifc.i.InterfaceReference = WdfDeviceInterfaceReferenceNoOp; - p_fdo->bus_ib_ifc.i.InterfaceDereference = WdfDeviceInterfaceDereferenceNoOp; +void fix_bus_ifc(struct pci_dev *pdev) +{ + PFDO_DEVICE_DATA p_fdo; - p_fdo->bus_ib_ifc.pdev = &p_fdo->pci_dev; + p_fdo = CONTAINING_RECORD(pdev, FDO_DEVICE_DATA, pci_dev); p_fdo->bus_ib_ifc.p_ibdev = p_fdo->pci_dev.ib_dev; p_fdo->bus_ib_ifc.pmlx4_dev = to_mdev(p_fdo->pci_dev.ib_dev)->dev; p_fdo->bus_ib_ifc.is_livefish = mlx4_is_livefish(p_fdo->pci_dev.dev); - - status = STATUS_SUCCESS; - -err: - MLX4_EXIT( MLX4_DBG_DRV ); - return status; } NTSTATUS @@ -611,7 +668,7 @@ EvtReleaseHardware( MLX4_ENTER(MLX4_DBG_DRV); - mlx4_remove_one( &p_fdo->pci_dev ); + __stop_card( p_fdo ); __put_resources( p_fdo ); MLX4_EXIT( MLX4_DBG_DRV ); @@ -894,7 +951,7 @@ __read_registry(WDFDRIVER *hDriver) // "Ports L2 type (ib/eth/auto, entry per port, comma seperated, default ib for all)" DECLARE_CONST_UNICODE_STRING(PortType, L"PortType"); - + ULONG value; WDFKEY hKey = NULL; NTSTATUS status = STATUS_SUCCESS; @@ -984,6 +1041,7 @@ __read_registry(WDFDRIVER *hDriver) else g.mod_interrupt_from_first = 1; + uvalue.Buffer = uvalue_data; uvalue.MaximumLength = MAX_UVALUE; uvalue.Length = 0; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.h b/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.h index 18037801..2873a243 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.h +++ b/branches/winverbs/hw/mlx4/kernel/bus/drv/drv.h @@ -62,24 +62,18 @@ typedef struct _FDO_DEVICE_DATA WDFWAITLOCK ChildLock; WDFDEVICE FdoDevice; struct pci_dev pci_dev; + int card_started; int pci_bus_ifc_taken; WDFDMAENABLER dma_enabler; int dma_adapter_taken; res_interrupt_t interrupt[MLX4_MAX_INTERRUPTS]; MLX4_BUS_IB_INTERFACE bus_ib_ifc; + int children_created; // Data for the Ethernet device -#ifdef _WIN64 - UCHAR pad[0x8]; -#endif - struct VipBusIfc mtnic_Ifc; } FDO_DEVICE_DATA, *PFDO_DEVICE_DATA; -#ifdef _WIN64 -C_ASSERT((FIELD_OFFSET(FDO_DEVICE_DATA, mtnic_Ifc) % 16) == 0); -#endif - WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(FDO_DEVICE_DATA, FdoGetData) // diff --git a/branches/winverbs/hw/mlx4/kernel/bus/drv/mlx4_bus.inx b/branches/winverbs/hw/mlx4/kernel/bus/drv/mlx4_bus.inx index fa4fbcac..0af59759 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/drv/mlx4_bus.inx +++ b/branches/winverbs/hw/mlx4/kernel/bus/drv/mlx4_bus.inx @@ -75,6 +75,7 @@ wdfcoinstaller01007.dll = 1,, %MT25448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6368 %MT26418.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6732 %MT26428.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_673c +%MT26448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6750 %MT00401.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_0191 [MLX4BUS.DeviceSection.ntamd64] @@ -84,6 +85,7 @@ wdfcoinstaller01007.dll = 1,, %MT26418.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6732 %MT25448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6368 %MT26428.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_673c +%MT26448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6750 %MT00401.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_0191 [MLX4BUS.DeviceSection.ntia64] @@ -93,42 +95,9 @@ wdfcoinstaller01007.dll = 1,, %MT25448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6368 %MT26418.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6732 %MT26428.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_673c +%MT26448.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_6750 %MT00401.DeviceDesc%=MLX4BUS.DDInstall, PCI\VEN_15B3&DEV_0191 -[MLX4BUS.DDInstall.ntx86.hw] -AddReg = MLX4BUS.HwReg - -[MLX4BUS.DDInstall.ntamd64.hw] -AddReg = MLX4BUS.HwReg - -[MLX4BUS.DDInstall.ntia64.hw] -AddReg = MLX4BUS.HwReg - -[MLX4BUS.HwReg] -HKR,"Interrupt Management", 0x00000010 -HKR,"Interrupt Management\MessageSignaledInterruptProperties",0x00000010 - -; MSI/MSI-X support -HKR,"Interrupt Management\MessageSignaledInterruptProperties",MSISupported,0x00010001,0 -HKR,"Interrupt Management\MessageSignaledInterruptProperties",MessageNumberLimit,0x00010001,8 -HKR,"Interrupt Management\Affinity Policy",0x00000010 - -; AssignmentSetOverride - processors KAFFINITY mask -HKR,"Interrupt Management\Affinity Policy",AssignmentSetOverride,0x00000001,0x0 - -; IrqPolicyMachineDefault (0) - use default policy for the computer -; IrqPolicyAllCloseProcessors (1) - connect interrupts to all processors of the near NUMA node -; IrqPolicyOneCloseProcessor (2) - connect interrupts to one processor -; IrqPolicyAllProcessorsInMachine (3) - connect interrupts to all processors in the machine -; IrqPolicySpecifiedProcessors (4) - connects interrupts according to AssignmentSetOverride -HKR,"Interrupt Management\Affinity Policy",DevicePolicy,0x00010001,0x0 - -; IrqArbPriorityUndefined (0) - no interrupt priority policy. -; IrqArbPriorityLow (1) - device can tolerate low IRQL -; IrqArbPriorityNormal (2) - device expects normal interrupt latencies -; IrqArbPriorityHigh (3) - device requires the lowest possible interrupt latency -HKR,"Interrupt Management\Affinity Policy",DevicePriority,0x00010001,0x0 - [MLX4BUS.DDInstall.ntx86] CopyFiles = MLX4BUS.CopyFiles @@ -244,6 +213,7 @@ MT25428.DeviceDesc="ConnectX (MT25428) - Mellanox ConnectX QDR Channel Adapter" MT25448.DeviceDesc="ConnectX (MT25448) - Mellanox ConnectX Condor Ethernet Adapter" MT26418.DeviceDesc="ConnectX (MT26418) - Mellanox ConnectX DDR_G2 Channel Adapter" MT26428.DeviceDesc="ConnectX (MT26428) - Mellanox ConnectX QDR_G2 Channel Adapter" +MT26448.DeviceDesc="ConnectX (MT26448) - Mellanox ConnectX Condor_G2 Ethernet Adapter" MT00401.DeviceDesc="ConnectX (MT00401) - Mellanox ConnectX Channel Adapter in Burning Mode" DiskId = "Mellanox Mlx4 Bus installation disk" SPSVCINST_NULL = 0x0 diff --git a/branches/winverbs/hw/mlx4/kernel/bus/drv/pdo.c b/branches/winverbs/hw/mlx4/kernel/bus/drv/pdo.c index 430a6956..920cd476 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/drv/pdo.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/drv/pdo.c @@ -12,6 +12,28 @@ #define MAX_ID_LEN 80 +NTSTATUS +EvtDeviceProcessQueryInterfaceRequest( + IN WDFDEVICE Device, + IN LPGUID InterfaceType, + IN OUT PINTERFACE ExposedInterface, + IN OUT PVOID ExposedInterfaceSpecificData + ) +{ + PPDO_DEVICE_DATA p_pdo = PdoGetData(Device); + PFDO_DEVICE_DATA p_fdo = p_pdo->p_fdo; + PMLX4_BUS_IB_INTERFACE p_ib_ifc = (PMLX4_BUS_IB_INTERFACE)ExposedInterface; + + UNUSED_PARAM(InterfaceType); + UNUSED_PARAM(ExposedInterfaceSpecificData); + + p_ib_ifc->p_ibdev = p_fdo->bus_ib_ifc.p_ibdev; + p_ib_ifc->pmlx4_dev = p_fdo->bus_ib_ifc.pmlx4_dev; + p_ib_ifc->is_livefish = p_fdo->bus_ib_ifc.is_livefish; + + return STATUS_SUCCESS; +} + NTSTATUS create_pdo( __in WDFDEVICE Device, @@ -186,9 +208,6 @@ Return Value: p_fdo->bus_ib_ifc.port_id = (u8) SerialNo; p_fdo->bus_ib_ifc.pVipBusIfc = &p_fdo->mtnic_Ifc; - p_fdo->bus_ib_ifc.pVipBusIfc->ulAllocatePortObjSize = MAX_PORT_SIZE; - p_fdo->bus_ib_ifc.register_interface = mlx4_register_interface; - p_fdo->bus_ib_ifc.unregister_interface = mlx4_unregister_interface; p_fdo->bus_ib_ifc.mlx4_interface.mlx4_pd_alloc = mlx4_pd_alloc; p_fdo->bus_ib_ifc.mlx4_interface.mlx4_pd_free = mlx4_pd_free; p_fdo->bus_ib_ifc.mlx4_interface.mlx4_uar_alloc = mlx4_uar_alloc; @@ -228,8 +247,10 @@ Return Value: p_fdo->bus_ib_ifc.mlx4_interface.mlx4_CLOSE_PORT = mlx4_CLOSE_PORT; p_fdo->bus_ib_ifc.mlx4_interface.mlx4_add_eq = mlx4_add_eq; p_fdo->bus_ib_ifc.mlx4_interface.mlx4_remove_eq = mlx4_remove_eq; - p_fdo->bus_ib_ifc.mlx4_interface.mlx4_register_ev_cb = ib_register_event_handler; - p_fdo->bus_ib_ifc.mlx4_interface.mlx4_unregister_ev_cb = ib_unregister_event_handler; + p_fdo->bus_ib_ifc.mlx4_interface.mlx4_register_ev_cb = mlx4_reset_cb_register; + p_fdo->bus_ib_ifc.mlx4_interface.mlx4_unregister_ev_cb = mlx4_reset_cb_unregister; + p_fdo->bus_ib_ifc.mlx4_interface.mlx4_reset_request = mlx4_reset_request; + p_fdo->bus_ib_ifc.mlx4_interface.mlx4_reset_execute = mlx4_reset_execute; // // Create a custom interface so that other drivers can @@ -239,7 +260,7 @@ Return Value: WDF_QUERY_INTERFACE_CONFIG_INIT( &p_pdo->qiMlx4Bus, (PINTERFACE) &p_fdo->bus_ib_ifc, - &MLX4_BUS_IB_INTERFACE_GUID, NULL); + &MLX4_BUS_IB_INTERFACE_GUID, EvtDeviceProcessQueryInterfaceRequest); status = WdfDeviceAddQueryInterface( hChild, &p_pdo->qiMlx4Bus ); if (!NT_SUCCESS(status)) @@ -252,6 +273,8 @@ Return Value: (PINTERFACE) &p_fdo->pci_dev.bus_pci_ifc, &GUID_BUS_INTERFACE_STANDARD, NULL); + // TODO: Soft Reset - how tobar getting interface during RESET_IN_PROGRESS + // maybe - using EvtDeviceProcessQueryInterfaceRequest status = WdfDeviceAddQueryInterface( hChild, &p_pdo->qiPciBus ); if (!NT_SUCCESS(status)) goto Cleanup; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/ah.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/ah.c index d281e629..85a0da13 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/ah.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/ah.c @@ -37,6 +37,9 @@ struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) struct mlx4_dev *dev = to_mdev(pd->device)->dev; struct mlx4_ib_ah *ah; + if (mlx4_is_barred(pd->device->dma_device)) + return ERR_PTR(-EFAULT); + ah = kmalloc(sizeof *ah, GFP_ATOMIC); if (!ah) return ERR_PTR(-ENOMEM); @@ -70,6 +73,9 @@ int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) { struct mlx4_ib_ah *ah = to_mah(ibah); + if (mlx4_is_barred(ibah->device->dma_device)) + return -EFAULT; + memset(ah_attr, 0, sizeof *ah_attr); ah_attr->dlid = be16_to_cpu(ah->av.dlid); ah_attr->sl = (u8)(be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28); @@ -105,6 +111,9 @@ int mlx4_ib_modify_ah( struct ib_ah *ibah, struct ib_ah_attr *ah_attr ) struct mlx4_av *av = &to_mah(ibah)->av; struct mlx4_dev *dev = to_mdev(ibah->pd->device)->dev; + if (mlx4_is_barred(dev)) + return -EFAULT; + // taken from mthca_create_av av->port_pd = cpu_to_be32(to_mpd(ibah->pd)->pdn | (ah_attr->port_num << 24)); av->g_slid = ah_attr->src_path_bits; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/cq.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/cq.c index 23e49dc7..5be4388f 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/cq.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/cq.c @@ -1,613 +1,620 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include -#include "mlx4_ib.h" -#include "cq.h" -#include "qp.h" -#include "user.h" - -static void mlx4_ib_cq_comp(struct mlx4_cq *cq) -{ - struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; - ibcq->comp_handler(ibcq->cq_context); -} - -static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) -{ - ib_event_rec_t event; - struct ib_cq *ibcq; - - if (type != MLX4_EVENT_TYPE_CQ_ERROR) { - printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " - "on CQ %06x\n", type, cq->cqn); - return; - } - - ibcq = &to_mibcq(cq)->ibcq; - event.type = IB_EVENT_CQ_ERR; - event.context = ibcq->cq_context; - event.vendor_specific = type; - ibcq->event_handler(&event); -} - -static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n) -{ - int offset = n * sizeof (struct mlx4_cqe); - - if (buf->buf.nbufs == 1) - return buf->buf.u.direct.buf + offset; - else - return buf->buf.u.page_list[offset >> PAGE_SHIFT].buf + - (offset & (PAGE_SIZE - 1)); -} - -static void *get_cqe(struct mlx4_ib_cq *cq, int n) -{ - return get_cqe_from_buf(&cq->buf, n); -} - -static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) -{ - struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); - - return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ - !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; -} - -static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) -{ - return get_sw_cqe(cq, cq->mcq.cons_index); -} - -int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) -{ - struct mlx4_ib_cq *mcq = to_mcq(cq); - struct mlx4_ib_dev *dev = to_mdev(cq->device); - struct mlx4_cq_context *context; - int err; - - context = kzalloc(sizeof *context, GFP_KERNEL); - if (!context) - return -ENOMEM; - - context->cq_period = cpu_to_be16(cq_period); - context->cq_max_count = cpu_to_be16(cq_count); - err = mlx4_cq_modify(dev->dev, &mcq->mcq, context, 1); - - kfree(context); - return err; -} - -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct mlx4_ib_cq *cq; - struct mlx4_uar *uar; - int buf_size; - int err; - - UNUSED_PARAM(vector); - - if (entries < 1 || entries > dev->dev->caps.max_cqes) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof *cq, GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); - - entries = roundup_pow_of_two(entries + 1); - cq->ibcq.cqe = entries - 1; - buf_size = entries * sizeof (struct mlx4_cqe); - spin_lock_init(&cq->lock); - - if (context) { - struct mlx4_ib_create_cq ucmd; - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - err = -EFAULT; - goto err_cq; - } - - cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size, - IB_ACCESS_LOCAL_WRITE, FALSE); - if (IS_ERR(cq->umem)) { - err = PTR_ERR(cq->umem); - goto err_cq; - } - - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem), - ilog2(cq->umem->page_size), &cq->buf.mtt); - if (err) - goto err_buf; - - err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem); - if (err) - goto err_mtt; - - err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr, - &cq->db); - if (err) - goto err_mtt; - - // add mapping to user's arm_sn variable - // we have no way pass the completion event to provider library - // so we'll increment user's arm_sn in kernel - err = ib_umem_map( ucmd.arm_sn_addr, sizeof(int), - IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn ); - if (err) - goto err_dbmap; - - uar = &to_mucontext(context)->uar; - } else { - err = mlx4_ib_db_alloc(dev, &cq->db, 1); - if (err) - goto err_cq; - - cq->mcq.set_ci_db = cq->db.db; - cq->mcq.arm_db = cq->db.db + 1; - *cq->mcq.set_ci_db = 0; - *cq->mcq.arm_db = 0; - - if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) { - err = -ENOMEM; - goto err_db; - } - - err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift, - &cq->buf.mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf); - if (err) - goto err_mtt; - - cq->mcq.p_u_arm_sn = NULL; - uar = &dev->priv_uar; - } - - err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, - cq->db.dma.da, &cq->mcq, 0, 0); - if (err) - goto err_dbmap; - - cq->mcq.comp = mlx4_ib_cq_comp; - cq->mcq.event = mlx4_ib_cq_event; - - if (context) - if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { - err = -EFAULT; - goto err_dbmap; - } - - return &cq->ibcq; - -err_dbmap: - ib_umem_unmap( cq->mcq.mdl, cq->mcq.p_u_arm_sn ); - if (context) - mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); - -err_buf: - if (context) - ib_umem_release(cq->umem); - else - mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe), - &cq->buf.buf); - -err_db: - if (!context) - mlx4_ib_db_free(dev, &cq->db); - -err_cq: - kfree(cq); - - return ERR_PTR(err); -} - -int mlx4_ib_destroy_cq(struct ib_cq *cq) -{ - struct mlx4_ib_dev *dev = to_mdev(cq->device); - struct mlx4_ib_cq *mcq = to_mcq(cq); - - mlx4_cq_free(dev->dev, &mcq->mcq); - mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt); - - if (cq->p_uctx) { - ib_umem_unmap( mcq->mcq.mdl, mcq->mcq.p_u_arm_sn ); - mlx4_ib_db_unmap_user(to_mucontext(cq->p_uctx), &mcq->db); - ib_umem_release(mcq->umem); - } else { - mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe), - &mcq->buf.buf); - mlx4_ib_db_free(dev, &mcq->db); - } - - kfree(mcq); - - return 0; -} - -static void dump_cqe(void *cqe) -{ - __be32 *buf = cqe; - - printk(KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", - be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]), - be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]), - be32_to_cpu(buf[6]), be32_to_cpu(buf[7])); -} - -static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe, - ib_wc_t *wc) -{ - if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) { - printk(KERN_DEBUG "local QP operation err " - "(QPN %06x, WQE index %x, vendor syndrome %02x, " - "opcode = %02x)\n", - be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index), - cqe->vendor_err_syndrome, - cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); - dump_cqe(cqe); - } - - switch (cqe->syndrome) { - case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: - wc->status = IB_WCS_LOCAL_LEN_ERR; - break; - case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: - wc->status = IB_WCS_LOCAL_OP_ERR; - break; - case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: - wc->status = IB_WCS_LOCAL_PROTECTION_ERR; - break; - case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: - wc->status = IB_WCS_WR_FLUSHED_ERR; - break; - case MLX4_CQE_SYNDROME_MW_BIND_ERR: - wc->status = IB_WCS_MEM_WINDOW_BIND_ERR; - break; - case MLX4_CQE_SYNDROME_BAD_RESP_ERR: - wc->status = IB_WCS_BAD_RESP_ERR; - break; - case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: - wc->status = IB_WCS_LOCAL_ACCESS_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: - wc->status = IB_WCS_REM_INVALID_REQ_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: - wc->status = IB_WCS_REM_ACCESS_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: - wc->status = IB_WCS_REM_OP_ERR; - break; - case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: - wc->status = IB_WCS_TIMEOUT_RETRY_ERR; - break; - case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: - wc->status = IB_WCS_RNR_RETRY_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: - wc->status = IB_WCS_REM_ABORT_ERR; - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } - - wc->vendor_specific = cqe->vendor_err_syndrome; -} - -static uint8_t mlx4_ib_ipoib_csum_ok(__be32 status, __be16 checksum) { - - #define CSUM_VALID_NUM 0xffff - uint8_t res = 0; - - // Verify that IP_OK bit is set and the packet is pure IPv4 packet - if ((status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 | - MLX4_CQE_IPOIB_STATUS_IPV4 | - MLX4_CQE_IPOIB_STATUS_IPV4OPT | - MLX4_CQE_IPOIB_STATUS_IPV6 | - MLX4_CQE_IPOIB_STATUS_IPOK)) == - cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 | - MLX4_CQE_IPOIB_STATUS_IPOK)) - { - // IP checksum calculated by MLX4 matched the checksum in the receive packet's - res |= MLX4_NdisPacketIpChecksumSucceeded; - if (checksum == CSUM_VALID_NUM) { - // TCP or UDP checksum calculated by MLX4 matched the checksum in the receive packet's - res |= (MLX4_NdisPacketUdpChecksumSucceeded | - MLX4_NdisPacketTcpChecksumSucceeded ); - ASSERT( status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_TCP | MLX4_CQE_IPOIB_STATUS_UDP)); - } - } - return res; -} - -static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, - struct mlx4_ib_qp **cur_qp, - ib_wc_t *wc) -{ - struct mlx4_cqe *cqe; - struct mlx4_qp *mqp; - struct mlx4_ib_wq *wq; - struct mlx4_ib_srq *srq; - int is_send; - int is_error; - u16 wqe_ctr; - - cqe = next_cqe_sw(cq); - if (!cqe) - return -EAGAIN; - - ++cq->mcq.cons_index; - - /* - * Make sure we read CQ entry contents after we've checked the - * ownership bit. - */ - rmb(); - - is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; - is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == - MLX4_CQE_OPCODE_ERROR; - - if (!*cur_qp || (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (u32)(*cur_qp)->mqp.qpn) { - /* - * We do not have to take the QP table lock here, - * because CQs will be locked while QPs are removed - * from the table. - */ -#if 1 - // radix_tree_insert in current implementation seems like - // can cause radix_tree_lookup to miss an existing QP - // so we call qp_lookup under the spinlock - mqp = mlx4_qp_lookup_locked( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn)); -#else - mqp = __mlx4_qp_lookup( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn)); -#endif - - if (unlikely(!mqp)) { - printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n", - cq->mcq.cqn, be32_to_cpu(cqe->my_qpn) & 0xffffff); - return -EINVAL; - } - - *cur_qp = to_mibqp(mqp); - } - - if (is_send) { - wq = &(*cur_qp)->sq; - wqe_ctr = be16_to_cpu(cqe->wqe_index); - wq->tail += (u16) (wqe_ctr - (u16) wq->tail); - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; - ++wq->tail; - } else if ((*cur_qp)->ibqp.srq) { - srq = to_msrq((*cur_qp)->ibqp.srq); - wqe_ctr = be16_to_cpu(cqe->wqe_index); - wc->wr_id = srq->wrid[wqe_ctr]; - mlx4_ib_free_srq_wqe(srq, wqe_ctr); - } else { - wq = &(*cur_qp)->rq; - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; - ++wq->tail; - } - - if (is_send) { - wc->recv.ud.recv_opt = 0; - switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { - case MLX4_OPCODE_RDMA_WRITE_IMM: - wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE; - case MLX4_OPCODE_RDMA_WRITE: - wc->wc_type = IB_WC_RDMA_WRITE; - break; - case MLX4_OPCODE_SEND_IMM: - wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE; - case MLX4_OPCODE_SEND: - wc->wc_type = IB_WC_SEND; - break; - case MLX4_OPCODE_RDMA_READ: - wc->wc_type = IB_WC_RDMA_READ; - wc->length = be32_to_cpu(cqe->byte_cnt); - break; - case MLX4_OPCODE_ATOMIC_CS: - wc->wc_type = IB_WC_COMPARE_SWAP; - wc->length = 8; - break; - case MLX4_OPCODE_ATOMIC_FA: - wc->wc_type = IB_WC_FETCH_ADD; - wc->length = 8; - break; - case MLX4_OPCODE_BIND_MW: - wc->wc_type = IB_WC_MW_BIND; - break; - default: - wc->wc_type = IB_WC_SEND; - break; - } - } else { - wc->length = be32_to_cpu(cqe->byte_cnt); - - switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { - case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: - wc->wc_type = IB_WC_RECV_RDMA_WRITE; - wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE; - wc->recv.ud.immediate_data = cqe->immed_rss_invalid; - break; - case MLX4_RECV_OPCODE_SEND: - wc->wc_type = IB_WC_RECV; - wc->recv.ud.recv_opt = 0; - break; - case MLX4_RECV_OPCODE_SEND_IMM: - wc->wc_type = IB_WC_RECV; - wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE; - wc->recv.ud.immediate_data = cqe->immed_rss_invalid; - break; - default: - wc->recv.ud.recv_opt = 0; - wc->wc_type = IB_WC_RECV; - break; - } - - wc->recv.ud.remote_lid = cqe->rlid; - wc->recv.ud.remote_sl = cqe->sl >> 4; - wc->recv.ud.remote_qp = cqe->g_mlpath_rqpn & 0xffffff00; - wc->recv.ud.path_bits = (u8)(cqe->g_mlpath_rqpn & 0x7f); - wc->recv.ud.recv_opt |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0; - wc->recv.ud.pkey_index = (u16)(be32_to_cpu(cqe->immed_rss_invalid) & 0x7f); - wc->csum_ok = mlx4_ib_ipoib_csum_ok(cqe->ipoib_status,cqe->checksum); - } - if (!is_send && cqe->rlid == 0){ - MLX4_PRINT(TRACE_LEVEL_INFORMATION,MLX4_DBG_CQ,("found rlid == 0 \n ")); - wc->recv.ud.recv_opt |= IB_RECV_OPT_FORWARD; - } - - if (unlikely(is_error)) - mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); - else - wc->status = IB_WCS_SUCCESS; - - return 0; -} - -int mlx4_ib_poll_cq( - IN struct ib_cq *ibcq, - IN OUT ib_wc_t** const pp_free_wclist, - OUT ib_wc_t** const pp_done_wclist ) -{ - struct mlx4_ib_cq *cq = to_mcq(ibcq); - struct mlx4_ib_qp *cur_qp = NULL; - unsigned long flags; - int err = 0; - int npolled = 0; - ib_wc_t *wc_p, **next_pp; - - spin_lock_irqsave(&cq->lock, &flags); - - // loop through CQ - next_pp = pp_done_wclist; - wc_p = *pp_free_wclist; - while( wc_p ) { - // poll one CQE - err = mlx4_ib_poll_one(cq, &cur_qp, wc_p); - if (err) - break; - - // prepare for the next loop - *next_pp = wc_p; - next_pp = &wc_p->p_next; - wc_p = wc_p->p_next; - ++npolled; - } - - // prepare the results - *pp_free_wclist = wc_p; /* Set the head of the free list. */ - *next_pp = NULL; /* Clear the tail of the done list. */ - - // update consumer index - if (npolled) - mlx4_cq_set_ci(&cq->mcq); - - spin_unlock_irqrestore(&cq->lock, flags); - return (err == 0 || err == -EAGAIN)? npolled : err; -} - -int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) -{ - mlx4_cq_arm(&to_mcq(ibcq)->mcq, - (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? - MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT, - to_mdev(ibcq->device)->uar_map, - MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock)); - - return 0; -} - -void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) -{ - u32 prod_index; - int nfreed = 0; - struct mlx4_cqe *cqe, *dest; - u8 owner_bit; - - /* - * First we need to find the current producer index, so we - * know where to start cleaning from. It doesn't matter if HW - * adds new entries after this loop -- the QP we're worried - * about is already in RESET, so the new entries won't come - * from our QP and therefore don't need to be checked. - */ - for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) - if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) - break; - - /* - * Now sweep backwards through the CQ, removing CQ entries - * that match our QP by copying older entries on top of them. - */ - while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { - cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); - if ((be32_to_cpu(cqe->my_qpn) & 0xffffff) == qpn) { - if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) - mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index)); - ++nfreed; - } else if (nfreed) { - dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); - owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; - memcpy(dest, cqe, sizeof *cqe); - dest->owner_sr_opcode = owner_bit | - (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); - } - } - - if (nfreed) { - cq->mcq.cons_index += nfreed; - /* - * Make sure update of buffer contents is done before - * updating consumer index. - */ - wmb(); - mlx4_cq_set_ci(&cq->mcq); - } -} - -void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) -{ - spin_lock_irq(&cq->lock); - __mlx4_ib_cq_clean(cq, qpn, srq); - spin_unlock_irq(&cq->lock); -} +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include +#include "mlx4_ib.h" +#include "cq.h" +#include "qp.h" +#include "user.h" + +static void mlx4_ib_cq_comp(struct mlx4_cq *cq) +{ + struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; + ibcq->comp_handler(ibcq->cq_context); +} + +static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) +{ + ib_event_rec_t event; + struct ib_cq *ibcq; + + if (type != MLX4_EVENT_TYPE_CQ_ERROR) { + printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " + "on CQ %06x\n", type, cq->cqn); + return; + } + + ibcq = &to_mibcq(cq)->ibcq; + event.type = IB_EVENT_CQ_ERR; + event.context = ibcq->cq_context; + event.vendor_specific = type; + ibcq->event_handler(&event); +} + +static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n) +{ + int offset = n * sizeof (struct mlx4_cqe); + + if (buf->buf.nbufs == 1) + return buf->buf.u.direct.buf + offset; + else + return buf->buf.u.page_list[offset >> PAGE_SHIFT].buf + + (offset & (PAGE_SIZE - 1)); +} + +static void *get_cqe(struct mlx4_ib_cq *cq, int n) +{ + return get_cqe_from_buf(&cq->buf, n); +} + +static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) +{ + struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); + + return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ + !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; +} + +static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) +{ + return get_sw_cqe(cq, cq->mcq.cons_index); +} + +int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) +{ + struct mlx4_ib_cq *mcq = to_mcq(cq); + struct mlx4_ib_dev *dev = to_mdev(cq->device); + struct mlx4_cq_context *context; + int err; + + if (mlx4_is_barred(dev->dev)) + return -EFAULT; + + context = kzalloc(sizeof *context, GFP_KERNEL); + if (!context) + return -ENOMEM; + + context->cq_period = cpu_to_be16(cq_period); + context->cq_max_count = cpu_to_be16(cq_count); + err = mlx4_cq_modify(dev->dev, &mcq->mcq, context, 1); + + kfree(context); + return err; +} + +struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct mlx4_ib_cq *cq; + struct mlx4_uar *uar; + int buf_size; + int err; + + UNUSED_PARAM(vector); + + if (mlx4_is_barred(ibdev->dma_device)) + return ERR_PTR(-EFAULT); + + if (entries < 1 || entries > dev->dev->caps.max_cqes) + return ERR_PTR(-EINVAL); + + cq = kzalloc(sizeof *cq, GFP_KERNEL); + if (!cq) + return ERR_PTR(-ENOMEM); + + entries = roundup_pow_of_two(entries + 1); + cq->ibcq.cqe = entries - 1; + buf_size = entries * sizeof (struct mlx4_cqe); + spin_lock_init(&cq->lock); + + if (context) { + struct mlx4_ib_create_cq ucmd; + + if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { + err = -EFAULT; + goto err_cq; + } + + cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size, + IB_ACCESS_LOCAL_WRITE, FALSE); + if (IS_ERR(cq->umem)) { + err = PTR_ERR(cq->umem); + goto err_cq; + } + + err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem), + ilog2(cq->umem->page_size), &cq->buf.mtt); + if (err) + goto err_buf; + + err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem); + if (err) + goto err_mtt; + + err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr, + &cq->db); + if (err) + goto err_mtt; + + // add mapping to user's arm_sn variable + // we have no way pass the completion event to provider library + // so we'll increment user's arm_sn in kernel + err = ib_umem_map( ucmd.arm_sn_addr, sizeof(int), + IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn ); + if (err) + goto err_dbmap; + + uar = &to_mucontext(context)->uar; + } else { + err = mlx4_ib_db_alloc(dev, &cq->db, 1); + if (err) + goto err_cq; + + cq->mcq.set_ci_db = cq->db.db; + cq->mcq.arm_db = cq->db.db + 1; + *cq->mcq.set_ci_db = 0; + *cq->mcq.arm_db = 0; + + if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) { + err = -ENOMEM; + goto err_db; + } + + err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift, + &cq->buf.mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf); + if (err) + goto err_mtt; + + cq->mcq.p_u_arm_sn = NULL; + uar = &dev->priv_uar; + } + + err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, + cq->db.dma.da, &cq->mcq, 0, 0); + if (err) + goto err_dbmap; + + cq->mcq.comp = mlx4_ib_cq_comp; + cq->mcq.event = mlx4_ib_cq_event; + + if (context) + if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { + err = -EFAULT; + goto err_dbmap; + } + + return &cq->ibcq; + +err_dbmap: + ib_umem_unmap( cq->mcq.mdl, cq->mcq.p_u_arm_sn ); + if (context) + mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); + +err_buf: + if (context) + ib_umem_release(cq->umem); + else + mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe), + &cq->buf.buf); + +err_db: + if (!context) + mlx4_ib_db_free(dev, &cq->db); + +err_cq: + kfree(cq); + + return ERR_PTR(err); +} + +int mlx4_ib_destroy_cq(struct ib_cq *cq) +{ + struct mlx4_ib_dev *dev = to_mdev(cq->device); + struct mlx4_ib_cq *mcq = to_mcq(cq); + + mlx4_cq_free(dev->dev, &mcq->mcq); + mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt); + + if (cq->p_uctx) { + ib_umem_unmap( mcq->mcq.mdl, mcq->mcq.p_u_arm_sn ); + mlx4_ib_db_unmap_user(to_mucontext(cq->p_uctx), &mcq->db); + ib_umem_release(mcq->umem); + } else { + mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe), + &mcq->buf.buf); + mlx4_ib_db_free(dev, &mcq->db); + } + + kfree(mcq); + + return 0; +} + +static void dump_cqe(void *cqe) +{ + __be32 *buf = cqe; + + printk(KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", + be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]), + be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]), + be32_to_cpu(buf[6]), be32_to_cpu(buf[7])); +} + +static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe, + ib_wc_t *wc) +{ + if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) { + printk(KERN_DEBUG "local QP operation err " + "(QPN %06x, WQE index %x, vendor syndrome %02x, " + "opcode = %02x)\n", + be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index), + cqe->vendor_err_syndrome, + cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); + dump_cqe(cqe); + } + + switch (cqe->syndrome) { + case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: + wc->status = IB_WCS_LOCAL_LEN_ERR; + break; + case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: + wc->status = IB_WCS_LOCAL_OP_ERR; + break; + case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: + wc->status = IB_WCS_LOCAL_PROTECTION_ERR; + break; + case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: + wc->status = IB_WCS_WR_FLUSHED_ERR; + break; + case MLX4_CQE_SYNDROME_MW_BIND_ERR: + wc->status = IB_WCS_MEM_WINDOW_BIND_ERR; + break; + case MLX4_CQE_SYNDROME_BAD_RESP_ERR: + wc->status = IB_WCS_BAD_RESP_ERR; + break; + case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: + wc->status = IB_WCS_LOCAL_ACCESS_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: + wc->status = IB_WCS_REM_INVALID_REQ_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: + wc->status = IB_WCS_REM_ACCESS_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: + wc->status = IB_WCS_REM_OP_ERR; + break; + case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: + wc->status = IB_WCS_TIMEOUT_RETRY_ERR; + break; + case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: + wc->status = IB_WCS_RNR_RETRY_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: + wc->status = IB_WCS_REM_ABORT_ERR; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + wc->vendor_specific = cqe->vendor_err_syndrome; +} + +static uint8_t mlx4_ib_ipoib_csum_ok(__be32 status, __be16 checksum) { + + #define CSUM_VALID_NUM 0xffff + uint8_t res = 0; + + // Verify that IP_OK bit is set and the packet is pure IPv4 packet + if ((status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 | + MLX4_CQE_IPOIB_STATUS_IPV4 | + MLX4_CQE_IPOIB_STATUS_IPV4OPT | + MLX4_CQE_IPOIB_STATUS_IPV6 | + MLX4_CQE_IPOIB_STATUS_IPOK)) == + cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4 | + MLX4_CQE_IPOIB_STATUS_IPOK)) + { + // IP checksum calculated by MLX4 matched the checksum in the receive packet's + res |= MLX4_NdisPacketIpChecksumSucceeded; + if (checksum == CSUM_VALID_NUM) { + // TCP or UDP checksum calculated by MLX4 matched the checksum in the receive packet's + res |= (MLX4_NdisPacketUdpChecksumSucceeded | + MLX4_NdisPacketTcpChecksumSucceeded ); + ASSERT( status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_TCP | MLX4_CQE_IPOIB_STATUS_UDP)); + } + } + return res; +} + +static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, + struct mlx4_ib_qp **cur_qp, + ib_wc_t *wc) +{ + struct mlx4_cqe *cqe; + struct mlx4_qp *mqp; + struct mlx4_ib_wq *wq; + struct mlx4_ib_srq *srq; + int is_send; + int is_error; + u16 wqe_ctr; + + cqe = next_cqe_sw(cq); + if (!cqe) + return -EAGAIN; + + ++cq->mcq.cons_index; + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; + is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR; + + if (!*cur_qp || (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (u32)(*cur_qp)->mqp.qpn) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ +#if 1 + // radix_tree_insert in current implementation seems like + // can cause radix_tree_lookup to miss an existing QP + // so we call qp_lookup under the spinlock + mqp = mlx4_qp_lookup_locked( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn)); +#else + mqp = __mlx4_qp_lookup( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn)); +#endif + + if (unlikely(!mqp)) { + printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n", + cq->mcq.cqn, be32_to_cpu(cqe->my_qpn) & 0xffffff); + return -EINVAL; + } + + *cur_qp = to_mibqp(mqp); + } + + if (is_send) { + wq = &(*cur_qp)->sq; + wqe_ctr = be16_to_cpu(cqe->wqe_index); + wq->tail += (u16) (wqe_ctr - (u16) wq->tail); + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } else if ((*cur_qp)->ibqp.srq) { + srq = to_msrq((*cur_qp)->ibqp.srq); + wqe_ctr = be16_to_cpu(cqe->wqe_index); + wc->wr_id = srq->wrid[wqe_ctr]; + mlx4_ib_free_srq_wqe(srq, wqe_ctr); + } else { + wq = &(*cur_qp)->rq; + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } + + if (is_send) { + wc->recv.ud.recv_opt = 0; + switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { + case MLX4_OPCODE_RDMA_WRITE_IMM: + wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE; + case MLX4_OPCODE_RDMA_WRITE: + wc->wc_type = IB_WC_RDMA_WRITE; + break; + case MLX4_OPCODE_SEND_IMM: + wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE; + case MLX4_OPCODE_SEND: + wc->wc_type = IB_WC_SEND; + break; + case MLX4_OPCODE_RDMA_READ: + wc->wc_type = IB_WC_RDMA_READ; + wc->length = be32_to_cpu(cqe->byte_cnt); + break; + case MLX4_OPCODE_ATOMIC_CS: + wc->wc_type = IB_WC_COMPARE_SWAP; + wc->length = 8; + break; + case MLX4_OPCODE_ATOMIC_FA: + wc->wc_type = IB_WC_FETCH_ADD; + wc->length = 8; + break; + case MLX4_OPCODE_BIND_MW: + wc->wc_type = IB_WC_MW_BIND; + break; + default: + wc->wc_type = IB_WC_SEND; + break; + } + } else { + wc->length = be32_to_cpu(cqe->byte_cnt); + + switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { + case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: + wc->wc_type = IB_WC_RECV_RDMA_WRITE; + wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE; + wc->recv.ud.immediate_data = cqe->immed_rss_invalid; + break; + case MLX4_RECV_OPCODE_SEND: + wc->wc_type = IB_WC_RECV; + wc->recv.ud.recv_opt = 0; + break; + case MLX4_RECV_OPCODE_SEND_IMM: + wc->wc_type = IB_WC_RECV; + wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE; + wc->recv.ud.immediate_data = cqe->immed_rss_invalid; + break; + default: + wc->recv.ud.recv_opt = 0; + wc->wc_type = IB_WC_RECV; + break; + } + + wc->recv.ud.remote_lid = cqe->rlid; + wc->recv.ud.remote_sl = cqe->sl >> 4; + wc->recv.ud.remote_qp = cqe->g_mlpath_rqpn & 0xffffff00; + wc->recv.ud.path_bits = (u8)(cqe->g_mlpath_rqpn & 0x7f); + wc->recv.ud.recv_opt |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0; + wc->recv.ud.pkey_index = (u16)(be32_to_cpu(cqe->immed_rss_invalid) & 0x7f); + wc->csum_ok = mlx4_ib_ipoib_csum_ok(cqe->ipoib_status,cqe->checksum); + } + if (!is_send && cqe->rlid == 0){ + MLX4_PRINT(TRACE_LEVEL_INFORMATION,MLX4_DBG_CQ,("found rlid == 0 \n ")); + wc->recv.ud.recv_opt |= IB_RECV_OPT_FORWARD; + } + + if (unlikely(is_error)) + mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); + else + wc->status = IB_WCS_SUCCESS; + + return 0; +} + +int mlx4_ib_poll_cq( + IN struct ib_cq *ibcq, + IN OUT ib_wc_t** const pp_free_wclist, + OUT ib_wc_t** const pp_done_wclist ) +{ + struct mlx4_ib_cq *cq = to_mcq(ibcq); + struct mlx4_ib_qp *cur_qp = NULL; + unsigned long flags; + int err = 0; + int npolled = 0; + ib_wc_t *wc_p, **next_pp; + + spin_lock_irqsave(&cq->lock, &flags); + + // loop through CQ + next_pp = pp_done_wclist; + wc_p = *pp_free_wclist; + while( wc_p ) { + // poll one CQE + err = mlx4_ib_poll_one(cq, &cur_qp, wc_p); + if (err) + break; + + // prepare for the next loop + *next_pp = wc_p; + next_pp = &wc_p->p_next; + wc_p = wc_p->p_next; + ++npolled; + } + + // prepare the results + *pp_free_wclist = wc_p; /* Set the head of the free list. */ + *next_pp = NULL; /* Clear the tail of the done list. */ + + // update consumer index + if (npolled) + mlx4_cq_set_ci(&cq->mcq); + + spin_unlock_irqrestore(&cq->lock, flags); + return (err == 0 || err == -EAGAIN)? npolled : err; +} + +int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + if (!mlx4_is_barred(ibcq->device->dma_device)) + mlx4_cq_arm(&to_mcq(ibcq)->mcq, + (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT, + to_mdev(ibcq->device)->uar_map, + MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock)); + + return 0; +} + +void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) +{ + u32 prod_index; + int nfreed = 0; + struct mlx4_cqe *cqe, *dest; + u8 owner_bit; + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) + if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) + break; + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { + cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); + if ((be32_to_cpu(cqe->my_qpn) & 0xffffff) == qpn) { + if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) + mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index)); + ++nfreed; + } else if (nfreed) { + dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); + owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; + memcpy(dest, cqe, sizeof *cqe); + dest->owner_sr_opcode = owner_bit | + (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); + } + } + + if (nfreed) { + cq->mcq.cons_index += nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + mlx4_cq_set_ci(&cq->mcq); + } +} + +void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) +{ + spin_lock_irq(&cq->lock); + __mlx4_ib_cq_clean(cq, qpn, srq); + spin_unlock_irq(&cq->lock); +} diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c index 513b8da3..01ba48cb 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c @@ -202,6 +202,9 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, u16 slid; int err; + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + slid = in_wc ? be16_to_cpu(in_wc->recv.ud.remote_lid) : be16_to_cpu(XIB_LID_PERMISSIVE); if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) { diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/main.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/main.c index e5115ecc..5faa22b6 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/main.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/main.c @@ -55,6 +55,9 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, struct ib_smp *out_mad = NULL; int err = -ENOMEM; + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) @@ -132,6 +135,9 @@ static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, struct ib_smp *out_mad = NULL; int err = -ENOMEM; + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) @@ -182,6 +188,9 @@ static int mlx4_ib_query_gid_chunk(struct ib_device *ibdev, u8 port, int index, __be64 subnet_prefix; int err = -ENOMEM; + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) @@ -227,6 +236,9 @@ static int mlx4_ib_query_pkey_chunk(struct ib_device *ibdev, u8 port, u16 index, struct ib_smp *out_mad = NULL; int err = -ENOMEM; + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) @@ -256,6 +268,9 @@ out: static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props) { + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) return -EOPNOTSUPP; @@ -274,6 +289,9 @@ static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, u32 cap_mask; int err; + if (mlx4_is_barred(ibdev->dma_device)) + return -EFAULT; + mutex_lock(&to_mdev(ibdev)->cap_mask_mutex); err = mlx4_ib_query_port(ibdev, port, &attr); @@ -300,6 +318,9 @@ static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev, struct mlx4_ib_alloc_ucontext_resp resp; int err; + if (mlx4_is_barred(ibdev->dma_device)) + return ERR_PTR(-EFAULT); + resp.qp_tab_size = dev->dev->caps.num_qps; resp.bf_reg_size = (__u16)dev->dev->caps.bf_reg_size; resp.bf_regs_per_page = (__u16)dev->dev->caps.bf_regs_per_page; @@ -381,6 +402,9 @@ static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev, struct mlx4_ib_pd *pd; int err; + if (mlx4_is_barred(ibdev->dma_device)) + return ERR_PTR(-EFAULT); + pd = kmalloc(sizeof *pd, GFP_KERNEL); if (!pd) return ERR_PTR(-ENOMEM); @@ -417,6 +441,8 @@ static int mlx4_ib_dealloc_pd(struct ib_pd *pd) static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { UNUSED_PARAM(lid); + if (mlx4_is_barred(ibqp->device->dma_device)) + return -EFAULT; return mlx4_multicast_attach(to_mdev(ibqp->device)->dev, &to_mqp(ibqp)->mqp, gid->raw); } @@ -424,6 +450,8 @@ static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { UNUSED_PARAM(lid); + if (mlx4_is_barred(ibqp->device->dma_device)) + return -EFAULT; return mlx4_multicast_detach(to_mdev(ibqp->device)->dev, &to_mqp(ibqp)->mqp, gid->raw); } @@ -591,8 +619,9 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) mlx4_uar_free(dev, &ibdev->priv_uar); mlx4_pd_free(dev, ibdev->priv_pdn); dealloc_dev: - ib_dealloc_device(&ibdev->ib_dev); mlx4_dbg(ibdev->dev, "MLX4_BUS: IB interface is REMOVED ! \n"); + ibdev->ib_dev.reg_state = IB_DEV_UNINITIALIZED; + ib_dealloc_device(&ibdev->ib_dev); } static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr, diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/mr.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/mr.c index 1475d9e8..5b63f210 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/mr.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/mr.c @@ -46,6 +46,9 @@ struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) struct mlx4_ib_mr *mr; int err; + if (mlx4_is_barred(pd->device->dma_device)) + return ERR_PTR(-EFAULT); + mr = kmalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); @@ -124,6 +127,9 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, int n; UNUSED_PARAM(udata); + + if (mlx4_is_barred(pd->device->dma_device)) + return ERR_PTR(-EFAULT); mr = kmalloc(sizeof *mr, GFP_KERNEL); if (!mr) @@ -188,6 +194,9 @@ struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, struct mlx4_ib_fmr *fmr; int err = -ENOMEM; + if (mlx4_is_barred(pd->device->dma_device)) + return ERR_PTR(-EFAULT); + fmr = kmalloc(sizeof *fmr, GFP_KERNEL); if (!fmr) return ERR_PTR(-ENOMEM); @@ -221,6 +230,9 @@ int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device); + if (mlx4_is_barred(ifmr->ibfmr.device->dma_device)) + return -EFAULT; + return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); } @@ -228,7 +240,7 @@ int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int mlx4_ib_unmap_fmr(struct list_head *fmr_list) { struct ib_fmr *ibfmr; - int err; + int err = 0; struct mlx4_dev *mdev = NULL; list_for_each_entry(ibfmr, fmr_list, list, struct ib_fmr) { @@ -252,7 +264,8 @@ int mlx4_ib_unmap_fmr(struct list_head *fmr_list) */ wmb(); - err = mlx4_SYNC_TPT(mdev); + if (!mlx4_is_barred(mdev)) + err = mlx4_SYNC_TPT(mdev); if (err) printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when " "unmapping FMRs\n", err); diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/qp.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/qp.c index 671fcb3e..40a51ece 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/qp.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/qp.c @@ -1,1737 +1,1752 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "mlx4_ib.h" -#include "ib_cache.h" -#include "ib_pack.h" -#include "qp.h" -#include "user.h" - -enum { - MLX4_IB_ACK_REQ_FREQ = 8, -}; - -enum { - MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83, - MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f -}; - -enum { - /* - * Largest possible UD header: send with GRH and immediate data. - */ - MLX4_IB_UD_HEADER_SIZE = 72 -}; - -struct mlx4_ib_sqp { - struct mlx4_ib_qp qp; - int pkey_index; - u32 qkey; - u32 send_psn; - struct ib_ud_header ud_header; - u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; -}; - -enum { - MLX4_IB_MIN_SQ_STRIDE = 6 -}; - -static const __be32 mlx4_ib_opcode[] = { - __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), /* [IB_WR_RDMA_WRITE] */ - __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), /* [IB_WR_RDMA_WRITE_WITH_IMM] */ - __constant_cpu_to_be32(MLX4_OPCODE_SEND), /* [IB_WR_SEND] */ - __constant_cpu_to_be32(MLX4_OPCODE_SEND_IMM), /* [IB_WR_SEND_WITH_IMM] */ - __constant_cpu_to_be32(MLX4_OPCODE_RDMA_READ), /* [IB_WR_RDMA_READ] */ - __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), /* [IB_WR_ATOMIC_CMP_AND_SWP] */ - __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), /* [IB_WR_ATOMIC_FETCH_AND_ADD]*/ -}; - -static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp) -{ - return container_of(mqp, struct mlx4_ib_sqp, qp); -} - -static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - return qp->mqp.qpn >= dev->dev->caps.sqp_start && - qp->mqp.qpn <= dev->dev->caps.sqp_start + 3; -} - -static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - return qp->mqp.qpn >= dev->dev->caps.sqp_start && - qp->mqp.qpn <= dev->dev->caps.sqp_start + 1; -} - -static void *get_wqe(struct mlx4_ib_qp *qp, int offset) -{ - if (qp->buf.nbufs == 1) - return qp->buf.u.direct.buf + offset; - else - return qp->buf.u.page_list[offset >> PAGE_SHIFT].buf + - (offset & (PAGE_SIZE - 1)); -} - -static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n) -{ - return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); -} - -static void *get_send_wqe(struct mlx4_ib_qp *qp, int n) -{ - return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); -} - -/* - * Stamp a SQ WQE so that it is invalid if prefetched by marking the - * first four bytes of every 64 byte chunk with 0xffffffff, except for - * the very first chunk of the WQE. - */ -static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n) -{ - u32 *wqe = get_send_wqe(qp, n); - int i; - - for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16) - wqe[i] = 0xffffffff; -} - -static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) -{ - ib_event_rec_t event; - struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; - - if (type == MLX4_EVENT_TYPE_PATH_MIG) - to_mibqp(qp)->port = to_mibqp(qp)->alt_port; - - switch (type) { - case MLX4_EVENT_TYPE_PATH_MIG: - event.type = IB_EVENT_PATH_MIG; - break; - case MLX4_EVENT_TYPE_COMM_EST: - event.type = IB_EVENT_COMM_EST; - break; - case MLX4_EVENT_TYPE_SQ_DRAINED: - event.type = IB_EVENT_SQ_DRAINED; - break; - case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: - event.type = IB_EVENT_QP_LAST_WQE_REACHED; - break; - case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: - event.type = IB_EVENT_QP_FATAL; - break; - case MLX4_EVENT_TYPE_PATH_MIG_FAILED: - event.type = IB_EVENT_PATH_MIG_ERR; - break; - case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: - event.type = IB_EVENT_QP_REQ_ERR; - break; - case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: - event.type = IB_EVENT_QP_ACCESS_ERR; - break; - default: - printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " - "on QP %06x\n", type, qp->qpn); - return; - } - - event.context = ibqp->qp_context; - ibqp->event_handler(&event); -} - -static int send_wqe_overhead(enum ib_qp_type type) -{ - /* - * UD WQEs must have a datagram segment. - * RC and UC WQEs might have a remote address segment. - * MLX WQEs need two extra inline data segments (for the UD - * header and space for the ICRC). - */ - switch (type) { - case IB_QPT_UD: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_datagram_seg); - case IB_QPT_UC: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_raddr_seg); - case IB_QPT_RC: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_atomic_seg) + - sizeof (struct mlx4_wqe_raddr_seg); - case IB_QPT_SMI: - case IB_QPT_GSI: - return sizeof (struct mlx4_wqe_ctrl_seg) + - ALIGN(MLX4_IB_UD_HEADER_SIZE + - DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE, - MLX4_INLINE_ALIGN) * - sizeof (struct mlx4_wqe_inline_seg), - sizeof (struct mlx4_wqe_data_seg)) + - ALIGN(4 + - sizeof (struct mlx4_wqe_inline_seg), - sizeof (struct mlx4_wqe_data_seg)); - default: - return sizeof (struct mlx4_wqe_ctrl_seg); - } -} - -static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, - int is_user, int has_srq, struct mlx4_ib_qp *qp) -{ - /* Sanity check RQ size before proceeding */ - if ((int)cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || - (int)cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) - return -EINVAL; - - if (has_srq) { - /* QPs attached to an SRQ should have no RQ */ - if (cap->max_recv_wr) - return -EINVAL; - - qp->rq.wqe_cnt = qp->rq.max_gs = 0; - } else { - /* HW requires >= 1 RQ entry with >= 1 gather entry */ - if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) - return -EINVAL; - - qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr)); - qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); - qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg)); - } - - /* leave userspace return values as they were, so as not to break ABI */ - if (is_user) { - cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; - cap->max_recv_sge = qp->rq.max_gs; - } else { - cap->max_recv_wr = qp->rq.max_post = - min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); - cap->max_recv_sge = min(qp->rq.max_gs, - min(dev->dev->caps.max_sq_sg, - dev->dev->caps.max_rq_sg)); - } - /* We don't support inline sends for kernel QPs (yet) */ - - return 0; -} - -static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, - enum ib_qp_type type, struct mlx4_ib_qp *qp) -{ - /* Sanity check SQ size before proceeding */ - if ((int)cap->max_send_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || - (int)cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || - (int)cap->max_inline_data + send_wqe_overhead(type) + - (int)sizeof(struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) - return -EINVAL; - - /* - * For MLX transport we need 2 extra S/G entries: - * one for the header and one for the checksum at the end - */ - if ((type == IB_QPT_SMI || type == IB_QPT_GSI) && - (int)cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg) - return -EINVAL; - - qp->sq.wqe_shift = ilog2(roundup_pow_of_two(max(cap->max_send_sge * - sizeof (struct mlx4_wqe_data_seg), - cap->max_inline_data + - sizeof (struct mlx4_wqe_inline_seg)) + - send_wqe_overhead(type))); - qp->sq.wqe_shift = max(MLX4_IB_SQ_MIN_WQE_SHIFT, qp->sq.wqe_shift); - qp->sq.max_gs = ((1 << qp->sq.wqe_shift) - send_wqe_overhead(type)) / - sizeof (struct mlx4_wqe_data_seg); - - /* - * We need to leave 2 KB + 1 WQE of headroom in the SQ to - * allow HW to prefetch. - */ - qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift); - qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr + qp->sq_spare_wqes); - - qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << qp->sq.wqe_shift); - if (qp->rq.wqe_shift > qp->sq.wqe_shift) { - qp->rq.offset = 0; - qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; - } else { - qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift; - qp->sq.offset = 0; - } - - cap->max_send_wr = qp->sq.max_post = - min(qp->sq.wqe_cnt - qp->sq_spare_wqes, - dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE); - cap->max_send_sge = min(qp->sq.max_gs, - min(dev->dev->caps.max_sq_sg, - dev->dev->caps.max_rq_sg)); - /* We don't support inline sends for kernel QPs (yet) */ - cap->max_inline_data = 0; - - return 0; -} - -static int set_user_sq_size(struct mlx4_ib_dev *dev, - struct mlx4_ib_qp *qp, - struct mlx4_ib_create_qp *ucmd) -{ - /* Sanity check SQ size before proceeding */ - if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes || - ucmd->log_sq_stride > - ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) || - ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE) - return -EINVAL; - - qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; - qp->sq.wqe_shift = ucmd->log_sq_stride; - - qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << qp->sq.wqe_shift); - - return 0; -} - -static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata, u32 sqpn, struct mlx4_ib_qp *qp) -{ - int err; - - mutex_init(&qp->mutex); - spin_lock_init(&qp->sq.lock); - spin_lock_init(&qp->rq.lock); - - qp->state = XIB_QPS_RESET; - qp->atomic_rd_en = 0; - qp->resp_depth = 0; - - qp->rq.head = 0; - qp->rq.tail = 0; - qp->sq.head = 0; - qp->sq.tail = 0; - - err = set_rq_size(dev, &init_attr->cap, !!pd->p_uctx, !!init_attr->srq, qp); - if (err) - goto err; - - if (pd->p_uctx) { - struct mlx4_ib_create_qp ucmd; - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - err = -EFAULT; - goto err; - } - - qp->sq_no_prefetch = ucmd.sq_no_prefetch; - - err = set_user_sq_size(dev, qp, &ucmd); - if (err) - goto err; - - qp->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr, - qp->buf_size, 0, FALSE); - if (IS_ERR(qp->umem)) { - err = PTR_ERR(qp->umem); - goto err; - } - - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem), - ilog2(qp->umem->page_size), &qp->mtt); - if (err) - goto err_buf; - - err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); - if (err) - goto err_mtt; - - if (!init_attr->srq) { - err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx), - ucmd.db_addr, &qp->db); - if (err) - goto err_mtt; - } - } else { - qp->sq_no_prefetch = 0; - - err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp); - if (err) - goto err; - - if (!init_attr->srq) { - err = mlx4_ib_db_alloc(dev, &qp->db, 0); - if (err) - goto err; - - *qp->db.db = 0; - } - - if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) { - err = -ENOMEM; - goto err_db; - } - - err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift, - &qp->mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf); - if (err) - goto err_mtt; - - qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL); - qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL); - - if (!qp->sq.wrid || !qp->rq.wrid) { - err = -ENOMEM; - goto err_wrid; - } - } - - if (!sqpn) - err = mlx4_qp_reserve_range(dev->dev, 1, 1, &sqpn); - if (err) - goto err_wrid; - - err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp); - if (err) - goto err_wrid; - - /* - * Hardware wants QPN written in big-endian order (after - * shifting) for send doorbell. Precompute this value to save - * a little bit when posting sends. - */ - qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); - - if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) - qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - else - qp->sq_signal_bits = 0; - - qp->mqp.event = mlx4_ib_qp_event; - - return 0; - -err_wrid: - if (pd->p_uctx) { - if (!init_attr->srq) - mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx), - &qp->db); - } else { - kfree(qp->sq.wrid); - kfree(qp->rq.wrid); - } - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &qp->mtt); - -err_buf: - if (pd->p_uctx) - ib_umem_release(qp->umem); - else - mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); - -err_db: - if (!pd->p_uctx && !init_attr->srq) - mlx4_ib_db_free(dev, &qp->db); - -err: - return err; -} - -static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state) -{ - switch (state) { - case XIB_QPS_RESET: return MLX4_QP_STATE_RST; - case XIB_QPS_INIT: return MLX4_QP_STATE_INIT; - case XIB_QPS_RTR: return MLX4_QP_STATE_RTR; - case XIB_QPS_RTS: return MLX4_QP_STATE_RTS; - case XIB_QPS_SQD: return MLX4_QP_STATE_SQD; - case XIB_QPS_SQE: return MLX4_QP_STATE_SQER; - case XIB_QPS_ERR: return MLX4_QP_STATE_ERR; - default: return -1; - } -} - -static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) -{ - if (send_cq == recv_cq) - spin_lock_irq(&send_cq->lock); - else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { - spin_lock_irq(&send_cq->lock); - spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); - } else { - spin_lock_irq(&recv_cq->lock); - spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); - } -} - -static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) -{ - if (send_cq == recv_cq) - spin_unlock_irq(&send_cq->lock); - else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { - spin_unlock(&recv_cq->lock); - spin_unlock_irq(&send_cq->lock); - } else { - spin_unlock(&send_cq->lock); - spin_unlock_irq(&recv_cq->lock); - } -} - -static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, - int is_user) -{ - struct mlx4_ib_cq *send_cq, *recv_cq; - - if (qp->state != XIB_QPS_RESET) - if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), - MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) - printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n", - qp->mqp.qpn); - - send_cq = to_mcq(qp->ibqp.send_cq); - recv_cq = to_mcq(qp->ibqp.recv_cq); - - mlx4_ib_lock_cqs(send_cq, recv_cq); - - if (!is_user) { - __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, - qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL); - if (send_cq != recv_cq) - __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); - } - - mlx4_qp_remove(dev->dev, &qp->mqp); - - mlx4_ib_unlock_cqs(send_cq, recv_cq); - - mlx4_qp_free(dev->dev, &qp->mqp); - mlx4_mtt_cleanup(dev->dev, &qp->mtt); - - if (is_user) { - if (!qp->ibqp.srq) - mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.p_uctx), - &qp->db); - ib_umem_release(qp->umem); - } else { - kfree(qp->sq.wrid); - kfree(qp->rq.wrid); - mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); - if (!qp->ibqp.srq) - mlx4_ib_db_free(dev, &qp->db); - } -} - -struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_sqp *sqp; - struct mlx4_ib_qp *qp; - int err; - - switch (init_attr->qp_type) { - case IB_QPT_RC: - case IB_QPT_UC: - case IB_QPT_UD: - { - qp = kzalloc(sizeof *qp, GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - - err = create_qp_common(dev, pd, init_attr, udata, 0, qp); - if (err) { - kfree(qp); - return ERR_PTR(err); - } - - qp->ibqp.qp_num = qp->mqp.qpn; - - break; - } - case IB_QPT_SMI: - case IB_QPT_GSI: - { - /* Userspace is not allowed to create special QPs: */ - if (pd->p_uctx) - return ERR_PTR(-EINVAL); - - sqp = kzalloc(sizeof *sqp, GFP_KERNEL); - if (!sqp) - return ERR_PTR(-ENOMEM); - - qp = &sqp->qp; - - err = create_qp_common(dev, pd, init_attr, udata, - dev->dev->caps.sqp_start + - (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) + - init_attr->port_num - 1, - qp); - if (err) { - kfree(sqp); - return ERR_PTR(err); - } - - qp->port = init_attr->port_num; - qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; - - break; - } - default: - /* Don't support raw QPs */ - return ERR_PTR(-EINVAL); - } - - return &qp->ibqp; -} - -int mlx4_ib_destroy_qp(struct ib_qp *qp) -{ - struct mlx4_ib_dev *dev = to_mdev(qp->device); - struct mlx4_ib_qp *mqp = to_mqp(qp); - - if (is_qp0(dev, mqp)) - mlx4_CLOSE_PORT(dev->dev, mqp->port); - - destroy_qp_common(dev, mqp, !!qp->pd->p_uctx); - - if (is_sqp(dev, mqp)) - kfree(to_msqp(mqp)); - else - kfree(mqp); - - return 0; -} - -static int to_mlx4_st(enum ib_qp_type type) -{ - switch (type) { - case IB_QPT_RC: return MLX4_QP_ST_RC; - case IB_QPT_UC: return MLX4_QP_ST_UC; - case IB_QPT_UD: return MLX4_QP_ST_UD; - case IB_QPT_SMI: - case IB_QPT_GSI: return MLX4_QP_ST_MLX; - default: return -1; - } -} - -static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, - int attr_mask) -{ - u8 dest_rd_atomic; - u32 access_flags; - u32 hw_access_flags = 0; - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - dest_rd_atomic = attr->max_dest_rd_atomic; - else - dest_rd_atomic = qp->resp_depth; - - if (attr_mask & IB_QP_ACCESS_FLAGS) - access_flags = attr->qp_access_flags; - else - access_flags = qp->atomic_rd_en; - - if (!dest_rd_atomic) - access_flags &= IB_ACCESS_REMOTE_WRITE; - - if (access_flags & IB_ACCESS_REMOTE_READ) - hw_access_flags |= MLX4_QP_BIT_RRE; - if (access_flags & IB_ACCESS_REMOTE_ATOMIC) - hw_access_flags |= MLX4_QP_BIT_RAE; - if (access_flags & IB_ACCESS_REMOTE_WRITE) - hw_access_flags |= MLX4_QP_BIT_RWE; - - return cpu_to_be32(hw_access_flags); -} - -static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr, - int attr_mask) -{ - if (attr_mask & IB_QP_PKEY_INDEX) - sqp->pkey_index = attr->pkey_index; - if (attr_mask & IB_QP_QKEY) - sqp->qkey = attr->qkey; - if (attr_mask & IB_QP_SQ_PSN) - sqp->send_psn = attr->sq_psn; -} - -static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port) -{ - path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6); -} - -static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, - struct mlx4_qp_path *path, u8 port) -{ - path->grh_mylmc = ah->src_path_bits & 0x7f; - path->rlid = cpu_to_be16(ah->dlid); - if (ah->static_rate) { - path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET; - while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && - !(1 << path->static_rate & dev->dev->caps.stat_rate_support)) - --path->static_rate; - } else - path->static_rate = 0; - path->counter_index = 0xff; - - if (ah->ah_flags & IB_AH_GRH) { - if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) { - printk(KERN_ERR "sgid_index (%u) too large. max is %d\n", - ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1); - return -1; - } - - path->grh_mylmc |= 1 << 7; - path->mgid_index = ah->grh.sgid_index; - path->hop_limit = ah->grh.hop_limit; - path->tclass_flowlabel = - cpu_to_be32((ah->grh.traffic_class << 20) | - (ah->grh.flow_label)); - memcpy(path->rgid, ah->grh.dgid.raw, 16); - } - - path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | - ((port - 1) << 6) | ((ah->sl & 0xf) << 2); - - return 0; -} - -static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, - enum ib_qp_state cur_state, enum ib_qp_state new_state) -{ - struct mlx4_ib_dev *dev = to_mdev(ibqp->device); - struct mlx4_ib_qp *qp = to_mqp(ibqp); - struct mlx4_qp_context *context; - enum mlx4_qp_optpar optpar = 0; - int sqd_event; - int err = -EINVAL; - - context = kzalloc(sizeof *context, GFP_KERNEL); - if (!context) - return -ENOMEM; - - context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) | - (to_mlx4_st(ibqp->qp_type) << 16)); - context->flags |= cpu_to_be32(1 << 8); /* DE? */ - - if (!(attr_mask & IB_QP_PATH_MIG_STATE)) - context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); - else { - optpar |= MLX4_QP_OPTPAR_PM_STATE; - switch (attr->path_mig_state) { - case IB_MIG_MIGRATED: - context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); - break; - case IB_MIG_REARM: - context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11); - break; - case IB_MIG_ARMED: - context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11); - break; - } - } - - if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || - ibqp->qp_type == IB_QPT_UD) - context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; - else if (attr_mask & IB_QP_PATH_MTU) { - if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { - printk(KERN_ERR "path MTU (%u) is invalid\n", - attr->path_mtu); - goto out; - } - context->mtu_msgmax = (u8)((attr->path_mtu << 5) | - ilog2(dev->dev->caps.max_msg_sz)); - } - - if (qp->rq.wqe_cnt) - context->rq_size_stride = (u8)(ilog2(qp->rq.wqe_cnt) << 3); - context->rq_size_stride |= qp->rq.wqe_shift - 4; - - if (qp->sq.wqe_cnt) - context->sq_size_stride = (u8)(ilog2(qp->sq.wqe_cnt) << 3); - context->sq_size_stride |= qp->sq.wqe_shift - 4; - - if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) - context->sq_size_stride |= !!qp->sq_no_prefetch << 7; - - if (qp->ibqp.p_uctx) - context->usr_page = cpu_to_be32(to_mucontext(ibqp->p_uctx)->uar.index); - else - context->usr_page = cpu_to_be32(dev->priv_uar.index); - - if (attr_mask & IB_QP_DEST_QPN) - context->remote_qpn = cpu_to_be32(attr->dest_qp_num); - - if (attr_mask & IB_QP_PORT) { - if (cur_state == XIB_QPS_SQD && new_state == XIB_QPS_SQD && - !(attr_mask & IB_QP_AV)) { - mlx4_set_sched(&context->pri_path, attr->port_num); - optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE; - } - } - - if (attr_mask & IB_QP_PKEY_INDEX) { - context->pri_path.pkey_index = (u8)attr->pkey_index; - optpar |= MLX4_QP_OPTPAR_PKEY_INDEX; - } - - if (attr_mask & IB_QP_AV) { - if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path, - attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) - goto out; - - optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | - MLX4_QP_OPTPAR_SCHED_QUEUE); - } - - if (attr_mask & IB_QP_TIMEOUT) { - context->pri_path.ackto = attr->timeout << 3; - optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT; - } - - if (attr_mask & IB_QP_ALT_PATH) { - if (attr->alt_port_num == 0 || - attr->alt_port_num > dev->dev->caps.num_ports) - goto out; - - if (attr->alt_pkey_index >= - dev->dev->caps.pkey_table_len[attr->alt_port_num]) - goto out; - - if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path, - attr->alt_port_num)) - goto out; - - context->alt_path.pkey_index = (u8)attr->alt_pkey_index; - context->alt_path.ackto = attr->alt_timeout << 3; - optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH; - } - - context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn); - context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28); - - if (attr_mask & IB_QP_RNR_RETRY) { - context->params1 |= cpu_to_be32(attr->rnr_retry << 13); - optpar |= MLX4_QP_OPTPAR_RNR_RETRY; - } - - if (attr_mask & IB_QP_RETRY_CNT) { - context->params1 |= cpu_to_be32(attr->retry_cnt << 16); - optpar |= MLX4_QP_OPTPAR_RETRY_COUNT; - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { - if (attr->max_rd_atomic) - context->params1 |= - cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); - optpar |= MLX4_QP_OPTPAR_SRA_MAX; - } - - if (attr_mask & IB_QP_SQ_PSN) - context->next_send_psn = cpu_to_be32(attr->sq_psn); - - context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn); - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { - if (attr->max_dest_rd_atomic) - context->params2 |= - cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); - optpar |= MLX4_QP_OPTPAR_RRA_MAX; - } - - if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) { - context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask); - optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE; - } - - if (ibqp->srq) - context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC); - - if (attr_mask & IB_QP_MIN_RNR_TIMER) { - context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); - optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT; - } - if (attr_mask & IB_QP_RQ_PSN) - context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); - - context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn); - - if (attr_mask & IB_QP_QKEY) { - context->qkey = cpu_to_be32(attr->qkey); - optpar |= MLX4_QP_OPTPAR_Q_KEY; - } - - if (ibqp->srq) - context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn); - - if (!ibqp->srq && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) - context->db_rec_addr = cpu_to_be64(qp->db.dma.da); - - if (cur_state == XIB_QPS_INIT && - new_state == XIB_QPS_RTR && - (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || - ibqp->qp_type == IB_QPT_UD)) { - context->pri_path.sched_queue = (qp->port - 1) << 6; - if (is_qp0(dev, qp)) - context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE; - else - context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE; - } - - if (cur_state == XIB_QPS_RTS && new_state == XIB_QPS_SQD && - attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) - sqd_event = 1; - else - sqd_event = 0; - - /* - * Before passing a kernel QP to the HW, make sure that the - * ownership bits of the send queue are set and the SQ - * headroom is stamped so that the hardware doesn't start - * processing stale work requests. - */ - if (!ibqp->p_uctx && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) { - struct mlx4_wqe_ctrl_seg *ctrl; - int i; - - for (i = 0; i < qp->sq.wqe_cnt; ++i) { - ctrl = get_send_wqe(qp, i); - ctrl->owner_opcode = cpu_to_be32(1 << 31); - - stamp_send_wqe(qp, i); - } - } - - err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state), - to_mlx4_state(new_state), context, optpar, - sqd_event, &qp->mqp); - if (err) - goto out; - - qp->state = new_state; - - if (attr_mask & IB_QP_ACCESS_FLAGS) - qp->atomic_rd_en = (u8)attr->qp_access_flags; - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - qp->resp_depth = attr->max_dest_rd_atomic; - if (attr_mask & IB_QP_PORT) - qp->port = attr->port_num; - if (attr_mask & IB_QP_ALT_PATH) - qp->alt_port = attr->alt_port_num; - - if (is_sqp(dev, qp)) - store_sqp_attrs(to_msqp(qp), attr, attr_mask); - - /* - * If we moved QP0 to RTR, bring the IB link up; if we moved - * QP0 to RESET or ERROR, bring the link back down. - */ - if (is_qp0(dev, qp)) { - if (cur_state != XIB_QPS_RTR && new_state == XIB_QPS_RTR) - if (mlx4_INIT_PORT(dev->dev, qp->port)) - printk(KERN_WARNING "INIT_PORT failed for port %d\n", - qp->port); - - if (cur_state != XIB_QPS_RESET && cur_state != XIB_QPS_ERR && - (new_state == XIB_QPS_RESET || new_state == XIB_QPS_ERR)) - mlx4_CLOSE_PORT(dev->dev, qp->port); - } - - /* - * If we moved a kernel QP to RESET, clean up all old CQ - * entries and reinitialize the QP. - */ - if (new_state == XIB_QPS_RESET && !ibqp->p_uctx) { - mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn, - ibqp->srq ? to_msrq(ibqp->srq): NULL); - if (ibqp->send_cq != ibqp->recv_cq) - mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL); - - qp->rq.head = 0; - qp->rq.tail = 0; - qp->sq.head = 0; - qp->sq.tail = 0; - if (!ibqp->srq) - *qp->db.db = 0; - } - -out: - kfree(context); - return err; -} - -static struct ib_qp_attr mlx4_ib_qp_attr; -static int mlx4_ib_qp_attr_mask_table[IB_QPT_UD + 1]; - -void mlx4_ib_qp_init() -{ - memset( &mlx4_ib_qp_attr, 0, sizeof(mlx4_ib_qp_attr) ); - mlx4_ib_qp_attr.port_num = 1; - - memset( &mlx4_ib_qp_attr_mask_table, 0, sizeof(mlx4_ib_qp_attr_mask_table) ); - mlx4_ib_qp_attr_mask_table[IB_QPT_UD] = (IB_QP_PKEY_INDEX | - IB_QP_PORT | - IB_QP_QKEY); - mlx4_ib_qp_attr_mask_table[IB_QPT_UC] = (IB_QP_PKEY_INDEX | - IB_QP_PORT | - IB_QP_ACCESS_FLAGS); - mlx4_ib_qp_attr_mask_table[IB_QPT_RC] = (IB_QP_PKEY_INDEX | - IB_QP_PORT | - IB_QP_ACCESS_FLAGS); - mlx4_ib_qp_attr_mask_table[IB_QPT_SMI] = (IB_QP_PKEY_INDEX | - IB_QP_QKEY); - mlx4_ib_qp_attr_mask_table[IB_QPT_GSI] = (IB_QP_PKEY_INDEX | - IB_QP_QKEY); -} - -int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibqp->device); - struct mlx4_ib_qp *qp = to_mqp(ibqp); - enum ib_qp_state cur_state, new_state; - int err = -EINVAL; - - UNUSED_PARAM(udata); - - mutex_lock(&qp->mutex); - - cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; - new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; - - if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) - goto out; - - if ((attr_mask & IB_QP_PORT) && - (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) { - goto out; - } - - if (attr_mask & IB_QP_PKEY_INDEX) { - int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; - if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) - goto out; - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && - attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) { - goto out; - } - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && - attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) { - goto out; - } - - if (cur_state == new_state && cur_state == XIB_QPS_RESET) { - err = 0; - goto out; - } - - if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_ERR) { - err = __mlx4_ib_modify_qp(ibqp, &mlx4_ib_qp_attr, - mlx4_ib_qp_attr_mask_table[ibqp->qp_type], - XIB_QPS_RESET, XIB_QPS_INIT); - if (err) - goto out; - cur_state = XIB_QPS_INIT; - } - - err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); - -out: - mutex_unlock(&qp->mutex); - return err; -} - -static enum ib_wr_opcode to_wr_opcode(struct _ib_send_wr *wr) -{ - - enum ib_wr_opcode opcode = -1; //= wr->wr_type; - - switch (wr->wr_type) { - case WR_SEND: - opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_SEND_WITH_IMM : IB_WR_SEND; - break; - case WR_RDMA_WRITE: - opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_RDMA_WRITE_WITH_IMM : IB_WR_RDMA_WRITE; - break; - case WR_RDMA_READ: opcode = IB_WR_RDMA_READ; break; - case WR_COMPARE_SWAP: opcode = IB_WR_ATOMIC_CMP_AND_SWP; break; - case WR_FETCH_ADD: opcode = IB_WR_ATOMIC_FETCH_AND_ADD; break; - } - return opcode; -} - -static int build_mlx_header(struct mlx4_ib_sqp *sqp, ib_send_wr_t *wr, - void *wqe) -{ - enum ib_wr_opcode opcode = to_wr_opcode(wr); - struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev; - struct mlx4_wqe_mlx_seg *mlx = wqe; - struct mlx4_wqe_inline_seg *inl = (void*)((u8*)wqe + sizeof *mlx); - struct mlx4_ib_ah *ah = to_mah((struct ib_ah *)wr->dgrm.ud.h_av); - __be16 pkey; - int send_size; - int header_size; - int spc; - u32 i; - - send_size = 0; - for (i = 0; i < wr->num_ds; ++i) - send_size += wr->ds_array[i].length; - - ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->ud_header); - - sqp->ud_header.lrh.service_level = - (u8)(be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28); - sqp->ud_header.lrh.destination_lid = ah->av.dlid; - sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.g_slid & 0x7f); - if (mlx4_ib_ah_grh_present(ah)) { - sqp->ud_header.grh.traffic_class = - (u8)((be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20) & 0xff); - sqp->ud_header.grh.flow_label = - ah->av.sl_tclass_flowlabel & cpu_to_be32(0xfffff); - sqp->ud_header.grh.hop_limit = ah->av.hop_limit; - ib_get_cached_gid(ib_dev, (u8)(be32_to_cpu(ah->av.port_pd) >> 24), - ah->av.gid_index, &sqp->ud_header.grh.source_gid); - memcpy(sqp->ud_header.grh.destination_gid.raw, - ah->av.dgid, 16); - } - - mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | - (sqp->ud_header.lrh.destination_lid == - XIB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | - (sqp->ud_header.lrh.service_level << 8)); - mlx->rlid = sqp->ud_header.lrh.destination_lid; - - switch (opcode) { - case IB_WR_SEND: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; - sqp->ud_header.immediate_present = 0; - break; - case IB_WR_SEND_WITH_IMM: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; - sqp->ud_header.immediate_present = 1; - sqp->ud_header.immediate_data = wr->immediate_data; - break; - default: - return -EINVAL; - } - - sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; - if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) - sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; - sqp->ud_header.bth.solicited_event = (u8)(!!(wr->send_opt & IB_SEND_OPT_SOLICITED)); - if (!sqp->qp.ibqp.qp_num) - ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); - else - ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->dgrm.ud.pkey_index, &pkey); - sqp->ud_header.bth.pkey = pkey; - sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp; - sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); - sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ? - cpu_to_be32(sqp->qkey) : wr->dgrm.ud.remote_qkey; - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); - - header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); - -#if 0 - { - printk(KERN_ERR "built UD header of size %d:\n", header_size); - for (i = 0; i < header_size / 4; ++i) { - if (i % 8 == 0) - printk(" [%02x] ", i * 4); - printk(" %08x", - be32_to_cpu(((__be32 *) sqp->header_buf)[i])); - if ((i + 1) % 8 == 0) - printk("\n"); - } - printk("\n"); - } -#endif - - /* - * Inline data segments may not cross a 64 byte boundary. If - * our UD header is bigger than the space available up to the - * next 64 byte boundary in the WQE, use two inline data - * segments to hold the UD header. - */ - spc = MLX4_INLINE_ALIGN - - ((u32)(ULONG_PTR)(inl + 1) & (MLX4_INLINE_ALIGN - 1)); - if (header_size <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | header_size); - memcpy(inl + 1, sqp->header_buf, header_size); - i = 1; - } else { - inl->byte_count = cpu_to_be32(1 << 31 | spc); - memcpy(inl + 1, sqp->header_buf, spc); - - inl = (void*)((u8*)(inl + 1) + spc); - memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); - /* - * Need a barrier here to make sure all the data is - * visible before the byte_count field is set. - * Otherwise the HCA prefetcher could grab the 64-byte - * chunk with this inline segment and get a valid (!= - * 0xffffffff) byte count but stale data, and end up - * generating a packet with bad headers. - * - * The first inline segment's byte_count field doesn't - * need a barrier, because it comes after a - * control/MLX segment and therefore is at an offset - * of 16 mod 64. - */ - wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); - i = 2; - } - - return ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); -} - -static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq) -{ - unsigned cur; - struct mlx4_ib_cq *cq; - - cur = wq->head - wq->tail; - if (likely((int)cur + nreq < wq->max_post)) - return 0; - - cq = to_mcq(ib_cq); - spin_lock(&cq->lock); - cur = wq->head - wq->tail; - spin_unlock(&cq->lock); - - return (int)cur + nreq >= wq->max_post; -} - -static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg, - u64 remote_addr, __be32 rkey) -{ - rseg->raddr = cpu_to_be64(remote_addr); - rseg->rkey = rkey; - rseg->reserved = 0; -} - -static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, ib_send_wr_t *wr) -{ - if (wr->wr_type == WR_COMPARE_SWAP) { - aseg->swap_add = wr->remote_ops.atomic2; - aseg->compare = wr->remote_ops.atomic1; - } else { - aseg->swap_add = wr->remote_ops.atomic1; - aseg->compare = 0; - } - -} - -static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, - ib_send_wr_t *wr) -{ - memcpy(dseg->av, &to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, sizeof (struct mlx4_av)); - dseg->dqpn = wr->dgrm.ud.remote_qp; - dseg->qkey = wr->dgrm.ud.remote_qkey; -} - -static void set_mlx_icrc_seg(void *dseg) -{ - u32 *t = dseg; - struct mlx4_wqe_inline_seg *iseg = dseg; - - t[1] = 0; - - /* - * Need a barrier here before writing the byte_count field to - * make sure that all the data is visible before the - * byte_count field is set. Otherwise, if the segment begins - * a new cacheline, the HCA prefetcher could grab the 64-byte - * chunk and get a valid (!= * 0xffffffff) byte count but - * stale data, and end up sending the wrong data. - */ - wmb(); - - iseg->byte_count = cpu_to_be32((1 << 31) | 4); -} - -static void set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg) -{ - dseg->lkey = cpu_to_be32(sg->lkey); - dseg->addr = cpu_to_be64(sg->vaddr); - - /* - * Need a barrier here before writing the byte_count field to - * make sure that all the data is visible before the - * byte_count field is set. Otherwise, if the segment begins - * a new cacheline, the HCA prefetcher could grab the 64-byte - * chunk and get a valid (!= * 0xffffffff) byte count but - * stale data, and end up sending the wrong data. - */ - wmb(); - - dseg->byte_count = cpu_to_be32(sg->length); -} - -static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg) -{ - dseg->byte_count = cpu_to_be32(sg->length); - dseg->lkey = cpu_to_be32(sg->lkey); - dseg->addr = cpu_to_be64(sg->vaddr); -} - -int mlx4_ib_post_send(struct ib_qp *ibqp, ib_send_wr_t *wr, - ib_send_wr_t **bad_wr) -{ - enum ib_wr_opcode opcode; - struct mlx4_ib_qp *qp = to_mqp(ibqp); - u8 *wqe; - struct mlx4_wqe_ctrl_seg *ctrl; - struct mlx4_wqe_data_seg *dseg; - unsigned long flags; - int nreq; - int err = 0; - int ind; - int size; - int i; - - spin_lock_irqsave(&qp->sq.lock, &flags); - - ind = qp->sq.head; - - for (nreq = 0; wr; ++nreq, wr = wr->p_next) { - if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - err = -ENOMEM; - if (bad_wr) - *bad_wr = wr; - goto out; - } - - if (unlikely(wr->num_ds > (u32)qp->sq.max_gs)) { - err = -EINVAL; - if (bad_wr) - *bad_wr = wr; - goto out; - } - - wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); - ctrl = (void*)wqe; - qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id; - opcode = to_wr_opcode(wr); - - ctrl->srcrb_flags = - (wr->send_opt & IB_SEND_OPT_SIGNALED ? - cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) | - (wr->send_opt & IB_SEND_OPT_SOLICITED ? - cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) | - (wr->send_opt & IB_SEND_OPT_TX_IP_CSUM ? - cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM) : 0) | - (wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM ? - cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) | - qp->sq_signal_bits; - - if (opcode == IB_WR_SEND_WITH_IMM || - opcode == IB_WR_RDMA_WRITE_WITH_IMM) - ctrl->imm = wr->immediate_data; - else - ctrl->imm = 0; - - wqe += sizeof *ctrl; - size = sizeof *ctrl / 16; - - switch (ibqp->qp_type) { - case IB_QPT_RC: - case IB_QPT_UC: - switch (opcode) { - case IB_WR_ATOMIC_CMP_AND_SWP: - case IB_WR_ATOMIC_FETCH_AND_ADD: - set_raddr_seg((void*)wqe, wr->remote_ops.vaddr, - wr->remote_ops.rkey); - wqe += sizeof (struct mlx4_wqe_raddr_seg); - - set_atomic_seg((void*)wqe, wr); - wqe += sizeof (struct mlx4_wqe_atomic_seg); - - size += (sizeof (struct mlx4_wqe_raddr_seg) + - sizeof (struct mlx4_wqe_atomic_seg)) / 16; - - break; - - case IB_WR_RDMA_READ: - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - set_raddr_seg((void*)wqe, wr->remote_ops.vaddr, - wr->remote_ops.rkey); - wqe += sizeof (struct mlx4_wqe_raddr_seg); - size += sizeof (struct mlx4_wqe_raddr_seg) / 16; - break; - - default: - /* No extra segments required for sends */ - break; - } - break; - - case IB_QPT_UD: - set_datagram_seg((void*)wqe, wr); - wqe += sizeof (struct mlx4_wqe_datagram_seg); - size += sizeof (struct mlx4_wqe_datagram_seg) / 16; - break; - - case IB_QPT_SMI: - case IB_QPT_GSI: - err = build_mlx_header(to_msqp(qp), wr, ctrl); - if (err < 0) { - if (bad_wr) - *bad_wr = wr; - goto out; - } - wqe += err; - size += err / 16; - - err = 0; - break; - - default: - break; - } - - /* - * Write data segments in reverse order, so as to - * overwrite cacheline stamp last within each - * cacheline. This avoids issues with WQE - * prefetching. - */ - - dseg = (void*)wqe; - dseg += wr->num_ds - 1; - size += wr->num_ds * (sizeof (struct mlx4_wqe_data_seg) / 16); - - /* Add one more inline data segment for ICRC for MLX sends */ - if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI || - qp->ibqp.qp_type == IB_QPT_GSI)) { - set_mlx_icrc_seg(dseg + 1); - size += sizeof (struct mlx4_wqe_data_seg) / 16; - } - - for (i = wr->num_ds - 1; i >= 0; --i, --dseg) - set_data_seg(dseg, wr->ds_array + i); - - ctrl->fence_size = (u8)((wr->send_opt & IB_SEND_OPT_FENCE ? - MLX4_WQE_CTRL_FENCE : 0) | size); - - /* - * Make sure descriptor is fully written before - * setting ownership bit (because HW can start - * executing as soon as we do). - */ - wmb(); - - if (opcode < 0 || opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { - err = -EINVAL; - goto out; - } - - ctrl->owner_opcode = mlx4_ib_opcode[opcode] | - (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0); - - /* - * We can improve latency by not stamping the last - * send queue WQE until after ringing the doorbell, so - * only stamp here if there are still more WQEs to post. - */ - if (wr->p_next) - stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) & - (qp->sq.wqe_cnt - 1)); - - ++ind; - } - -out: - if (likely(nreq)) { - qp->sq.head += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - - writel(qp->doorbell_qpn, - (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL); - -#if 0 - if (qp->mqp.qpn == 0x41) - DbgPrint( "[MLX4_BUS] mlx4_ib_post_send : qtype %d, qpn %#x, nreq %d, sq.head %#x, wqe_ix %d, db %p \n", - ibqp->qp_type, qp->mqp.qpn, nreq, qp->sq.head, ind, - (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL ); -#endif - /* - * Make sure doorbells don't leak out of SQ spinlock - * and reach the HCA out of order. - */ - mmiowb(); - - stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) & - (qp->sq.wqe_cnt - 1)); - } - - spin_unlock_irqrestore(&qp->sq.lock, flags); - - return err; -} - -int mlx4_ib_post_recv(struct ib_qp *ibqp, ib_recv_wr_t *wr, - ib_recv_wr_t **bad_wr) -{ - struct mlx4_ib_qp *qp = to_mqp(ibqp); - struct mlx4_wqe_data_seg *scat; - unsigned long flags; - int err = 0; - int nreq; - int ind; - int i; - - spin_lock_irqsave(&qp->rq.lock, &flags); - - ind = qp->rq.head & (qp->rq.wqe_cnt - 1); - - for (nreq = 0; wr; ++nreq, wr = wr->p_next) { - if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.send_cq)) { - err = -ENOMEM; - if (bad_wr) - *bad_wr = wr; - goto out; - } - - if (unlikely(wr->num_ds > (u32)qp->rq.max_gs)) { - err = -EINVAL; - if (bad_wr) - *bad_wr = wr; - goto out; - } - - scat = get_recv_wqe(qp, ind); - - for (i = 0; i < (int)wr->num_ds; ++i) - __set_data_seg(scat + i, wr->ds_array + i); - - if (i < qp->rq.max_gs) { - scat[i].byte_count = 0; - scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); - scat[i].addr = 0; - } - - qp->rq.wrid[ind] = wr->wr_id; - - ind = (ind + 1) & (qp->rq.wqe_cnt - 1); - } - -out: - if (likely(nreq)) { - qp->rq.head += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - - *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); - -#if 0 - if (qp->mqp.qpn == 0x41) - DbgPrint( "[MLX4_BUS] mlx4_ib_post_recv : qtype %d, qpn %#x, nreq %d, rq.head %#x, wqe_ix %d, db_obj %p, db %p \n", - ibqp->qp_type, qp->mqp.qpn, nreq, qp->rq.head, ind, &qp->db, qp->db.db ); -#endif - } - - spin_unlock_irqrestore(&qp->rq.lock, flags); - - return err; -} - -static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state) -{ - switch (mlx4_state) { - case MLX4_QP_STATE_RST: return XIB_QPS_RESET; - case MLX4_QP_STATE_INIT: return XIB_QPS_INIT; - case MLX4_QP_STATE_RTR: return XIB_QPS_RTR; - case MLX4_QP_STATE_RTS: return XIB_QPS_RTS; - case MLX4_QP_STATE_SQ_DRAINING: - case MLX4_QP_STATE_SQD: return XIB_QPS_SQD; - case MLX4_QP_STATE_SQER: return XIB_QPS_SQE; - case MLX4_QP_STATE_ERR: return XIB_QPS_ERR; - default: return -1; - } -} - -static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state) -{ - switch (mlx4_mig_state) { - case MLX4_QP_PM_ARMED: return IB_MIG_ARMED; - case MLX4_QP_PM_REARM: return IB_MIG_REARM; - case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED; - default: return -1; - } -} - -static int to_ib_qp_access_flags(int mlx4_flags) -{ - int ib_flags = 0; - - if (mlx4_flags & MLX4_QP_BIT_RRE) - ib_flags |= IB_ACCESS_REMOTE_READ; - if (mlx4_flags & MLX4_QP_BIT_RWE) - ib_flags |= IB_ACCESS_REMOTE_WRITE; - if (mlx4_flags & MLX4_QP_BIT_RAE) - ib_flags |= IB_ACCESS_REMOTE_ATOMIC; - - return ib_flags; -} - -static void to_ib_ah_attr(struct mlx4_dev *dev, struct ib_ah_attr *ib_ah_attr, - struct mlx4_qp_path *path) -{ - memset(ib_ah_attr, 0, sizeof *ib_ah_attr); - ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1; - - if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports) - return; - - ib_ah_attr->dlid = be16_to_cpu(path->rlid); - ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf; - ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f; - ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; - ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0; - if (ib_ah_attr->ah_flags) { - ib_ah_attr->grh.sgid_index = path->mgid_index; - ib_ah_attr->grh.hop_limit = path->hop_limit; - ib_ah_attr->grh.traffic_class = - (u8)((be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff); - ib_ah_attr->grh.flow_label = - be32_to_cpu(path->tclass_flowlabel) & 0xfffff; - memcpy(ib_ah_attr->grh.dgid.raw, - path->rgid, sizeof ib_ah_attr->grh.dgid.raw); - } -} - -int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct mlx4_ib_dev *dev = to_mdev(ibqp->device); - struct mlx4_ib_qp *qp = to_mqp(ibqp); - struct mlx4_qp_context context; - int mlx4_state; - int err; - - UNUSED_PARAM(qp_attr_mask); - - if (qp->state == XIB_QPS_RESET) { - qp_attr->qp_state = XIB_QPS_RESET; - goto done; - } - - err = mlx4_qp_query(dev->dev, &qp->mqp, &context); - if (err) - return -EINVAL; - - mlx4_state = be32_to_cpu(context.flags) >> 28; - - qp_attr->qp_state = to_ib_qp_state(mlx4_state); - qp_attr->path_mtu = context.mtu_msgmax >> 5; - qp_attr->path_mig_state = - to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3); - qp_attr->qkey = be32_to_cpu(context.qkey); - qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff; - qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff; - qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff; - qp_attr->qp_access_flags = - to_ib_qp_access_flags(be32_to_cpu(context.params2)); - - if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { - to_ib_ah_attr(dev->dev, &qp_attr->ah_attr, &context.pri_path); - to_ib_ah_attr(dev->dev, &qp_attr->alt_ah_attr, &context.alt_path); - qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f; - qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; - } - - qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f; - if (qp_attr->qp_state == XIB_QPS_INIT) - qp_attr->port_num = qp->port; - else - qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1; - - /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ - qp_attr->sq_draining = (u8)(mlx4_state == MLX4_QP_STATE_SQ_DRAINING); - - qp_attr->max_rd_atomic = (u8)(1 << ((be32_to_cpu(context.params1) >> 21) & 0x7)); - - qp_attr->max_dest_rd_atomic = - (u8)(1 << ((be32_to_cpu(context.params2) >> 21) & 0x7)); - qp_attr->min_rnr_timer = - (u8)((be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f); - qp_attr->timeout = context.pri_path.ackto >> 3; - qp_attr->retry_cnt = (u8)((be32_to_cpu(context.params1) >> 16) & 0x7); - qp_attr->rnr_retry = (u8)((be32_to_cpu(context.params1) >> 13) & 0x7); - qp_attr->alt_timeout = context.alt_path.ackto >> 3; - -done: - qp_attr->cur_qp_state = qp_attr->qp_state; - qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; - qp_attr->cap.max_recv_sge = qp->rq.max_gs; - - if (!ibqp->p_uctx) { - qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; - qp_attr->cap.max_send_sge = qp->sq.max_gs; - } else { - qp_attr->cap.max_send_wr = 0; - qp_attr->cap.max_send_sge = 0; - } - - /* - * We don't support inline sends for kernel QPs (yet), and we - * don't know what userspace's value should be. - */ - qp_attr->cap.max_inline_data = 0; - - qp_init_attr->cap = qp_attr->cap; - - return 0; -} - +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "mlx4_ib.h" +#include "ib_cache.h" +#include "ib_pack.h" +#include "qp.h" +#include "user.h" + +enum { + MLX4_IB_ACK_REQ_FREQ = 8, +}; + +enum { + MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83, + MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f +}; + +enum { + /* + * Largest possible UD header: send with GRH and immediate data. + */ + MLX4_IB_UD_HEADER_SIZE = 72 +}; + +struct mlx4_ib_sqp { + struct mlx4_ib_qp qp; + int pkey_index; + u32 qkey; + u32 send_psn; + struct ib_ud_header ud_header; + u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; +}; + +enum { + MLX4_IB_MIN_SQ_STRIDE = 6 +}; + +static const __be32 mlx4_ib_opcode[] = { + __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), /* [IB_WR_RDMA_WRITE] */ + __constant_cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), /* [IB_WR_RDMA_WRITE_WITH_IMM] */ + __constant_cpu_to_be32(MLX4_OPCODE_SEND), /* [IB_WR_SEND] */ + __constant_cpu_to_be32(MLX4_OPCODE_SEND_IMM), /* [IB_WR_SEND_WITH_IMM] */ + __constant_cpu_to_be32(MLX4_OPCODE_RDMA_READ), /* [IB_WR_RDMA_READ] */ + __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), /* [IB_WR_ATOMIC_CMP_AND_SWP] */ + __constant_cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), /* [IB_WR_ATOMIC_FETCH_AND_ADD]*/ +}; + +static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp) +{ + return container_of(mqp, struct mlx4_ib_sqp, qp); +} + +static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + return qp->mqp.qpn >= dev->dev->caps.sqp_start && + qp->mqp.qpn <= dev->dev->caps.sqp_start + 3; +} + +static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + return qp->mqp.qpn >= dev->dev->caps.sqp_start && + qp->mqp.qpn <= dev->dev->caps.sqp_start + 1; +} + +static void *get_wqe(struct mlx4_ib_qp *qp, int offset) +{ + if (qp->buf.nbufs == 1) + return qp->buf.u.direct.buf + offset; + else + return qp->buf.u.page_list[offset >> PAGE_SHIFT].buf + + (offset & (PAGE_SIZE - 1)); +} + +static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); +} + +static void *get_send_wqe(struct mlx4_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); +} + +/* + * Stamp a SQ WQE so that it is invalid if prefetched by marking the + * first four bytes of every 64 byte chunk with 0xffffffff, except for + * the very first chunk of the WQE. + */ +static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n) +{ + u32 *wqe = get_send_wqe(qp, n); + int i; + + for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16) + wqe[i] = 0xffffffff; +} + +static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) +{ + ib_event_rec_t event; + struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; + + if (type == MLX4_EVENT_TYPE_PATH_MIG) + to_mibqp(qp)->port = to_mibqp(qp)->alt_port; + + switch (type) { + case MLX4_EVENT_TYPE_PATH_MIG: + event.type = IB_EVENT_PATH_MIG; + break; + case MLX4_EVENT_TYPE_COMM_EST: + event.type = IB_EVENT_COMM_EST; + break; + case MLX4_EVENT_TYPE_SQ_DRAINED: + event.type = IB_EVENT_SQ_DRAINED; + break; + case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: + event.type = IB_EVENT_QP_LAST_WQE_REACHED; + break; + case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: + event.type = IB_EVENT_QP_FATAL; + break; + case MLX4_EVENT_TYPE_PATH_MIG_FAILED: + event.type = IB_EVENT_PATH_MIG_ERR; + break; + case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.type = IB_EVENT_QP_REQ_ERR; + break; + case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: + event.type = IB_EVENT_QP_ACCESS_ERR; + break; + default: + printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " + "on QP %06x\n", type, qp->qpn); + return; + } + + event.context = ibqp->qp_context; + ibqp->event_handler(&event); +} + +static int send_wqe_overhead(enum ib_qp_type type) +{ + /* + * UD WQEs must have a datagram segment. + * RC and UC WQEs might have a remote address segment. + * MLX WQEs need two extra inline data segments (for the UD + * header and space for the ICRC). + */ + switch (type) { + case IB_QPT_UD: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_datagram_seg); + case IB_QPT_UC: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_raddr_seg); + case IB_QPT_RC: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_atomic_seg) + + sizeof (struct mlx4_wqe_raddr_seg); + case IB_QPT_SMI: + case IB_QPT_GSI: + return sizeof (struct mlx4_wqe_ctrl_seg) + + ALIGN(MLX4_IB_UD_HEADER_SIZE + + DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE, + MLX4_INLINE_ALIGN) * + sizeof (struct mlx4_wqe_inline_seg), + sizeof (struct mlx4_wqe_data_seg)) + + ALIGN(4 + + sizeof (struct mlx4_wqe_inline_seg), + sizeof (struct mlx4_wqe_data_seg)); + default: + return sizeof (struct mlx4_wqe_ctrl_seg); + } +} + +static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, + int is_user, int has_srq, struct mlx4_ib_qp *qp) +{ + /* Sanity check RQ size before proceeding */ + if ((int)cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || + (int)cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) + return -EINVAL; + + if (has_srq) { + /* QPs attached to an SRQ should have no RQ */ + if (cap->max_recv_wr) + return -EINVAL; + + qp->rq.wqe_cnt = qp->rq.max_gs = 0; + } else { + /* HW requires >= 1 RQ entry with >= 1 gather entry */ + if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) + return -EINVAL; + + qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr)); + qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); + qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg)); + } + + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; + cap->max_recv_sge = qp->rq.max_gs; + } else { + cap->max_recv_wr = qp->rq.max_post = + min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); + cap->max_recv_sge = min(qp->rq.max_gs, + min(dev->dev->caps.max_sq_sg, + dev->dev->caps.max_rq_sg)); + } + /* We don't support inline sends for kernel QPs (yet) */ + + return 0; +} + +static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, + enum ib_qp_type type, struct mlx4_ib_qp *qp) +{ + /* Sanity check SQ size before proceeding */ + if ((int)cap->max_send_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || + (int)cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || + (int)cap->max_inline_data + send_wqe_overhead(type) + + (int)sizeof(struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) + return -EINVAL; + + /* + * For MLX transport we need 2 extra S/G entries: + * one for the header and one for the checksum at the end + */ + if ((type == IB_QPT_SMI || type == IB_QPT_GSI) && + (int)cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg) + return -EINVAL; + + qp->sq.wqe_shift = ilog2(roundup_pow_of_two(max(cap->max_send_sge * + sizeof (struct mlx4_wqe_data_seg), + cap->max_inline_data + + sizeof (struct mlx4_wqe_inline_seg)) + + send_wqe_overhead(type))); + qp->sq.wqe_shift = max(MLX4_IB_SQ_MIN_WQE_SHIFT, qp->sq.wqe_shift); + qp->sq.max_gs = ((1 << qp->sq.wqe_shift) - send_wqe_overhead(type)) / + sizeof (struct mlx4_wqe_data_seg); + + /* + * We need to leave 2 KB + 1 WQE of headroom in the SQ to + * allow HW to prefetch. + */ + qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift); + qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr + qp->sq_spare_wqes); + + qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + + (qp->sq.wqe_cnt << qp->sq.wqe_shift); + if (qp->rq.wqe_shift > qp->sq.wqe_shift) { + qp->rq.offset = 0; + qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + } else { + qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift; + qp->sq.offset = 0; + } + + cap->max_send_wr = qp->sq.max_post = + min(qp->sq.wqe_cnt - qp->sq_spare_wqes, + dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE); + cap->max_send_sge = min(qp->sq.max_gs, + min(dev->dev->caps.max_sq_sg, + dev->dev->caps.max_rq_sg)); + /* We don't support inline sends for kernel QPs (yet) */ + cap->max_inline_data = 0; + + return 0; +} + +static int set_user_sq_size(struct mlx4_ib_dev *dev, + struct mlx4_ib_qp *qp, + struct mlx4_ib_create_qp *ucmd) +{ + /* Sanity check SQ size before proceeding */ + if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes || + ucmd->log_sq_stride > + ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) || + ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE) + return -EINVAL; + + qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; + qp->sq.wqe_shift = ucmd->log_sq_stride; + + qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + + (qp->sq.wqe_cnt << qp->sq.wqe_shift); + + return 0; +} + +static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, u32 sqpn, struct mlx4_ib_qp *qp) +{ + int err; + + mutex_init(&qp->mutex); + spin_lock_init(&qp->sq.lock); + spin_lock_init(&qp->rq.lock); + + qp->state = XIB_QPS_RESET; + qp->atomic_rd_en = 0; + qp->resp_depth = 0; + + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.head = 0; + qp->sq.tail = 0; + + err = set_rq_size(dev, &init_attr->cap, !!pd->p_uctx, !!init_attr->srq, qp); + if (err) + goto err; + + if (pd->p_uctx) { + struct mlx4_ib_create_qp ucmd; + + if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { + err = -EFAULT; + goto err; + } + + qp->sq_no_prefetch = ucmd.sq_no_prefetch; + + err = set_user_sq_size(dev, qp, &ucmd); + if (err) + goto err; + + qp->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr, + qp->buf_size, 0, FALSE); + if (IS_ERR(qp->umem)) { + err = PTR_ERR(qp->umem); + goto err; + } + + err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem), + ilog2(qp->umem->page_size), &qp->mtt); + if (err) + goto err_buf; + + err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); + if (err) + goto err_mtt; + + if (!init_attr->srq) { + err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx), + ucmd.db_addr, &qp->db); + if (err) + goto err_mtt; + } + } else { + qp->sq_no_prefetch = 0; + + err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp); + if (err) + goto err; + + if (!init_attr->srq) { + err = mlx4_ib_db_alloc(dev, &qp->db, 0); + if (err) + goto err; + + *qp->db.db = 0; + } + + if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) { + err = -ENOMEM; + goto err_db; + } + + err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift, + &qp->mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf); + if (err) + goto err_mtt; + + qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL); + qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL); + + if (!qp->sq.wrid || !qp->rq.wrid) { + err = -ENOMEM; + goto err_wrid; + } + } + + if (!sqpn) + err = mlx4_qp_reserve_range(dev->dev, 1, 1, &sqpn); + if (err) + goto err_wrid; + + err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp); + if (err) + goto err_wrid; + + /* + * Hardware wants QPN written in big-endian order (after + * shifting) for send doorbell. Precompute this value to save + * a little bit when posting sends. + */ + qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); + + if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) + qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); + else + qp->sq_signal_bits = 0; + + qp->mqp.event = mlx4_ib_qp_event; + + return 0; + +err_wrid: + if (pd->p_uctx) { + if (!init_attr->srq) + mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx), + &qp->db); + } else { + kfree(qp->sq.wrid); + kfree(qp->rq.wrid); + } + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &qp->mtt); + +err_buf: + if (pd->p_uctx) + ib_umem_release(qp->umem); + else + mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); + +err_db: + if (!pd->p_uctx && !init_attr->srq) + mlx4_ib_db_free(dev, &qp->db); + +err: + return err; +} + +static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state) +{ + switch (state) { + case XIB_QPS_RESET: return MLX4_QP_STATE_RST; + case XIB_QPS_INIT: return MLX4_QP_STATE_INIT; + case XIB_QPS_RTR: return MLX4_QP_STATE_RTR; + case XIB_QPS_RTS: return MLX4_QP_STATE_RTS; + case XIB_QPS_SQD: return MLX4_QP_STATE_SQD; + case XIB_QPS_SQE: return MLX4_QP_STATE_SQER; + case XIB_QPS_ERR: return MLX4_QP_STATE_ERR; + default: return -1; + } +} + +static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) +{ + if (send_cq == recv_cq) + spin_lock_irq(&send_cq->lock); + else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { + spin_lock_irq(&send_cq->lock); + spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&recv_cq->lock); + spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) +{ + if (send_cq == recv_cq) + spin_unlock_irq(&send_cq->lock); + else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { + spin_unlock(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else { + spin_unlock(&send_cq->lock); + spin_unlock_irq(&recv_cq->lock); + } +} + +static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + int is_user) +{ + struct mlx4_ib_cq *send_cq, *recv_cq; + + if (qp->state != XIB_QPS_RESET) + if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), + MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) + printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n", + qp->mqp.qpn); + + send_cq = to_mcq(qp->ibqp.send_cq); + recv_cq = to_mcq(qp->ibqp.recv_cq); + + mlx4_ib_lock_cqs(send_cq, recv_cq); + + if (!is_user) { + __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, + qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL); + if (send_cq != recv_cq) + __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); + } + + mlx4_qp_remove(dev->dev, &qp->mqp); + + mlx4_ib_unlock_cqs(send_cq, recv_cq); + + mlx4_qp_free(dev->dev, &qp->mqp); + mlx4_mtt_cleanup(dev->dev, &qp->mtt); + + if (is_user) { + if (!qp->ibqp.srq) + mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.p_uctx), + &qp->db); + ib_umem_release(qp->umem); + } else { + kfree(qp->sq.wrid); + kfree(qp->rq.wrid); + mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); + if (!qp->ibqp.srq) + mlx4_ib_db_free(dev, &qp->db); + } +} + +struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_sqp *sqp; + struct mlx4_ib_qp *qp; + int err; + + if (mlx4_is_barred(pd->device->dma_device)) + return ERR_PTR(-EFAULT); + + switch (init_attr->qp_type) { + case IB_QPT_RC: + case IB_QPT_UC: + case IB_QPT_UD: + { + qp = kzalloc(sizeof *qp, GFP_KERNEL); + if (!qp) + return ERR_PTR(-ENOMEM); + + err = create_qp_common(dev, pd, init_attr, udata, 0, qp); + if (err) { + kfree(qp); + return ERR_PTR(err); + } + + qp->ibqp.qp_num = qp->mqp.qpn; + + break; + } + case IB_QPT_SMI: + case IB_QPT_GSI: + { + /* Userspace is not allowed to create special QPs: */ + if (pd->p_uctx) + return ERR_PTR(-EINVAL); + + sqp = kzalloc(sizeof *sqp, GFP_KERNEL); + if (!sqp) + return ERR_PTR(-ENOMEM); + + qp = &sqp->qp; + + err = create_qp_common(dev, pd, init_attr, udata, + dev->dev->caps.sqp_start + + (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) + + init_attr->port_num - 1, + qp); + if (err) { + kfree(sqp); + return ERR_PTR(err); + } + + qp->port = init_attr->port_num; + qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; + + break; + } + default: + /* Don't support raw QPs */ + return ERR_PTR(-EINVAL); + } + + return &qp->ibqp; +} + +int mlx4_ib_destroy_qp(struct ib_qp *qp) +{ + struct mlx4_ib_dev *dev = to_mdev(qp->device); + struct mlx4_ib_qp *mqp = to_mqp(qp); + + if (!mlx4_is_barred(dev->dev) && is_qp0(dev, mqp)) + mlx4_CLOSE_PORT(dev->dev, mqp->port); + + destroy_qp_common(dev, mqp, !!qp->pd->p_uctx); + + if (is_sqp(dev, mqp)) + kfree(to_msqp(mqp)); + else + kfree(mqp); + + return 0; +} + +static int to_mlx4_st(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_RC: return MLX4_QP_ST_RC; + case IB_QPT_UC: return MLX4_QP_ST_UC; + case IB_QPT_UD: return MLX4_QP_ST_UD; + case IB_QPT_SMI: + case IB_QPT_GSI: return MLX4_QP_ST_MLX; + default: return -1; + } +} + +static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, + int attr_mask) +{ + u8 dest_rd_atomic; + u32 access_flags; + u32 hw_access_flags = 0; + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + dest_rd_atomic = attr->max_dest_rd_atomic; + else + dest_rd_atomic = qp->resp_depth; + + if (attr_mask & IB_QP_ACCESS_FLAGS) + access_flags = attr->qp_access_flags; + else + access_flags = qp->atomic_rd_en; + + if (!dest_rd_atomic) + access_flags &= IB_ACCESS_REMOTE_WRITE; + + if (access_flags & IB_ACCESS_REMOTE_READ) + hw_access_flags |= MLX4_QP_BIT_RRE; + if (access_flags & IB_ACCESS_REMOTE_ATOMIC) + hw_access_flags |= MLX4_QP_BIT_RAE; + if (access_flags & IB_ACCESS_REMOTE_WRITE) + hw_access_flags |= MLX4_QP_BIT_RWE; + + return cpu_to_be32(hw_access_flags); +} + +static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr, + int attr_mask) +{ + if (attr_mask & IB_QP_PKEY_INDEX) + sqp->pkey_index = attr->pkey_index; + if (attr_mask & IB_QP_QKEY) + sqp->qkey = attr->qkey; + if (attr_mask & IB_QP_SQ_PSN) + sqp->send_psn = attr->sq_psn; +} + +static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port) +{ + path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6); +} + +static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, + struct mlx4_qp_path *path, u8 port) +{ + path->grh_mylmc = ah->src_path_bits & 0x7f; + path->rlid = cpu_to_be16(ah->dlid); + if (ah->static_rate) { + path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET; + while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && + !(1 << path->static_rate & dev->dev->caps.stat_rate_support)) + --path->static_rate; + } else + path->static_rate = 0; + path->counter_index = 0xff; + + if (ah->ah_flags & IB_AH_GRH) { + if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) { + printk(KERN_ERR "sgid_index (%u) too large. max is %d\n", + ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1); + return -1; + } + + path->grh_mylmc |= 1 << 7; + path->mgid_index = ah->grh.sgid_index; + path->hop_limit = ah->grh.hop_limit; + path->tclass_flowlabel = + cpu_to_be32((ah->grh.traffic_class << 20) | + (ah->grh.flow_label)); + memcpy(path->rgid, ah->grh.dgid.raw, 16); + } + + path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | + ((port - 1) << 6) | ((ah->sl & 0xf) << 2); + + return 0; +} + +static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state) +{ + struct mlx4_ib_dev *dev = to_mdev(ibqp->device); + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_qp_context *context; + enum mlx4_qp_optpar optpar = 0; + int sqd_event; + int err = -EINVAL; + + context = kzalloc(sizeof *context, GFP_KERNEL); + if (!context) + return -ENOMEM; + + context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) | + (to_mlx4_st(ibqp->qp_type) << 16)); + context->flags |= cpu_to_be32(1 << 8); /* DE? */ + + if (!(attr_mask & IB_QP_PATH_MIG_STATE)) + context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); + else { + optpar |= MLX4_QP_OPTPAR_PM_STATE; + switch (attr->path_mig_state) { + case IB_MIG_MIGRATED: + context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); + break; + case IB_MIG_REARM: + context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11); + break; + case IB_MIG_ARMED: + context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11); + break; + } + } + + if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || + ibqp->qp_type == IB_QPT_UD) + context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; + else if (attr_mask & IB_QP_PATH_MTU) { + if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { + printk(KERN_ERR "path MTU (%u) is invalid\n", + attr->path_mtu); + goto out; + } + context->mtu_msgmax = (u8)((attr->path_mtu << 5) | + ilog2(dev->dev->caps.max_msg_sz)); + } + + if (qp->rq.wqe_cnt) + context->rq_size_stride = (u8)(ilog2(qp->rq.wqe_cnt) << 3); + context->rq_size_stride |= qp->rq.wqe_shift - 4; + + if (qp->sq.wqe_cnt) + context->sq_size_stride = (u8)(ilog2(qp->sq.wqe_cnt) << 3); + context->sq_size_stride |= qp->sq.wqe_shift - 4; + + if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) + context->sq_size_stride |= !!qp->sq_no_prefetch << 7; + + if (qp->ibqp.p_uctx) + context->usr_page = cpu_to_be32(to_mucontext(ibqp->p_uctx)->uar.index); + else + context->usr_page = cpu_to_be32(dev->priv_uar.index); + + if (attr_mask & IB_QP_DEST_QPN) + context->remote_qpn = cpu_to_be32(attr->dest_qp_num); + + if (attr_mask & IB_QP_PORT) { + if (cur_state == XIB_QPS_SQD && new_state == XIB_QPS_SQD && + !(attr_mask & IB_QP_AV)) { + mlx4_set_sched(&context->pri_path, attr->port_num); + optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE; + } + } + + if (attr_mask & IB_QP_PKEY_INDEX) { + context->pri_path.pkey_index = (u8)attr->pkey_index; + optpar |= MLX4_QP_OPTPAR_PKEY_INDEX; + } + + if (attr_mask & IB_QP_AV) { + if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path, + attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) + goto out; + + optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | + MLX4_QP_OPTPAR_SCHED_QUEUE); + } + + if (attr_mask & IB_QP_TIMEOUT) { + context->pri_path.ackto = attr->timeout << 3; + optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT; + } + + if (attr_mask & IB_QP_ALT_PATH) { + if (attr->alt_port_num == 0 || + attr->alt_port_num > dev->dev->caps.num_ports) + goto out; + + if (attr->alt_pkey_index >= + dev->dev->caps.pkey_table_len[attr->alt_port_num]) + goto out; + + if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path, + attr->alt_port_num)) + goto out; + + context->alt_path.pkey_index = (u8)attr->alt_pkey_index; + context->alt_path.ackto = attr->alt_timeout << 3; + optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH; + } + + context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn); + context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28); + + if (attr_mask & IB_QP_RNR_RETRY) { + context->params1 |= cpu_to_be32(attr->rnr_retry << 13); + optpar |= MLX4_QP_OPTPAR_RNR_RETRY; + } + + if (attr_mask & IB_QP_RETRY_CNT) { + context->params1 |= cpu_to_be32(attr->retry_cnt << 16); + optpar |= MLX4_QP_OPTPAR_RETRY_COUNT; + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + if (attr->max_rd_atomic) + context->params1 |= + cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); + optpar |= MLX4_QP_OPTPAR_SRA_MAX; + } + + if (attr_mask & IB_QP_SQ_PSN) + context->next_send_psn = cpu_to_be32(attr->sq_psn); + + context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn); + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (attr->max_dest_rd_atomic) + context->params2 |= + cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); + optpar |= MLX4_QP_OPTPAR_RRA_MAX; + } + + if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) { + context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask); + optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE; + } + + if (ibqp->srq) + context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC); + + if (attr_mask & IB_QP_MIN_RNR_TIMER) { + context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); + optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT; + } + if (attr_mask & IB_QP_RQ_PSN) + context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); + + context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn); + + if (attr_mask & IB_QP_QKEY) { + context->qkey = cpu_to_be32(attr->qkey); + optpar |= MLX4_QP_OPTPAR_Q_KEY; + } + + if (ibqp->srq) + context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn); + + if (!ibqp->srq && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) + context->db_rec_addr = cpu_to_be64(qp->db.dma.da); + + if (cur_state == XIB_QPS_INIT && + new_state == XIB_QPS_RTR && + (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || + ibqp->qp_type == IB_QPT_UD)) { + context->pri_path.sched_queue = (qp->port - 1) << 6; + if (is_qp0(dev, qp)) + context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE; + else + context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE; + } + + if (cur_state == XIB_QPS_RTS && new_state == XIB_QPS_SQD && + attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) + sqd_event = 1; + else + sqd_event = 0; + + /* + * Before passing a kernel QP to the HW, make sure that the + * ownership bits of the send queue are set and the SQ + * headroom is stamped so that the hardware doesn't start + * processing stale work requests. + */ + if (!ibqp->p_uctx && cur_state == XIB_QPS_RESET && new_state == XIB_QPS_INIT) { + struct mlx4_wqe_ctrl_seg *ctrl; + int i; + + for (i = 0; i < qp->sq.wqe_cnt; ++i) { + ctrl = get_send_wqe(qp, i); + ctrl->owner_opcode = cpu_to_be32(1 << 31); + + stamp_send_wqe(qp, i); + } + } + + err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state), + to_mlx4_state(new_state), context, optpar, + sqd_event, &qp->mqp); + if (err) + goto out; + + qp->state = new_state; + + if (attr_mask & IB_QP_ACCESS_FLAGS) + qp->atomic_rd_en = (u8)attr->qp_access_flags; + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) + qp->port = attr->port_num; + if (attr_mask & IB_QP_ALT_PATH) + qp->alt_port = attr->alt_port_num; + + if (is_sqp(dev, qp)) + store_sqp_attrs(to_msqp(qp), attr, attr_mask); + + /* + * If we moved QP0 to RTR, bring the IB link up; if we moved + * QP0 to RESET or ERROR, bring the link back down. + */ + if (is_qp0(dev, qp)) { + if (cur_state != XIB_QPS_RTR && new_state == XIB_QPS_RTR) + if (mlx4_INIT_PORT(dev->dev, qp->port)) + printk(KERN_WARNING "INIT_PORT failed for port %d\n", + qp->port); + + if (cur_state != XIB_QPS_RESET && cur_state != XIB_QPS_ERR && + (new_state == XIB_QPS_RESET || new_state == XIB_QPS_ERR)) + mlx4_CLOSE_PORT(dev->dev, qp->port); + } + + /* + * If we moved a kernel QP to RESET, clean up all old CQ + * entries and reinitialize the QP. + */ + if (new_state == XIB_QPS_RESET && !ibqp->p_uctx) { + mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn, + ibqp->srq ? to_msrq(ibqp->srq): NULL); + if (ibqp->send_cq != ibqp->recv_cq) + mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL); + + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.head = 0; + qp->sq.tail = 0; + if (!ibqp->srq) + *qp->db.db = 0; + } + +out: + kfree(context); + return err; +} + +static struct ib_qp_attr mlx4_ib_qp_attr; +static int mlx4_ib_qp_attr_mask_table[IB_QPT_UD + 1]; + +void mlx4_ib_qp_init() +{ + memset( &mlx4_ib_qp_attr, 0, sizeof(mlx4_ib_qp_attr) ); + mlx4_ib_qp_attr.port_num = 1; + + memset( &mlx4_ib_qp_attr_mask_table, 0, sizeof(mlx4_ib_qp_attr_mask_table) ); + mlx4_ib_qp_attr_mask_table[IB_QPT_UD] = (IB_QP_PKEY_INDEX | + IB_QP_PORT | + IB_QP_QKEY); + mlx4_ib_qp_attr_mask_table[IB_QPT_UC] = (IB_QP_PKEY_INDEX | + IB_QP_PORT | + IB_QP_ACCESS_FLAGS); + mlx4_ib_qp_attr_mask_table[IB_QPT_RC] = (IB_QP_PKEY_INDEX | + IB_QP_PORT | + IB_QP_ACCESS_FLAGS); + mlx4_ib_qp_attr_mask_table[IB_QPT_SMI] = (IB_QP_PKEY_INDEX | + IB_QP_QKEY); + mlx4_ib_qp_attr_mask_table[IB_QPT_GSI] = (IB_QP_PKEY_INDEX | + IB_QP_QKEY); +} + +int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibqp->device); + struct mlx4_ib_qp *qp = to_mqp(ibqp); + enum ib_qp_state cur_state, new_state; + int err = -EINVAL; + + UNUSED_PARAM(udata); + + if (mlx4_is_barred(dev->dev)) + return -EFAULT; + + mutex_lock(&qp->mutex); + + cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) + goto out; + + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) { + goto out; + } + + if (attr_mask & IB_QP_PKEY_INDEX) { + int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; + if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) + goto out; + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && + attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) { + goto out; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && + attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) { + goto out; + } + + if (cur_state == new_state && cur_state == XIB_QPS_RESET) { + err = 0; + goto out; + } + + if (cur_state == XIB_QPS_RESET && new_state == XIB_QPS_ERR) { + err = __mlx4_ib_modify_qp(ibqp, &mlx4_ib_qp_attr, + mlx4_ib_qp_attr_mask_table[ibqp->qp_type], + XIB_QPS_RESET, XIB_QPS_INIT); + if (err) + goto out; + cur_state = XIB_QPS_INIT; + } + + err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); + +out: + mutex_unlock(&qp->mutex); + return err; +} + +static enum ib_wr_opcode to_wr_opcode(struct _ib_send_wr *wr) +{ + + enum ib_wr_opcode opcode = -1; //= wr->wr_type; + + switch (wr->wr_type) { + case WR_SEND: + opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_SEND_WITH_IMM : IB_WR_SEND; + break; + case WR_RDMA_WRITE: + opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? IB_WR_RDMA_WRITE_WITH_IMM : IB_WR_RDMA_WRITE; + break; + case WR_RDMA_READ: opcode = IB_WR_RDMA_READ; break; + case WR_COMPARE_SWAP: opcode = IB_WR_ATOMIC_CMP_AND_SWP; break; + case WR_FETCH_ADD: opcode = IB_WR_ATOMIC_FETCH_AND_ADD; break; + } + return opcode; +} + +static int build_mlx_header(struct mlx4_ib_sqp *sqp, ib_send_wr_t *wr, + void *wqe) +{ + enum ib_wr_opcode opcode = to_wr_opcode(wr); + struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev; + struct mlx4_wqe_mlx_seg *mlx = wqe; + struct mlx4_wqe_inline_seg *inl = (void*)((u8*)wqe + sizeof *mlx); + struct mlx4_ib_ah *ah = to_mah((struct ib_ah *)wr->dgrm.ud.h_av); + __be16 pkey; + int send_size; + int header_size; + int spc; + u32 i; + + send_size = 0; + for (i = 0; i < wr->num_ds; ++i) + send_size += wr->ds_array[i].length; + + ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->ud_header); + + sqp->ud_header.lrh.service_level = + (u8)(be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28); + sqp->ud_header.lrh.destination_lid = ah->av.dlid; + sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.g_slid & 0x7f); + if (mlx4_ib_ah_grh_present(ah)) { + sqp->ud_header.grh.traffic_class = + (u8)((be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20) & 0xff); + sqp->ud_header.grh.flow_label = + ah->av.sl_tclass_flowlabel & cpu_to_be32(0xfffff); + sqp->ud_header.grh.hop_limit = ah->av.hop_limit; + ib_get_cached_gid(ib_dev, (u8)(be32_to_cpu(ah->av.port_pd) >> 24), + ah->av.gid_index, &sqp->ud_header.grh.source_gid); + memcpy(sqp->ud_header.grh.destination_gid.raw, + ah->av.dgid, 16); + } + + mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); + mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | + (sqp->ud_header.lrh.destination_lid == + XIB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | + (sqp->ud_header.lrh.service_level << 8)); + mlx->rlid = sqp->ud_header.lrh.destination_lid; + + switch (opcode) { + case IB_WR_SEND: + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->ud_header.immediate_present = 0; + break; + case IB_WR_SEND_WITH_IMM: + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + sqp->ud_header.immediate_present = 1; + sqp->ud_header.immediate_data = wr->immediate_data; + break; + default: + return -EINVAL; + } + + sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; + if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) + sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; + sqp->ud_header.bth.solicited_event = (u8)(!!(wr->send_opt & IB_SEND_OPT_SOLICITED)); + if (!sqp->qp.ibqp.qp_num) + ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); + else + ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->dgrm.ud.pkey_index, &pkey); + sqp->ud_header.bth.pkey = pkey; + sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp; + sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); + sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ? + cpu_to_be32(sqp->qkey) : wr->dgrm.ud.remote_qkey; + sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); + + header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); + +#if 0 + { + printk(KERN_ERR "built UD header of size %d:\n", header_size); + for (i = 0; i < header_size / 4; ++i) { + if (i % 8 == 0) + printk(" [%02x] ", i * 4); + printk(" %08x", + be32_to_cpu(((__be32 *) sqp->header_buf)[i])); + if ((i + 1) % 8 == 0) + printk("\n"); + } + printk("\n"); + } +#endif + + /* + * Inline data segments may not cross a 64 byte boundary. If + * our UD header is bigger than the space available up to the + * next 64 byte boundary in the WQE, use two inline data + * segments to hold the UD header. + */ + spc = MLX4_INLINE_ALIGN - + ((u32)(ULONG_PTR)(inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (header_size <= spc) { + inl->byte_count = cpu_to_be32(1 << 31 | header_size); + memcpy(inl + 1, sqp->header_buf, header_size); + i = 1; + } else { + inl->byte_count = cpu_to_be32(1 << 31 | spc); + memcpy(inl + 1, sqp->header_buf, spc); + + inl = (void*)((u8*)(inl + 1) + spc); + memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); + /* + * Need a barrier here to make sure all the data is + * visible before the byte_count field is set. + * Otherwise the HCA prefetcher could grab the 64-byte + * chunk with this inline segment and get a valid (!= + * 0xffffffff) byte count but stale data, and end up + * generating a packet with bad headers. + * + * The first inline segment's byte_count field doesn't + * need a barrier, because it comes after a + * control/MLX segment and therefore is at an offset + * of 16 mod 64. + */ + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); + i = 2; + } + + return ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); +} + +static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq) +{ + unsigned cur; + struct mlx4_ib_cq *cq; + + cur = wq->head - wq->tail; + if (likely((int)cur + nreq < wq->max_post)) + return 0; + + cq = to_mcq(ib_cq); + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return (int)cur + nreq >= wq->max_post; +} + +static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg, + u64 remote_addr, __be32 rkey) +{ + rseg->raddr = cpu_to_be64(remote_addr); + rseg->rkey = rkey; + rseg->reserved = 0; +} + +static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, ib_send_wr_t *wr) +{ + if (wr->wr_type == WR_COMPARE_SWAP) { + aseg->swap_add = wr->remote_ops.atomic2; + aseg->compare = wr->remote_ops.atomic1; + } else { + aseg->swap_add = wr->remote_ops.atomic1; + aseg->compare = 0; + } + +} + +static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, + ib_send_wr_t *wr) +{ + memcpy(dseg->av, &to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, sizeof (struct mlx4_av)); + dseg->dqpn = wr->dgrm.ud.remote_qp; + dseg->qkey = wr->dgrm.ud.remote_qkey; +} + +static void set_mlx_icrc_seg(void *dseg) +{ + u32 *t = dseg; + struct mlx4_wqe_inline_seg *iseg = dseg; + + t[1] = 0; + + /* + * Need a barrier here before writing the byte_count field to + * make sure that all the data is visible before the + * byte_count field is set. Otherwise, if the segment begins + * a new cacheline, the HCA prefetcher could grab the 64-byte + * chunk and get a valid (!= * 0xffffffff) byte count but + * stale data, and end up sending the wrong data. + */ + wmb(); + + iseg->byte_count = cpu_to_be32((1 << 31) | 4); +} + +static void set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg) +{ + dseg->lkey = cpu_to_be32(sg->lkey); + dseg->addr = cpu_to_be64(sg->vaddr); + + /* + * Need a barrier here before writing the byte_count field to + * make sure that all the data is visible before the + * byte_count field is set. Otherwise, if the segment begins + * a new cacheline, the HCA prefetcher could grab the 64-byte + * chunk and get a valid (!= * 0xffffffff) byte count but + * stale data, and end up sending the wrong data. + */ + wmb(); + + dseg->byte_count = cpu_to_be32(sg->length); +} + +static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, ib_local_ds_t *sg) +{ + dseg->byte_count = cpu_to_be32(sg->length); + dseg->lkey = cpu_to_be32(sg->lkey); + dseg->addr = cpu_to_be64(sg->vaddr); +} + +int mlx4_ib_post_send(struct ib_qp *ibqp, ib_send_wr_t *wr, + ib_send_wr_t **bad_wr) +{ + enum ib_wr_opcode opcode; + struct mlx4_ib_qp *qp = to_mqp(ibqp); + u8 *wqe; + struct mlx4_wqe_ctrl_seg *ctrl; + struct mlx4_wqe_data_seg *dseg; + unsigned long flags; + int nreq; + int err = 0; + int ind; + int size; + int i; + + if (mlx4_is_barred(ibqp->device->dma_device)) + return -EFAULT; + + spin_lock_irqsave(&qp->sq.lock, &flags); + + ind = qp->sq.head; + + for (nreq = 0; wr; ++nreq, wr = wr->p_next) { + if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { + err = -ENOMEM; + if (bad_wr) + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_ds > (u32)qp->sq.max_gs)) { + err = -EINVAL; + if (bad_wr) + *bad_wr = wr; + goto out; + } + + wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); + ctrl = (void*)wqe; + qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id; + opcode = to_wr_opcode(wr); + + ctrl->srcrb_flags = + (wr->send_opt & IB_SEND_OPT_SIGNALED ? + cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) | + (wr->send_opt & IB_SEND_OPT_SOLICITED ? + cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) | + (wr->send_opt & IB_SEND_OPT_TX_IP_CSUM ? + cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM) : 0) | + (wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM ? + cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) | + qp->sq_signal_bits; + + if (opcode == IB_WR_SEND_WITH_IMM || + opcode == IB_WR_RDMA_WRITE_WITH_IMM) + ctrl->imm = wr->immediate_data; + else + ctrl->imm = 0; + + wqe += sizeof *ctrl; + size = sizeof *ctrl / 16; + + switch (ibqp->qp_type) { + case IB_QPT_RC: + case IB_QPT_UC: + switch (opcode) { + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + set_raddr_seg((void*)wqe, wr->remote_ops.vaddr, + wr->remote_ops.rkey); + wqe += sizeof (struct mlx4_wqe_raddr_seg); + + set_atomic_seg((void*)wqe, wr); + wqe += sizeof (struct mlx4_wqe_atomic_seg); + + size += (sizeof (struct mlx4_wqe_raddr_seg) + + sizeof (struct mlx4_wqe_atomic_seg)) / 16; + + break; + + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + set_raddr_seg((void*)wqe, wr->remote_ops.vaddr, + wr->remote_ops.rkey); + wqe += sizeof (struct mlx4_wqe_raddr_seg); + size += sizeof (struct mlx4_wqe_raddr_seg) / 16; + break; + + default: + /* No extra segments required for sends */ + break; + } + break; + + case IB_QPT_UD: + set_datagram_seg((void*)wqe, wr); + wqe += sizeof (struct mlx4_wqe_datagram_seg); + size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + break; + + case IB_QPT_SMI: + case IB_QPT_GSI: + err = build_mlx_header(to_msqp(qp), wr, ctrl); + if (err < 0) { + if (bad_wr) + *bad_wr = wr; + goto out; + } + wqe += err; + size += err / 16; + + err = 0; + break; + + default: + break; + } + + /* + * Write data segments in reverse order, so as to + * overwrite cacheline stamp last within each + * cacheline. This avoids issues with WQE + * prefetching. + */ + + dseg = (void*)wqe; + dseg += wr->num_ds - 1; + size += wr->num_ds * (sizeof (struct mlx4_wqe_data_seg) / 16); + + /* Add one more inline data segment for ICRC for MLX sends */ + if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI || + qp->ibqp.qp_type == IB_QPT_GSI)) { + set_mlx_icrc_seg(dseg + 1); + size += sizeof (struct mlx4_wqe_data_seg) / 16; + } + + for (i = wr->num_ds - 1; i >= 0; --i, --dseg) + set_data_seg(dseg, wr->ds_array + i); + + ctrl->fence_size = (u8)((wr->send_opt & IB_SEND_OPT_FENCE ? + MLX4_WQE_CTRL_FENCE : 0) | size); + + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + wmb(); + + if (opcode < 0 || opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { + err = -EINVAL; + goto out; + } + + ctrl->owner_opcode = mlx4_ib_opcode[opcode] | + (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0); + + /* + * We can improve latency by not stamping the last + * send queue WQE until after ringing the doorbell, so + * only stamp here if there are still more WQEs to post. + */ + if (wr->p_next) + stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) & + (qp->sq.wqe_cnt - 1)); + + ++ind; + } + +out: + if (likely(nreq)) { + qp->sq.head += nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + writel(qp->doorbell_qpn, + (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL); + +#if 0 + if (qp->mqp.qpn == 0x41) + DbgPrint( "[MLX4_BUS] mlx4_ib_post_send : qtype %d, qpn %#x, nreq %d, sq.head %#x, wqe_ix %d, db %p \n", + ibqp->qp_type, qp->mqp.qpn, nreq, qp->sq.head, ind, + (u8*)to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL ); +#endif + /* + * Make sure doorbells don't leak out of SQ spinlock + * and reach the HCA out of order. + */ + mmiowb(); + + stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) & + (qp->sq.wqe_cnt - 1)); + } + + spin_unlock_irqrestore(&qp->sq.lock, flags); + + return err; +} + +int mlx4_ib_post_recv(struct ib_qp *ibqp, ib_recv_wr_t *wr, + ib_recv_wr_t **bad_wr) +{ + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_wqe_data_seg *scat; + unsigned long flags; + int err = 0; + int nreq; + int ind; + int i; + + if (mlx4_is_barred(ibqp->device->dma_device)) + return -EFAULT; + + spin_lock_irqsave(&qp->rq.lock, &flags); + + ind = qp->rq.head & (qp->rq.wqe_cnt - 1); + + for (nreq = 0; wr; ++nreq, wr = wr->p_next) { + if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.send_cq)) { + err = -ENOMEM; + if (bad_wr) + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_ds > (u32)qp->rq.max_gs)) { + err = -EINVAL; + if (bad_wr) + *bad_wr = wr; + goto out; + } + + scat = get_recv_wqe(qp, ind); + + for (i = 0; i < (int)wr->num_ds; ++i) + __set_data_seg(scat + i, wr->ds_array + i); + + if (i < qp->rq.max_gs) { + scat[i].byte_count = 0; + scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); + scat[i].addr = 0; + } + + qp->rq.wrid[ind] = wr->wr_id; + + ind = (ind + 1) & (qp->rq.wqe_cnt - 1); + } + +out: + if (likely(nreq)) { + qp->rq.head += nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); + +#if 0 + if (qp->mqp.qpn == 0x41) + DbgPrint( "[MLX4_BUS] mlx4_ib_post_recv : qtype %d, qpn %#x, nreq %d, rq.head %#x, wqe_ix %d, db_obj %p, db %p \n", + ibqp->qp_type, qp->mqp.qpn, nreq, qp->rq.head, ind, &qp->db, qp->db.db ); +#endif + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return err; +} + +static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state) +{ + switch (mlx4_state) { + case MLX4_QP_STATE_RST: return XIB_QPS_RESET; + case MLX4_QP_STATE_INIT: return XIB_QPS_INIT; + case MLX4_QP_STATE_RTR: return XIB_QPS_RTR; + case MLX4_QP_STATE_RTS: return XIB_QPS_RTS; + case MLX4_QP_STATE_SQ_DRAINING: + case MLX4_QP_STATE_SQD: return XIB_QPS_SQD; + case MLX4_QP_STATE_SQER: return XIB_QPS_SQE; + case MLX4_QP_STATE_ERR: return XIB_QPS_ERR; + default: return -1; + } +} + +static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state) +{ + switch (mlx4_mig_state) { + case MLX4_QP_PM_ARMED: return IB_MIG_ARMED; + case MLX4_QP_PM_REARM: return IB_MIG_REARM; + case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED; + default: return -1; + } +} + +static int to_ib_qp_access_flags(int mlx4_flags) +{ + int ib_flags = 0; + + if (mlx4_flags & MLX4_QP_BIT_RRE) + ib_flags |= IB_ACCESS_REMOTE_READ; + if (mlx4_flags & MLX4_QP_BIT_RWE) + ib_flags |= IB_ACCESS_REMOTE_WRITE; + if (mlx4_flags & MLX4_QP_BIT_RAE) + ib_flags |= IB_ACCESS_REMOTE_ATOMIC; + + return ib_flags; +} + +static void to_ib_ah_attr(struct mlx4_dev *dev, struct ib_ah_attr *ib_ah_attr, + struct mlx4_qp_path *path) +{ + memset(ib_ah_attr, 0, sizeof *ib_ah_attr); + ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1; + + if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports) + return; + + ib_ah_attr->dlid = be16_to_cpu(path->rlid); + ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf; + ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f; + ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; + ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0; + if (ib_ah_attr->ah_flags) { + ib_ah_attr->grh.sgid_index = path->mgid_index; + ib_ah_attr->grh.hop_limit = path->hop_limit; + ib_ah_attr->grh.traffic_class = + (u8)((be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff); + ib_ah_attr->grh.flow_label = + be32_to_cpu(path->tclass_flowlabel) & 0xfffff; + memcpy(ib_ah_attr->grh.dgid.raw, + path->rgid, sizeof ib_ah_attr->grh.dgid.raw); + } +} + +int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct mlx4_ib_dev *dev = to_mdev(ibqp->device); + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_qp_context context; + int mlx4_state; + int err; + + UNUSED_PARAM(qp_attr_mask); + + if (mlx4_is_barred(dev->dev)) + return -EFAULT; + + if (qp->state == XIB_QPS_RESET) { + qp_attr->qp_state = XIB_QPS_RESET; + goto done; + } + + err = mlx4_qp_query(dev->dev, &qp->mqp, &context); + if (err) + return -EINVAL; + + mlx4_state = be32_to_cpu(context.flags) >> 28; + + qp_attr->qp_state = to_ib_qp_state(mlx4_state); + qp_attr->path_mtu = context.mtu_msgmax >> 5; + qp_attr->path_mig_state = + to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3); + qp_attr->qkey = be32_to_cpu(context.qkey); + qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff; + qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff; + qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff; + qp_attr->qp_access_flags = + to_ib_qp_access_flags(be32_to_cpu(context.params2)); + + if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { + to_ib_ah_attr(dev->dev, &qp_attr->ah_attr, &context.pri_path); + to_ib_ah_attr(dev->dev, &qp_attr->alt_ah_attr, &context.alt_path); + qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f; + qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; + } + + qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f; + if (qp_attr->qp_state == XIB_QPS_INIT) + qp_attr->port_num = qp->port; + else + qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1; + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + qp_attr->sq_draining = (u8)(mlx4_state == MLX4_QP_STATE_SQ_DRAINING); + + qp_attr->max_rd_atomic = (u8)(1 << ((be32_to_cpu(context.params1) >> 21) & 0x7)); + + qp_attr->max_dest_rd_atomic = + (u8)(1 << ((be32_to_cpu(context.params2) >> 21) & 0x7)); + qp_attr->min_rnr_timer = + (u8)((be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f); + qp_attr->timeout = context.pri_path.ackto >> 3; + qp_attr->retry_cnt = (u8)((be32_to_cpu(context.params1) >> 16) & 0x7); + qp_attr->rnr_retry = (u8)((be32_to_cpu(context.params1) >> 13) & 0x7); + qp_attr->alt_timeout = context.alt_path.ackto >> 3; + +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = qp->rq.max_gs; + + if (!ibqp->p_uctx) { + qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + /* + * We don't support inline sends for kernel QPs (yet), and we + * don't know what userspace's value should be. + */ + qp_attr->cap.max_inline_data = 0; + + qp_init_attr->cap = qp_attr->cap; + + return 0; +} + diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/srq.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/srq.c index 34c5f1ab..9067e1fc 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/srq.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/srq.c @@ -1,360 +1,373 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "mlx4_ib.h" -#include "qp.h" -#include "srq.h" -#include "user.h" - -static void *get_wqe(struct mlx4_ib_srq *srq, int n) -{ - int offset = n << srq->msrq.wqe_shift; - - if (srq->buf.nbufs == 1) - return srq->buf.u.direct.buf + offset; - else - return srq->buf.u.page_list[offset >> PAGE_SHIFT].buf + - (offset & (PAGE_SIZE - 1)); -} - -static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) -{ - ib_event_rec_t event; - struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; - - switch (type) { - case MLX4_EVENT_TYPE_SRQ_LIMIT: - event.type = IB_EVENT_SRQ_LIMIT_REACHED; - break; - case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: - event.type = IB_EVENT_SRQ_ERR; - break; - default: - printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " - "on SRQ %06x\n", type, srq->srqn); - return; - } - - event.context = ibsrq->srq_context; - ibsrq->event_handler(&event); -} - -struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_srq *srq; - struct mlx4_wqe_srq_next_seg *next; - int desc_size; - int buf_size; - int err; - int i; - u32 cqn = 0; - u16 xrcd = 0; - - /* Sanity check SRQ size before proceeding */ - if ((int)init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || - (int)init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) - return ERR_PTR(-EINVAL); - - srq = kzalloc(sizeof *srq, GFP_KERNEL); - if (!srq) - return ERR_PTR(-ENOMEM); - - mutex_init(&srq->mutex); - spin_lock_init(&srq->lock); - srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); - srq->msrq.max_gs = init_attr->attr.max_sge; - - desc_size = max(32UL, - roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + - srq->msrq.max_gs * - sizeof (struct mlx4_wqe_data_seg))); - srq->msrq.wqe_shift = ilog2(desc_size); - - buf_size = srq->msrq.max * desc_size; - - if (pd->p_uctx) { - struct mlx4_ib_create_srq ucmd; - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - err = -EFAULT; - goto err_srq; - } - - srq->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr, - buf_size, 0, FALSE); - if (IS_ERR(srq->umem)) { - err = PTR_ERR(srq->umem); - goto err_srq; - } - - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), - ilog2(srq->umem->page_size), &srq->mtt); - if (err) - goto err_buf; - - err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); - if (err) - goto err_mtt; - - err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx), - ucmd.db_addr, &srq->db); - if (err) - goto err_mtt; - } else { - err = mlx4_ib_db_alloc(dev, &srq->db, 0); - if (err) - goto err_srq; - - *srq->db.db = 0; - - if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) { - err = -ENOMEM; - goto err_db; - } - - srq->head = 0; - srq->tail = srq->msrq.max - 1; - srq->wqe_ctr = 0; - - for (i = 0; i < srq->msrq.max; ++i) { - next = get_wqe(srq, i); - next->next_wqe_index = - cpu_to_be16((i + 1) & (srq->msrq.max - 1)); - } - - err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, - &srq->mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); - if (err) - goto err_mtt; - - srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); - if (!srq->wrid) { - err = -ENOMEM; - goto err_mtt; - } - } - err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcd, &srq->mtt, - srq->db.dma.da, &srq->msrq); - if (err) - goto err_wrid; - - srq->msrq.event = mlx4_ib_srq_event; - - if (pd->p_uctx) - if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { - err = -EFAULT; - goto err_wrid; - } - - init_attr->attr.max_wr = srq->msrq.max - 1; - - return &srq->ibsrq; - -err_wrid: - if (pd->p_uctx) - mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx), &srq->db); - else - kfree(srq->wrid); - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &srq->mtt); - -err_buf: - if (pd->p_uctx) - ib_umem_release(srq->umem); - else - mlx4_buf_free(dev->dev, buf_size, &srq->buf); - -err_db: - if (!pd->p_uctx) - mlx4_ib_db_free(dev, &srq->db); - -err_srq: - kfree(srq); - - return ERR_PTR(err); -} - -int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, - enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); - struct mlx4_ib_srq *srq = to_msrq(ibsrq); - int ret; - - UNUSED_PARAM(udata); - - /* We don't support resizing SRQs (yet?) */ - if (attr_mask & XIB_SRQ_MAX_WR) - return -ENOSYS; - - if (attr_mask & XIB_SRQ_LIMIT) { - if ((int)attr->srq_limit >= srq->msrq.max) - return -ERANGE; - - mutex_lock(&srq->mutex); - ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); - mutex_unlock(&srq->mutex); - - if (ret) - return ret; - } - - return 0; -} - -int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) -{ - struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); - struct mlx4_ib_srq *srq = to_msrq(ibsrq); - int ret; - int limit_watermark; - - ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); - if (ret) - return ret; - - srq_attr->srq_limit = limit_watermark; - srq_attr->max_wr = srq->msrq.max - 1; - srq_attr->max_sge = srq->msrq.max_gs; - - return 0; -} - -int mlx4_ib_destroy_srq(struct ib_srq *srq) -{ - struct mlx4_ib_dev *dev = to_mdev(srq->device); - struct mlx4_ib_srq *msrq = to_msrq(srq); - - mlx4_srq_invalidate(dev->dev, &msrq->msrq); - mlx4_srq_remove(dev->dev, &msrq->msrq); - - mlx4_srq_free(dev->dev, &msrq->msrq); - mlx4_mtt_cleanup(dev->dev, &msrq->mtt); - - if (srq->p_uctx) { - mlx4_ib_db_unmap_user(to_mucontext(srq->p_uctx), &msrq->db); - ib_umem_release(msrq->umem); - } else { - kfree(msrq->wrid); - mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, - &msrq->buf); - mlx4_ib_db_free(dev, &msrq->db); - } - - kfree(msrq); - - return 0; -} - -void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) -{ - struct mlx4_wqe_srq_next_seg *next; - - /* always called with interrupts disabled. */ - spin_lock(&srq->lock); - - next = get_wqe(srq, srq->tail); - next->next_wqe_index = cpu_to_be16(wqe_index); - srq->tail = wqe_index; - - spin_unlock(&srq->lock); -} - -int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, ib_recv_wr_t *wr, - ib_recv_wr_t **bad_wr) -{ - struct mlx4_ib_srq *srq = to_msrq(ibsrq); - struct mlx4_wqe_srq_next_seg *next; - struct mlx4_wqe_data_seg *scat; - unsigned long flags; - int err = 0; - int nreq; - int i; - - spin_lock_irqsave(&srq->lock, &flags); - - for (nreq = 0; wr; ++nreq, wr = wr->p_next) { - if (unlikely(wr->num_ds > (u32)srq->msrq.max_gs)) { - err = -EINVAL; - *bad_wr = wr; - break; - } - - if (unlikely(srq->head == srq->tail)) { - err = -ENOMEM; - *bad_wr = wr; - break; - } - - srq->wrid[srq->head] = wr->wr_id; - - next = get_wqe(srq, srq->head); - srq->head = be16_to_cpu(next->next_wqe_index); - scat = (struct mlx4_wqe_data_seg *) (next + 1); - - for (i = 0; i < (int)wr->num_ds; ++i) { - scat[i].byte_count = cpu_to_be32(wr->ds_array[i].length); - scat[i].lkey = cpu_to_be32(wr->ds_array[i].lkey); - scat[i].addr = cpu_to_be64(wr->ds_array[i].vaddr); - } - - if (i < srq->msrq.max_gs) { - scat[i].byte_count = 0; - scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); - scat[i].addr = 0; - } - } - - if (likely(nreq)) { - srq->wqe_ctr = (u16)(srq->wqe_ctr + nreq); - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - - *srq->db.db = cpu_to_be32(srq->wqe_ctr); - } - - spin_unlock_irqrestore(&srq->lock, flags); - - return err; -} +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "mlx4_ib.h" +#include "qp.h" +#include "srq.h" +#include "user.h" + +static void *get_wqe(struct mlx4_ib_srq *srq, int n) +{ + int offset = n << srq->msrq.wqe_shift; + + if (srq->buf.nbufs == 1) + return srq->buf.u.direct.buf + offset; + else + return srq->buf.u.page_list[offset >> PAGE_SHIFT].buf + + (offset & (PAGE_SIZE - 1)); +} + +static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) +{ + ib_event_rec_t event; + struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; + + switch (type) { + case MLX4_EVENT_TYPE_SRQ_LIMIT: + event.type = IB_EVENT_SRQ_LIMIT_REACHED; + break; + case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: + event.type = IB_EVENT_SRQ_ERR; + break; + default: + printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " + "on SRQ %06x\n", type, srq->srqn); + return; + } + + event.context = ibsrq->srq_context; + ibsrq->event_handler(&event); +} + +struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_srq *srq; + struct mlx4_wqe_srq_next_seg *next; + int desc_size; + int buf_size; + int err; + int i; + u32 cqn = 0; + u16 xrcd = 0; + + if (mlx4_is_barred(pd->device->dma_device)) + return ERR_PTR(-EFAULT); + + /* Sanity check SRQ size before proceeding */ + if ((int)init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || + (int)init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) + return ERR_PTR(-EINVAL); + + srq = kzalloc(sizeof *srq, GFP_KERNEL); + if (!srq) + return ERR_PTR(-ENOMEM); + + mutex_init(&srq->mutex); + spin_lock_init(&srq->lock); + srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); + srq->msrq.max_gs = init_attr->attr.max_sge; + + desc_size = max(32UL, + roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + + srq->msrq.max_gs * + sizeof (struct mlx4_wqe_data_seg))); + srq->msrq.wqe_shift = ilog2(desc_size); + + buf_size = srq->msrq.max * desc_size; + + if (pd->p_uctx) { + struct mlx4_ib_create_srq ucmd; + + if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { + err = -EFAULT; + goto err_srq; + } + + srq->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr, + buf_size, 0, FALSE); + if (IS_ERR(srq->umem)) { + err = PTR_ERR(srq->umem); + goto err_srq; + } + + err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), + ilog2(srq->umem->page_size), &srq->mtt); + if (err) + goto err_buf; + + err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); + if (err) + goto err_mtt; + + err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx), + ucmd.db_addr, &srq->db); + if (err) + goto err_mtt; + } else { + err = mlx4_ib_db_alloc(dev, &srq->db, 0); + if (err) + goto err_srq; + + *srq->db.db = 0; + + if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) { + err = -ENOMEM; + goto err_db; + } + + srq->head = 0; + srq->tail = srq->msrq.max - 1; + srq->wqe_ctr = 0; + + for (i = 0; i < srq->msrq.max; ++i) { + next = get_wqe(srq, i); + next->next_wqe_index = + cpu_to_be16((i + 1) & (srq->msrq.max - 1)); + } + + err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, + &srq->mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); + if (err) + goto err_mtt; + + srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); + if (!srq->wrid) { + err = -ENOMEM; + goto err_mtt; + } + } + err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcd, &srq->mtt, + srq->db.dma.da, &srq->msrq); + if (err) + goto err_wrid; + + srq->msrq.event = mlx4_ib_srq_event; + + if (pd->p_uctx) + if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { + err = -EFAULT; + goto err_wrid; + } + + init_attr->attr.max_wr = srq->msrq.max - 1; + + return &srq->ibsrq; + +err_wrid: + if (pd->p_uctx) + mlx4_ib_db_unmap_user(to_mucontext(pd->p_uctx), &srq->db); + else + kfree(srq->wrid); + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &srq->mtt); + +err_buf: + if (pd->p_uctx) + ib_umem_release(srq->umem); + else + mlx4_buf_free(dev->dev, buf_size, &srq->buf); + +err_db: + if (!pd->p_uctx) + mlx4_ib_db_free(dev, &srq->db); + +err_srq: + kfree(srq); + + return ERR_PTR(err); +} + +int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); + struct mlx4_ib_srq *srq = to_msrq(ibsrq); + int ret; + + UNUSED_PARAM(udata); + + if (mlx4_is_barred(ibsrq->device->dma_device)) + return -EFAULT; + + /* We don't support resizing SRQs (yet?) */ + if (attr_mask & XIB_SRQ_MAX_WR) + return -ENOSYS; + + if (attr_mask & XIB_SRQ_LIMIT) { + if ((int)attr->srq_limit >= srq->msrq.max) + return -ERANGE; + + mutex_lock(&srq->mutex); + ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); + mutex_unlock(&srq->mutex); + + if (ret) + return ret; + } + + return 0; +} + +int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); + struct mlx4_ib_srq *srq = to_msrq(ibsrq); + int ret; + int limit_watermark; + + if (mlx4_is_barred(ibsrq->device->dma_device)) + return -EFAULT; + + ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); + if (ret) + return ret; + + srq_attr->srq_limit = limit_watermark; + srq_attr->max_wr = srq->msrq.max - 1; + srq_attr->max_sge = srq->msrq.max_gs; + + return 0; +} + +int mlx4_ib_destroy_srq(struct ib_srq *srq) +{ + struct mlx4_ib_dev *dev = to_mdev(srq->device); + struct mlx4_ib_srq *msrq = to_msrq(srq); + + if (!mlx4_is_barred(dev->dev)) + mlx4_srq_invalidate(dev->dev, &msrq->msrq); + mlx4_srq_remove(dev->dev, &msrq->msrq); + + mlx4_srq_free(dev->dev, &msrq->msrq); + mlx4_mtt_cleanup(dev->dev, &msrq->mtt); + + if (srq->p_uctx) { + mlx4_ib_db_unmap_user(to_mucontext(srq->p_uctx), &msrq->db); + ib_umem_release(msrq->umem); + } else { + kfree(msrq->wrid); + mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, + &msrq->buf); + mlx4_ib_db_free(dev, &msrq->db); + } + + kfree(msrq); + + return 0; +} + +void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) +{ + struct mlx4_wqe_srq_next_seg *next; + + /* always called with interrupts disabled. */ + spin_lock(&srq->lock); + + next = get_wqe(srq, srq->tail); + next->next_wqe_index = cpu_to_be16(wqe_index); + srq->tail = wqe_index; + + spin_unlock(&srq->lock); +} + +int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, ib_recv_wr_t *wr, + ib_recv_wr_t **bad_wr) +{ + struct mlx4_ib_srq *srq = to_msrq(ibsrq); + struct mlx4_wqe_srq_next_seg *next; + struct mlx4_wqe_data_seg *scat; + unsigned long flags; + int err = 0; + int nreq; + int i; + + if (mlx4_is_barred(ibsrq->device->dma_device)) + return -EFAULT; + + spin_lock_irqsave(&srq->lock, &flags); + + for (nreq = 0; wr; ++nreq, wr = wr->p_next) { + if (unlikely(wr->num_ds > (u32)srq->msrq.max_gs)) { + err = -EINVAL; + *bad_wr = wr; + break; + } + + if (unlikely(srq->head == srq->tail)) { + err = -ENOMEM; + *bad_wr = wr; + break; + } + + srq->wrid[srq->head] = wr->wr_id; + + next = get_wqe(srq, srq->head); + srq->head = be16_to_cpu(next->next_wqe_index); + scat = (struct mlx4_wqe_data_seg *) (next + 1); + + for (i = 0; i < (int)wr->num_ds; ++i) { + scat[i].byte_count = cpu_to_be32(wr->ds_array[i].length); + scat[i].lkey = cpu_to_be32(wr->ds_array[i].lkey); + scat[i].addr = cpu_to_be64(wr->ds_array[i].vaddr); + } + + if (i < srq->msrq.max_gs) { + scat[i].byte_count = 0; + scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); + scat[i].addr = 0; + } + } + + if (likely(nreq)) { + srq->wqe_ctr = (u16)(srq->wqe_ctr + nreq); + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *srq->db.db = cpu_to_be32(srq->wqe_ctr); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return err; +} diff --git a/branches/winverbs/hw/mlx4/kernel/bus/inc/bus_intf.h b/branches/winverbs/hw/mlx4/kernel/bus/inc/bus_intf.h index f507d929..5d5c5e5c 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/inc/bus_intf.h +++ b/branches/winverbs/hw/mlx4/kernel/bus/inc/bus_intf.h @@ -115,6 +115,8 @@ typedef void (*MLX4_REMOVE_EQ) (struct mlx4_dev *dev, u8 eq_num); typedef int (*MLX4_REGISTER_EVENT_HANDLER) (struct ib_event_handler *event_handler); typedef int (*MLX4_UNREGISTER_EVENT_HANDLER)(struct ib_event_handler *event_handler); +typedef int (*MLX4_RESET_REQUEST) (struct ib_event_handler *event_handler); +typedef int (*MLX4_RESET_EXECUTE) (struct ib_event_handler *event_handler); struct mlx4_interface_ex { MLX4_PD_ALLOC mlx4_pd_alloc; @@ -165,6 +167,8 @@ struct mlx4_interface_ex { MLX4_REGISTER_EVENT_HANDLER mlx4_register_ev_cb; MLX4_UNREGISTER_EVENT_HANDLER mlx4_unregister_ev_cb; + MLX4_RESET_REQUEST mlx4_reset_request; + MLX4_RESET_EXECUTE mlx4_reset_execute; }; @@ -176,8 +180,6 @@ typedef struct _MLX4_BUS_IB_INTERFACE{ struct mlx4_dev * pmlx4_dev; struct mlx4_interface_ex mlx4_interface; int is_livefish; - MLX4_REGISTER_INTERFACE register_interface; - MLX4_UNREGISTER_INTERFACE unregister_interface; u8 port_id; struct VipBusIfc *pVipBusIfc; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/inc/device.h b/branches/winverbs/hw/mlx4/kernel/bus/inc/device.h index 29d7d725..09b0c77b 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/inc/device.h +++ b/branches/winverbs/hw/mlx4/kernel/bus/inc/device.h @@ -36,7 +36,10 @@ enum { MLX4_FLAG_MSI_X = 1 << 0, MLX4_FLAG_OLD_PORT_CMDS = 1 << 1, - MLX4_FLAG_LIVEFISH = 1 << 10 + MLX4_FLAG_LIVEFISH = 1 << 10, + MLX4_FLAG_RESET_CLIENT = 1 << 11, + MLX4_FLAG_RESET_DRIVER = 1 << 12, + MLX4_FLAG_RESET_STARTED = 1 << 13 }; enum { diff --git a/branches/winverbs/hw/mlx4/kernel/bus/inc/ib_verbs.h b/branches/winverbs/hw/mlx4/kernel/bus/inc/ib_verbs.h index 7e40518f..d05558da 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/inc/ib_verbs.h +++ b/branches/winverbs/hw/mlx4/kernel/bus/inc/ib_verbs.h @@ -272,7 +272,10 @@ enum ib_event_type { IB_EVENT_LID_CHANGE = IB_AE_UNKNOWN + 1, IB_EVENT_PKEY_CHANGE, IB_EVENT_SM_CHANGE, - IB_EVENT_CLIENT_REREGISTER + IB_EVENT_CLIENT_REREGISTER, + IB_EVENT_RESET_DRIVER, // device will be reset upon fatal error + IB_EVENT_RESET_CLIENT, // device will be upon client request + IB_EVENT_RESET_END // device has been reset }; struct ib_event { @@ -285,21 +288,32 @@ struct ib_event { } element; enum ib_event_type event; struct ib_event_ex x; - }; +}; + +enum ib_event_handler_flags { + IB_IVH_RESET_CB = (1 << 0), + IB_IVH_NOTIFIED = (1 << 1), + IB_IVH_RESET_READY = (1 << 2) +}; + struct ib_event_handler { struct ib_device *device; void (*handler)(struct ib_event_handler *, struct ib_event *); - void * ctx; struct list_head list; + void * ctx; + void * rsrv_ptr; + u32 flags; }; -#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler, _ctx) \ +#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler, _ctx, _rptr, _flags) \ { \ (_ptr)->device = _device; \ (_ptr)->handler = _handler; \ + INIT_LIST_HEAD(&(_ptr)->list); \ (_ptr)->ctx = _ctx; \ - INIT_LIST_HEAD(&(_ptr)->list); \ + (_ptr)->rsrv_ptr = _rptr; \ + (_ptr)->flags = _flags; \ } struct ib_global_route { diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/alloc.c b/branches/winverbs/hw/mlx4/kernel/bus/net/alloc.c index 16b1d79e..62a074e6 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/alloc.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/alloc.c @@ -395,6 +395,9 @@ int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, { int err; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + err = mlx4_db_alloc(dev, &wqres->db, 1); if (err) return err; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/catas.c b/branches/winverbs/hw/mlx4/kernel/bus/net/catas.c index 9658bdd9..564d5549 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/catas.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/catas.c @@ -43,6 +43,58 @@ static LIST_HEAD(catas_list); // "Reset device on internal errors if non-zero (default 1)") int g_internal_err_reset = 1; +static void dispatch_event(struct ib_device *ibdev, enum ib_event_type type) +{ + unsigned long flags; + struct ib_event event; + struct ib_event_handler *handler; + + event.device = ibdev; + event.event = type; + + spin_lock_irqsave(&ibdev->event_handler_lock, &flags); + + list_for_each_entry(handler, &ibdev->event_handler_list, list, struct ib_event_handler) + { + // notify only those, that are not notified + if ( handler->flags & IB_IVH_RESET_CB ) + if ( !(handler->flags & IB_IVH_NOTIFIED) ) { + handler->flags |= IB_IVH_NOTIFIED; + handler->handler(handler, &event); + } + } + + spin_unlock_irqrestore(&ibdev->event_handler_lock, flags); +} + +/** + * get_event_handlers - return list of handlers of the device + * @device:device + * @tlist:list + * + * get_event_handlers() remove all the device event handlers and put them in 'tlist' + */ +static void get_event_handlers(struct ib_device *device, struct list_head *tlist) +{ + unsigned long flags; + struct ib_event_handler *handler, *thandler; + + spin_lock_irqsave(&device->event_handler_lock, &flags); + + list_for_each_entry_safe(handler, thandler, &device->event_handler_list, + list, struct ib_event_handler, struct ib_event_handler) + { + // take out only reset callbacks + if ( handler->flags & IB_IVH_RESET_CB ) { + list_del( &handler->list ); + list_add_tail( &handler->list, tlist ); + } + } + + spin_unlock_irqrestore(&device->event_handler_lock, flags); +} + + static void dump_err_buf(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); @@ -97,6 +149,12 @@ static void poll_catas(struct mlx4_dev *dev) mlx4_dispatch_event(dev, MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR, 0, 0); + // bar the device + dev->flags |= MLX4_FLAG_RESET_DRIVER; + + // notify the clients + dispatch_event(dev->pdev->ib_dev, IB_EVENT_RESET_DRIVER); + if (g_internal_err_reset) { PIO_WORKITEM catas_work = IoAllocateWorkItem( dev->pdev->p_self_do ); @@ -163,6 +221,10 @@ void mlx4_stop_catas_poll(struct mlx4_dev *dev) struct mlx4_priv *priv = mlx4_priv(dev); spin_lock_irq(&catas_lock); + if (priv->catas_err.stop) { + spin_unlock_irq(&catas_lock); + return; + } priv->catas_err.stop = 1; spin_unlock_irq(&catas_lock); @@ -177,4 +239,126 @@ void mlx4_stop_catas_poll(struct mlx4_dev *dev) spin_unlock_irq(&catas_lock); } +static int wait4reset(struct ib_event_handler *event_handler) +{ + int n_not_ready = 0; + unsigned long flags; + struct ib_event_handler *handler; + struct ib_device *ibdev = event_handler->device; + + spin_lock_irqsave(&ibdev->event_handler_lock, &flags); + + // mark this handler (=client) reset-ready + event_handler->flags |= IB_IVH_RESET_READY; + + // check the number of still not ready client + + list_for_each_entry(handler, &ibdev->event_handler_list, list, struct ib_event_handler) + if ( handler->flags & IB_IVH_RESET_CB ) + if ( !(handler->flags & IB_IVH_RESET_READY) ) + ++n_not_ready; + + spin_unlock_irqrestore(&ibdev->event_handler_lock, flags); + + return n_not_ready; +} + +int mlx4_reset_execute( struct ib_event_handler *event_handler ) +{ + int err; + struct ib_event event; + struct list_head tlist; + struct ib_event_handler *handler, *thandler; + struct ib_device *ibdev = event_handler->device; + struct pci_dev *pdev = ibdev->dma_device->pdev; + + // mark client as "ready for reset" and check whether we can do reset + if (wait4reset(event_handler)) + return 0; + + // fully bar the device + ibdev->dma_device->flags |= MLX4_FLAG_RESET_STARTED; + + // get old handler list + INIT_LIST_HEAD(&tlist); + get_event_handlers(ibdev, &tlist); + + // restart the device + err = mlx4_restart_one(pdev); + + // recreate interfaces + fix_bus_ifc(pdev); + + // notify the clients + event.event = IB_EVENT_RESET_END; + list_for_each_entry_safe(handler, thandler, &tlist, + list, struct ib_event_handler, struct ib_event_handler) + { + // because 'handler' will be re-registered during the next call + list_del( &handler->list ); + handler->handler(handler, &event); + } + + return err; +} + +static void +card_reset_wi( + IN DEVICE_OBJECT* p_dev_obj, + IN struct ib_event_handler * event_handler ) +{ + NTSTATUS status; + struct ib_device *ibdev = event_handler->device; + struct mlx4_dev *dev = ibdev->dma_device; + + UNUSED_PARAM(p_dev_obj); + IoFreeWorkItem( event_handler->rsrv_ptr ); + + // reset the card + mlx4_stop_catas_poll( dev ); + status = mlx4_reset( dev ); + if ( !NT_SUCCESS( status ) ) + mlx4_err( dev, "Failed to reset HCA, aborting. (status %#x)\n", status ); + + // notify the clients + dispatch_event(ibdev, IB_EVENT_RESET_CLIENT); +} + +int mlx4_reset_request( struct ib_event_handler *event_handler ) +{ + struct ib_device *ibdev = event_handler->device; + struct mlx4_dev *dev = ibdev->dma_device; + + // set device to RESET_PENDING mode + if (!mlx4_is_barred(dev)) { + PIO_WORKITEM reset_work; + + // bar the device + dev->flags |= MLX4_FLAG_RESET_CLIENT; + + // delay reset to a system thread + // to allow for end of operations that are in progress + reset_work = IoAllocateWorkItem( dev->pdev->p_self_do ); + if (!reset_work) + return -EFAULT; + event_handler->rsrv_ptr = reset_work; + IoQueueWorkItem( reset_work, card_reset_wi, DelayedWorkQueue, event_handler ); + } + + return 0; +} + +int mlx4_reset_cb_register( struct ib_event_handler *event_handler ) +{ + if (mlx4_is_in_reset(event_handler->device->dma_device)) + return -EBUSY; + + return ib_register_event_handler(event_handler); +} + +int mlx4_reset_cb_unregister( struct ib_event_handler *event_handler ) +{ + return ib_unregister_event_handler(event_handler); +} + diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/cmd.c b/branches/winverbs/hw/mlx4/kernel/bus/net/cmd.c index 5dac0dd8..a49a5f10 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/cmd.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/cmd.c @@ -491,6 +491,9 @@ struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev) { struct mlx4_cmd_mailbox *mailbox; + if ( mlx4_is_barred(dev) ) + return ERR_PTR(-EFAULT); + mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL); if (!mailbox) return ERR_PTR(-ENOMEM); @@ -520,6 +523,9 @@ EXPORT_SYMBOL_GPL(mlx4_free_cmd_mailbox); int imlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { + if ( mlx4_is_barred(dev) ) + return -EFAULT; + return __mlx4_cmd(dev, in_param, out_param, out_is_imm, in_modifier, op_modifier, op, timeout); } diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/cq.c b/branches/winverbs/hw/mlx4/kernel/bus/net/cq.c index d810b351..9fed0ade 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/cq.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/cq.c @@ -130,6 +130,9 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, #define COLLAPSED_SHIFT 18 #define ENTRIES_SHIFT 24 + if ( mlx4_is_barred(dev) ) + return -EFAULT; + cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap); if (cq->cqn == -1) return -ENOMEM; @@ -212,6 +215,9 @@ int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, struct mlx4_cmd_mailbox *mailbox; int err; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); @@ -228,9 +234,10 @@ void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cq_table *cq_table = &priv->cq_table; - int err; + int err = 0; - err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn); + if (!mlx4_is_barred(dev)) + err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn); if (err) mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn); @@ -242,7 +249,8 @@ void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq) if (atomic_dec_and_test(&cq->refcount)) complete(&cq->free); - wait_for_completion(&cq->free); + if (!mlx4_is_barred(dev)) + wait_for_completion(&cq->free); mlx4_table_put(dev, &cq_table->table, cq->cqn); mlx4_bitmap_free(&cq_table->bitmap, cq->cqn); diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/eq.c b/branches/winverbs/hw/mlx4/kernel/bus/net/eq.c index 5296160a..bd8741c4 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/eq.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/eq.c @@ -663,6 +663,9 @@ int mlx4_add_eq(struct mlx4_dev *dev, int nent, UNREFERENCED_PARAMETER(intr); + if ( mlx4_is_barred(dev) ) + return -EFAULT; + for (i = MLX4_NUM_EQ; i < MLX4_NUM_EQ + MLX4_MAX_EXTRA_EQS ; i++) { if(priv->eq_table.eq[MLX4_NUM_EQ].isr == NULL) { new_eq = i; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/fw.c b/branches/winverbs/hw/mlx4/kernel/bus/net/fw.c index ac61f25c..b945ab38 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/fw.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/fw.c @@ -780,6 +780,9 @@ int mlx4_INIT_PORT(struct mlx4_dev *dev, int port) u32 flags; u16 field; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { #define INIT_PORT_IN_SIZE 256 #define INIT_PORT_FLAGS_OFFSET 0x00 @@ -828,6 +831,9 @@ EXPORT_SYMBOL_GPL(mlx4_INIT_PORT); int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port) { + if ( mlx4_is_barred(dev) ) + return -EFAULT; + return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000); } EXPORT_SYMBOL_GPL(mlx4_CLOSE_PORT); diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/main.c b/branches/winverbs/hw/mlx4/kernel/bus/net/main.c index 907574e9..dbd377c1 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/main.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/main.c @@ -100,7 +100,8 @@ mlx4_pci_table[] = { HCA(MELLANOX, QDR, HERMON), HCA(MELLANOX, DDR_G2, HERMON), HCA(MELLANOX, QDR_G2, HERMON), - HCA(MELLANOX, ETH_CONDOR, HERMON), + HCA(MELLANOX, ETH_CONDOR, HERMON), + HCA(MELLANOX, ETH_CONDOR_G2, HERMON), HCA(MELLANOX, BD, LIVEFISH), }; #define MLX4_PCI_TABLE_SIZE (sizeof(mlx4_pci_table)/sizeof(struct pci_device_id)) @@ -898,6 +899,7 @@ int mlx4_init_one(struct pci_dev *pdev) struct mlx4_priv *priv; struct mlx4_dev *dev; int err; + NTSTATUS status; #ifdef FORCE_LIVEFISH if (pdev) @@ -963,9 +965,10 @@ run_as_livefish: * attempt a firmware command, since a boot ROM may have left * the HCA in an undefined state. */ - err = mlx4_reset(dev); - if (err) { - mlx4_err(dev, "Failed to reset HCA, aborting.\n"); + status = mlx4_reset(dev); + if ( !NT_SUCCESS( status ) ) { + mlx4_err(dev, "Failed to reset HCA, aborting.(status %#x)\n", status); + err = -EFAULT; goto err_free_dev; } diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/mlx4.h b/branches/winverbs/hw/mlx4/kernel/bus/net/mlx4.h index 9ece1bf9..0027c707 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/mlx4.h +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/mlx4.h @@ -427,4 +427,17 @@ int mlx4_add_eq(struct mlx4_dev *dev, int nent, u8* p_eq_num, struct mlx4_eq ** p_eq); void mlx4_remove_eq(struct mlx4_dev *dev, u8 eq_num); + +int mlx4_reset_execute( struct ib_event_handler *event_handler ); + +int mlx4_reset_request( struct ib_event_handler *event_handler ); + +int mlx4_reset_cb_register( struct ib_event_handler *event_handler ); + +int mlx4_reset_cb_unregister( struct ib_event_handler *event_handler ); + +void fix_bus_ifc(struct pci_dev *pdev); + + + #endif /* MLX4_H */ diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/mr.c b/branches/winverbs/hw/mlx4/kernel/bus/net/mr.c index 6a6f68a8..1036e70d 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/mr.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/mr.c @@ -256,6 +256,9 @@ int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, u32 index; int err; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + index = mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); if (index == -1) return -ENOMEM; @@ -280,7 +283,7 @@ void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) struct mlx4_priv *priv = mlx4_priv(dev); int err; - if (mr->enabled) { + if (!mlx4_is_barred(dev) && mr->enabled) { err = mlx4_HW2SW_MPT(dev, NULL, key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); @@ -300,6 +303,9 @@ int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) struct mlx4_mpt_entry *mpt_entry; int err; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + err = mlx4_table_get(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key)); if (err) return err; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/pd.c b/branches/winverbs/hw/mlx4/kernel/bus/net/pd.c index 3009a9e8..ed036c9d 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/pd.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/pd.c @@ -39,6 +39,9 @@ int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) { struct mlx4_priv *priv = mlx4_priv(dev); + if ( mlx4_is_barred(dev) ) + return -EFAULT; + *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); if (*pdn == -1) return -ENOMEM; @@ -69,6 +72,9 @@ void mlx4_cleanup_pd_table(struct mlx4_dev *dev) int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) { + if ( mlx4_is_barred(dev) ) + return -EFAULT; + uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); if (uar->index == -1) return -ENOMEM; diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/port.c b/branches/winverbs/hw/mlx4/kernel/bus/net/port.c index fda61f47..12547084 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/port.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/port.c @@ -1,289 +1,293 @@ -/* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include "mlx4.h" -#include "cmd.h" - - -void mlx4_init_mac_table(struct mlx4_dev *dev, u8 port) -{ - struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table; - int i; - - sema_init(&table->mac_sem, 1); - for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { - table->entries[i] = 0; - table->refs[i] = 0; - } - table->max = 1 << dev->caps.log_num_macs; - table->total = 0; -} - -void mlx4_init_vlan_table(struct mlx4_dev *dev, u8 port) -{ - struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; - int i; - - sema_init(&table->vlan_sem, 1); - for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { - table->entries[i] = 0; - table->refs[i] = 0; - } - table->max = 1 << dev->caps.log_num_vlans; - table->total = 0; -} - -static int mlx4_SET_PORT_mac_table(struct mlx4_dev *dev, u8 port, - __be64 *entries) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 in_mod; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE); - - in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port; - err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index) -{ - struct mlx4_mac_table *table = - &mlx4_priv(dev)->port[port - 1].mac_table; - int i, err = 0; - int free = -1; - u64 valid = 1; - - mlx4_dbg(dev, "Registering mac : 0x%llx\n", mac); - down(&table->mac_sem); - for (i = 0; i < MLX4_MAX_MAC_NUM - 1; i++) { - if (free < 0 && !table->refs[i]) { - free = i; - continue; - } - - if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) { - /* Mac already registered, increase refernce count */ - *index = i; - ++table->refs[i]; - goto out; - } - } - mlx4_dbg(dev, "Free mac index is %d\n", free); - - if (table->total == table->max) { - /* No free mac entries */ - err = -ENOSPC; - goto out; - } - - /* Register new MAC */ - table->refs[free] = 1; - table->entries[free] = cpu_to_be64(mac | valid << MLX4_MAC_VALID_SHIFT); - - err = mlx4_SET_PORT_mac_table(dev, port, table->entries); - if (unlikely(err)) { - mlx4_err(dev, "Failed adding mac: 0x%llx\n", mac); - table->refs[free] = 0; - table->entries[free] = 0; - goto out; - } - - *index = free; - ++table->total; -out: - up(&table->mac_sem); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_register_mac); - -void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index) -{ - struct mlx4_mac_table *table = - &mlx4_priv(dev)->port[port - 1].mac_table; - - down(&table->mac_sem); - if (!table->refs[index]) { - mlx4_warn(dev, "No mac entry for index %d\n", index); - goto out; - } - if (--table->refs[index]) { - mlx4_warn(dev, "Have more references for index %d," - "no need to modify mac table\n", index); - goto out; - } - table->entries[index] = 0; - mlx4_SET_PORT_mac_table(dev, port, table->entries); - --table->total; -out: - up(&table->mac_sem); -} -EXPORT_SYMBOL_GPL(mlx4_unregister_mac); - -static int mlx4_SET_PORT_vlan_table(struct mlx4_dev *dev, u8 port, - __be32 *entries) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 in_mod; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE); - in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port; - err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); - - mlx4_free_cmd_mailbox(dev, mailbox); - - return err; -} - -int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index) -{ - struct mlx4_vlan_table *table = - &mlx4_priv(dev)->port[port - 1].vlan_table; - int i, err = 0; - int free = -1; - - down(&table->vlan_sem); - for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { - if (free < 0 && (table->refs[i] == 0)) { - free = i; - continue; - } - - if (table->refs[i] && - (vlan == (MLX4_VLAN_MASK & - be32_to_cpu(table->entries[i])))) { - /* Vlan already registered, increase refernce count */ - *index = i; - ++table->refs[i]; - goto out; - } - } - - if (table->total == table->max) { - /* No free vlan entries */ - err = -ENOSPC; - goto out; - } - - /* Register new MAC */ - table->refs[free] = 1; - table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID); - - err = mlx4_SET_PORT_vlan_table(dev, port, table->entries); - if (unlikely(err)) { - mlx4_warn(dev, "Failed adding vlan: %u\n", vlan); - table->refs[free] = 0; - table->entries[free] = 0; - goto out; - } - - *index = free; - ++table->total; -out: - up(&table->vlan_sem); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_register_vlan); - -void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index) -{ - struct mlx4_vlan_table *table = - &mlx4_priv(dev)->port[port - 1].vlan_table; - - down(&table->vlan_sem); - if (!table->refs[index]) { - mlx4_warn(dev, "No vlan entry for index %d\n", index); - goto out; - } - if (--table->refs[index]) { - mlx4_dbg(dev, "Have more references for index %d," - "no need to modify vlan table\n", index); - goto out; - } - table->entries[index] = 0; - mlx4_SET_PORT_vlan_table(dev, port, table->entries); - --table->total; -out: - up(&table->vlan_sem); -} -EXPORT_SYMBOL_GPL(mlx4_unregister_vlan); - -int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, - int reset_qkey_viols, u32 cap_mask) -{ - struct mlx4_cmd_mailbox *mailbox; - int err; - u8 is_eth = (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) ? 1 : 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memset(mailbox->buf, 0, 256); - if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { +/* + * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include "mlx4.h" +#include "cmd.h" + + +void mlx4_init_mac_table(struct mlx4_dev *dev, u8 port) +{ + struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table; + int i; + + sema_init(&table->mac_sem, 1); + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + table->entries[i] = 0; + table->refs[i] = 0; + } + table->max = 1 << dev->caps.log_num_macs; + table->total = 0; +} + +void mlx4_init_vlan_table(struct mlx4_dev *dev, u8 port) +{ + struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; + int i; + + sema_init(&table->vlan_sem, 1); + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + table->entries[i] = 0; + table->refs[i] = 0; + } + table->max = 1 << dev->caps.log_num_vlans; + table->total = 0; +} + +static int mlx4_SET_PORT_mac_table(struct mlx4_dev *dev, u8 port, + __be64 *entries) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 in_mod; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE); + + in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port; + err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index) +{ + struct mlx4_mac_table *table = + &mlx4_priv(dev)->port[port - 1].mac_table; + int i, err = 0; + int free = -1; + u64 valid = 1; + + if ( mlx4_is_barred(dev) ) + return -EFAULT; + + mlx4_dbg(dev, "Registering mac : 0x%llx\n", mac); + down(&table->mac_sem); + for (i = 0; i < MLX4_MAX_MAC_NUM - 1; i++) { + if (free < 0 && !table->refs[i]) { + free = i; + continue; + } + + if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) { + /* Mac already registered, increase refernce count */ + *index = i; + ++table->refs[i]; + goto out; + } + } + mlx4_dbg(dev, "Free mac index is %d\n", free); + + if (table->total == table->max) { + /* No free mac entries */ + err = -ENOSPC; + goto out; + } + + /* Register new MAC */ + table->refs[free] = 1; + table->entries[free] = cpu_to_be64(mac | valid << MLX4_MAC_VALID_SHIFT); + + err = mlx4_SET_PORT_mac_table(dev, port, table->entries); + if (unlikely(err)) { + mlx4_err(dev, "Failed adding mac: 0x%llx\n", mac); + table->refs[free] = 0; + table->entries[free] = 0; + goto out; + } + + *index = free; + ++table->total; +out: + up(&table->mac_sem); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_register_mac); + +void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index) +{ + struct mlx4_mac_table *table = + &mlx4_priv(dev)->port[port - 1].mac_table; + + down(&table->mac_sem); + if (!table->refs[index]) { + mlx4_warn(dev, "No mac entry for index %d\n", index); + goto out; + } + if (--table->refs[index]) { + mlx4_warn(dev, "Have more references for index %d," + "no need to modify mac table\n", index); + goto out; + } + table->entries[index] = 0; + if ( !mlx4_is_barred(dev) ) + mlx4_SET_PORT_mac_table(dev, port, table->entries); + --table->total; +out: + up(&table->mac_sem); +} +EXPORT_SYMBOL_GPL(mlx4_unregister_mac); + +static int mlx4_SET_PORT_vlan_table(struct mlx4_dev *dev, u8 port, + __be32 *entries) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 in_mod; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE); + in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port; + err = mlx4_cmd(dev, mailbox->dma.da, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B); + + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + +int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index) +{ + struct mlx4_vlan_table *table = + &mlx4_priv(dev)->port[port - 1].vlan_table; + int i, err = 0; + int free = -1; + + down(&table->vlan_sem); + for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { + if (free < 0 && (table->refs[i] == 0)) { + free = i; + continue; + } + + if (table->refs[i] && + (vlan == (MLX4_VLAN_MASK & + be32_to_cpu(table->entries[i])))) { + /* Vlan already registered, increase refernce count */ + *index = i; + ++table->refs[i]; + goto out; + } + } + + if (table->total == table->max) { + /* No free vlan entries */ + err = -ENOSPC; + goto out; + } + + /* Register new MAC */ + table->refs[free] = 1; + table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID); + + err = mlx4_SET_PORT_vlan_table(dev, port, table->entries); + if (unlikely(err)) { + mlx4_warn(dev, "Failed adding vlan: %u\n", vlan); + table->refs[free] = 0; + table->entries[free] = 0; + goto out; + } + + *index = free; + ++table->total; +out: + up(&table->vlan_sem); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_register_vlan); + +void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index) +{ + struct mlx4_vlan_table *table = + &mlx4_priv(dev)->port[port - 1].vlan_table; + + down(&table->vlan_sem); + if (!table->refs[index]) { + mlx4_warn(dev, "No vlan entry for index %d\n", index); + goto out; + } + if (--table->refs[index]) { + mlx4_dbg(dev, "Have more references for index %d," + "no need to modify vlan table\n", index); + goto out; + } + table->entries[index] = 0; + mlx4_SET_PORT_vlan_table(dev, port, table->entries); + --table->total; +out: + up(&table->vlan_sem); +} +EXPORT_SYMBOL_GPL(mlx4_unregister_vlan); + +int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, + int reset_qkey_viols, u32 cap_mask) +{ + struct mlx4_cmd_mailbox *mailbox; + int err; + u8 is_eth = (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) ? 1 : 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memset(mailbox->buf, 0, 256); + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { *(u8 *) mailbox->buf = (u8)(!!reset_qkey_viols << 6); ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask); } else { ((u8 *) mailbox->buf)[3] = (u8)!!reset_qkey_viols; ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask); } - - if (is_eth) { - ((u8 *) mailbox->buf)[3] = 7; - ((__be16 *) mailbox->buf)[3] = - cpu_to_be16(dev->caps.eth_mtu_cap[port] + - ETH_HLEN + ETH_FCS_LEN); - ((__be16 *) mailbox->buf)[4] = cpu_to_be16(1 << 15); - ((__be16 *) mailbox->buf)[6] = cpu_to_be16(1 << 15); - } - err = mlx4_cmd(dev, mailbox->dma.da, port, is_eth, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - + + if (is_eth) { + ((u8 *) mailbox->buf)[3] = 7; + ((__be16 *) mailbox->buf)[3] = + cpu_to_be16(dev->caps.eth_mtu_cap[port] + + ETH_HLEN + ETH_FCS_LEN); + ((__be16 *) mailbox->buf)[4] = cpu_to_be16(1 << 15); + ((__be16 *) mailbox->buf)[6] = cpu_to_be16(1 << 15); + } + err = mlx4_cmd(dev, mailbox->dma.da, port, is_eth, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/qp.c b/branches/winverbs/hw/mlx4/kernel/bus/net/qp.c index 18807eec..71ce754a 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/qp.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/qp.c @@ -74,6 +74,9 @@ int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, static u16 op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE]; static int op_inited = 0; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + if (!op_inited) { op[MLX4_QP_STATE_RST][MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP; op[MLX4_QP_STATE_RST][MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP; @@ -174,6 +177,9 @@ int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp) struct mlx4_qp_table *qp_table = &priv->qp_table; int err; + if ( mlx4_is_barred(dev) ) + return -EFAULT; + if (!qpn) return -EINVAL; @@ -259,7 +265,8 @@ void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp) if (atomic_dec_and_test(&qp->refcount)) complete(&qp->free); - wait_for_completion(&qp->free); + if (!mlx4_is_barred(dev)) + wait_for_completion(&qp->free); mlx4_table_put(dev, &qp_table->cmpt_table, qp->qpn); mlx4_table_put(dev, &qp_table->rdmarc_table, qp->qpn); diff --git a/branches/winverbs/hw/mlx4/kernel/bus/net/srq.c b/branches/winverbs/hw/mlx4/kernel/bus/net/srq.c index 5ff433d0..9c45f103 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/net/srq.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/net/srq.c @@ -117,6 +117,9 @@ int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, int err; UNREFERENCED_PARAMETER(xrcd); + if ( mlx4_is_barred(dev) ) + return -EFAULT; + srq->srqn = mlx4_bitmap_alloc(&srq_table->bitmap); if (srq->srqn == -1) return -ENOMEM; @@ -190,6 +193,9 @@ void mlx4_srq_invalidate(struct mlx4_dev *dev, struct mlx4_srq *srq) { int err; + if ( mlx4_is_barred(dev) ) + return; + err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn); if (err) mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn); @@ -212,7 +218,8 @@ void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq) if (atomic_dec_and_test(&srq->refcount)) complete(&srq->free); - wait_for_completion(&srq->free); + if (!mlx4_is_barred(dev)) + wait_for_completion(&srq->free); mlx4_table_put(dev, &srq_table->table, srq->srqn); mlx4_bitmap_free(&srq_table->bitmap, srq->srqn); diff --git a/branches/winverbs/hw/mlx4/kernel/hca/SOURCES b/branches/winverbs/hw/mlx4/kernel/hca/SOURCES index 96817641..ab9c2973 100644 --- a/branches/winverbs/hw/mlx4/kernel/hca/SOURCES +++ b/branches/winverbs/hw/mlx4/kernel/hca/SOURCES @@ -6,7 +6,7 @@ TARGETTYPE=DRIVER # WDK build only - transform .inx --> .inf adding date & version stamp. # see .\makefile.inc INF_NAME=$(TARGETNAME) -INF_TARGET==..\..\..\..\bin\kernel\$(O)\$(INF_NAME).inf +INF_TARGET=..\..\..\..\bin\kernel\$(O)\$(INF_NAME).inf NTTARGETFILES=$(INF_TARGET) !endif @@ -42,7 +42,7 @@ NTTARGETFILE0=mofcomp KMDF_VERSION=1 -C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUSE_WDM_FRAMEWORK +C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUSE_WDM_FRAMEWORK -DUSE_WDM_INTERRUPTS TARGETLIBS= \ $(TARGETPATH)\*\complib.lib \ diff --git a/branches/winverbs/hw/mlx4/kernel/hca/data.c b/branches/winverbs/hw/mlx4/kernel/hca/data.c index 9a51a6f2..a3cea329 100644 --- a/branches/winverbs/hw/mlx4/kernel/hca/data.c +++ b/branches/winverbs/hw/mlx4/kernel/hca/data.c @@ -383,8 +383,7 @@ to_av( &p_ib_ah_attr->grh.traffic_class, &p_ib_ah_attr->grh.flow_label ); err = p_ib_dev->x.find_cached_gid((struct ib_device *)p_ib_dev, (union ib_gid *)p_ib_av_attr->grh.src_gid.raw, &port_num, &gid_index); - if (err) { - + if (err) { HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_SHIM , ("ib_find_cached_gid failed %d (%#x). Using default: sgid_index = 0\n", err, err)); gid_index = 0; @@ -435,8 +434,7 @@ int from_av( err = p_ib_dev->x.get_cached_gid((struct ib_device *)p_ib_dev, p_ib_ah_attr->port_num, p_ib_ah_attr->grh.sgid_index, (union ib_gid*)p_ib_av_attr->grh.src_gid.raw ); - if (err) { - + if (err) { HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_SHIM , ("ib_get_cached_gid failed %d (%#x). Using default: sgid_index = 0\n", err, err)); } diff --git a/branches/winverbs/hw/mlx4/kernel/hca/drv.c b/branches/winverbs/hw/mlx4/kernel/hca/drv.c index fb1209b8..485576ba 100644 --- a/branches/winverbs/hw/mlx4/kernel/hca/drv.c +++ b/branches/winverbs/hw/mlx4/kernel/hca/drv.c @@ -33,6 +33,7 @@ #include "precomp.h" #include #include +#include #if defined(EVENT_TRACING) #ifdef offsetof @@ -44,6 +45,9 @@ #define DRV_VERSION "1.0" #define DRV_RELDATE "02/01/2008" +#define MLX_VERBS_MIN_VERSION 2 +#define MLX_VERBS_MAX_VERSION 2 + GLOBALS g; /* @@ -88,6 +92,10 @@ static int __get_dev_info(PFDO_DEVICE_DATA p_fdo, __be64 *node_guid, u32 *hw_id) #ifndef USE_WDM_FRAMEWORK +// +// TODO: add support for Hibernate/Standby as in WDM version below +// + #ifdef ALLOC_PRAGMA #pragma alloc_text (INIT, DriverEntry) #pragma alloc_text (PAGE, EvtDeviceAdd) @@ -657,7 +665,8 @@ EvtDevicePrepareHardware( /* get node GUID */ err = __get_dev_info( p_fdo, &p_fdo->hca.guid, &p_fdo->hca.hw_ver ); - if (err) { + if (err) { + HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_LOW , ("can't get guid - ib_query_device() failed (%08X)\n", err )); //TODO: no cleanup on error @@ -1037,6 +1046,12 @@ hca_query_capabilities( IN IRP* const p_irp, OUT cl_irp_action_t* const p_action ); +static NTSTATUS +hca_query_interface( + IN DEVICE_OBJECT* const p_dev_obj, + IN IRP* const p_irp, + OUT cl_irp_action_t* const p_action ); + static NTSTATUS hca_query_pnp_state( IN DEVICE_OBJECT* const p_dev_obj, @@ -1109,6 +1124,7 @@ __pnp_notify_ifc( #pragma alloc_text (PAGE, hca_cancel_remove) #pragma alloc_text (PAGE, hca_surprise_remove) #pragma alloc_text (PAGE, hca_query_capabilities) +#pragma alloc_text (PAGE, hca_query_interface) #pragma alloc_text (PAGE, hca_query_pnp_state) #pragma alloc_text (PAGE, hca_query_bus_relations) #pragma alloc_text (PAGE, hca_query_removal_relations) @@ -1616,12 +1632,13 @@ __unmap_hca_memory( HCA_ENTER( HCA_DBG_PNP ); - for( i = 0; i < HCA_BAR_TYPE_MAX; i++ ) { - if (pdev->bar[i].virt) { - MmUnmapIoSpace( pdev->bar[i].virt, pdev->bar[i].size ); - cl_memclr( &pdev->bar[i], sizeof(hca_bar_t) ); + if ( pdev ) + for( i = 0; i < HCA_BAR_TYPE_MAX; i++ ) { + if (pdev->bar[i].virt) { + MmUnmapIoSpace( pdev->bar[i].virt, pdev->bar[i].size ); + cl_memclr( &pdev->bar[i], sizeof(hca_bar_t) ); + } } - } HCA_EXIT( HCA_DBG_PNP ); } @@ -1766,10 +1783,13 @@ hca_start( p_fdo->bus_ib_ifc_taken = TRUE; p_fdo->bus_ib_ifc.p_ibdev->x.p_fdo = p_fdo; + InitializeListHead(&p_fdo->hca.event_list); + KeInitializeSpinLock(&p_fdo->hca.event_list_lock); /* get node GUID */ err = __get_dev_info( p_fdo, &p_fdo->hca.guid, &p_fdo->hca.hw_ver ); - if (err) { + if (err) { + HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_LOW , ("can't get guid - ib_query_device() failed (%08X)\n", err )); //TODO: no cleanup on error @@ -2041,6 +2061,101 @@ hca_query_capabilities( } +static VOID +__hca_noop( VOID *context ) +{ + UNREFERENCED_PARAMETER(context); +} + + +static NTSTATUS +__query_ci_ifc( + IN DEVICE_OBJECT* const p_dev_obj, + IN IO_STACK_LOCATION* const p_io_stack ) +{ + RDMA_INTERFACE_VERBS *p_ifc; + PFDO_DEVICE_DATA p_fdo; + ci_interface_t *p_hca_ifc; + NTSTATUS status; + UINT8 version; + + HCA_ENTER( HCA_DBG_PNP ); + + version = VerbsVersionMajor(p_io_stack->Parameters.QueryInterface.Version); + if( version < MLX_VERBS_MIN_VERSION || version > MLX_VERBS_MAX_VERSION ) + { + status = STATUS_NOT_SUPPORTED; + goto exit; + } + + if( p_io_stack->Parameters.QueryInterface.Size < sizeof(RDMA_INTERFACE_VERBS) ) + { + status = STATUS_BUFFER_TOO_SMALL; + goto exit; + } + + p_fdo = (PFDO_DEVICE_DATA)p_dev_obj->DeviceExtension; + p_hca_ifc = __alloc_hca_ifc( p_fdo ); + if( !p_hca_ifc ) + { + status = STATUS_NO_MEMORY; + goto exit; + } + + p_ifc = (RDMA_INTERFACE_VERBS *) p_io_stack->Parameters.QueryInterface.Interface; + + p_ifc->InterfaceHeader.Size = sizeof(RDMA_INTERFACE_VERBS); + p_ifc->InterfaceHeader.Version = VerbsVersion(version, 0); + p_ifc->InterfaceHeader.Context = p_dev_obj; + p_ifc->InterfaceHeader.InterfaceReference = __hca_noop; + p_ifc->InterfaceHeader.InterfaceDereference = __hca_noop; + p_ifc->Verbs = *p_hca_ifc; + p_ifc->Verbs.p_hca_dev = &p_fdo->hca; + + ExFreePool( p_hca_ifc ); + status = STATUS_SUCCESS; + +exit: + HCA_EXIT( HCA_DBG_PNP ); + return status; +} + + +static NTSTATUS +hca_query_interface( + IN DEVICE_OBJECT* const p_dev_obj, + IN IRP* const p_irp, + OUT cl_irp_action_t* const p_action ) +{ + NTSTATUS status; + IO_STACK_LOCATION *p_io_stack; + + HCA_ENTER( HCA_DBG_PNP ); + +#pragma warning( push, 3 ) + PAGED_CODE(); +#pragma warning( pop ) + + p_io_stack = IoGetCurrentIrpStackLocation( p_irp ); + + /* Compare requested GUID with our supported interface GUIDs. */ + if( IsEqualGUID( p_io_stack->Parameters.QueryInterface.InterfaceType, + &GUID_RDMA_INTERFACE_VERBS ) ) + { + status = __query_ci_ifc( p_dev_obj, p_io_stack ); + *p_action = IrpComplete; + } + else + { + status = p_irp->IoStatus.Status; + *p_action = IrpSkip; + } + + HCA_EXIT( HCA_DBG_PNP ); + return status; +} + + static NTSTATUS hca_query_pnp_state( IN DEVICE_OBJECT* const p_dev_obj, @@ -2231,6 +2346,7 @@ __DevicePowerUpCompletionWorkItem( IN DEVICE_OBJECT* p_dev_obj, IN void* context ) { + int err; NTSTATUS status; IO_STACK_LOCATION *pIoStack; PFDO_DEVICE_DATA p_fdo; @@ -2250,6 +2366,26 @@ __DevicePowerUpCompletionWorkItem( HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_PO, ("***** Restart the HCA, IRQL %d\n", KeGetCurrentIrql())); + /* get MLX4_BUS IB interface */ + status = __get_ifc( p_dev_obj, &MLX4_BUS_IB_INTERFACE_GUID, + sizeof(MLX4_BUS_IB_INTERFACE), MLX4_BUS_IB_INTERFACE_VERSION, NULL, (PINTERFACE)&p_fdo->bus_ib_ifc); + if( !NT_SUCCESS( status ) ) { + HCA_PRINT(TRACE_LEVEL_ERROR, HCA_DBG_PNP, ("Getting MLX4 BUS interface failed: status=0x%x\n", status)); + goto err_hca_reg; + } + p_fdo->bus_ib_ifc_taken = TRUE; + p_fdo->bus_ib_ifc.p_ibdev->x.p_fdo = p_fdo; + + /* get node GUID */ + err = __get_dev_info( p_fdo, &p_fdo->hca.guid, &p_fdo->hca.hw_ver ); + if (err) { + + HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_LOW , + ("can't get guid - ib_query_device() failed (%08X)\n", err )); + //TODO: no cleanup on error + goto err_hca_reg; + } + if( p_fdo->p_al_dev ) { status = __hca_register( p_dev_obj ); if( !NT_SUCCESS( status ) ) { @@ -2384,6 +2520,12 @@ __DevicePowerDownWorkItem( { __hca_deregister( p_fdo ); + + // release MLX4_BUS resources + if(p_fdo->bus_ib_ifc_taken) { + p_fdo->bus_ib_ifc_taken = FALSE; + __put_ifc( (PINTERFACE)&p_fdo->bus_ib_ifc ); + } } IoCopyCurrentIrpStackLocationToNext( p_irp ); @@ -2521,7 +2663,7 @@ hca_init_vfptr( void ) vfptrHcaPnp.pfn_query_resources = cl_irp_ignore; vfptrHcaPnp.pfn_query_res_req = cl_irp_ignore; vfptrHcaPnp.pfn_query_bus_info = cl_irp_ignore; - vfptrHcaPnp.pfn_query_interface = cl_irp_ignore; + vfptrHcaPnp.pfn_query_interface = hca_query_interface; vfptrHcaPnp.pfn_read_config = cl_irp_ignore; vfptrHcaPnp.pfn_write_config = cl_irp_ignore; vfptrHcaPnp.pfn_eject = cl_irp_ignore; diff --git a/branches/winverbs/hw/mlx4/kernel/inc/l2w.h b/branches/winverbs/hw/mlx4/kernel/inc/l2w.h index 79beda2b..a67c670a 100644 --- a/branches/winverbs/hw/mlx4/kernel/inc/l2w.h +++ b/branches/winverbs/hw/mlx4/kernel/inc/l2w.h @@ -153,6 +153,8 @@ struct pci_dev USHORT dev_id; DMA_ADAPTER * p_dma_adapter; /* HCA adapter object */ DEVICE_OBJECT * p_self_do; /* mlx4 FDO */ + // mlx4_ib: various objects and info + struct ib_device * ib_dev; // mlx4_net: various objects and info struct mlx4_dev * dev; volatile long dpc_lock; @@ -160,8 +162,6 @@ struct pci_dev PKINTERRUPT int_obj; /* HCA interrupt object */ KSPIN_LOCK isr_lock; /* lock for the ISR */ #endif - // mlx4_ib: various objects and info - struct ib_device * ib_dev; }; /* DPC */ @@ -304,4 +304,14 @@ static inline int mlx4_is_livefish(struct mlx4_dev *dev) return dev->flags & MLX4_FLAG_LIVEFISH; } +static inline int mlx4_is_barred(struct mlx4_dev *dev) +{ + return dev->flags & (MLX4_FLAG_RESET_CLIENT | MLX4_FLAG_RESET_DRIVER); +} + +static inline int mlx4_is_in_reset(struct mlx4_dev *dev) +{ + return dev->flags & MLX4_FLAG_RESET_STARTED; +} + #endif diff --git a/branches/winverbs/hw/mlx4/kernel/inc/l2w_pci.h b/branches/winverbs/hw/mlx4/kernel/inc/l2w_pci.h index 38d986ba..0d943451 100644 --- a/branches/winverbs/hw/mlx4/kernel/inc/l2w_pci.h +++ b/branches/winverbs/hw/mlx4/kernel/inc/l2w_pci.h @@ -10,6 +10,7 @@ #define DEVID_HERMON_ETH_CONDOR 0x6368 /* 25448 */ #define DEVID_HERMON_DDR_G2 0x6732 /* 26418 */ #define DEVID_HERMON_QDR_G2 0x673c /* 26428 */ +#define DEVID_HERMON_ETH_CONDOR_G2 0x6750 /* 26448 */ /* livefish */ #define DEVID_HERMON_BD 0x0191 /* 401 */ diff --git a/branches/winverbs/hw/mlx4/kernel/inc/vip_dev.h b/branches/winverbs/hw/mlx4/kernel/inc/vip_dev.h index cae7adfa..83729a33 100644 --- a/branches/winverbs/hw/mlx4/kernel/inc/vip_dev.h +++ b/branches/winverbs/hw/mlx4/kernel/inc/vip_dev.h @@ -25,7 +25,6 @@ Notes: #define MTNIC_MAX_PORTS 2 -#define MAX_PORT_SIZE 250000 #define MXE_INTERFACE_VERSION 2 enum mtnic_state { @@ -35,13 +34,11 @@ enum mtnic_state { CARD_DISABLED }; +struct _MP_PORT; + typedef struct { enum mtnic_state state; - NDIS_WORK_ITEM PortStateWorkItem; - NDIS_WORK_ITEM ResetWorkItem; KEVENT ConfigChangeEvent; - KTIMER HeardBeatTimer; - LONG ResetCount; // Objects that are needed in order to work with the hw @@ -59,22 +56,12 @@ struct VipBusIfc { PVOID Context; LONG NoOfConnectedPorts; - ULONG ulAllocatePortObjSize; + NicData_t NicData; -#ifdef _WIN64 - UCHAR pad[0x8]; -#else - UCHAR pad[0x8]; -#endif -#ifdef MTNIC - C_ASSERT(MAX_PORT_SIZE >= sizeof(struct _MP_PORT)); - MP_PORT ports[MTNIC_MAX_PORTS]; -#else - UCHAR ports[MAX_PORT_SIZE * MTNIC_MAX_PORTS]; -#endif -}; -C_ASSERT((FIELD_OFFSET(struct VipBusIfc, ports) % 16) == 0); + struct _MP_PORT *ports[MTNIC_MAX_PORTS]; + +}; diff --git a/branches/winverbs/hw/mthca/kernel/ib_bus32.cdf b/branches/winverbs/hw/mthca/kernel/ib_bus32.cdf new file mode 100644 index 00000000..de22a915 --- /dev/null +++ b/branches/winverbs/hw/mthca/kernel/ib_bus32.cdf @@ -0,0 +1,13 @@ +[CatalogHeader] +Name=ib_bus.cat +PublicVersion=0x0000001 +EncodingType=0x00010001 +CATATTR1=0x10010001:OSAttr:2:6.0 +[CatalogFiles] +ib_bus.inf=ib_bus.inf +ibbus.sys=ibbus.sys +ibiou.sys=ibiou.sys +ibal.dll=ibal.dll +complib.dll=complib.dll +ibald.dll=ibald.dll +complibd.dll=complibd.dll diff --git a/branches/winverbs/hw/mthca/kernel/mt_cache.c b/branches/winverbs/hw/mthca/kernel/mt_cache.c index 11a4f282..39b45e21 100644 --- a/branches/winverbs/hw/mthca/kernel/mt_cache.c +++ b/branches/winverbs/hw/mthca/kernel/mt_cache.c @@ -302,10 +302,6 @@ static void ib_cache_event(struct ib_event_handler *handler, struct ib_event *event) { struct ib_update_work *work; - static int temp_skip = 10; - - if (temp_skip-- <= 0) - return; if (event->event == IB_EVENT_PORT_ERR || event->event == IB_EVENT_PORT_ACTIVE || diff --git a/branches/winverbs/ulp/dirs b/branches/winverbs/ulp/dirs index b4b69cc7..3410deb6 100644 --- a/branches/winverbs/ulp/dirs +++ b/branches/winverbs/ulp/dirs @@ -5,4 +5,5 @@ DIRS=\ srp \ wsd \ qlgcvnic \ - libibverbs + libibverbs \ + nd diff --git a/branches/winverbs/ulp/ipoib/kernel/ipoib.cdf b/branches/winverbs/ulp/ipoib/kernel/ipoib.cdf index 3710847c..44a8ab99 100644 --- a/branches/winverbs/ulp/ipoib/kernel/ipoib.cdf +++ b/branches/winverbs/ulp/ipoib/kernel/ipoib.cdf @@ -8,3 +8,5 @@ CATATTR1=0x10010001:OSAttr:2:6.0 ipoib.sys=ipoib.sys ibwsd.dll=ibwsd.dll ibwsd32.dll=ibwsd32.dll +ibndprov.dll=ibndprov.dll +ibndprov32.dll=ibndprov32.dll diff --git a/branches/winverbs/ulp/ipoib/kernel/ipoib-xp32.cdf b/branches/winverbs/ulp/ipoib/kernel/ipoib32-xp.cdf similarity index 100% rename from branches/winverbs/ulp/ipoib/kernel/ipoib-xp32.cdf rename to branches/winverbs/ulp/ipoib/kernel/ipoib32-xp.cdf diff --git a/branches/winverbs/ulp/ipoib/kernel/ipoib32.cdf b/branches/winverbs/ulp/ipoib/kernel/ipoib32.cdf new file mode 100644 index 00000000..8e984a15 --- /dev/null +++ b/branches/winverbs/ulp/ipoib/kernel/ipoib32.cdf @@ -0,0 +1,10 @@ +[CatalogHeader] +Name=ipoib.cat +PublicVersion=0x0000001 +EncodingType=0x00010001 +CATATTR1=0x10010001:OSAttr:2:6.0 +[CatalogFiles] +netipoib.inf=netipoib.inf +ipoib.sys=ipoib.sys +ibwsd.dll=ibwsd.dll +ibndprov.dll=ibndprov.dll diff --git a/branches/winverbs/ulp/ipoib/kernel/ipoib_port.c b/branches/winverbs/ulp/ipoib/kernel/ipoib_port.c index 93bb1941..e438999a 100644 --- a/branches/winverbs/ulp/ipoib/kernel/ipoib_port.c +++ b/branches/winverbs/ulp/ipoib/kernel/ipoib_port.c @@ -2100,6 +2100,7 @@ __recv_mgr_filter( } else { + ip_stat_sel_t ip_stat; p_desc->len = len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t); if( p_dst->h_mcast) @@ -2110,17 +2111,22 @@ __recv_mgr_filter( p_dst->dgid.multicast.raw_group_id[13] == 0xFF ) { p_desc->type = PKT_TYPE_BCAST; + ip_stat = IP_STAT_BCAST_BYTES; } else { p_desc->type = PKT_TYPE_MCAST; + ip_stat = IP_STAT_MCAST_BYTES; } } else { p_desc->type = PKT_TYPE_UCAST; + ip_stat = IP_STAT_UCAST_BYTES; + } cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item ); + ipoib_inc_recv_stat( p_port->p_adapter,ip_stat , len ); } } diff --git a/branches/winverbs/ulp/ipoib/kernel/netipoib-xp32.inf b/branches/winverbs/ulp/ipoib/kernel/netipoib-xp32.inf index 917c84f6..a4d6694e 100644 --- a/branches/winverbs/ulp/ipoib/kernel/netipoib-xp32.inf +++ b/branches/winverbs/ulp/ipoib/kernel/netipoib-xp32.inf @@ -37,15 +37,17 @@ CopyFiles = IpoibCopyFiles Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL AddReg = IpoibAddReg CopyFiles = IpoibCopyFiles -CopyFiles = WsdCopyFiles -CopyFiles = WOW64CopyFiles +CopyFiles = WsdCopyFiles +CopyFiles = NdCopyFiles +CopyFiles = WOW64CopyFiles [Ipoib.DDInstall.ntia64] Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL AddReg = IpoibAddReg CopyFiles = IpoibCopyFiles -CopyFiles = WsdCopyFiles -CopyFiles = WOW64CopyFiles +CopyFiles = WsdCopyFiles +CopyFiles = NdCopyFiles +CopyFiles = WOW64CopyFiles [Ipoib.DDInstall.ntx86.Services] AddService = ipoib, 2, IpoibService, IpoibEventLog @@ -57,6 +59,7 @@ AddService = ipoib, 2, IpoibService, IpoibEventLog AddService = ipoib, 2, IpoibService, IpoibEventLog [IpoibAddReg] +HKR, ,RDMACapable, %REG_DWORD%, 1 HKR, Ndi, Service, 0, "ipoib" HKR, Ndi\Interfaces, UpperRange, 0, "ndis5" HKR, Ndi\Interfaces, LowerRange, 0, "ethernet" @@ -125,6 +128,12 @@ HKR, Ndi\Params\PayloadMtu, Default, 0, "2044" HKR, Ndi\Params\PayloadMtu, Min, 0, "60" HKR, Ndi\Params\PayloadMtu, Max, 0, "4092" +HKR, Ndi\Params\MCLeaveRescan, ParamDesc, 0, "MC leave rescan (sec)" +HKR, Ndi\Params\MCLeaveRescan, Type, 0, "dword" +HKR, Ndi\Params\MCLeaveRescan, Default, 0, "260" +HKR, Ndi\Params\MCLeaveRescan, Optional, 0, "0" +HKR, Ndi\Params\MCLeaveRescan, Min, 0, "1" +HKR, Ndi\Params\MCLeaveRescan, Max, 0, "3600" [IpoibService] DisplayName = %IpoibServiceDispName% ServiceType = 1 ;%SERVICE_KERNEL_DRIVER% @@ -153,8 +162,12 @@ ipoib.sys,,,2 [WsdCopyFiles] ibwsd.dll,,,0x00000002 +[NdCopyFiles] +ibndprov.dll,,,0x00000002 + [WOW64CopyFiles] ibwsd.dll,ibwsd32.dll,,0x00000002 +ibndprov.dll,ibndprov32.dll,,0x00000002 [SourceDisksNames.x86] 1 = %IcsDisk1%,,,"" @@ -173,15 +186,20 @@ ipoib.sys = 1 ipoib.sys = 1 ibwsd.dll = 1 ibwsd32.dll = 1 +ibndprov.dll = 1 +ibndprov32.dll = 1 [SourceDisksFiles.ia64] ipoib.sys = 1 ibwsd.dll = 1 ibwsd32.dll = 1 +ibndprov.dll = 1 +ibndprov32.dll = 1 [DestinationDirs] IpoibCopyFiles = %DIRID_DRIVERS% WsdCopyFiles = %DIRID_SYSTEM% +NdCopyFiles = %DIRID_SYSTEM% WOW64CopyFiles = %DIRID_SYSTEM_X86% DefaultDestDir = %DIRID_SYSTEM% @@ -193,4 +211,5 @@ IcsDisk1 = "OpenIB IPoIB Disk #1" DIRID_SYSTEM = 11 DIRID_DRIVERS = 12 DIRID_SYSTEM_X86 = 16425 +REG_DWORD = 0x00010001 REG_DWORD_NO_CLOBBER = 0x00010003 diff --git a/branches/winverbs/ulp/ipoib/kernel/netipoib.inx b/branches/winverbs/ulp/ipoib/kernel/netipoib.inx index 7cf7b312..7a769ae9 100644 --- a/branches/winverbs/ulp/ipoib/kernel/netipoib.inx +++ b/branches/winverbs/ulp/ipoib/kernel/netipoib.inx @@ -32,21 +32,24 @@ ExcludeFromSelect = IBA\IPoIB Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL AddReg = IpoibAddReg CopyFiles = IpoibCopyFiles -CopyFiles = WsdCopyFiles +CopyFiles = WsdCopyFiles +CopyFiles = NdCopyFiles [Ipoib.DDInstall.ntamd64] Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL AddReg = IpoibAddReg CopyFiles = IpoibCopyFiles -CopyFiles = WsdCopyFiles -CopyFiles = WOW64CopyFiles +CopyFiles = WsdCopyFiles +CopyFiles = NdCopyFiles +CopyFiles = WOW64CopyFiles [Ipoib.DDInstall.ntia64] Characteristics = 0x81 ; NCF_HAS_UI | NCF_VIRTUAL AddReg = IpoibAddReg CopyFiles = IpoibCopyFiles -CopyFiles = WsdCopyFiles -CopyFiles = WOW64CopyFiles +CopyFiles = WsdCopyFiles +CopyFiles = NdCopyFiles +CopyFiles = WOW64CopyFiles [Ipoib.DDInstall.ntx86.Services] AddService = ipoib, 2, IpoibService, IpoibEventLog @@ -161,8 +164,12 @@ ipoib.sys,,,2 [WsdCopyFiles] ibwsd.dll,,,0x00000002 +[NdCopyFiles] +ibndprov.dll,,,0x00000002 + [WOW64CopyFiles] ibwsd.dll,ibwsd32.dll,,0x00000002 +ibndprov.dll,ibndprov32.dll,,0x00000002 [SourceDisksNames.x86] 1 = %IcsDisk1%,,,"" @@ -176,20 +183,26 @@ ibwsd.dll,ibwsd32.dll,,0x00000002 [SourceDisksFiles.x86] ipoib.sys = 1 ibwsd.dll = 1 +ibndprov.dll = 1 [SourceDisksFiles.amd64] ipoib.sys = 1 ibwsd.dll = 1 ibwsd32.dll = 1 +ibndprov.dll = 1 +ibndprov32.dll = 1 [SourceDisksFiles.ia64] ipoib.sys = 1 ibwsd.dll = 1 ibwsd32.dll = 1 +ibndprov.dll = 1 +ibndprov32.dll = 1 [DestinationDirs] IpoibCopyFiles = %DIRID_DRIVERS% WsdCopyFiles = %DIRID_SYSTEM% +NdCopyFiles = %DIRID_SYSTEM% WOW64CopyFiles = %DIRID_SYSTEM_X86% DefaultDestDir = %DIRID_SYSTEM% diff --git a/branches/winverbs/ulp/nd/dirs b/branches/winverbs/ulp/nd/dirs new file mode 100644 index 00000000..db5a8974 --- /dev/null +++ b/branches/winverbs/ulp/nd/dirs @@ -0,0 +1,2 @@ +DIRS=\ + user diff --git a/branches/winverbs/ulp/nd/user/SOURCES b/branches/winverbs/ulp/nd/user/SOURCES new file mode 100644 index 00000000..95cbf93b --- /dev/null +++ b/branches/winverbs/ulp/nd/user/SOURCES @@ -0,0 +1,15 @@ +TARGETNAME=fake +TARGETPATH=..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=PROGRAM +UMTYPE=console +USE_MSVCRT=1 + +SOURCES=fake.c + +INCLUDES=..;..\..\..\inc;..\..\..\inc\kernel; + +ND_TARGET1=..\..\..\bin\user\$(O)\ibndprov.dll +ND_TARGET2=..\..\..\bin\user\$(O)\ndinstall.exe + +NTTARGETFILES=$(ND_TARGET1) $(ND_TARGET2) + diff --git a/branches/winverbs/ulp/nd/user/fake.c b/branches/winverbs/ulp/nd/user/fake.c new file mode 100644 index 00000000..34ad6498 --- /dev/null +++ b/branches/winverbs/ulp/nd/user/fake.c @@ -0,0 +1,2 @@ + +int __cdecl main() { return 0; } diff --git a/branches/winverbs/ulp/nd/user/makefile b/branches/winverbs/ulp/nd/user/makefile new file mode 100644 index 00000000..a28a5610 --- /dev/null +++ b/branches/winverbs/ulp/nd/user/makefile @@ -0,0 +1,8 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# +MINIMUM_NT_TARGET_VERSION=0x502 + +!INCLUDE ..\..\..\inc\openib.def diff --git a/branches/winverbs/ulp/nd/user/makefile.inc b/branches/winverbs/ulp/nd/user/makefile.inc new file mode 100644 index 00000000..924855ba --- /dev/null +++ b/branches/winverbs/ulp/nd/user/makefile.inc @@ -0,0 +1,31 @@ + +# files copied to the $(ND_TARGET*) folder +# When ND sources hit svn, this hack will be removed. +# recreate dir structure and nd binaries are obj*\ folders tend to get +# removed; don't want to lose the ONLY copies of the ND binaries we have. +# +# ND on ia64 is not supported [8-07-08]. Building the ia64\ folders is a +# blantant hack to keep the build happy; WIX installer does not use the +# ia64 .exe or .dll. If someone knows how to force the build to skip for ia64, +# please enlighten. + +.\$(O)\ibndprov.dll : + xcopy objfre_svr-03_amd64 objfre_wnet_amd64 /E/I/K/Y + xcopy objfre_svr-03_amd64 objchk_wnet_amd64 /E/I/K/Y + xcopy objfre_svr-03_x86 objfre_wnet_x86 /E/I/K/Y + xcopy objfre_svr-03_x86 objchk_wnet_x86 /E/I/K/Y + xcopy objfre_svr-08_amd64 objfre_wlh_amd64 /E/I/K/Y + xcopy objfre_svr-08_amd64 objchk_wlh_amd64 /E/I/K/Y + xcopy objfre_svr-08_x86 objfre_wlh_x86 /E/I/K/Y + xcopy objfre_svr-08_x86 objchk_wlh_x86 /E/I/K/Y + xcopy objfre_svr-08_ia64 objfre_wlh_ia64 /E/I/K/Y + xcopy objfre_svr-08_ia64 objchk_wlh_ia64 /E/I/K/Y + xcopy objfre_svr-08_ia64 objfre_wnet_ia64 /E/I/K/Y + xcopy objfre_svr-08_ia64 objchk_wnet_ia64 /E/I/K/Y + +$(ND_TARGET1) : .\$(O)\ibndprov.dll + copy /B/Y .\$(O)\$(@B).dll $@ + +$(ND_TARGET2) : .\$(O)\ndinstall.exe + copy /B/Y .\$(O)\$(@B).exe $@ + diff --git a/branches/winverbs/ulp/nd/user/objfre_svr-03_amd64/amd64/ibndprov.dll b/branches/winverbs/ulp/nd/user/objfre_svr-03_amd64/amd64/ibndprov.dll new file mode 100644 index 0000000000000000000000000000000000000000..9d879320a5dbe43fc6c304c5e01d9bd0bf821b39 GIT binary patch literal 43520 zcmeIb3v^V~^*??lnF&K4%z%l8mkc;)1k^~-1_L^mOyG`8AV5G`!6YOT5(&weOfV?c zi4!!F%h+hKQtP);2rZy((XW+|+A>o6;XvUZpAsA$=8Ib-g{P&;yXc!wetz{g0a_kG! zn>F?qrstNFR+*fY6&ouHHk%3y%F8QUrVS2LWp%l!wA^G}zRI+@qR4T1LPD%rZF=h` z51AG=eqI#{lh*vG>N>=iCj6*s9#8+GY7*e2H9xJI&!M+!0nh&_FE?q;y*xf?&E35G zq&4+wd8=xPxW{EE5}54cwgwp;Mi2l~1-e`Z9yBswxWEMP#AHt2s9@#IVCUkpwQGu%f&rj2! z4t*| zD*Q@?@%M@HKUd?ItMLUYyk3RHDs-!`QH8%$;m0Z-uc`45RQRn5C#m^U)cDgX4B`FS zka2vg#>YYaBL9k2Jg-rqTZO4AT&co~)q1Hz>PNN}(k;ZYHzK4@MlgsHrpGP(!LP!Q zi26nLVga}spvT2hX0A1Ban?$T;MukY`b%1won=Mbw#=4G>DJ{pEF(yN%PLH(Tm>$N zDYK-Ye51pZvTS+IrAS{@y`kLUGG!N(7i@HFc9fIsudJwcIVzENYpJWSglaQJ)yf@Z zCaI{vNtvscEnP;vuvgHL{OLAEygh|znU<1VT5 z#U~3Bi*IXM^5`35Kfdkt_jj#Zclo-~4FzRHN3}e<(5^rN{hEr zD~z>E77j!B+ZT!WV=7F|6!CbgfbJ{-yA}&rmn~qZ{^x2ucbP~p8Umead`gbU|8RwX z@+twhdMEpq=F4!RA9V$%wW!8kbv#wd}#y}#s%&ROMu_RW9m zu|m;*2)`*sBEF8TW0xb8vJI?&kyRo7Ob!cw2$gSCkRajCf;{7Ugx&IcYfNH>s;9SZd$jYw0vE4m7{W9 z#fF=UD;;^a!t}~3*j#k=ymjKYbVGR&w#{3xYubQDM)-4|i*`+NpNnxNxz8D0M)$c1 zt~hs3s{7ouEzxQIW6jj}u@3QvvXlHXT4i6tQ|E#~!&81)(YC6YhNoKH=d>+46gMn+ z21R~9tVoAtmF)S@B>R$%eh~~x&G9HJE3XHW7^2(#hPnqR!I$&^&(-qW=YvV_d``L5 z+C!S&TajBaIQMucH>&sh$hGlY!a`P1_m`pE=-v$EUc_^0$H;T*L%G`CsmMKtsMM?v z<=)J54G;LWI@xpFbYqAL9UaBBXYLFZE1{uxr1{-v-0pL3!%ly=5YpxcBN7@Uji=iX z&hBs*X!_jm81ySN18H)b=36&#^RCc`6J=$ffUHR2jL!@eQHH}-bBbL#^~80b8Sa$y zpGk%aeX&=X)nGNJ7F)fxlzQn(DwB?ETQYLy8lJM6EybPxo~LWeiVASGNXY!U+x1_lD+wZ*YG4(ERksd_|gLXm1VZXM#bQp*^DA zE5gr`pEZ1RekS3-1tuvmnUh_-@>vj0w ziPo=}6h^%RW0}+%hz=(h4olkNnRf<52@NZ>BepN(m-qx?TfZW@K64BPmxFP7GZTpb z`6nbzt?bQArtv#ZNKu-!=Zd|VCW_V+dzWhKm&O20kbO(dEpI~5kU7Kd&D3L@OO3&# z!+pV^tmK+YWS;^-U{bRMzg+ryKEuXQ)kCsRvV?FgZha2aZf)%z7*5C8)-kTQ)}Cme zq#^b@hDRzEA+a4DXd^w;MuTJyJ$5LPc_ti|1GJD#gXY;?VCvJaFony5mNO&y0`b|A zad*LNgZ~|^e#Pu?70AA+?}Rj+g;G$*G^ox7lfi#RtG|Ar8=AA}gUd`8Wu}K(>F8*X zrVqh&I&d8*fHp3QY{T%>ii;$!;afk@O0zp%i|A*%nDE8Q)F~>p~#&AZ=A^ovKF~<(+M$FS5I~0AanY39=m|oDxJg5%JfJOra+y!ZF_dC<{E7HQ>KSXp$)8G{J)Uk&+A|r?NU&gBF>bTIuUC&>gXHhlTJ_q{g3HT4gYtoa z@_yIYhIqZ_jG>_we6Q6<(h#jhMxleX_2bnc%iB*|l~a=8q)RU;&$I@n)JJ7+_q)bO ztsf+2)f=<7Kj#`-dxjZcj(Eqabg6(98_K+*0Hz&ap*w1u9${av!On! zwMR=$#wx8?2!l-WGHNxZwI@-k4>j3gIEhwk-=T#Njm33>1pd}{^l_brlgcw{zYVPe ztU&FLX=wEav;yH6_Zfp!AEHo4hkJS-sk+cGI~p{x&j6XQ0n_k|;i>1)TXo0+HZ~XBgGo7M0H>$gga0)7&!`{Uuzsy9w^$u*GF1N)b*#h^&||a6=7olc8*@~ zZx|)2!_#O=iW4-f7c1^~JyJK$;2r+ML-8derkpJ!1r&U~05}z-3C0 zRN`|XnMPuCNFx_hy;QFup^Ao#hMI)#YK7Q z+}?U7+VIi*43`1wLKamW3CKOkH&R5 zEtbith zXsz0X7G6_p{#=FSD!fT;p`5pHE{bV@Ydw0e#d8#M)ezJ&R|!EqbFC$4WGh^kQ@&*6F~`GhOx+ znN7@UIDE2YHo6OJyNMf@VdC4BH?zD;O(?q7<|zy?TU%x~SuHkSW{S<5*#&5#+DPxD z?I~{s8SR0&qNTMF{dw1!lh@oJtww7(*wGnf1s4A6vU1XnB8~T z+W@wV7hI;Z(v&D{$G1+@Rl=aQ8SOsZqaOr=#VtuJa18{PMv%(zlftsnp3Cw{(b#Ew zG2j-U<8QN?O=MODlRkcr%Jv6gi>Das?!_jaYNmrZ?B*J~55~FdwVP9ULxL)@(k>n0 zgNkPv3@tfhc}wws4F)~^8pC#*+DIz1`YciCb^^Ku5}}?`P~0+rLD>KXsfN1v0SuI- zDYDmUw#dMOqhgVcfO)a&#g-`op_UZj5lq^Fc6k5KQ5ofYY}^C-1jgVmsQ*1rdOk2` zf`~tgc*Eie^_eDFS&Y3D?L)?R`lDRec=|7J#WqVC=I^cmW4!)Iz9W=dOmx#6&<0fnuKvV*r{Gm?N~ z?*1OY5^i{|xFr@l9ZXK(p>Kl0G=Hy-jBdZIJZyFoHlE`-()zrc!ptLscYsxy8Th!& z$u~&3(hcM}u`6ee>_-na<(y6Fla&twdlsVlcC^%G_bogEPH|<*o_r&klvBU}aw>jy zm!Jw#b~{yAqe8a|*AldQ)7vb1pv8Kl(%z7_zfnD=`2rx>`$N0(qFs5`ro2L=pNRo% z_bYu{H^`pf^9JqS42UhVJ|0iK{GRtFQg#}ULi6o87j(r0lk#C|Af6hE=ZLsbCes9) z;9w4<b<9eLA&x? zcE*o)1B>eMc4eg=k^L0GUba^^+3)EhFH(%ntA|@lR$^@#O7bKsD=XLQ5iwC@P+48J zSKoWOIm*J^q291>q`Bd<0crNU6zns!Zq<9*g4G*o6rjoxL1v$BRu4w&)wpzlSD>>8 zQ|7jDgDJE4%wWn~E#e`{j1R8llzBS?lxZW9fQH9SAtDUr7vdF(v`m+*d?hPAP&9Yc zqC=l<3xSOG5f#m%_es&jvB0UXNO@d@smo5a(&U3&e+nku1&@g`Ms$w(NT>O*M4~}! zUD%2XMi(vW$9n#Rf}mh}FlqnW z#5EXQd)nz>QUhW_ma1d9eD?erC3$PI(n-ygpgGR`$-$&DQJ4B@4<@Za3@m%6l4(8? z3MhQGv!WB`*$NyTG|;i=4gKnvgoKv~J8gj3mAR*uWwwa_pn zP%utlSYzVrl!*cokMswFM`!^EjGxaJw2#3{f|jny zszS<7h79DJplDJ?Rx}08pj=SS=-)EtC=|TE_d?mb32dJQMtlLwrSd0I4v!`AIq2Vd zne4ee8LiD0x@HcP6?98TVnVv)*C3YCM!Muep-WOpm#Dm}Y!ft+)`so~dm~3OEt0xzT9zYR$4zXWmV52wJ-A z#o(5u(CV7QmsIJ933=Jd@s^!pVNXd$+={X}c5ji{6imAGEv!-*C#uK%9!U6m*Lk*? zbCRpev*yn;SC`n7)yY&?rq(U=lz#RZ4MkG_5RLNL$_rTZQ{L2MFq?NXreIlWwjt}q ztTewJ3z#VozZ$mdvr3n1Ji365ue=nP2m&a_dMC*Wl>UpDSt#fP%3#Rgl4tj=Gw0is z8^Ms9Qfx|9s!h2)-KM-N-6XA-)`cgFbPPDSj2l|HWN6ux09SdWUmCXY+3^q7$CA`rM*X#a}s4hN?0VaFVC0-s|m~b5>rc)N(5ZEe;bZA7v(CTBHNlfHz8R~L`v}q4kk#6|nnpbCYD}AQA;=&FSAo}w z)eJf=d8emSD`q!L0rkQrx3=6Pj@w0f8OY} z%#kdL^JVYt7%P^Nm&gkCZUR$}Ae|A)3s!u1J;8*NO+>hd1@V#FIO4kqjDmO(G<0#x zcoz8VGaxQwk^+u-jKS2&zyi?kg2d?b9P2#vkO!2;)aRLsy?Ha*mheG%^UU>mDZSCM z^0l1tHH3xaws*Yjy%l4DS^8pdCB||GrYWRfXk{6)QV$;oY&%jdIoKDTlekd_D-Uzg zfpcY`^ellBNn*o0y3<`lq7NW>&c%2^^iJ5@x~tIx~CB} zNrw>R>;1FD`QRY3_=>&TAjPlERJHC8?7qw?c5j;GNoTHk>ZXF!E~LhS9ZSkEGakNV zCUzFVq@!IVX2X=U9$1qGs59>Z3Tfb+WX{J*+h83)@;_kt4W}jO2%d+2{#5XDeGSp@ zvRVS8?5?t*`-)r6V}Un60cySn>!D^1N26%da4~Hf#_&x;?+>NsDL~CWKe|e1_eS?# zK;}R}4$y|hmtm|LC$~i#QGi<-y*YF^hfM?2TY^^Ye8`639*|&r91{flkl=vs2)6(>oR8T6Iq@yTKY+!5f{UurZoE5x3&gUx@ zSD}KpNR4)rJteoyQ}w8snx~ynFlp~=>PGK0bR86^#cbH#MrzOV98tjcw3#fOPk$qS zc?$an4Hiyr3LZd_BPNdJPF_M*THV($SFG%T>huIO)t^8TTrks4Nk>qO$+J7<*(YUR zT_UyTx*FTGTrh#`+c_R8;!gW)Ke>Mfjh-~%+}N9fU1pMjlUjn)Gi z>Bwztj2p*?C%`lHo5^?qrKr_llH*mxAc0;F?XC=U%Xx0nSx}c7j0;7+q3&vAfL(M= zl;c~KPEEMc1lXsJKLk5z(+-^u;y*~Wl+1J*(PUY%W2NwUXgeqSa?LrxPHc*|{}`3& zEG}|JK;aTTEkyksbzQ?#meSFZj;Wxf29mpu)-ygC=c6O+x~nHz=VK+Ros(b-4R;nxVZ{kE>KQIVq+(SgPltIm#j3 zU3r9tg6UMwvt#UQX<;dWq~yceQI^P@$adu!8jVF(UL*2i0|I{Y_v}VvA&y@Tju%rE zCai_%48I2T*|I?8braPQ@S!CpxcefFrcu^zZ=pR zLT~aF6vKi?Dj~lLnE7@XF_uqwQe?$bPX&P46bzT#EPT>b2BKNwTT7JV3>YRA9`ZShtR01;tjbF* zc9mpYGZQAQ~qQ zCJhFZXP1gZY>1VrZ~HDbqS2^qlrx--ay!XUro%!! zlaB)d+_`ZX#li$%9AA^55#ffQQ#PrYb_cXz@d5dC_knk85K0(*LC0R3e>jJxKH~se)@0;ZO`cZ zA>j>hBp5YCi3SJ#21QX81_S9-z+pOs0gJ4E=?V>PuCMILQ0o5o4sY6 z(C-ldp;LJ0GkE9FNc`x-!h3^%={j;JDLc6WQe7mSaFHaCCjAF!Dpq*Xh{2>PKB(m; zVvSi?IE{9tz8*|P=XjI&x(ri~wSAFm3a7$YZcouNA90EhR?OHhLpRvGH|lM0q1n7u{0xDP zqA&;v20=}-t&KYsNDJ7Bv1~@n&suN^EG^5~HjVT&FlO)$C?7y~y-B<3rc?vz08PP02QSzKN0y zJC94sD^jpgPm83I@Y>BZnYxztBlcQI!{|_2KHJ`tQ9+^$k=%1?t5Vn1(jE-$^+(bk_-=Q&=O?nx|jA zhOY!^Z{RpM>@zjgz07HnZ6ZE0VH|vMR`~^kwu5)XlCV?okmqAfu#sWpWVinWm208% z-MkU#ec2oTcC{YtFi*yf0T|yF&pF+li(K=-$d}Lo@cDFPdok^W7R!qE?KWn@`Btx0 zJZo6GzZb4R?b{v`>%BrGVukYeUWA)m+Y&wJ#{6)F=iJyIqSaW}6`pf(E)X#O?KaI{ zQAqY2)5zP;8n$0X`xbI!gpaE7^KM#?IQ56-+`VuT&~p@~xEE@Lo);1xPm7~hZX*cM zul-Prjimph5d{U_&$MACgXR|W!%^FBpiTP z^SFA22R9S5(6|do)22C==#25_RBs8XQsEhbUQsHXu5cC42~6AyQ5&*DQzMDHa4`OlYw{XOd!P8ID;I1 zwncVSJ{_KufDRm%hs_<=o=Dk)=YU&6v46+$e9F5Zmb0~=j zN?udMWr`E6R0>!qzMJC6gjx=%7qTJ{5!58|ftsekXX2E-e=D4#;PdBk{EI^@?jiN_ zHj>5mJ|8O;*3wPt7XNna72xp?^OLaME1&pxZ1+HpgeF>=SFtGoc4M9~f!gr8T%XQTrjyi*yt#S&&bD zAUoqF*wbJLu>1mId0Y%((jQ4)vol^a+|!K*+2?}R_Y~^sqK&*g=#pgLY8(PB#u3({ zew)|wxqbdMhI`&cQ6@You9cpz#~SL!fds;STq(D;596{ z{bF_MpXXCg6Fp?=(G8{^w<9q1Z9}L2Y%5L>)u|^HK};N)ctr*yfE&Oz>2MKP1y_I_ zR8GLjn$i_`98wD=_#QC0DcxS;t~ftf?X zHp_6&uTV-*oMcKBxIQ6pByA*RFXF-gY?1*ZT!;!CoTdw2FSH>CWSsn1A3{)p-zi!-s|q|NZ0>>Y3S+Vo0ha@res$%%6&&P_34 zi$VTnq@GBti3JT7-U3jOjARRGS>jvD#cZrIQ^*k-EaMn>G(dK3fBb-5%k_*==$Wzn z_S~nyDdgUUBvfJji*Rob+|hD!w+Zbx?pViZwS~YawA$b&TH&h3DLQ2#%`fb8Y}7*< z8y6_?n96@;QsFsak)%xaoo1XyvP6D?=tPFN6L=!m)Km_4N7;D`vXZnB|W zj;=EsDH}E6!N8nnQzJPx2>Y8_bAt=OC>Oh z6itD`=NCj2_*uBG&YsZ?H9{iWvj_tst|eT{!;(O5AX&0i{81 zVTS|k2AOQi5)-lIHt%C*H{z6C&(jy1yic0B<&UIB%A(Du4?9t%o(>Yom5@Wc>kKxg zpd*NUEcej$EGGoW#D%sVI#UG zR|F(TunqN0J7_LFXadfMwTHzH+6aFVzthW4(MkqR(MkmRAPbz+p)mv-Q-P}$j*5~j zm#$#5I%BzmqT% zD{!XZY!m4lSxcO>Rz22o4?otr<9A|e$KZ4{N9)pK=GV?5Zc!2*S`C0}E*4JOYdC4= z@reeKoVH^$(MeDbh_Z%a~3 zIRC5t&`6|Fbb7)-C=Z4dIcl z{uX&IQ|}B~pc`Zp=$X%o@ms{|?dEl$3Ek8sD+6O#0@e!q194F$ z*q9FXQpHihMK1;ZKoH$ZcK~w2jw)zl3-KPQW!$oWkV!`frIzE|waw+M@4qn_0nL|P zI3vY){lu6waAM3Um=#*nNv6P{M4|8^0H2cjo{8{FkzC_b13oG7o(gA3;b~gMT~avH>$Li=k5N5N$!Lrp@p z^gOQp6@xYwG+2x`4GeHK<33L+DePsKoZ4^5Q)RekVR)iKlcefAm3AT(i^t_MLf!QW9nr*VutPBB~$E5(hV!g1Mr$FRXV!} zFXCJ(xo?AhnqgC}hqBk5g6vt9c77p!KFkBEhe|V038O%7`Eb{Fwe}i|PN<6T(UXbHF-5q!t|}a`^~RJzqLPShiSIs*`E2QvKCK{P^8T6xUXl^4mf1 z=1pjhh0~ygUsS=qFYB4!0w{>?9Kk_zx#pU~HP^**dw@;=M2mvQudNnjN8W^nXyRarylMdE!H2w(UZ9EVu# z{Qk~?UqhC0+|7-T(B)ka#uOrqJd$q_NabYn-IiG8CJT%bzXPMj$BAILgIs~rp$0l{ zK4z9ZZByzq|6#)^I?iOZW^vPd`RBCLhWLJNIs2?eHe~e)#Nlw}hpg@5j*0RrWK$+` z(OL!O*k(>8TW|+vpFAF>fNv%U#20RK08t{RCQj&UFua_wM^5nk^$-6$Jl_TPQJCku zDe}*Gp3m<{e&UOXFSBB; zawXw}6wZVjK+oRFv|!q4%iOcQ7ul5Of=PLgl6-&*`h!U~^5>T<4Sm6+>+q~}p3R)3 zam~PMCU|ibw=E$?-uYx3;ia7yy)vAEnMQApzIO`BMY-rQ8gTVvuDa-cI}SG{dRI*$ zYnAgAyiSAg6*PbCp?`^_g~ zTPWgSV5<%!K1SFMv7?1d^2^2{=Esp#!t|wF7AFxHMHY)8V*EU+8eUpl@b)FlL8s*G zBbEhAw_tXnh;`j~094;&z4{=DF5Y8R`V4hvpxb>*rXbe>46z8{^&NF_RJwvme?CC8 zB&nvixt>N$&VZ8ZGSuxy#eww=$5rCwm)12!ta{QBBU(g4N{ik!0}}Hh@hGf^1R#*n zW~d|EM~PEQXS8pb>D|G%hPZb(sM~DB)KaqYww&>{p)Q|xO?W*ZsetlUY5xd^Aj|j@ zj_<$HxDtGef-n)})-yUe`V-l6!f4BwXWlX+m=q`4RbMdsVm}=k&NFlOL(iLNAzRU! zDf5NEU#DYHE4)!l`PtEL;2nhdw`-UAtZ(UYU(j&x{X{F@se<2&9gCgEfR&f4+TjHOz`j;$aCE+B%I{$SE93<>k44lofhdQpq- zBS~hUr^-TlD!e*(;I0<;Td3}3Y&w1{qjxe?VYk3t5ps~yU8#D?^66>`jKY{NV1BuQ z`X)T3vgd1~;jVWu9K1ISHp?eABlBz~%rnDw{syw=>nVo2$n_6`EzzH_nu}s!u362F z-gsq+mf~7qatj$&@DutHtkC)3^U+y+)vv-V9(v`rL^aE>d|IR<#4g+hqvce#!i-EI z{>^VOhBy`C0zRa6lov@7`hN-Ii!Khq_+qXkDlKO#9a!v$U4DLyCV{`#KxSDD?N-XC zqL13+ZgfSLup>6mLQh@(lYd7~HR6S*@Y=A8BL94C_zpcaA2y}XQ)}#`r%nOeVR}l` z$87|hBSGsMHClIXu}#r~5AGvtC_`U8&ZfwEIYZVDMkm838v`rQCbT&8y)WSxRNkv^ z6R$->J-!VMr{Q<5ywNi4`o8R(IS#J?fSOSj<{qRm@shDPq{Od^6MgFLs(k}WikFvz zGsh!sq_Sg2DEm=UMA?CL6tC6tZCSztS-{_?yO+QyEdOUP3g5%@{E;{&4#nR_Pn1?d!N=V-_&~$wbU=%CBXdvvRB7}Z=u3tQ{Th= zI6JIQ=nEM$2YVjs1EvC3ANWl{BGdrbanhIA0nqD%P_!?@Y}C{JVGG|11(SZghtQo& z@9jShwC_c$a>iSRIyy4r!`y~pYAd&p8G~WIh6(3h+P^-{hj|mqMjDalxb+gn~FW;ds0!D{sl(snKd-6K{ADr#7~c9F66? zVZ6b16K|OOOJt_fCXW0ak@t`Xc%4MM3pO{tYe6*sK;tTNZ5D-!)j|Uw-hj`J-UlPg6Lk6w#4XVJdvOTJdgrweW`Po@ zWJm(GsKi-GdDT+d~*vy6R6iL3E zD01Q;Ej7z0$76%njw?id#QAk1IC zuV_4)HlZmynnJ7RQ;&fgMROrv(NMor$X}&mHR$^&dUS*4z5!E&{8;UQxY61x;diS- zek^=WQpK%Jd}xI_fIn*t8hi{55^g9$)Gz_pDTp@*eoLhG94}Fo!9T-w&|R=j72D-# z91YT4u=c=BqQTJMp`pcEIO9kE&RW<{Tch#nw=;89nt8kTmm#^TC{{lU}(^oLucaa~y zawd)-K3T}|`GCME_?*Dx@HIM%@S&}z=ST|a9zz`t(!l3O$ghmpdm=_IZNy#^F^h<` zix{0-AdhxfsFyEdkBL~fi2Yo|0wUJPW5<&5W-C0TcTr4HUnRxhaNGMOT6APeJcLaC zE*h;wi0|ixUlnT@;=3u1!l{(GOO0UNS5LB#V+vf~MXo}nzXpp5cG1}A>&cPsM`^n< z+3ualw*~aFC%>LO_{v-O*q_!1?#DK%$C?r663zv{Lin;7hG6Q{0WqK|FBl2q%za&2eb`AIl3IDjzv(Yd~>-g!^LgB;y4N_#h08oRD z((AKKIGDt)!rk*TN>{Rz<&1MUD*QV}*He!q!{It}35sT=1&gJOHfz0ji>O+sNGUi5 z$BP`r-mDqq(v@pl$$MPvxi+d8yWodFF1j)SI~4bN;LN3MAH7H5urAjANy@PHr*C%jnZiuA=ily z4dMPjzwW{3ncx4X&p`+WRQ!3kviBPxYB(B0Td9qFEA`^HaHab!I$H)_zi357_GQlj zj)4Re$5F`o%jp^qwpiKo@M+kG`MD=t&_6||`LL^gdkx=zxm9QR4l}Vc_nJS9haMpm z*voA%rlyH#o+Cu8D8u$#q@dlHG{4%O1;nFT0J{Zf0zhL~bAU+Jo^7gBoF{k0^n$14DbZH48Wf8vTM0;)yu+Fk2n5@x#}fMtYmne<`;c} zo>s6mwtKqY{R}Fez9cnH_1-tb8nlDeh_7Vd4)fFK5f^R)9oS|OU)h!QMi?FV)PTF(SDU?)aYR^wU)9Zl4k-DnKAK9WJA2l2Qrl0as`?u=qvn3&2D!fqd=5VGh2Baz`@ua?r5;$fLX!<4BSo<`+<)eQ>v9 zlPvCb(8c3{&uefWsOgNSL&n z(9Hnt$BSd%8D~jOs*?x4HA3f9ccKo+{uVYH$utA%i$L_ckJ1i;IYD1cC0_{$2UW@N z=^%?L?2qmQ^=U6=3Vg$9iB&{BeE~5Cf1@OPF9jYRTJhlFdHiYsp}y%DRfhUj)i(FP zLv7P-xSydEKBc!4V6;!^Id>6-vEBGP?d)s6CYBB9erde@w_=O4+965UtA^7 zyNu7yC46?KgOnw9xMK3vDQbU^@H|gUP1+YUHFrcz%`zaRA>E9rQD4&%Z)k-lCMP^G zxE?@mAE3o&g|E+G8zCVi7#-oM*?@c1QpRzm>khmUy9`rLE_b`PIGKE^JH+)XxQa64 zFp%2gG&lU>eT1`!)?mK;4qOTNVjfBlEWLc^hRZ}G5+5K7Z!*3MFxs1pN8xs&EnDDP zaySg0utzX?$`Z3@qZtmHvlw7LXkFw=q=|80QfM_f8jmtGS|wqd+RY`L@Y6ZrvFYc# zytU}Yicg*3D~o8y6lnhx=lXmNI=F1{@lLj9i zH0LAe(C;EcW0(vH;1>E+<0q&c{{wkaKfLo^5TGkPyWWtcnpuA@}h)gCIHYc2{E-A+YoGj^i(WE@gt)SjV+w?DU4 zUwe+8s;A4_KXSzzp28J0!UnExI*N{?gmXM02EHkX4H=x|VU34E#Jtng^6wW$h)wA1 zDtylqD+ewSVv3>KUI&HzSE8=(vLggW`Hsy4Kc*?h2QL!&F&5v}!4b!=&X)0`DcAzR zMqvu*tKa7i7<@!R`BUI2?8vtJYZe<0XU-`0E{Q^i%hjF+tsz!rInL9<-?qGn_n6Gc zD&bP|NBF$NahE0HonX)L)R4=MkJCi&PKIv`y_QPU;kQhPP<+sT7V6=7%JS)s6Bxw| z-+ey~8XskthOS5ATN2c?kGS?z`L~QK*%aR$Mu;tJS)7@Y)rx|oGldWjaPM+VTE??5 zoVIO+FFk)93~XY+-c1eAG468~*I4(teA$ro4GuN9F_LlZ7QOU)&Vw-Ttma~}#{)i3x-iba%m6;>3V#o&<_aR|Ww#I*g^#lDBa-6#3>`R+A%U$vPv}ze zaV>!DtbVxy<^+8PM_4Dc_Xrs!13dPw?7Lx(`YolYbid^^S^R@Pp>*LG_#M}c@Cl{s zL8j0rlv*zr_|D@B;&QIKQw)cfz%*Z;8o`nGHL5o9WYtEVgE4$JM~ZKu7}cw=WU+}i z6d(gjsaM}rEAM<{i_L@%i(97P`-lAOB|AAQ?IYM8A+mhI$&y8DM}+9TD{M~jI0fSVWS(yJ=531tisoAfDA5)1$rP-p8nhLk;JQ^uw9A4q%W1QZxtWiJcAe< zQzcLNCyq=Hu#z2^#$JTqXHked72#IyPv&gF> zvf*$WOsO$$oEt1_gV!v4<`G)s>E#%1i7C2QXJej)0;KsDet&XMkUGR0P~v0JwAlL+ z&h7jsM~QmEg`bvpRZqot9TA1@!k6VaI=Iv;8I~RE$)qI++D1#Ai7Ux9*z7kFmi71Mi81kU2)_rZcQK-tcz#~W$6@67yXf3>Hb=c$A51#u zg$w^Tzz^s#hvDKs7)Nw%>ot|7dP_I zC}$+Rekpeex3(IKzr^Wtb<}6aH$a#-V+O%0;e#*I z`&`?buv_w=Rr%=LJ#Sz3Q{pWUX*}*h;62(eJ8}558M(csG6FA*Rpq^mb(8EJC21>^hEL* zDFW~t^KU0u$g=TW^%GtOcO$5OSy}kQS*QaQkdNgCS|FIj`^OkE)QJU0J@0N7Z*Fpdkxlx3)_!Vv#H6{i&ytJ?_9Ss1^9cX z-VT7{syZaL>3)F7SS`M&uFJ&}CpQ?W2IQTCcp5B1#DYnGyN9Tg(1_lGNv9DBCLPCb z3sr?O3MT!A3V7f*zbSN;CO9(+rN#6HXMTJ4!1GCz4W*xmc>Xlv`Qz}jI8)A2pm^~n z>vNLl-G1pT)6$oT&XW|2KD#hS`(>9s`jlMQZolZ1t?=0ON+OmbT*Pja1Vw{6mP!p?VvGepSc5aMQ zqt`NhrPdjpAC;l{7I-gC>O=6ShC9=t%iX4q8ftg%WnW<zvyB=$xojsz=@p%>+H0kUO4Dm}g)U?jPrm zcgE#w;T8%1+I%ibovv23Etz$O-rV-s@|*u;79Y~uY%elZ@5DRMkC z*2<{ViIkh`uxroKK7__B8l}^TK%0&&Uj?!3$I4*;H2&n_85}rskf< zrp}wdrk)lvs!5Jz$wfw%oNHjo7;p0ZDj$s3vhm=H@t?*zMVSdGGahA7_6fC&K|9a~ zr%xQ{13Kl71%32v;!4nm<27F3#h>}OR2(OuKJX+QIbUF`sGEqoi7UYue7=l>=ZlX8 zV<^HHfX`ImGxdpy-4}GCFOCEHE<)d!kBQ)~iF1=W2Y3$csysJw2JuPKFg|IddEt2% zv9W!Fd5H7gY>Y8?Hrl?FO~kxS=reW-PKua6qCa@V03I>ikFoJ_#|RnbIF1L7MzjNd z0?j8rk>nRVeIam49>mklz-NRx>Lw}HB*cJ!j4a`Pa0aA6jME4j8ZgdrEb$4H;q!|* z23?au*W|g0eN=bi)EGAI)4}7bL0^48!jJm56U0a!57~)GOUKyzjoLWiv) zv!dKpSy8sQw9LUCiC*a{TT@!;sxBy7U4Cn6c@bkDMIp^zQL(AoxwyK#&{bMdo`o7) z8GBWOoJ{=R^3|0NYQ<6GV&yLUhs69pC0@faD;@aX>#1Rhea#kEIvn=W4V49`Iw4AS z6gZ`_vWh~M%%3Tjv3(l+zrZ<_6@`wfDk{O)Pbg*B{}j9@R8A@?szh1*AHz}hih?4w zeF>LX*^u$Et664QMU_J?C@;cDzaN3!4eVZKEv+ipQ0B-j0nVXjG7HMes@Nk@E2$}O*{dv6^dt8S5`X8U7{fy=(ji`RUI0VJ1BM4 z)+(1{b8hKohg3ymq9;}qo$ILFTv`s&sl~B%9Nq?VMXxR|;e#p4sws3hX=2a_A;xl| zS2Cd8!-oysV|5v`)S_JS%`R%K-c#yF25Zgb*4m(C?L**$EPqr6C3nZrt* zC1uV6XQ>$96*_9n34Xnjo~j(KNWHUh!!DWP%YF*|3izNJN}!{TKfsu27eX0hMVoN; z3%DO2=hzARG=Q+p#8?S}9`@`jNKXe$oC1A^=R|rQ5@3Qc8G#|pLzprXGlh_kD|Oun zZiM{^p|}&zeF%*R%?P2m3w@K#T!&DEzz~cGeHSt|XBy5D5g6Ki1;K^Tj*xOOW1j;i z0yY6^0gb@HgkUiP4}>WQZiGgJd;}-L9E8U(#%6>R;Lw433lMfAe275tGBv&c`ETMm z2jLI`Q{&szIQE9D1~@h%uu83-#7HX_`C@F2pY z2!4c4gdT)%5R5-#Y!-qQ;rj@;Ab1cSL3j$G1K|w9IfOVILR^AyEkZ8B7KHl{{u|*% zg!d7?K}h&7#%3T~gb*rgd+$qBK!?Ow-@4z za4kX}!tDt6BRq<54B=e_O%r3&5UdC#2=xg6h42JICqe*W{65C!A}m92Al!}c2*Oc> zR}p#;f(R3T$=Ga!g$TI_B?xyS+=uWBgkK~05#B+F`W4t3VFAKw1kz|xEShPVj>WJs zOwY!$ScW|gOJL)e0j3)EKqT1A<}(!(I$w zPGOg_S!_0&!!E=B6MqFuWmmGfEDf6YDy**GW9e)@u3==b1#BU^mR-jzOk#^zCbP0E zwwNtpGPAMk*-~a_*=!kG&T`lawvw%4xokCC!){<}u`2(7-H88zem%R1<*|HLfd7fU zkQFfpD`p#62`gncvrVjwZD!@Hf;rhOtddpXf2yx$TiC6vhHYiHvD?`;b_e?*yOX(D zExU_(SRLEW?#B91&%CUGDQpL-B#rQ~F#8}bS%C$F?BuX3}KClqbY z+lc=tzN)Okl~-1gS5iu)kcj1HNFo(0aJee;DvFC~aYDIpQdQ|~j!?qpf}*?))y1J` zc}0;U&$ZR*$b(TAN_3Vx^RT3rqmPKhib{&Zp2#b%tN;olW<{jlLiSD`@1W9A7;0Oj zItyGSdBaoDMQM3)#qbt|7iXoTxU?pe z0rGA}W2J>K0zzp-&mu>0L8z0;65iexN2NL^#bua2qA7By@bDZ$yi!b8kz-4qKqMm3 zSy@rUDNW5pq~GE!ACiK>I`dpG;_|i>lnu#qRTen&%BwdINycOq4enBmXb2vZI1(O| zJ{S*584VB08vzeWj>My)koqcyK`MB-dPCLLs_!Z>N@Z#n6ZsvLhxJ^|37l6{TCfq* z8YV)8nqI-J+&pkGk&z^#w7k@nSMK0b9FEiYE6S_5=3l$H-`hSOw7r{T1bOOMzouM_TcV@Pbm zEmKOkU5*$@PDpz5HoK}vOhg^dxLnS{rNOp&6d7K{xe4m6v{(#%P!FM;kz1?`k!w&L zN*%d;6~~IJB(7&@?yz|*D9dw_c2nhQa6WNmMWrfS?ve4T@N=C?#T9Br1D6W(?3|C4Vjw_{(ge#>F z!<8pSXs8GjXpM=$l}bk7%6WulqtwY2c$jCnpp8u&Jrwj+*lgk0xvL1UKQ*r!5V{*p((4V7VJJKV@uh{V?#Q60275J ze_-syn4@!wVB66+g>wHXr-TcA2OA72GmH&*{!lic#9?zxiK9;?<&HF!l#WTtgXb&n zJExQvB$|&lrQrf2Oey7!Fr_@1Thiqfd4=%4!C^Lt(WtNT;6@NB31`7p+N|U`g>_$5hTdG{7^C*7)rb_e%_?{qVYBnv zm8BJxrLL{K;6QSiGF6)eo~i)XlQ*gRZ%)8ou;0bl{{-Oa&s7hv%f@Y(v{%LV)hVTXe9srbL% zaZNQ9=???`b}oE+cy0z3moWuYONLRqMdlaTeIMo!?PRk)MEI$@!SO%B?D)) zqX&G*hPIKu8}Rn)MLTtXM-l2!rX6t3Qk(`cJz<(HGtc);fDkcp95wu6Zkj* z6LUa2l%E3l?h4?A@*e_9tKct0x(ryA3tunNw*g+e8g=k&1vK6O|1-)Y1Fk}dNBTOz z2i51@fOq}??P0w2fN$LhpEI7{158>EeDOR5a18?CzYcH@f)>yF0iVbRM46NV@DG9= z&rbr*+aSii0Pq9?&1nbVFAL!Zr#6cigV%)99J>HE(E*&%W+Px50`X@jU``2q>L|Y! z&|iu%f%npHW~>5%%G3a!Q=j8EF?Iz4rKbbls6Nv<%%n2W7oE#&Mp&z09s!q>15eOA z2QaBZl$io}Cj!k;J>ZK7G~QPLFU0vGJ(~e6rOztS}upTAL7Yw<7HBDzL1KO1%iuc95{0r zi@KR8vl(Xq1t`B2*c2cJC4;|gz%u-!oC=%@(4P~h7@BY&R+Om1$pZE51P-Mr2c?0( z<%m1*yqKRoP<|mwn$W%j}atFZ7$+CFIW4a8KvGsb4oEWJKu%7jFTvhO3Nag=@4xr=XJEXxdGS^Jt{CiRU8+? zuy%&dz||`KujZq^60s|BekJAtO<;bBKi2^7%J3)#I1_psJdG1M6L|DygE>3b}DH94E!YT_wCb?*!8>)V&QKYhPtzq~(Z|Jwce`%Csa_t)%q z@2}tAxPRCFzWx3C*#Z3l%Pb{`BJ o>^sE1MMc?0h2i~u&%HOLytwV3eYVf< zGz-fs0c|N^?E9Kb-;nq+f?=g zqbut?Atn&?FAX~D8JE-N^YhFSjtPc+jK{}VXIC)wem6HsuTM5AqW`kWWqav_erO^; zo*5YWGRlu<)(m|}g07((B-k+Y6~MjsH4NPf_|l1#q3=rN8ijwQ0z(SCTY+H-&h@zJ zsE#XpV*h7g9LN?QZF%xoltHTjUnG3LK$8jRHR#C)XcSV7mfe zQsAQsyib8{1v(U1rN~Lxb&Eo$a~Y{=s8asBGN(=|Fr?JWfRG0i`jZOWtiV?kc%uUE zQJ_zOwF<0KV5I^}6*x(OLlu~!K(zvU6hCz-<%B&67{8v7Yc1Op>_?QI&Rkue*vse= zsY0HBU2j4%5Tu%1h5uF=_G@Zd(-difprkcTYgRQ4(Q0irTXS=>7Rx)k$1Y&W`oM1whT+0A+vz< zWI4EkWEsjyL^;|uv+%c>ks9AX^L1w&PpB@V{#V=64O$m{^;E1Skxyk3vf=i*2>L)J`e5B1mp2IoIG3l^!@0jfgBR>}Pl&&&I@D|Mf{GHGFI?{k`gy<0?+s;*V}cwX z4*DRJ3kLl`CM(lDE}5{LGXN*}U0|qndc2%l7I9_!D_hq>LXdNM84g;wpp*B&v9d^M zm@>1cWV@M(OlE~FZ!WhKqch*>4Rf|eMH92!!`CsHlExB$*z0EKJ!2`yGft+?>2rHI zI$Fv&LfcYwIk=*)7D6g93<>u$xU@LEu*$sx*A!`(65Sz6xjMl+l|c`cv4oSP;Zw}Q zEKr<)p1^J^=f(7qOuyQ2NU?H?@P8!dT70IQt2HVnRXC;;&7#Wvq5PhGTApwW8y5%k;+D~L4+>4K zS`zkn-Jzxs}gmW8X8fu@EoBg<*<*yRcU@j06W8 z*~?=OW+VF~5dBmYeor(U>XEE59fCcgVFyYq%yh8PcBc9}i}!TOiEXJe>gKbN(`Tpz zL0dT-zyzPHHXaAXbd-gS4mM^Oe*pr(Y-U@m#ypB;+e{rcQ!Bf#4UA3utj4TSxGkZA z5e<5%E7-%vZXJUAU6^A9yU}5d%}j!zF%X1il%L-J*cP5P&`1r!dWlmse0&ZaA1xXT zlG*y3s3T*B1%C&Kw$)gLR?#qm(x_a~uuI|o1dtW16W(u8tK_Y;by>ntBubC zzzGz634rzs=&K5fb~}Iqw924e2K!~u1;7e#u`%ua=u+BNjhc<_R@WBnscl(3b~@Kq!^@)KYw8$1ia(W#pG*E( zD~&8`>WnYP;mZn78c7mhq@7NI?HD z&8^{?(rBP3qTxoFk{HclDALiK4v6ttNX>{DEObUCM$;~tCXMD7f2Gy(GI$5mF}`hp z+{Z!6+Q^bRGe(jkvu&YluU)}5PGY-2bw=B>?@G-0&Q590(|MP%0299pgO7=S7cMW# z#BY@-?+IT}E125tYPD#{m3qXF5Dlv&^Ymx2@PsbW@Do&=I)YJ3+luihRHrr5xmDX1 z*HN5KjOf?udq=Xubr^=*FbhRP9W^PYFGJzfdN8m;g^rDOh*(R)`7gJy=MJ*bQz|z4 zm6}g(nmUo!Hcg!z-kG*@hDx?0+T-D;M|-?{L9{2xkCH_ng*&kBWYM2V1%g8NSg41x z=Rx5$tS~b@KKg5rQ{rm4r&}ApAbIIlbjsA)JybM|m%CSp?umw-C`9*T%+?wur(n*M zV6NC>Pa3J)%7nYp%Y+b9CY-2N+>2S0Cuo;sV0j4o1pxM#HXEQtdk20o$8VQ_!6*T} zo&q|NMS~i;TC_tbA=;jo7%e(XI<{q{G2PzcHfC7n(P@eOM3b2o(PUuu*u-?n6}Nuz7YrB7$f9LMq$b!S@pnQZgIe3oF6XeGI#G z`Zh#3h%_r&ie?SN`JnwH_Cct;0rUO|DlevA1rVPM-TSn^K_%TjY|SUb127A|uYC;^ zHd;gh!3bmgfW^O$u0VVmR-h&3P@6hL!#dFIg7yHNy@KIQ071VEzwu!>Kw2bvwCFhy zNLWYDfmh*&Xwi0wkRlphKvB>>i(>c07VXo3(V~@7VAchtG%nB-2I4J1ih3FYQbb61RPH<$^Mv6Z9P!!7-@fys# z*{n-H^dssP?OUL-N&6%DG-47yk+#*6Q8W;v$&V-+2ub0Crq<|ox(%l`49}1B0O?Kz zX$7C!Fd#n?M>*}WR%t_T(f-N+N6diJXkUK(B?Jq40YK1ppr$PPd(qx;^024q9 zOx7=v7=8hU7FGP0P#G;+M1wOpn_~JWP!jn!@lZvnr%K^`<$%2m z)YjHMEaivU0{tn3R>FhWwzl-zb8Vs+R-fI&wsoYR+6XDVt)zXJu?ZQh@Sa`hutZPv zSpG&dcHtm@okcL~%3Ti4xg++ZgRE=6eeA(9*Pa=|kaSikNf%fhn>17RH%n3)pOW@# zOH%as8FsxrX}{g{yE37J4#{1o!dT_Ky6Cu+3zj@q0K>s}1a;f!I_ld|%7dYen{mU83$&{!jHrz^R;k1Nw_&AVY0^@X zG>dq()RR7arspt@U}?Y59!;ea&27vk?5>=jeK7Hf@hBJU<5SpkFh|>t>h1;>x8`io zFp4V1Wftg2pjP2Xndzu#$moL)p)BCeOH>X7!Hzb0cHC*V!B`|L{IN&aZ6-yXz2F0T zn)L`agIUm4cJ3`;c4`)yndy2-9CDaXY>OS|Qsgwf2x1D{fE@z&&`+sdTHSS^;Z#^i zZ%VjA)Xe5KJrcngb*Wggl?fEAa1R7n>uGsUv774F9dL2<@Tueo*s z5?NesEB3>?CYl{{c_Zf0R|m0v?mh- zKY$qAz~>I?n#9QVm*YJAIgal*GKET;Y)Qs6l=~-<{?n2i>TL8}mY1Dq`z~apk47v7 znZf~7?-)W7^%2lsMOLAI5=_>RgIjno?*OPbnXvX3Cl5 z_ImMeD0Oedjbuo8X^`Uxc}6vz;}JkYh~rjQmsBfZ zMpv~QY=e%uq`@e(uj&eSde^3xhp-c zWfB2rCoLm7`?Z(w^SWN#{A5Zm?%0#oiyQwkIDt3)X#LOx&g|GZ zT*fuKI<9{(m*N{g?R@k4-EYk@GiRQ9Eiiub3vc|?WA0jfG4taQ56&Lm9W~$mqr;Dt zq-ta1UVO^@)vgWJPa}gLimcjh-c|U{ray>fqZ)U=W%lnqzV=j8$Mn~KxY@F!dD*<1 z^JC`%$5&3<@&1FyzWV8f?)#pbQ?l_-4JSIk95;D&A`LiD{ zE_!2ef$-F$i^Rp7&TM2_E1y|=yYt{hVUchob8*X8N0&bLSkKv;cl~Vt;`f_;_Ku%C z&ip_& zXnE)VoGhMfzw5*D_MPh<`%_2pqwN(1Ve5e*JAeFQ@v)_^ugkZ+aO2B=SXXQuyZ56V zuNO>l3ilK{=iS+M-^&#%2mhwJ`0!_wWBqnM+HAOBEjAWEb1MItoBw?O=39#IeSYJ~ z9qyH-UGLAk!}Yy>ontGPN5w}XMUR}HI{Y42X33P+=L<)-9L!ra{qb$&;^BfnKCn=C z{mbzaYc?en>|Je}*8kT}gl|7OX{zb&k%z_}&H1eC;YInIzQ40|!F$iF+Vr#6az3hi zWze&8pSIq)HGM*6*~y8w<<73y^5(M4hwjw;qV?GBdoRTI-uTof4=h^nNZ~F=bWFyq zncR|#(T85_x9;9SCnp|z`prZ8HScdaf`~QJa|h;jjoVrJ(B6-42!!`YpFtA++n@iJ z9$1f<;UkEI>N4>@2=F;X?9Uhpc?mFzw?J{=YXLt(7%l@m^gY59e9IMro(K3R8j=N0 zzw0!3lVAl--zFSEZURo}${wV<0S;YaD<$7q3ig5+0n8`Si;Hzczd+RNy{78#nA~z*(deI8DFReEzlk1+OL{ zdeGuwnWOaXVa34c`KrO6ijw=JO={Yr0&hNoc>m!eOCimJ_aYo5`$#QbhY+$#;>jkt zc&wI>oQwA(=m*}L3_#lw@YLgFi4*)QV2u+cd{7{j1JbuAw8l>wC0ZlNLYssRD>y=U zaYA(mV1oyIK8f!izfPfc1qq|rNbUB(CaT8`DlOAjF-}Rl{CcI3r1sG}A9{n}xuh=# zFJ-c&oDE2^9Q4&hrobAxE%gU|X%hsbk}vg?YQ0?NB=9vNV@;7;pw|w3snm*kD*!*y z9xug9JRUq((|(Cwj#1BjSkmLa+1YSENx*^jl^>?{^a8eNdQom7FU_ z;5Nnow@JO8g3=U{C*gFa(f-hRIv<*Xy}h6#BF|7ljF{Vak%C$}3qt6t6P~yuUlu0YrF7x}YedbJNG6U-G z@&EsRpU(}fIp^%>+UvE~UVERJlG}E%D8`ry;Po;#fGho(_~+$6JjkAQ^>3!JXD7We ze;{Jz8}nB;Gd$74#eJ`gTW)zOhALw5meCrM1p}6Maqp z`P$d-er(y3NeZmDe3A4ho`2Kxm!u!_^e2*D!gal6Ea@Shejw=w{Q6g4xz~N^fA&4U z$FJp$n;NK1Wdv3;V=E(6?A5wQEK1r57CC=PgqpE?XdrUxJ{zt&z;1dr$@h_r#o&n` zn&E-|jsQTkNgp%N&lJrcs2)}3SNYO?2VR} zce!Vjf95jzHwyOc0`i-?@j-aeSc%yIe|p9?+*nsD)FN#i;kFwmbt-gRlxHaMbjqUd4Rup^{V`6Rq5$}FLz@Le+n*$#I`}Y3_3JmOE>;mAs zfDZw00)7ghzeC#?dl&E#Us96Rz6<-vjIcJc_hG;yMb5L0i`V z(gC*s>H*o%iQACp2GHL%c&5MGaJ|KM--kBZ0QG=%fWSXRheMMj=!LBcCsBPJ7*DoqvWxXEh6^@N9 zc0pfK+fuvPzQx``x>Mfj5bW*9+tw&-YG912wAh>VOY3UeD5J7;MJY8~G~L9EfFwX7 zKnEDpnb;YC2XF##3~(4gWpCd0fk?Ds4^ z^5(Of;_B~sV%gKjC;j7&KY#eh@|$heQmbv*@*-QIWohX$43GKJR)$HdsJyws0{29Lbg-iV@{x*@e zw1~?^;5d*H&?hh8e=d|O1U&P;sL;A~H?C{kSlbMCuzxNwvHTrJL9jfNO6svH#34vBfuzy?9W!v2+sr`1G3E~_M>9-yA1tVOl;^D6FUp|ru1=re$6~{LwV8G zts9eW_ufl} z&s9fXe0tiE<2#=JMbRsE&rNwJZ@BWUPY=x%PL161O!XTzg`Fo}=v%n!rGnoqE&B1v zoXrn^@L|L5-!9nQvG(Ocm)`KNv8T(lvwvTs`K#^2WxqbMxaA#J)_dP{9N)D3&u*2+ zIfCmqtvl(LdlEi*t@-yqw7+6*x8C(%zrIaw%Ta4$pW}Xbor$f(zZ(IKY$K~>0*M^+ z*2v|4fpYHwiq@NWehaH(ZLFQOvQE^gW6k(Sf8P|Zv!naob6-*Sqc2DQaW>9TQ{1{q z?yq{QiM@mSygC#6Az-uJ#D3FYV*kxiatJ^4dBY7*O z(SEZ!H%geD8zaof)d|ybX9$y%l5?-_R2keOQ3kiEX0M7JraxY~yGtF_t{O6{nf7_J zT2l3{#N-rGMAb<}qJ2d+gcPZg(w@M?RxNd@n=}nH?^S#M=-jXAbvsn4?vaKjRRc|g zw{63 z%F$r0(;_=1wGrRH8PxyR*F7hupzc(2n+bQ-t$`HYB~@5|LV>={~7V zwnWtIRR@q3m824lbAyVB7ToEE83~JM(bVkK1Qsh%OA%6;YC_9MU83ZC?MO0CyEUy|7WNqHQ@}y{)jLWrFTHS886 zqzLT+F{0nB>AM+qb-~oZ_~QmW6S>XhD=w^jm1v~O_&!TSe~GTo64_r8*Jp_WAFOJr zG65_gHpGZlO}$h}tezp2Bo46%@K+~V;%fHlgL>CMc+}NRF>9o;vrl$!kBWqV=xo%= zdWUMr6cy2JiRzvpo@RBw>B?)&n#E>~Ph|pyydUV9)$Wr~YRi=&;*YUjIp~R+NI#Mg zRj6K~oqki)rBstLRhP&FsryUxG`7BCn7DeeSk+e?gUbxboFEp*N#;bcI9@U@5{nb+ zC3A9LX)MN#d_A6wVkxvU4o}yJr66NTe8aooqpLX4IusKzR2B-cEaE^ znY@mqo*}3*tMU|9F$Oyavy&V|Xti^qZ9z|J(hBVhgM%kRN%B;z5Cc+~MxyXZZYk%NWkY5#{$@Qjq_x|^5sJ8a>p!H z9r;X@fC6x+G1k37W*xU3MoS?^;-=6 zm6$-gyH8zB;$jGCRw;3{`&4ZB5HU_)eA2!;*|IpKJ2bX9 zTw|!j5d|92ghu$9gP!P+?$F{j&J*F#xTgE$7+1S%!lAJsJTVt2#9R=r(ly~`xqizv z9Cvww@3VyME-f6_fqPmA8kmbB!(m34ODrN?q^U>xJk+Ph4N5WRB}~ z3+kSeOzZWEKq$zwDbX3+{dQk2x*0qXr#&zQb6BvitLZ%my%b{mk6?D_n~rH1IQAYC zCQC^8^qITn;$9Gz^?0hZ7t>5HS+vJaLhn&#*1LJnU0jrL&ANiTe&!BgsR)oV__iKfTbov21$ zU|3HFu9Ai_s0>`w5Ij(kyHd`R5fhyxA3^G|cIrRVe^mRzUYRb@3oimWN~#yjLPw^w z2v?ofR2}pzyKSB^0 zC)DE7P;N*jSfR^Til5LE)M-$9Yw#o8LPW6Gl)chhJ*RNP22bjyyaf?JYb4`)VIsJh z)RDm;i^ZSRius5n;}K}Q$YgU9*>4VSu}G6lz$Q?^~MK84R0BlMKCZJl7fAp%>1LV-m{wOE8ir$;5s zx`C-tuov5!Y#@22iJgwYEH$8~y9<=2qq|S(j?tZm?#|4>-CyYLUAjApyBi|)XgQ?W zN2t(#D)dXb+e3Fx(%qwU_b+tUM|TgbXGq92_kH(G0lh5mIV8%8vuS4f` z#5FDPIy7E~+H-aptvMkmfzs8)nj&511Z$JZWsbK_s4>WEEUMWXAFxQ}aik{In%C;) zAS~)W`*R1a=~h|2vsPne>=#{wJ4lFs z!geDVOCQ2j=S>NbLVWl3E27(HG~5ZoHcj!i#d+Iw9yk?V$*gngSrtZ$?yW$l%G;** z&N88KDF$5!j+m(kwsq*YdVQh^wG$BMtrB*K) zFI5q?RN3Xnp!JPm#YL~TLaLF&ECz42tK>o%I8vy#HmO~O@z#bXSPt`cCWu`Zyt5K8 zDg;PlhYQtTwK>n-RRoDOM~21oLm zBBa+nxkRy;DlbAoW}RfwRZDMmpNgnH|4wyv!@ChV%sR}X(A?a&KYx3{k@D&?Z>Qe# zBQs;R^=l6Xagk_H1soJgtx66@smqgKx{m~w|3cj1bX6e>5VgzKoZ+iEjn};G`oNkM zRn^j;FJFD8ulnyUc)hOw*nk0RUyFfB(>p!0K+mNSpl2&Qil&4$S?cyU3Q@lW1%nY)LBVrh zOM2n?ABg5r^e0xHp_b6E+GQTKHY_5U-I^dbV)-0l^Mx2XjqzZBb#smy!K+U$Kg2n{ zAujmR>upMQnGeHqP)tp_*McUcE{*iE|R_ULuLp z2ZN^JvN9;s<%tmQBf?c*Kz;dm;XF*2icRQ!a+&u?uUj-^}a=8)tPAd>9kqD=f;8$^Y#!NMzjL|8jUETkr77Ty}f!qp)xBrK~Ntbr^f9tCso zUf!Ntm@0Ly1ILJikioDF!@7&2-VK-t?yhpdiFojHuNMx51v?aQRhJy9fDoxt*9uFd zPnOWO(aW=`D7KB3oayxny8gJBoo@@1`gEq{t9lPQMtcw195I%c?tmvV1bXgU8#vuV zuq96CDV)f|F;>_jA=)#HNl#|z6t1ptgx~ah3?n8LC&0Q?OL>bRa6**T`zNVbj}-Z~ zQ{8*sCd9}S*!loVLbK7gfu=GW4Pr(JH*Z5XUB%>kB#{Lf>-9ROf?Sg@HNNTj51JZ> zCvFE`UN1F1rrvwr>%}?}J_GkDMiLPCaAp=^D(aZU^&am2a0DG7;bvY(D{z-tV>oWD z5NkA^e?YDCXgYD6fWs!`Wdgej@1KCDk?=f|b5^n0Zn>hh%JUmC1;=`icj>C&?oruf zD!XswF{v~WlRIr7mg=!}_#&}SwCJR&M9~^2Rp~`*JQjr7=Sw$;mPA}OVh>O%O~y8z z^R6PcEyj75$=ey@v?O~+Vx&@&Xi1i^eP=Nan=;`Bvtd(iCI&KWGChQg!F^hf4#r*9 z%M)y>k6@LbdA(Ilde0Zvkp*2NOpy+Fp3^{n&WuDF&i9_jcyUV0J#u|D5#&wsVQ-*p zu&iq;XV*%QTpAvUrJ*L1Q4RJozry>P=ELKSbA-ntPkR6!TjvMhF`46W`Xq)*o+izF z8gW|Zo< za{O`s?fg5k^Y$9~Zepg-duw84-VVLzuQ^!04GU^m4V5%D^0X9H;kDAv{xshFVl+QM zo!Ma0_Qri}7fk1)pmdffsPiOWJI=|^^Z^tM^*xe%;56Zxd^#pNi*V~QAF(ziQWf%J)Yb;0 z%o+bQqV1i9V$QB3iQ3*NyuvY;HKJc_+Ii6N&n0^ulezkz{tO&$coe3NJc?shj}y8I z6MKptytB4LPcdOTX?hjjZ0JKdq~`Ke;Z@g)MP<%Wopj;V`?1s^*V22WY0-L^>ZcUU z3m1Wxb6gV5o@K6*r{HtYDv#U%$99a%X+Mlf(Jc?myq`wJt5;bYGJT>}bHG0)M?#k4 z2&Tl61CrS-O5Cd|IV?)#M$xQztF$5sQ&pU+@~QHkebgs;P`typ{L>+pyfiDq!L+S6 zys_2;Q^A709N<+U{aGwT=+EjcQ(-MK8+`9|Ywv#s)WVdc^L8TDo9>Ch0N=e@xFW!LzpW|O^%D8@(aHVv17%j(PoM1VjPrKtJi}N|`TTTcJv->M zBzT<`z1MJXn9lg{Rn(DH3Y1)IA~!|f zictQWj_4OZAjxG|6jbWV=`*3+3K0tmDEu@7YOp^UMmHhAVtk{v!?e1Rx4H zpDd6Qcr2(>-Kkn}7I@*o$pp8^=*rnPd zsvR*>-&bABf*khGa{QxzD z;D~fA=dapci6})ixUDaFSCY+r$qBgW)ndsx9?VO2mFzET$}DRbfK-WHABbHZ(J?A^ zoe{gnLNp=S5iQlcE3Zb)ABe>|$?T!yND5j&>I$iQzl;>tr~AvDL-B}yj?(&c$zO%* zRtWP}Ks`n%kOd;UN9M@tp_3izKrMm91a2H$ei6&5ro`R8cN#W_E9xAssKil zv1&pjqFBe3f7e9ipRkVQP0r*^(t#>8=?g}TueNDPS*)8<95)=34E@1kvVdktS{jC1 z8A2hwsL~lLQ(-nNt_WIk9|Zb@b>A>zh?sE#H%7=KVcIv{adgC zixVyJ0kb>iRA^mDsJ$wg=2C>E11CK`|C*DyBbvw~I(mK|Qahg4#?H>7;E7mM{gr|G z+)GZGL2}VE+6`WSCUMS-G)v^tEU^HxRbe>pyr`$RT}Bm3(ndxCayW>%=BvSdlhA+cyY}I=~k7rRwG^^#Yfzx@?yO_hzgo4M_l_G zhU4eW@eyLp0kP68*6bH+j*2zM5cM`6Q=vBUA_bMRig(NS~?f**2Q{2Iq-iLK7i3bUFQcTW)YVAEwk(5-aqW0z- z(B8*IrZT~mJ3VI%w%{y@+x=9)oQMM;7M0;3UDO7z_7F}OYkwzZN~Jo(d5S-oHDa4e zmI*v51WsIV$7%Eq`!{CqT__<_&{`cWJO*wB+Rp5W`mUYXEv^HI$7T) zA*2yK@F@_Y0DIse=C2&P;(on`<3vCce$)RVrt2r78e$rPxzjUBt6O+2U2#sJ@QLWS zpcq;+Ns?Dpp}mi^+V$9-G}~3Gof}mMgvYU7Z|8EKsa>I#+F)ugFh#xpN_-tfg?20GF!16_>Jrq@$pglOVL=UgJFRXYJTkM4`uJF8t%&*XhwG*Yn>qvx` z=5^@3js&kG9&0kJahoFH6nGlYK~0bdg)baIg>ym*_l(AYY2!gFG8YIE!VM97&!F_> zrvN3emU;r~vIE{Mvksy_VtsmMBx70TI7RJL{4pMn=P36x^qFN&z$3y>?$aOQaez)& zWtsIz`<&9==0y_mI7W~A@L2ttnm%2<=YW{v1DC{!t^|@Z?CiT}=95Yi3~tet$ZzAt z-ZviO`?p#c$bTs#Y90>7tSKx*kG#syRxkjmjYF|f94P^}i{7mt= zmB<$#v+~c?_zZHk)mg7lHDU5_FZIPY!U#EzEDgZ>9HlF0K0)a5k}%?#*w({zHB-34 zzrD8u3IXcRbw?BRN1C9MU=<$8LXNDJ#dWn*a!#rolU}Jq!xSE_iWBcj2oQ%9zC>KY zw~iN9!Kw8=znZA@JQ1ZD8^_#9P}VNLM*s~@27+!p=5{!dVs62PPcyz0dX2{>hF_v4 zge!SRaeOU0t)gj$FZ}W9dqAv{7T?}o0~j&+tLGG{D8&{6wxrLAm17)btcntNGqX9q zy7ID$CS_+`1Jq~+YA)l(Sw3h@<_M!0pPOQ8)GDk0)Ptp~#7?Y_UK7uGjn8 z9#n>|kYFiPp|f8WAqB_sZ@7qD;kuC}z@y(3?|gC{im`gA}ksPh)B{6pOn(%4$K1VfVG`!DIwQ5e?{edAWNCn?` zw;A48i00huJ)4qzn|ZXxA+?++CML^XI#%}J#M(Y7CleU?p6 zA{JifNdsD80T<;Q!y{Ru5gQx;Y?>&+;4XayS=cub*BrOSWL%$-Bj-2un-iNNr8oPE z^?mE-ds4_0(B9W1tlwq;>k*toO2TR}`gX4d9>+d7-V+x@pLiyLP|613ZiKo*HwZBw zG?3@!If6F*$Al!lydxS-T;2t&?PwF}1~jT{W(}jWiDBD(3@?eowu>;>2@?+nOPgE| z(#R;5JK1+ihX-<#l|XJ%@EY$ji_{qEKWLFkFM7(W%8)hW(fLpqT7d>`|H!M=ORPG7 z%4^_wSdh8t(ySAV0Wvb5n}@kD50UM&Fv5m&Afi9k+5{8_mePu83OVew>1S|*Z`d7; z>2+5(W!kWwp*bcPgrZR@^QL^ijDkfJO~u7L7Wu{ z(;#h}ps|0G8|%q`2ud8LK{})1D%LqO*>H?%7S!b`m^V7A#Utfh{Y*jX7Zi{PHbFF( zjGG_|O~&KMlK32%22MSnsh$u<=EPIN$Z^NOU&Jz${S>^yhWW-ZE$0lzh>(E$^bK$i zdb)I^huZt*;=wBRk|lEWw&9b43B7a(fZvd%vf=<)LoS7=9-~M9WQoan=O3UH&R3|O zTHAYoKTuu7>|Vly)Z2^)+LD>d=a}CKOU^IFqHtmN4s5MlZI#)s@otUtZq<1@$ja~< za`Ae48&d2Q(1c?uy+oj!as-F4MQm}Jb=nsy~Oxq>4;-{ZuwS)7l|zkcfkxqPbSr93Q(tHu1<^jYzzC* zEv-;{pCf9>=*eVO)-stFh|DBXu#2&HM-i_ps&S6EuMj>y&Z~kDZk$);G4tpFERn}I z@0*@dYXLM0o|d@E zC~@+WM^P7X@1;6yqU(ftRpl;+x~d#oTvnX@ZwOM~^#=F2{U^(L`7RWn{Vv{x&N(0J zVcOa89ZM~f)KvkyJ0a&%FW8%esz(a)YgGW}PlXf6(rgIL(Rbe2*5ws;C zS;wT|#EA398t#V_md$m4eq_O*Rf=qa56Sm0UY|=RQN~Y93_3TcnR9~<@pv+BzIiTa zdW-Xv#*9yIhlmdob|q6Wy~We?Mhhw5d^Q=U_5$`Me@VK}r?doRN>f5NgK}^3=WxU* z(2$m^u@w{}MPSoK9e@mo0D&C$9uQb4hERtFaleK%CjGBSRU&) z8IBv=eP->*wM6P!I_wup$I2_jN*o_yc);N=1?>x^SDqh?!0FbzbnyHU zj}foHb`D-H&`MQ0@fHoGELD%pJ3kmHOp-cvh@?kK7yC=JqD?1O$MvncLR^ORAGy*& zF(ukC;2Y}IuVuoVj$ALrI_(#jt5nUWMLJX36k)Auu&psM=P<&sW8PUB!YLhd{qJc# z?jN?lAwGe8E-W6o3a+tKs*#RNc}vih!P{1$rn4SmT8|f)sD;0X{D8R4cMz;07dQ+@ zWX=W!`1!5?iTQe#>CVf`f^#&XqH0`Ko1)hW^SDKxlX>eEIGVl)J^P&-;Rx1d=c^jJ zVeOlus;U}rjci!Skt?E+jCrvGsRV3k@hjFK)n8ckMZ-P zU}HHKv1_E=*^dG^(}7cdjtegH2h~kFYlB*U4S4T6^9ZBgCdKBMFLchVuEN0?-UOWx z6CGpH#hi=6Pk2!}W+yFENe2vXFFDktSubdp3zJ-J-rLW|JBKvRA&8((VmW6zvtc<1 z3x3Fqj$CoijEjVZ$*bE>ht0I(>8_UX^_3hsj_qx0d4*@w9LBrCTYqnhDRA7xnOW5; zSJ6JzUAMVbO_B^^(OH%D)vaFVKUBi}(o#oG{V;aN6Ncaw;0mpWXfyi5Z{&Jfr5O77XW-Bg9G^}87MN+8SK9(2jeW& zbeO0q-erE7e|}(X(1HIrs!Q((fa6EKvxvD_T}OpESzX738D7T;DvX8JyPog7hc#0) z$Jw?XD@8i8I#iF_D~}sFyjIST zP1wj=&gVW|W<`_YNTi|Vkp(ybG9IW=wIfiY_yV(#@A>U`LjmQYshkqf`8(e5!pvyG zy9GX5D}RR%LXs4F?XU<{c$3TGSP?+X0*+D^N68>twYZV10}71$SSrQrDRG}J-k)jY zMvA2(JgWBGGK25q&TESB&_N}92@mr;gc~XVbEWoNj{@VK2>Dv%ZQh8nDcnJiDL((Y=YB4``CBIUH0|Wu`Lv7uJ`bt|r4{hsm-W}I=<)qDScs>4^M3HaXddHF` zwI>Db(%OF#4zku@bhQ0PZ|V$8!@=m0Y05iN(F2ht)*UchJ`kF-zIq^+^NaUW+#tvk zkqLRZI`4+9!w<8>4(K9O-6O>qt&Z;_D^nvzn}pF8VlAMIHq$p+tT4m~ejxBu@7UeZ zeB8=#19zgJjCk$Lzo&iaSJ3jO!)ETs&URpo0ksiOObGwdnugby@Y0%G;}UU z_oGJOF*ejd+|!R?xIO!y@_KmdIx)a<$T0#O{6>u62$_wqiV+m!y+8Q3bQnP$9C!;1 z82Cl*vTxn3yu9;5dQc5cDSGf@=+^|Dx||-kF&w{d^9X`ld?6M0zjGnQyHF<9G~UJk zzJ(NTT+!oWLmYoB`mmqw%^KK^iB`BXYk=Z1f{-;p;a8zKYaofPby))x2^ZF74NzQJ zSe-R+fvzjF1}G>Y6uGt{1m(!9z*;ZqQlUMo=T9T(6K96*)JihLRk)TUFO5Q1VNRnZirvDS+vKk_V{hIa=J<_yX) zsjlUDlhL8J*F-%o)A3i*9;RqjxrA?hi0~b7uO5!&%iFbTe#HGxz~G0btJmF z_R!tw?}7M4Ep6|cWGGjt>zywW{*l_=SCD{qsF?PJimU3y%7@UjiNZ{~Da_O(hnb$C zggun-Exp+Fa?s)99DevXjUPU~ssEJrcl|@wdZ|Z_Kvg3I#oOQ^D9*!azC*|_LT?d0 zcA4GY=horY>N$(u1Zjl^^o39D-HQDWl3Rqra3DGce8Yo$B4i*30VV$4HF%2G2S5X{ zWT%8vN+o@?1T48I=UJ+N;@Ce8{R2Uy2-KBH6C_Ggr{SL47pZ{~i z0mLe*hGRH~3)OQEe*V^iHxW5O(1OQKJhL%SoSWxC%&w_425vZ=o3I|_iN5WB{~`33 zCXq)QWTDHHle)@o7!h}(^Q1Uod-ei%hr0W&c!q@f~$n?c{TCZB$V&A|37?G}l>w132L zeTlEP_X&v{;j6fV8!vNEaUuV85LDpxdb;8sLhFikn2G`Cl=@8r?S9dA*2A<@RqlFt zC0)2Y5@qPLJ|Z9TrWZucMIu4+<{~L3L35G1_nY*6y|t8O##v~-&(+sU%QjcmF^OTn;N2WMF@QSJmh#QaXh`yAF2+++XupqiNEim$+C{tPcd#OdUF{*1n!$oE>z^(Y@4S@*3r z6+IkZTKQ*h&m~|1mwBSbE)#HLB0WX_N*MR!hfzC!$8C(P#i ziGFhbCW^?dX5FulNihl2<(mxxj;W2rHt9wpaYk0+!k44&)N-dsdYCMeIC`IZANK#kBq|F;}yA2eHp%-v)bFMaAf9j`M6ukiV? z1X^bX9!tR2apGG_Ei-WVyf5hRc~w)U=l)T7*-mfULjOW`aAr~z)PE=g+bVdmcEYX- zH|BVuhBg)RNrUU*%C9c-8I)T|<;>>{ z<}-#f(h$A`hgx}@PUNir{K(@vTz!k*oT-%Ph&;RR^kOhKAaIK}tsT=4JMzxjfW|!J z8SrnZR$j}a*hkN(9@=01xalVdyiLue&smxDInPL+Y031NmxRyy`)A$CFH=|Y%e+PW z5}n8|3-$bRLjo?3ekay$dZ;{czh)u<(bbb zPA%dkOh}rfV(w*PUJ-5)mH!D17gq{^wqYm@8bnRKvzqCaN{X+XxPZaUYCr-=SBHzu zM;-6N-FT8siFe_n0r!st+^dy){+b#onE^-U@Le(pA78Lmc*-Looro*n;aHUOS3!#j zPYJ|Qs1<@gp1}7Xj*SfEd@jrhQ#R-Gk=0G926tA@`pzuZig;!w)2ZfylaS4 z)Lm-C(=i4HBzKvRSx!C%uhL%y;=l@OJ9<0mAdxRX%6!~e+2Qf!<|YBV>@@?cZ`s>Cm=2E z4&)=v7*O{{r0qmnZa`Wc((XrExWE6Gey^q~xwogES9C zg!|Qwem0?9R>2C}?f8uw>R>4wW5w-u`^v_R?X~SYn2{AWw|3YqwJmk%eNkXfMRtEr zvm-6`+BT{O#_{{5&GfTQj}r#g_SQ}Ijt(k;--A+8me$p^BNfZ9$d#?Nb-wb{0_IPBO!`{Bbil0JhZQoMcvdLc9>L6OMzfwP|8#mp~ z6BsL|=R!w&JASHXbpwW5x4aI&&=ZjALlen|D79iohhX2bx^atrX$KL8?(phkQDaB# zMm+H*d`K78HaB;$^--(s?OPgKKuBLx_}Qw^_E}3*Wed;;IqNVGdz;YM+JX)UX?9Cg zgyN|u#UAcX8BL?2i8(QXQtm|0H?uo2yX%9)qZzdI0EwwIJ@Y%^WHQPvh zo7>qYFsiw6BS&;A^|i9LLom0uw_=P&hUevN_>q~_geKd@D(o$FOUtdSv8|!Gt+uUE z9$%{@4yd{q+i*9&C}2HfZ*O4iEnMH418Kw;!ZZSQ&SR_r z(2sjB(vRV4j>m6G0S@3^fgkzqzmJ!^dH@>0ZGhNB#+Cs}uxl8-2tRp->t2BJ+=%lQ}#eEWBGvFCKuf%mB zaL5C^cMW5o0e%L^TY$CzQvnMBuOjUgjPWf%ByiYvEqlq*Wwxt9AC%vbEIogE(7WFkhUM!kAU}br0oSF2T(3i!HK4B^PyqM^+J7E&+Cl)l zdjYGFwiS8%0Y3*2KLoT_4G;mun}1Mw@a$i@8G9P=3g7_XUBG96X+4Z30x|)&05$^d z06YTt72puyeE_?Mu{nU705=2H0~~<+0sjg(05}2o5}`~b+d+zz-O z@N>ZHfR6#w?uRY_iU2miJ%Gmn`vIeXnJ)bP9H10X54aETQ^4;4X8|+54ZQ?d0nLE# z0G-z3^4gSkT*a!pbT(3pd0WI;BmleGH4m z_moazli3uefy+6KY2gk{XEWGL7RP3>**F~&&#qusvbp#T$a(B4b~V$p`Rp3DfF-hp z>{@mmTg0wsi`f#E#BRXvCf&r6Sqe*K23X%Tmd-L*Cd*>kcqtC9IUKVpdkh%Hb)jW|f$stJ$q=4ZDr4W$RcCThDB41FOZa zIc;Kf%+BiBW_)vCBhFsjj$doq!dmd_O>Jx|e#NN+zxL$7FF9?)Z(QzRcd&1m>We(i#zjoBJE+BQ*^t+B(_ zzNM|rWaMdeTlmkabu_mMw&qq_L!&7rUCDsEWedzm9VPwi;D z!yb^frMAwt(NQ0e(t=+&wFx`g>^69R0qJcmZ8l8OE$A~iy|tZ^;9l73+grC>Iybo5 zR&tPRyfgfq8@`^0C_q@s8Ev&fgDq?Zx@&ByZw;FX&ka8S7t&Zq=cd5ojdi@E&F~_W z@%fTE_^&70+S={)joSnAz>_U#xN#HQg@9yYK%Kq5mikQbQNO)`H;&(=^YOF3xpkYu zw9Uw+N)vPCx-_hIncsExPMaUQfb_QZ);i7{9~J>wTiaSf6G4zRn*bNl)>+#enk}@~ zw%J-7TS9Zd=(-RL zZ0y+40Rji1KcUd2b7>?Hg1{z1`;j>)ccA1t@XLn~%DTqd%@Bz|=C%5=TDeDUgNpd5 zO{&+}(kR$k?AuHRSp@uPM4r}`4!a^(JOh?NNtUex+}e#0G*X+6O|5N;Voyw#)zjZp zK&56IiKnfhwM`Mxz`CT=3bJw~8hTS}V~ejvdGx+2oOT?G@X42~6NrS54CB3V%iWtnEY@GNVC8E1VQ$v6T4^xe01%# z{&{h{fPcWZ+jnpY-B=4FC<|2^rC}f>!wWwPRZ#R{l()^`_=V z@?qNTh#Z8}rUDJETe)u2tQodIb7RZxwmSUSFwOlwqvk^}s3%^oqs?xwv(;`A8apr3 znq11x<0Fbs!c}@)&8btlScgM#4kr}ZqA2EsblE~Q?6BF}+igNa_zt$TZ45hF2#;-3 zZTKEr?2VfnHo~+7NLUaF!0OKCjI9W&~%OyoExM-5%a0KziU|q0z@oafp8*r$#TKM^A*KN zc)}Ke@Ptbzu?UwQ#QJare5?u5VI6H30HrDFvvw{^~Oz?-^KN{%QMJ+Ap#0Zs;pCZGW9fGUT&zP)<$k0 z8Kt(`9kh(JwfUD`9nI)mphzVZ@bM@hk=pkWv;)y!_@=y__QuxsMq!5!qkx=173$a` z4RnGC=fy0vxq?=Y!~!L#~=_N4wH^U4TvdXk7f_z zw^2cr_=73Rh-(!hj5d6ld8~0c;*OpgtD8ZF}lb&)~5!Ix>Hbh|~{CyOmLwvE8xkB`&PtI8+oi18)}H*P7CO-3MDDh zI^ItzBEZQUl5wTGRE*Jph%n;Rh%ZNQItaX83vg);7-1u#x-ICb&DZOVxTo>b-*!Zp zBM>3BgQ8T6sMH4f&`2l(j7{n=X}`c2QaK)(xSL2nJtE9B(gse?X0*5&tq}$F=t)5D z*gN4kuk!Un9HrhBp3;-doy*~FK_orGN6R|Iq=~kHe7zKJs(}sBqZRzY<~4B9hO{%0 z1L;2cr}I&#;3_-41AoaBvkur*L$;93gWEwkF$R#Q3z*uX6?KpoYsrFET z|FeHO?4d?rFEYiNvO49~lyxZ^QtDC~QkqlRQiPQ4DOSU3!>xvOh7E=~LxZ8&&}I+} z+YNUbx(z!GPZ*vuJY(2nc-Amrc-i1K>^B@R95x&=BpZ#!Ok=LmWGpgTj4O>+<7(rr z#&yOG#yVqzvDw&W6pY)AcN%vYA2L2-eAKwx_=NE(<1@xR#%GNK#+Qw5<9_1-<6+|w z<5A-=Lr;?m;N?oZp5_E6d*X^*DuPJ1HlskCR(_M|6z)d>8A9ebW8e%^t$wh^yc)o^j+x>r9YDXX!`E-C(@rveap*H%e?ICvQx5ivd!76vW4usviq{X zoBe3^8`(#*k7u9CKAo+}nVEBC&itH3IkCBkxhr!Ub6awIa(|I~Ltbj0Iq&Yg?Ik__Vw&X3z_a>i7 zzAeR;(w_2a%2-N#>b%tC)a9waN}Xr8#*k*{F_?^v;KirL&yADQ>eJdl?FH$J(sRRu%~{=9hqDH=hqFJ+l&S2N$rOv)+FS&_3o=VZ>_+{(P$^1Ab0 z$a^_&U*6|=YS7P?Kb-%E{CD#Io)*0wWXETxWiQQMn_Zjzt?ciB^FPggD%+j?UUpL5?|RYtFZFp3HeQ=SWUO?vmUWa*yPGl>2dRWL|sT_PodQ{w?ox-p6^LZT|NByYpT7-_3tA|0T%6hxsG|_=ygB88l*rV%sn@06l$w!Rn7SslHnk;nSL&}*2U1^6-Jg0W^+@WUQvZ_rx75+p zv#DRCMi_L4S%$fW#fF;<8HPNA#b7n8gn zTC+S^`<%V^*=L`9_IciOlD}vp2_=L`0fIotPNXzw#rUTqAF{)5d^McBJnqo6oszsm z(~8X&m%`z+*Ex+1iW;NMX6F>kScTJVQ&?<@oP`EOgWbf=nK*HLY(Vwet?mnZd!}p) z!f8^wZ6D~<#J#OgOqXu!LMrKDcipzfMSkP9RxxF_b&F|TAgvM8B1?^#>I}9( zuOlQ+A|<;`n{$J_lO#kkK_VmMI&dcWx!b2AjRYt~S|Z{hgp2`6A&h{aVJQGk8`@3a zKW&RhD8QoyXn8;8M-aEh5n`g0mXK}K;a3JdA)6(H44^DE;m5*2)D?XL`Mw_$27eC6 zHgQNlos5XuMcYa{dsI-MHFHcx&IsOS+DIjU%G>}L6|{tO&2hS%HQVtkn)dE7hY1P`%Ruqk@)@xueLR(f_wj(27C%Q2N-h;Aul7PAr`n2paB#B zdS(#v65ugFE5HUQ2jl|g0ImZJ$72iwjso5UybPcLW3YK6@VfxD02gS=AA(*P=or9s zKn|cB@Fe)B0Ph3d1ndR84A>5M0I&|w3}^&c0l}cUj$jt~!^cyIM7Z%19pnes06LQ^ z0IF!fIDc*&F@R?v#Y9K)NfB9yUonXzm9TgffFmqvAPh+a$3m778)mhGIEkG!k~zSw z_|dQsTo$&gU&%&CwMjBDlk3#S%Ehq_mDz5mleKZ# zb{prkTMaB%X<(g=mKxSoS!S`h>@~|P^+qhVEXO)smBrb5h25qwWEGHw4%W6{t|H57 zwHR$REKwlLwjf$yb8$wiRl%8Ah0CvtuGozh6YCr=N;wf1u?_Y{Rul?o`Y~CJ4Yl8X zc@dX`t+CWvSksTxV1;;|#l>Bf?HW(p?sggL*mOmlOR;#i<+j;QB#x@8@gz=}pfo9F zE8+~}Ntq#AXLC2`op#P%W4F2#PL^{!ZK#rUI_*wHEx^edtqK-WtkcL@;F$P0(+t#* z%NiYUjJ+m)hN9MJv9hMC^&#sDjsC)LI(1)l&}HcgSQ79lj1k=!P>L2nF+wq19h6sK zgEU%UnQ0ZaL{S&5o@CqIR+ED6mvt53AYaT!eF?21~8%>*0K zPn6M!p+~)IuN|>9Cod0e`oXeRyU`Ss3wUTLLnJQ%sISH}H?M7OX~td^y0&@kz0FZ! zVLF|zt*tE#Ymu^9-`q@OB>onazO0#yC8unQDn$5JdyvGp2B~=lmGj_uYdl5iCN##2Ds8I zm$@xgldCc@jY+8FaP(PTxy)j#bYlWl+LzVWI@#)#HkPYyQl(T{7(_NxppWJ_Ov|pc z%X~f7yJ=T&d7*?qTPnC^eDc0XA~A28goW;Xv+o+3c=pMcRPB|LJ{m+x&slkCyKY$D zt{Y;$)t8of-9x^M;e>dG1osUsYli1z{p6wr_pn*^3uuU@Ix8y9P`?_=1osf15erVe zOgN?&q7^{-x%PQ_ zy^vzwK9UW)TY5P7R>(i+{)G9~P%73p74=mwUeeorsXZ-cn<3UgyMx*9)eRd;nG=Tg zw$|?jL2oeMNJ5CgU@&inH_VW07NTX)(bnazsE;(`Y$K@;HJFc6U%f($2+>l1c6gA+ zIZ4uggAg5$RNt`+v8PLG-UZ*F{auvYClAU-{h;i+%Vf(Aw5>x$_raguN7n=MKDtqi zHz7XIwiz)!CGDoRwZHxd$fprJ?I(c=p7wJzsSjCFA6d@yW_jC=gZ8$4NaIJh)42Tw zF+NP=SKbx*r^MJN#%INNK#YeFGhNjYJ2sZX0kz&uVZ-2r_SCB1aevX?)(k>l*e;r@ zH%C$rQloqy%Ei&LWU+fz%j;C=!mU`QCz?$oC$$8Whi664tX z;V9dBFPkDQXE$N?UK&zm{2F{6Z=JMrpf4@|#F)>rwalnLVDNkR{ys-$*pG zIjQD0_tA*mS#+@NM5Ue~O9hwU8M1P5o*^eU4vK~^2txnORIhuiIFkEAg_SbjW2m!V z?vtUK{xIJWBto|kaC8Ti@@^T*M4&Gs`S7-0E<`xu z>5;14a_8A4RsMugN4T16XH~OleuA*NpvS9WsZ!*m^pMtFUI3nN8a7IN%TNiij0o^eCB<#e;E;(}; zkvBOH<^{D)F~3U-_0d8Xu<2A(90;Sj;3JY&b;>Qnk{G=dxoOvL0-Kv`rirhbibDN( zLw$%rh^9OtnYu$rrv4C;sY85p+GFwP%mtBly#}#672UiI>UiB*UdE5}=AYCT`kF9h zDk{2p*2n8kiqg!9fc#jzfmvgqz&+$M{TfaJ*@CdTus~Unb=H|iV7im9c|gfO2~O-LUgN0m5WqIGbKo8 zEA`$T(Ba95SKJeJ(2DJ?vQV~&)kmg z7kMKQMiE~e8_DOyM)7&EF(pO%!a_4PQ;A7PMhgN{eV;ccR%Y_XQ4U0ve7OwLvzW=e zL+-hFkyArQlIOdNoK*eK#yRatjqR0xisUDFbLF19rNrH5mcVg*w#;|OSVH~`TqG(% z^uJ)*0Bg~#eKFNb7I#CfbTCT_JIPKe$1$i+#^=hIgFXhsv4l6r2oAa6h!h-AzT3gU zIH?k4&>>Seul7Os<0wPSP}1w4hUi~D=95A@Lw=p%n9Zw^DT59SzL|T z%74LsO*_lCmCiVt+lzcb_#8(FIH{gs4UV{hND6%sFdL>;p%!bbtf)}nVxxR7W3Et3 z_R20hP{i;$nSrm0D&<`=Mp?#>@eEz;(Fx>$?i_7h``wY;1inBT`Zgn12_CN?B*?bs`@DA!^NZ!43m3gQjQ1F??ku;oe@y!ypop|bE#~tcifZd^ z&v_`)s~g~nS2x5(@yC2}G`+10l}c7??#bqLLok6q?tZIzjX*esXwgsFa1L$dPgixr zWLi=GA(+*os&O@(bkFc)Y{amH8Z%9O_rgxkJ}IVm1fL&SFDVqFDc`eC#^*=*{sb5C zx~Q~rId=_TA}cE6-{BAXcKEq`{t!mX|8C6yh5~==OaV&03J}r^@`%2fmMmXuZhw&oUuyJE-795)Y8Dp|&N>zlvZ)Wobg+2&^CfV0vp?nqnqMA~s=3 z2<#2PvNZLVhoQNe(1Q^>vZ4&75j$3lVUD(aTh^WDI9Zz2#wt$*V{uMZQJw30-+J;EtJl^+i4}Wl+vc zUK{&{@5aHyo|C}^vR^^hS;7;hHv`kNOEjCVvw;Pa>GIV;ZO{`>jd((cdtwAP<3Ju< zM=(Ppz1G;scEHS&m{8&wB0)FI?ALo)A3Z(mh6Bnma@%zSrcP8qy7~M8lXoITE2{k? zvCv$Z0(NV$mvR-aB zQ-|fjAxsxK5nFkd=e)pars;;o4t3Au3pH&=SI4$|2SXxVQ`@EEy}DEW&gH`U$1+`7 z>K&#F+(0XQbD{3C!&5!#&qMuid92>I7TLhAi`b#9tj!1^?y&lh@~ZCIj8LG^hXJIM z1zHgJJc$&Y=3$4#el6Qm@VWLi^#LyO8G0wpjT=1N8HLvO8}Jn~gsnz}x_I|6;~o;C zHNcqxAzB+qbHo&GhFmU{)16~<1I&PW&>gmWA@q8asV4{D_a;-FOR7g7d|J-NW0~z- znuP6YVkM2E%V~UlF^!Y;G|tFFT)RHHNlYeM#pL>2F$v2NlQ^xI%+?@zWJ8z(39loa zCNY(Envu5aWStzP+fJfO4}{UrUkN(0gJyPGK^+LApT*;(r0YPKNz4swTKE7!kq&A- zz5;+!_tOb%ZjGXIc@{-!IU!mJ6n;oT5@=>$DF$rRgf4eHb`rWf%9*2sU-QT5MuT*K z?;bRPe~|B?Y3Pyu`M@i|DUOv>nImV8oH+8uk>eA3-RB0s;RhfU1qn=o!OugFIK$PD zTPbr!s3F@pu3#oIrWD6UF(-h@fXN3x7(DLzfbef(k^C#IXUX2k;MQ#@d`+oDKTilC zFj4NkQ|8T8+bVH z|6uq}y663i1h-b%R=bP6lU+qVqtraB+hwMdG~}{du`EK4h|;)m!C`(2LH1L{?5|mVN<~ILBngb?z@i=#Ak#C(EL1L|nW*B%PA^EQgCV+{U`71R*OZ zC(r2Obo3IEkUMon1-iVTo^PZMr^UwA67smTHh`XxR9o!J@M#go*{$}KtdrbNs;hAa z*kH8SjLte2iIP;;uuT?Rpf8hDi%31GF0XNL^DI`}`jQP&f9`wGWV3K!HDE2NHa58G z{Nm(YXmsIv6gN-}M%?>4VcEY(wb4}Xc5&6UCQBm@uaI$6SJ&7Z=nEaaj1cKsKXufv zq1xzhRCB8w$Zv2p);Kx8e^N;%trhlitX5l*RbIY`+yblpm)5oL%bi!<*WyKi@&Z?c z!P`mP8B*?5x0VFl#Ok0QXM`-Up+a2XE@w@;rW)Lp*0H2C)L>;<2bn%PH{0IeFgkHP z8W1AKLW($RiESlrFv%UExIQhh`L9~3BGy5#i35JUhxTQ@-Q>2ix#DFk#!_cUp__HC z(z8z7FE`MOb==!hTgcz2bg`voxyT@79;E}f_r+#<-@CwsTi#LJz`z0fP0ICO_!nCm z*en+;U@1nHgcb)K8xY5n+0{CTE%Zm5@q??uDCV<=iVn}%J#RzL>FXWtZWY~nU!1%r zCn$mCN(|y61|~gR2J^%eU#b#pLiD3*0zeB8u!%M!Z9z%;aW4ZX0QUe!!V@9d5&U4- zCHy(mlOX?n|P0$CrI?P1*ZSfmSi_ z*ei~i+y1!!QH%C))%S6yue)#IXCqDApXNR7P{A~Ze?{fOLzWdE!|0j3c_2An&w(r3G;ai@2V)AF5 z^Cq&_%H1t%-HpAu`Q>v@ZT)SUI(g?yUmEhap03K+U!~?Bdti}JwPm0~(N+BP`c&hQ z@A*ai@wh7Q`4e@|J~;HvEqk6kSoKk}EwA?v4=a8XQnYkmoEz;Whx!irx3swf&h)#;&<{oqliAp7;Nrx%d0;-!isdv$F5MPG#PfxA>F%eY-b3_<3*U1N#hW zcg~@x-M@c6^W(Z#H>K(Rc*BctZ_13lx#!beuc{M_{9T#Gl8QHe^`c?*q}Qr5k9~Dp z``b^r>w|sHWwp%mTZ0|U=%d~pl;g3pguep0n-_6A*=)!(;HB0G`BkvS2nw4ULOxRZ zmP>Q&q)Fs8k$98|){z4V7hZ5_-457bft*d0`{~zSYL^r1q2ESow*@xQdQ9L_n!Yj{ z#oGO^!5R`is@_u6>x9gu`V#T-tQ0|slwyf!D?<`sjlV3di@sbtky6cMaNSL|a^k4^JFPL=Rrhdj_pY6zxy}yV8#eyr$E4Z!;{h!xB3* zS-^2YHV+z2(3vB)cVVC&Y8|yX=yyu_d+Dca&!sN|#Q(BxB5u^kjcWC!_xX9ywoI(4 zMzo7usrswU_#Lob&}M@b3wcIMYh-U8DR^c~aS$v@U5@vNE|Vd4Fdr@Zdc`~3RPvQ7S(#yBdTXqFRFH^_NaPPM^z_OU#h-Qg{RF(W75*nGSePQ z+naVW?TfU3r2Q)`E&cZNob(0h1?h(L^7JL?wdt<(`_dmye<^)e`rGM$P5(Ikv-B|a zc=a#TG3q#Vidv)2R+p)()HQ00x>3Da-KKs@yaW!2)ESxunuVHD zO|^#AG-zC!yEHAD^_oqZ-)mmie5pCF3A;TeQ<1649H5Jh0{<q=gMk$|D?o!^6 zaBD(xLQ%p~3C|=PO!!+uZsM;KwF=NY*54 flPi;#Ca+Fz7H_Dj5kH5YJ@B&!e)hn>&jbGhcM2rP literal 0 HcmV?d00001 diff --git a/branches/winverbs/ulp/nd/user/objfre_svr-08_amd64/amd64/ibndprov.dll b/branches/winverbs/ulp/nd/user/objfre_svr-08_amd64/amd64/ibndprov.dll new file mode 100644 index 0000000000000000000000000000000000000000..347e84a67838a52e7a1309b0a66051551313101a GIT binary patch literal 44032 zcmeIb3w%`7wLiWmnS>!E%z%l8mkcmy|IZDdJ!`+t-fOSD_S$Q&wa+2v#%31H7}FyJgN(HT(w~+8e)ms5iYLx&o5-FW z|H`aZo#T~Rd8K7FW>l4$zKfU(GiotMG#T7_r{`;(o3pl*EVg`rbtC-56Sxcu>B=Ph`T6#8zYs!jBsjaav z&&_76_`VpHzhS2ws?*PAnTM6JA@v%Crw`)Le42-NQZgor;YV@o5F z{~P@$C~(U|;-POJ6mX)JwrFsP2G?jX)WyeI`Wp>)YVgpode1x{o;a?-H#PW?2EWkY zqZ+(W!y`k3vIcL~prXMht^RH;eMp0+HF&pHUai4m4PK|gCpA2ywE92LU}$_l7&eX! z?YRpz7^lI{n#B_V4Vtm8>2IC}MQZ48NcIp*--sYjLokXu=3iNP#IHe-%f$jkmxY$J zUlc5pvhr-%%d*!{5^?)=kZ;nOoNOD?_LcTL%D1h$ekDQrTUljZs}w3ub5?0#J~?!RenKDSzXq&fjEL=idE~e|3NVUvJ;{RrE6@r z-nwbQmf&R@|JCDRSFWu4MEyu!wQ}Y1ReAk!|LXp_KhFF2pPE=eQAE~%G~@cDPOX2SrSgK$SCO$>TL6 z(~&hJ^FwlpT8PXK$*mFjnp_*6A1T*D?X1YYAwNr&RxZ~t)biKn49N}myWDOYTA>!Y0;He9-)Y*S%5(fHEhatY-%D(M2YTp?h=l>(-GPr#My91e!Rl}_ch zs_HGavTA3Ma@AE?S+`zw)pDn@YOAxlywK%x7O!#E)Rrrw)>~Pnl$C6wRv7D+tQ>~$ zmzRq4J`FC&5@~}?z&o=Ad~=zA?i>L_&-ZKT+?67~U>Gdd(#g4^Jg{0o+gbtZ@&ru0 zUcmJle6~QOL;YlE^ggaZ$0kw!jt0}7%$~ex-c`#z@jV-7-uJVpzn?LV`*mDlR^!`0 zUh#ANn}?s8*!A|dL;G#NaRx5C`pm+quYY!Gn(}_%wNGFF%Eqj%Jo02f?|>0 zz&5Z;5z5#mR>(-J5PzvoE02W6vr9|&Xt1ed`Ez;j;P*FsBpiaOgw`=J;1Wdn6z|?yLtg9C=#J{cgiu9VbwzUtQUil+iXNTPK z>mTpCX-`8m>1s7sZP-*>R$g4QVbS8W%Qq-E3UAr4sjPBCZH=>fL)E5RN~)dtx0RRX z7giKsnZ7~%mTjsm#wmI$4pp1b#%O=;bJ5CF_qkXl$$ieGnB3Q42~ZkK%tht36q#zTHt)wgSf#zXDybNaRzR5va^iYm{KsM2X&D|(>$?r5RYOZV zL#0uDH=xwcO9>19>@P#5(S4aHox@A%M9E8=L#6t@87Mu6q||B%l~(go<0F23jO;mX zz9~e7&d!qhGxr2b)X>m7)BNr;ZudF2@nL_s67m)YBQlyKou|hbF79*}>iXU98x5;< zgL(1^9jT|XceNp$DXW7OWK{|md~U3YGCpUsq&U>qpSk99<2{n$bIDj`DDg`3nrxQT z5}Vha(kNX{buv(FPe#ci;~|^HTGHM@)zVQ-ZgN<1XP@%-#Kv_Xxn63#_DWhvh?;*=Im{lh=E?u1FM?r?MU~LP12Q~o@)l{ z7!OHTwg+Ov8BNmnhL?YDXnDr)@{GuGRa#)|Xb%|XfP==*J*2#rl$FAvya-YIvT-xo&NWu4XdYyQSZc9rnU#7 z!x_fsBz?);dxD{irq%jU+ZXGX_yo(gadmWK);J6<7vuD1B@zJ&UXpb6vNtQ4#_xJb ziqfT>Eb(TUDOp$IU7>GW5eqO`_N}nAy@Q2@!r2aQmI32jVG1U_)*lSYYM!N3_NiD1 zOlq~_m#@Cw|5L`8CWmC7WDVh3(tZ-uZg1}y98Txi_HoLD_TFfpq$Bn_hA35wSh1a* zXd@%kMw4U-MLSi=G8YESL0U-WA@l4mH1``;o5S@%%ej$!f%N>yw7YP=(f_{QuzG&@ z5v+Z4|4Y&=7Rtdh<{{5)G8_G8^oDB(yP-LoHMGtwQD;`DmCnv4Y1S}YX93s23TR_a zWE;jqtLI2u!nc2{m*#gVOX+8p++NTw9}&%wFWbszbTZeQrYS#Z9L?XRh(|Mk#Tg zbGq7O>0ngfdD6iNeG}d9Uz3f!#{E^4&1myWP5y8rP0}Sp#*6V;BFE=0vS5HmGsqNVKf=u!yO}e;+v4tO^ zXP_p8J5Y6?1S+6k+XQ-euxe*X`za_)tHYZ=-!Exn(e@+gIR?SMVU#*ihv(50l_cm` zA9mPpgH#JTLHJ4VzE;opod5NJ)*#P|KFur7)v=NZM@M~bD&>Z85IBZ+c~zP|WFM<9 zoDQ!XtAC$1xACi|hn_<{BE+;%+C~|Nhyd+9)3}xt92cUU6n+YGB$U)|N(Q^obdY zm^9-l^ctZos}feo$nxaDvAwRgw+BoTsKA2EHlYj2GbYdpsz&?Ae2p29OMEU?s)-mK ztCKHVgVd;_MHY=U8)_1=s~yV-#KO~9umX03i=h`v;r<jd=mf2yv|4Gc2v4Om5tvWMh-L_6yQQxaGF{WgCdUeXBv{}yb^y-zw z`g07&7UTBcqPFadG5i(!3t;zO1l*^=ou7zwsRmsH>wBZnyfTkkn?bFaL~FGQTKK*8 zpSJsUxUoN-04DQ`QqSF-0b5W=aM@D^pSkx|x!U zJ{fZ>2K4!$)htY*trxPd74B1m`F*BR-xl?WP z(On=upDUMP;ycuLvb`(JsJh!`HEsb zD@~8WaeUj9m})50c9X*w^VG+|U`bmN3tWYTOCv~S_(@^eX(zLNQZ!E5UJSSm==e|A zEM`)xf=M_32h|-2LKjalHv9mGczQAe%;B)qIebvgWv|1M${P|?k<||AFdtMr%VcQD z6~|k8_`_h(GoUkWw`+~0GMmpDg>EOKTObkYIR(|NgBZMn=QIpbjScaG7^o{!WUtL) zm4O9E#VQ@fic(%8lUG`F9#M(7U`#uZdx|E zG0QBg%W#&WbI3T)K$LQoXW#-Qu2s@8fA4+#p>eX;(}=m1)#a$@X%UHs;?PW*!;51FXs|z(=tpUoYiJ*OTSM zp`JOs4?Wn`b9S{~RzD8>=xRK^9W8Y`d{-X^rzlynr@(|Jj1yJ^&|-a2X>ZHh->x0kdI6B^yW63@>QG;_t8WtN=VAcc{c8WVO|s{C z-k`&qiDiqTzfPiFp69)ZoQI9bq51Zl3o5a}q+F;TNT-I}{1CAm>STv+#@(+6gAVm%PUb_qfJN;j zhq}gq#6C*kEZY||&F|?ZD^jf8Yk*lxR^#lMYRXiqE34NUkVvJ(kh(EBUPIsM)+j4; zhkC=gk>-Za23E7@wP3%oeVf5^B3QebMggiE7G(Ct%evDO$5)% z^><+xfsr(>6{bIT?*QUl-jS%wfJsJuHSp8~TA|&%f-SR~Huz(`e?Uc0Gb5Pv#e2j- z7+pu&>0r`Cj8Ckq+Bm*edw+qNyfsJ!Ud89)p91z0F2me)N@sT0|p7U9Xk>QS(IBKSpCkIUZmkT9hXFfL$K zXXg8qnM)OoOET|4TfUfd0%J^x39`AQErA6d9|#5y(*_WTSqmXbM|Ft)QGauyx!K2zY6UA+-$OY^%hxeu zu_vPEp$HUo>6LMOEPds+#dDO&p3F(D*p1wWX?(`)_P%vA;wX&Rpb)t1K z(Ylyu4IDsgTF^vkA}tl9ItEEjgExcTouGH%@;u^+4Hi)0CQxk)sP+MN%CYqd+8cx- zA*<&=G5HQoF^Rw!6q^CTSkh)#bb@FyD5WEL>fkjyls`_hO7uDE~{D={TWT>N?prb9fW4img_N4V^1#*-vN z2=YDtMdEyL5NUiR-aE01kIvPk?u`y#)^vwAP4Z+gC0#pIkl4klvEsy%!WT`%xuOlW zWF}4)!KCHgw9G~b>2DT6>x2You&d2JBp)!Eh|DfhKl9r$&cpmckGr`Yw zbwtC3w-OlRbd~ci(eOMLc;{1~=KFCY#H{hTC^|G;M2CiP{Ls*Mx70cvs5ur#*Tguy z(R~+?3NU^((1ymBX{wnhpNKZ00@p73a>+S|Lj%NH!WXxR&W9Zs?iUA!9}YV({FX#9 z9T*a|149|m8chjK4e5TgF6f+@C)a=C2L`elb1QY3hAqCq(#mt*lD&DPKPPOa5?@RK z$7yuUZwW9cX*-_$ng_E%gwFrHcF;TxSqA}XwHUXbAhG8;NfhwC zU?J`1vtPKt3Q}G`DQY#CwC;CEVFh|Ubh@4;|-ZU!Em5 z*o8y!_J{B=`QjpH1Qagi(?ZnG)%G>SvenME3`_-WHCVYDXg}kV@z|SuGHR#T7Go!> zolFh7&|fJALU%oZXKli63Cu3R($m$Nkl4+d* zbp<{Ds0Dfs^!MX_fXK~?uo)EfuE=Hmgn}o0F+U|RhG^XS7IEuwSh2 zi+NtqmSah$${9c>m(WRcsNmc@i`AjNL~Gitz5%S>Rwi?~bC^i%`8tZUizw({Pz6A@ zo06eGVVnOTBr$~CKC-HErUPmagjuD8 z+C(A`Mwe^c-b4c(A`U=WE(cHyh9#;Cn@^i9LZ8xTGv~Bmy#&aB_4?Xa&?&cXd>fGj zBhHQJdjk4)QQuKmBdSUZji?1f!}VxG_UxgKz+=!Qar&_i9lwb*2$wegYE;Ac35+#< z$hnfXDHuPjibrT(Ial$+@D%E8kR&Ox>S?43z-&5(OJ){6X&M93Eb-$c>Tw1PlM0an z&LZo_;3%8=TAM>78JEn&PP9@=gGqf|f^KGwpHgbi$N6Uc_F(OKw0aA$a_J1w7jr~# zS9l7i1CM46Ul<%YzMSWA(C_;4J#7WkQstxa9Cw8%$@d*Is#+R@m)x; zlD4TVP!3}ih{lP7NrM9A*{LBB7hdcU~hyl`=LLg?gTdo_A?I z6C-){P{X6KO{bN{S^$Rt?%XIQu`$6G$M+;iM3^4P>n63*>3}vYJ|JJr1K=Gy79|ZP z7it*EW@uGbie->Jnd;IEU@OXjeBx$hs*y@YTIgY?_|gb$l+mo`ctME#ov>FEM!7%- z*OzWYH)($RtgHZGMz(8?hx(Ot7}h3gydPrcvtN=m8=ELMXnR+5*}ZE!B;V>8=qB*> za^6F3xQ9;kLH4x^fBVna*Se=f*w=a}@&CZSwusg?4*lC;NASh?pe+dF+RSk@Sz(z7zrorj^KIaTKNG*MFu};f_fW3t=4ppS2qtWw zkjK!DliU1U<28Wh&;wrK`WwVg#Qm|+>@ReDP34BV0hz2~WTZTKZo{K?>9GpjB{z&1fJdA}MGIoii4?79}1gF+k8n<6BBWc zY~<5DeLBVD`TBfgLmU)CVL%wqbtl@=BRmFSA&j#z0Lnlc_@OuDw~x{6x7SN6j|b)f}_uA5oI=;p39} zrW9;8(1xfc{P7k5U(6c>z!X)SDatrg6lwf;Ke!v5m;qT%Ml^0v8uxWD*tZnV|5QBx zkoNp#{P|1pd{p%u?fK>6`GHsIc_Tfag3Zz#NP`KP+nCg2!%vv7FNLKa?Qe#4*Kw(z z_BLFWr%9nxjo4uGO<^_+{jXQ`47M!uMwz2}pA_98m>ER$9JdncF~zN|i*i0fE} zzYk_V{d*oW>$^;3VmI>l&B3j$I}<(U#@)Tzb8h_IXf;l`%yVvn0s_XrcS83cs3d!i z>E!KajoTN}8HJ1%VYgI4{>>YarvA{JyAK8dibr9J`yf^*el_9of^a`@eLyVz`cK5z zX!UmVk%b!Ht|F1Ws)0onwUe$zoXEr&>J45^&+9BOl^wuCNp_*Oya zVrkMsCxX*D;tTx{Sg)n1`vi1q1wpMUE zaeXQ^kJ{#7(yb?kuKm;@YoDuLkT*jIVZk6=+8~li4usBUtDw{Kc7;HgPXb{k{ha6d zIw@pQynu_tK9lg!4V4eO2(kL(Zh#OY^Lv|Cq3afJ=996Jz!;Np60(fjl{&}^qQW6g z;uOj@W}$f}kfsB29MKu$&#m1W5~acl1i7MFfrPxE9kCe#Q>bc0IP* z&Y{6(LAJYhaf;yvhD3_mkUbSLFh}_AW$wF|4xa}KkjI`hAp_zNB=?+xO~bc3nIvIg z;x!nF=eQg;CzBl2+Ncw4@XO$6BO}L~)k6;fb|IY_zC(b_4s_tBF_B`0bSo+yq>j)TnuzH`?V9#`C@)aV2tTHaFV7AD$11i zXu4jo90V?{LHv*}C9d@>?VOY+tIvwlcV<7bzohJ+Q1%cni;UHe8CaQIFfz-@#bO88_hLuCY36o6w_Qwl?0ez}i@|FwhErFc#;) zwm9c(lB8>u}-T#;-b6=H;?TJy>@5Mmf)E1+)`NN26xrc0lU}wg=6tm#``}&RW-`)Y1c_! z>o+JesB)s@HQ;8C-QzdOx-0{Xj>(S9*Q!r|tNW>C>@q2|)huCW+jYc7UEIEm;Z=$oDASG|0h8^(=^o}DGn6f{^5As~Huqoz_RMe9?yczh)xdA3 zVO71Y+(Mhg&7Qt^WsT?S@y3RUAc4>y*T^T@i6E^e>i)|=%SLNE)VaV7c#yA&e&hCU zzeV{4piiF#O^b$|C>I88Q|nbxC>q}i-f#=s(xci680n+SCo6rJFUOuMm|yJbTij#! znecgZ6aXP#Ogn&d1PVu_$g^c^n-abg_1tk1c{uf4^hXGEoJ}#?nSTn+@a3S-&>?>i z8gdB+z=s@myy~g4r$n3jw?k8JjhOmH#`{T`_|B#e<)rSM%vT)h z`#8piCWVe}cF#Gz@&3p8;INx$i|s1D(0psb%%R0L&v^gOQA<#q)|9s3`o)5y)kdrA zRa_2$PBN&3i={$N(i9>P(K{p99N<$N)Q1*~FPYP73(+c{XtkEpDvy>3r`6h_+!J0f z85G02(yi;ct)F;5a0Ll4-E zMb2+yOyWKnSVteW>>^rczHYqt*NE8EEQ2Gu-JaQHynip!w6)W=b*)d2+faVY)aN81 ze@M@uh%>SAl-+nz_D*tm?FKa~Iqhw{(8M_tK2l8BGLU~Ii6@e3Vnc(5w**uqCD}?+ zmiU%RF&i5!6mo?G%RB}i4Uk?t5I?BbayerXa%McgDfbz03YoI85^AvjMVPV&?_;^R z*@Vs;^&2>?ZYMAXt+x7!R=Ae&ALLOX$uIPC9MnS+8MSG= z^J`BWb@TfqEceFR{5Igh&{}?Hl+CXc<=?TCNkU^Or$Sn0fGM$Ie}HR={CY}QJ*7Lx zB+cEl3b-bS8?Q7fsDX6ML@dPtk#5ex-P=}DmIG&@3=Tb${I}Ww)mY$}5p7u4!iYhF zT4Sg>qOKZazX3q_i3^a!PaXpfa-P%4Ba=#T|BJvFt0)x$pI-)f9Cj9Nt8--bK#b6m z?Oloi5!aHoA`T*4y3FU<&sw-p<_D24sCFXHe%eBxX_7MoXyosd{SPF?QD$+2WdI2j z@Po^TR1bM%Q9<4SRqmoZA6$=Z#9hIpdZ5G=r#es?eiR&IuSAS<$&As0#NF$i}$CLd+?{V|>Fx z!209Fz2UWX^TtTKI@(O8Q?ymEa2+9l`yU*S(QnsRpkHq5Ab*j-ZG}15yq>xQE3I!c zU}V9hD?w$4x+0mDD-0Y77mjFBA68DM@+NVmr0j3Bn28lQQ*gG4w2iDMPFkP3IF0BsqBnWjU2W-urM9m6KP^ zN}Aj6F%fChn0UfK2oHuFa#_&c z8$;Q?m~Zn4ztL})e1&DnUuY0uwG^5|EIe^v@;b~VAAlNTT}$KO7l!(Q64{Al zj6ot_fi|xTQ3V&vE&{M;zB;Na>KUvNekgl@n^bU(jvPiWk4Y}-Fg|xop3p&lTgY87 z8$2+Co6GoIeIrIl_sWuSudKEjcYf-dk;Cu#G-3#S`J_dQO-PgeYCT;KL*ikS6Rt}H zu2%?L@!mua6Mb3#7w@Wt&Z-I`OVX^;-{eSG$WSuCFcQH-TWyo7@@)qd^;8cc61$_R+pV z2#61J1qUtSl4}8%To*Ywsj!qubmgdaT>pzHf;1o%op%!W`!^WC_OnWKiT6TlV-;*- zedo1eKK*_1!aEkz`l+@mISXS|&bydc_EEk{p{l`NCT_&u9KI1NY-URz|7UDw$z&#* zq`f<0ro?~l+a`n=UKo)KxJ;&5NU*-)yW?H(t+LrRTRUgGqn*B}w=ZHn$z)u@K-HCYcEt zAC^^-eu5>dFu%T-D+r7s$lGCTpKU>Ms`w@XL(3{QC!&PM+`H;jrELf0X z*Ek$o9sK^z$zNc7!v)S2lF;Q{U}O#vMjp-H2&8gz_z6txd=msl`tQJ~8RALkHt-lV zkq`QqMfRMS-k9}`9iDagY4sLy(|gqybo$1U{=$0pMV)NS?iYx|2+bX@9pa9O`a7&w znaD-^7ntKtODgGyJ3v~pf|vs{E&_r0W{wW9yvWcAuYDbcmm9XyiFI(*qyNl02*9Ki zUI+b@_|Mk?pI`E-nul@xMa1#5z;|aU}yrwWPKqqB=)oH=B)7C|2`{vlylfk6?r)YhE3kHHonLPfQwW&XtbPeKa z*V(M8I%PIqGr^0ixNV7LWpZ+s6&xn*g*8iI95#Mg{yi6~w-C*z>3)W62%3c6sABLm_ zvHLwSe}!-k1}b2Bg&P=nMMU;CavIWF^<3(Yq11NjQgl+VY$Tx zZ(qU!bV>$5Vp*_s8)hepSl5jRp!Oc?MF(i<;yqTi-`H>l;?uW$I!dj;5E}zt-_bTq zwL6%!2z3YF;BwHDnGkZ_#)f@(aB$Cq3rl!|X|GeoPADBVp+#h*v>D8^v0`2&9)(8n z$o^n3^MtX1^dWVERy(s}>s;>+eq0PDJ&ziL-M)g9R!dgjlQZ8lHWbiF3a|OKNBNIe=2)kGTAfJEn8;?lZr&U+Dm1%NZGyV7H*R0 zeFrV%sCo+(z7m)*2OC|{O?oQNiGCaJAS}K^ztU%W*MR$iM*cxs+$`vU@;6yo24HdBaGv0QNf^GVKTv=Rb{^261-?Y$7%+I@2(9Pf&fY z;e>cC8shOaNH`t8hvkiy=_L4N*W8JC1pw5HvNHD&iK*d1pSSzT-a!e$>&U^mzd)XF zY~nYfZpA2dw?)<++&l65EI*zlJOHFIye0x;u=z(|GJcl%`)`R$CZPJ)Xq8g~JKHGK zxSFW(JrZ3SHJ&G3GLjnY*M_Kp_o~OWazez*-f`S&&*{N;K5QdlJ<9V*3YO&=8^`yL z1jfMkN{%mX17-u?w3BK6e&eqaR-;4XBkhnfp5rMY2V?6NsQFrg(}vs7`L$Sj-+6|@ zJ6l-6J9}5T_-qyqd$Zhk;0p_UPsd5)B$f+z26V%oiAN;V-5$h#177e1VGH99MbF$; zJ28i6?k<8Qp1IEu^vwMQ&-TT9Mmvsgh6hP&icpI#Y+aD4-+qJi!VTltYFn6!U4p*xM<)&CXH zZbGYa=DWrQa-H#E-icxAtGANsf?>Xo3Fp?@Pha4}yb<;B&BV-im2F|7=)**LmeqpN1)_B&hu(bx!$5bK5KOw1P!R4Dj>m_x`mXGq5nReqfti4)J(W`%$4QRHO}t^e zwRQ_{nCwiXn)0&|CN}c!@gT3$a_)wnjc-{H%|F&DW%M%-TWBrpJ_$dd4t+8A^L{&& zY<{@Bh952$lfA2N4l0$2N}IGwgCE;~?T+37BW)9OYHJAw1Ft+UoPn$_y^Y8p&_lJw zO2AR|7|$Sg>JC`i`x4Vm2Rw+NfVQ&N0)K?S7D`_P>(kvL%?=d0TNH(6aexcoEs}h> zQDn?PUTU^aj>n;{16PUsNb_q&DC5_P$hreG_w3QG7A=DU2*M3sEsE$%vl;0{S7>I4 z&1e#uHlwNaXbP>KPdx^{#6Zzp$YwO${uH`bu3vW`B11m(^rGsZ_?f#Y+_i$W?+zIQ_ zu)PV5qd~e8))DxM=5BECkj>iu_rrf?|1(7D>8Aghp8g&7fGxD>N?;GLfYmoZA^dOK z15P|fD<2kOai$V4$&WA(RGcGKj97R*XfbFW7;<-h`Zb`GFD8k=7(!h41PL5mOmo9_ z0s8>d)&IVIAQ3446ZV0-=)RpVW-Ea)(A%_&(8DE?GccKcmwjOW&p?mwv=5B5R}_S= zo(WgPU#{l(d_-Ujd``m-#rNpdgby7zJ%>|B_81%BPy<^YCi^l{ABq&2wvqb1NLfXy zL!`)mfigN(;kg2ldRnA#YvXw#7?^B@ZO z`)9NhA-#`R{+-yvklsaUR8FPBomv9>zUI+Fi8-*en@oo4Kpi#{oSbpEH;_TykJ=7( zn!`JV9|h%jpKGv`qAETLQG$;o*czYZWUGx)r4+cs z@j^$5H+wdjc1v*F{V8Zv}I@goA>TI9My;J%44 z=V!hI+UMR?esvs>F>W`aYddT~5oRa;>Il6NV24%GnEfaICV)b30^qt14n!HWkwm-+ zz%PFAndWZ-&_^JI+3CsiaCz^Szz>+x0ctZpKy`i#qq@%;W6#8k7)OwheK`w&W8es? z!zr@yQo7KCgH=vCJ_!3L_kh9}{S$Or0L^OuRs7WD*39NREX2^7I znkKS&4ik-{jN8{B2kpkD`L*_}pd>v7uuFgz05q1p0ElE?YEw7TCzJ)x3OhVu)GF!{ z%fqwWz%c-zB?0g(0GtFm;0biUT8vcY8_FtSw3mg^9`E^&FxpF)TFLk_pJ5xV92*!L zhrF00{|7>zUI(A3+3^oSA9R4(h_hth4$BMZ5m#>mcTX0a6|YOEElsC73p*EaVu}QwCIdfz9fKsn zCm7L7*(u;SeAQvl0Gb1=zOkNt2_`_D?AvGoH+~Cl1P{ub2SFgtkX*jNg6G45mKB=_e#qV58p1%>*CXGB&5c zJ4UyldQWoFu1cm?zy2b8n|3oTYZtxi;6|zD-c%EQsCALb$v=;iA8#1ogF`~4y$#(A zQht(f|IR*3tE4u0@cSa~N2MxrEk5zPw&QV#ttN%(#W#5e7Du)ld z-HQ7$YT@&F%>ZM4KJU(*L}45^zNXWB{nx~>A$dH7TF$&XIV6n7^PZN03c`uZZ+Wa2 zRCcI0CzG?f%8qMr(Dk*u-v#P23&>F_HDxVqv{}jl7c_3LR3csx2!Mrv<)>E&^cM2j zxtPz+EReF)0rN|NHbrm#nD9JLOikL~X=?6@n3|P9N=K3zQ=`4ECEnBuPfTujVsK4> zOg%u0&kEn4!|Q{VMQ3mx^dfe1syr(fA&~SZ_4G3{w*w*#g&+!C~lx zJ&ws!ms>oWEilrY#Q+OH>tZF5CdSE)u)hz~U`QY?=$cTcb#pN%{47p*9QyewuNd9f z@Ocw_SrP4+10A2izt7L(__bo%XffDH;AGG~F(_at23nIvFYPo@^G!}oYU5x$DuQgJ zaFA7PA2wRL7a&tw+zTM(x}nXip1SsET8GU7WP(J;Nc?@3{x6210_Gb|J?D+E4h)E7%ehXSH04H%=5`KAco zq``*=&H0F%dBc1(f(*@JGVBMp(B~Q9kY&`fal4!5jpoENut9+jCE-tt1m<)2%*?>d zL{&o-PSmHl7Sq2h{KynFJ8O*isZ$a z4jfw`8yivLFfNU@tDS*jV%^1=eGBn0l$`?Q2tBNHglgwmibGWwb1u((7*9`wVM866By&$HBUT5Q;d&Xn7QkUAM0R<<5y_QxjPD) zK(JXT0*2a;xB&(qj!^#)IEoY5c7NS6<8xWFOT5dY(BTTLry+ZYOBx z8R;Z^)trLOOSrhK5$^_jj;Dr9eteu}dWSN6bLey`hBV0He9(Us;^71p`C|M8#!$nb zevk%@4>8O~*Q4<*acJ6Kxb)Nbx10;v6yIGYEL%8@;3p-WRjgxe5f5^2A0{pHMW{`8 zZo`F!f(;u_%{Q~X#Ni*o4%W2a0hkiEc^f=fZH;nSxq-#N@ z&}Wl=bE&{LoePLdx#&(YKDQi-`Kr_ij{I4(rXf$$G~^77;k!Aqvy*>0NTVRv-?Zzo zq`ir^7O);N2?O7ZbdJdA+M5}jm$XgC_YcX318q7x?Jv+nLL~Y-Cs8)-BN5B+1EGD2 zSA^ZXirq|7s~PdIM)w=K>jE4Adt!WMo7D6JdJcRC@@`K|@t*#qVUL5g`f#W6mvfrIh9q1q-REG+#I)uL!f)Cqr zhf~+G14hB(Y&ekcpAzA{{3SZ!rGrSZT8$6CL<7LB zPZBfauH&M{rk*6_y;O7CiFYM&{{`5>e>BpE@>LF$r`uA z!btgW~z^v5lUcMkQ};tiye62bK{VU^gjgYEtH$h z@c;G1_Wg|RG23XJ$3vFUtLeAH_>r9WDP-<&~yGkS zZyE-XP3~1AKikt2h=_XD=X8BmwZZZUGU=<-4Yap<2K0Bwcm`tc25JwB=CqyyH=;q$ zKvr!!KILU>cnq@4x(HSWfkz+}rQ7{xpY zcdrw9oWR;NYil=jT&CtbQ2&HjS;Snbj=l%rMohSKz4))2By6nsy53Sfafh`n-i&Jp z#4w~%noyH}J`>-+_HslT-hqpOrVly?%`KiCKDjiZ!K6`vcaUx+gHP{a zWQBH(5G*MvY389>&P-T*33u1Fx0_17gg3l4>hoiA`$<#pTfipt0KLURiB2>1fAfo`~{QF=UC48A)4#M;_RKWwA{>`DQJ;Ay6B2P?jaPIf-8;su`5if~|UmFo$77@QT z+>Y>PXRFv0@aFAF$@9U0be8Gqt4Zfcs@0HFl&k-;+Y$Y`T-4#1^SZt0*tKdRwnut+)cf!BN>_aF-Mg$_=ojh57cAK!`ZPj zIQJ08Qx~Uaap?vYw(6I>+X&fE}G{!Px@i=Bo2V6A4rJI?+ zW)@$_X6DUcGt;NDnY!~?a{u`~iCvTZCKu{Yi(}J@&u7!}lGwELL^ka+QKB;^Ftaj) znTw}0bKW#&PM?bJcAU?q^k2}EcYwbQ6nz#^4%P5Syzr;6G1zQ18;k zMzPpuqFg$>tg7FHdX!!il{&>To=w(W#3uBN?;7Wiap?=9bE8u6|G+mf`aTHLnGDR7 zK8cwgjPHu`i#DgsqH$fqjJnxO3ADrlqcTFgmGJgV_k~g1CL3>ls-sB!rmna!*Q-@6_K79g$9)$Frooi&#?nT$ZFWvxz;_vuMAc zVLUTqSVH*#AIF_ipj3x9Br6ScWpt?kJnj6kLCKh)Z zyu1DkTqoms;7hnB8dzd6Lf$x*2>wo7WO9l5GYy$PF(!0OhG$S(9V=aNnN7t@n{?EhuY@-fyTf!5x6F9tdJ3W?l6DAJsG$sFG}pEcBagTWfMOeihCXU-PepC?a%Q5V=0pu z>-k|Q2Uox0ohdM3FOJA_KZJ7R?TmQ#r^rjjv#k;JCc`YjKg2ZH+>qEU)DHf>MIZDx zQA=AixI}|%G$?8@_OX`!MuVLiJT$D{GZ+v39oOKS8vICuUuf`A4PL0>k)c6ZgEwnX z(O{ESf47!Cq`}h~yjv@;)?l#)uhZa@8lF*FeSY0hpvr%LFl-zd+H)6bFiwM?Y4i_h z&`f3^{LRy#NHO-8*czp}vZ%tv^tzRb&AGL#$Z4<2s;X3~tIC&^l{?u^{TikGy0U7e zwy=C%N?BE9HlEnVSaB3evI<>FZMBnHan>oUQo(VP`+gVn%vrutT!xCGmTUPCKI?6Ux7vj;~QL?kpC6$*~6|pCIoJtwHC<_07ac*^0 zk+Y_TYA}{WIV1i@ViL`994zITKf`|J~nK8%*L|Ht7@EbVP!E!S|5Sk z_3Ra9E2}BoRPM|x1`_XpxG^iMP6Ca z7M{V_zv+<9s;#beRw|+)JLvaxM6Nb8WUr-X*KVs(oE3Rx6;7#!$V4%gsn2s(SCmzP zbXs-Bo*LN(bLiJqmh!HUN!zt z>l~1=a5Ls$9reYtvMY%s%gPHk*RbouxsIyMo1N7f8JQ<)jdODuhEdHQSzcRKTqCis zqt;bckH}~2x^OF^ZHIHKvz$q`>!e(J2%p29Lx4X#NRfmU5)GU@r%3#Rp1SpdSw)>@Xt_AsI^ahxB0+KntBZ5iArR7K(A> zbv=R!!GE@Qa}F6h(!2+ats2%)qaafW_&0d^pS(hbO?GseRR3W5p2jKF3xwgjOZ zp#$xHh@c>JBV=5}m>$7`(1LU#pcy!%BIH`IM<6UfXhztD;6iXCEJ1hyW9&f401ltv zIS0Z)gl`ZieW#Xop!^-g3lRJWCM~@~OQ+6etnos$kKjVTDS#OWr3iHhGQxUxSL3nl+#x@7IBQzl1f>4L}62SFqi(sTt@yo zh?@W(MtLKk1F#fuALXGe8R>TrcL4_QOe^vhAl?C(j{K)V-v)$75e}lf2Q)f}&;y+N z5z3MG0?Ix_=tQ9TFrd62&n5!W+~9wJW2u<4o4|Kl5bi=~MED89FAKM+ihVjn@UA>4p)D}o2%afCw%od{*`oD#Awy|3FB1jIr4WS0LCCiV*HXcmUx5!fz3}5k5nReH?T| zn1^r;!g_>igdGS65c~*#MEEPh#9fSCf^ap$8iY~=H^L(bPa_;b_#Hwo!nX*CPhdU} zB!u+{TM+6I9zb{!;RS>@5I#Wo5+Qy!V{;L%L|B1Pif|i3Bf{edhY|jO@OOl%dw>T* zAwoUEj}Q(cyo&H=gqRk_W*~eYAs^umga;9xLO6!-0fKHXmLh@;p%kGJ;b#cXAao%F z5GFmz*dm0L2u_6i5FST3g77(}P4RSKbTxmekz z{30sJFRRI~u5j^;;)?vuP9?vlyh_P0ugWhiqgu$sCNwOQsue0qb$(Sz32jlR7tX3F zyWJVes3)}CJM)!oE@wWJ!%(KH(v^>`xe|RuWL8yE8hS~7Np%%a7_}%O z_g2!I@_7f<&Z1D;BG*-@l;)4jMHgk2B~>F!pykz;hX*CvsM%T+u2@#gN4uGJc^bKv zuBqKr3A(tdoh4;;p#qS%0*#dwL752U5j~5YC553*s!Mr$&?K}uDJjSFQOhWy$|Fk% z@oF($#m=qy0+EPJS9Mh}r!+Mak$&jQ44CZewEFV^;R2RDPD{Cu;Wn;36 zhjuAOGz<^Q91Rc3ABqR%jD-i4je-YdN8(XcM17S&u@yXAyQyYd&3Dxp<6&wS6Zstv zkLbCU6F9%PtZ*}?HB5vmEx(Fu%lY79A|tJcvdS_gztYL4IGm>OS5?;VrNc9^E+ey? z6`Q~xe0>&GxkR;Ld1ArvULs1$^NF4EORM-AhfW!8hn81FZ5r4uRb`bUn&C7V)@e9z z^xC6#%AW~$8eTTxmMJIPE=P=3PH6S!S17fkX5tynxO|<3YlCg`DKYX9*A|GkvJx@$ zAw7gjMsKk?M6MyvQ10mUYdBV1Byl-IbB9A{VR^obq?@*`hL#goR#j_@tFj7mL-2DE z9Rq175qX2W7LmExS?R1U;{q@ucgQk~$X9T_!U1(?yOFCcvM%KIu$mETNUKQGHyl^W z8-Xijj>MI6N5hr!N8rjcBP3J=3be;W;7Ta0zAw!e4&(Y;`0{H#KHl) zRFrY@hjaNF=CeutgtIjsjZBA_Bb*oFvLO$KGDp-7ab5WFNQMh%(g+DrK7#5(E?Nc) z#elXrw{iKgsZdB67iD07Dyc2!hhWGBjuJ5^XIMHB*MhslD1=Lhctz!9v}sp6arhqk z9F>+<-O3jk?JOfp%gZXaY;{~&8rcG`Fh*Go^eVv`Ly(~N0uJj#jFs#l9HsIyM*?=-f%rRw- zJ(W~C+Eh|LCMh2lwEXX!QeKg0KGu|mD~vLwR5r?#@@%e2S61a0!HNgN+Ym;h3Zu?B z6^xu)IxTQH64uzLIi5?x1QtBVFwQ+Zj~W?Lv!;+MK_nzxh1=+`lJ64QeN8!fQ^*cT z?Nh4}Ia-@FuvS86=TBFcRaKWM+jzyn>@a0&Dg>Uxd(pC~UM*j9R54N~bHebM7anJHi!~=j^ zFN0kc@kYS$sjvn6V4nwk@^aYCk^eN{5%?WGL4F5dc{*%)h%11P;$@2$5N`pz9`A+F zvju==ydp#K6u>PRuzjK(1+aQCY@>+R0os?q&Ww02V9V98W264lfGyugeToDA^%~ei z5&r`4ZA>xce+XD1;f*PbQ2~4nArEnRDeQs>G?smUbFyG(Mtyph9)>tpinsx=CL7Np zekb57%TR>)n}FXgM;z_wW!OILu+Jm^QNY`-745hIpGR1NI>!N1R=_5Xcq(9p1LHz{ z1+Xp$wprlN4`^R0@F@p0<-&f8`pJOrt%iMn2j~xY^;*Q6&^KUJ9_WnxI>4*e!LE+D z6)^sK*yT|t5pWekH}cm5KBUDT1-#=%*z1ws0QlBTpdaErfRi@DwvTu+;2H$Ne?8!1 z2*evLfWIsN?x>Sd2>wAxNBlUTZIc+g1Mv3 z&0T$$+kUw2wM1s;qtcr!MmwK8}%oM^cc-VQ8Jb~~?=3jazfJSzm3 z(m3FA8INbPP^SVukV4eo25btEf{?*q4q!R{QAriNBJ}5i7e^QF!-g6)@T^eZF5pmx zdJr1;TZOa}@nziiLghuMX-4}_)VmG!s^Nh#V=E~`%!yJ3sa(XX;DagSz8Z)G_<$xx z^miRrSgypNU=p2V;Ve4kooXW={Ow}{#+$+DUTRo8N5Q3=*gw^dMV;G zUizzphe`(zlM@uBr-(`}(1%7s9wdmUQS$a<42w7(8Hf$1pBWx18fht~XE|Dgkp?+L z7YAA_Mw?4H&a1S3Xhx~`(410A%+7b=F2fV2(`Z=?pBB+Ja$d)po9lrM(W46UgDW+_ z#Wu&$W2K#oUt8lG>8dVr`LIa$DB76ttAKxLWF3+%1hQ z%`H1ycD3wlIoQ(C($&)4($f-X>2DcmVS5dGO?#8~n)hbxweFSo=I&j;w_tDSUf15b zz3#n@dz<&}+k0?t>t6rfj=f!byZ83&4eagTJFu5MY27F9%iXtrU%|f8eXf0V``r5) z_cia^xo_9Lef!vc!+z8L%hJP2M@F!@E_Qfm{S)amG9>hKhV0z+TJFWlK*8eYGrPq7_ literal 0 HcmV?d00001 diff --git a/branches/winverbs/ulp/nd/user/objfre_svr-08_amd64/amd64/ndinstall.exe b/branches/winverbs/ulp/nd/user/objfre_svr-08_amd64/amd64/ndinstall.exe new file mode 100644 index 0000000000000000000000000000000000000000..5ac605e3e9348250cc89f4508a6346918dbc3b34 GIT binary patch literal 10752 zcmeHN4|G%4dB2h+u#IgQf)Ya^JU~vIKM-RPW-v~qSR&8i0R|HwU}6jD2`k8wDm{V8 z*n%Tqs1I*xmY$QYS^jN@H63ZwwWk4=1sl8o3CS@uYe?8x8DXblCMiwQK$13Zzwf>) zBmN}aIXgY4r_1xX@B8j|zx(~W-+k};ds?va5mv+)v!NLVW4i(A^6>kY-%etuTNgu8A(ZrWPEKh&?N-}u~=M_)~Hf49g`w4$+xIg>WYWd>E-36E+P7j z$@`kV;W=8FZ_jVrSE-}_{I*>Jey=ja#|JC-aQaJ?-{J6SG5)-O&vCdU66_>ddCOb; zjD@z@*}}HRVEhY3k>%4#vg;9~^C(mnZ*D2qlkm!>`UcT~oU? z3{{S&8yfizKMF62v5x7XfEEC;cqDGMXo$f@-+<%tFxEFcVKQ>tLEwB-01GY;Sf?jb z$slOlSNP0zx(b+pdN`jNjf3$n8tOo^qSc@kTplr8K>7;(KStox4W@vP1hm~~_OBJt zCE%bK&#UtYx+35T0Ur?X)J-Pm8v_25fKLke4FMAZt`yKG;7kF>3phf+oXClj0{%wO z^LFhObjo#Hn|F%-jyzq!lo&SwQclos6|i2wDgjFc{9O3=v4Cd5-X-Mc?a9OKUt{cQ+j`eTlP*oVay)CYRQT)-h2vilL z&Dg;bTL@fZ8^D!h>(KXMZW6R0s1hE1=KKietH2Ad){QAm?P_aECzEPSYl_FTWIWoc zYHh7*a(yJIrrMTAVySp=U0X{4FBw%+lc~0)O)XM9CbfDOvPB6s)_j-bjYcDZSWso@ zRA8+-N2*RqtENS6o0bGftX^F@tWv@?ErMHfETsjaQAz7mrIe|Q-el_|AvIabxstdq zQM=;nRn86sbPbcw>J4>>8_S!T#Et@^u#CTT-N>y)aw#-?~W8j|Q7uvXQi zfYcd?g`z60mRLt(!H_38iYWLOVM~B zl#he3?y7v@NGycdA(v_5dZaw9N<5sOEBO=!%dw=@*Sg#P+6(W=eNB6wuG#V@EkA6e zXGdyI+nRJF8cMa*&34ad({LPH*S01SYfB?}+u~~;2q)F{4bjf_Kv!ti%r^5L3kmP> zre8C>U4{`^ACR*vs_tF3T+W=8G9ATC>6I&$UQe<6nEP<q}j7&!>0e zDW{E*d&?VtYZ!*p_|!1Y?1U`6&E+wieV-VHq92g;Lwk!sG@MU?AaCuH_2X#=Ce`vu zuVT`Wrt~TDszhgM-Ol#c7V2|OPK&HUV7CAE@wco$`z!CG3vu&>8WpdBi zb0k5x4~zqFBPLs1CqZ$akYQts-0QsMGY9~)N6z|O^@P>>-G}||KKWoj7&jmExoRfi zw1WZ3aCTiVjD`bp@9kS4FRjYD;%f2r&M$(XDG-Ed#82lpIjhlx5TB4P=bVPqkM7W9 z!=-*(Y3E0DRdUuI&PT7x3T~*De4A zfzTHK$SKQ@f5ed+mp z?cj;w{Bx6%U(F5Z(rW$#AXantdA^!n%BcKm?jjmxiQ!yKT)pNt@L$JRt_JUX@^Q4- z8#UaShq)A!?RsLXzKrcA&bEr!G6S`%@)>{R2+w)SyQ<|#{6Q=}692Vp0nEhj>4S#zGYp(Lj#YBJi1nCjb!?^F3is!V38%!! zd~W@-39^0-Ce6YE7|ui7!SZHw&O8nVS#K?tGlvcACH>P6zAv9YDre4E{$eAYOiqiXp zi^A5duc~a;Q&O}3Zo3FCvZh|QzlxKZPcPpGp!C`w2gurY;*K1DiUa2k4$9L6D3J~4 z-#~HJzKIB#f%^Pv(J~dw*-frWB^z>8dGDpP#Ph^fUCSfaUF>(CcK4l=9Y1V5hyMw+ z339{1bm>0Y(LVhbLu2;jU@+qL5xjkilXd$(FoLTL>5RO&5qo=RwQhe7RHX5OY~xeF zl-_9x9*Wx+6Mz{tWM^~3uTvAjHBQzm%b8bTC3uGZ15fM9-Kan?o)y`~ zpWxqM-L9A_Ct@dk0#mRervP$K!MKC=TQEqE5C7KF=@MkYU)ZZbku!}17*-hX17NxH zV1jrDcA&Sn#p*t6IBlRSy8U)2k#*-R(p4_up1Te&kSwQX8pnXZ(#7;1D1+8a<77@K zGn{sGb-M%Ip=nwBNWe_v>1&9)gt*`T&@hf<8b9C^u>TBw-F^~XS=Z>*wFxgVJ;dKC zwO)FucxZtnS^sZYx5)ZYy*Rgvc2j0R8h+&*0Y^=^r}6GnhOuC!W<%aBl7r>z(M5i^ z2^1abnN1as^t+B1y;Y5)AVM48I128kkGlIZd+0G-v3}g_o(n*RDnQz(RjeqHzj=k&VprZA*Hj;r#el(0_@ATuyKK^Vrpf^C1jhGHO(hg-fPVqnZul!)8EO zxM%i)mFer`;NrPGb4&T^l!kjYl~#?iIv(rWUtlxMQi9&OI3FnbF{P;A>v+*K*6(<6 z#R&h_6ZiBgtV)}tFD_FWjwMTVZ*kFnrC~5x>CN=xq1RvIZ+I>FpYA@Kqxy%#dxXD_ zBt`GXB2=El&RGXX+$S0iq$z)w~#%Aj*Q~&^@mH+@$k^nUj_wD|c7A_!986 zQlcWXl&J0#J@t4M_BkFqpyQ90aP=^&Q9XF>Vr*U-#*ZK-7wv-%9ql@M+E zt-nosz3-Rs4*UN`oak~oxr)yCaHhM>njX0uxrePly`tUp^wX@|lLDXVy@yvG-=Fpq z4$9S#tCq?8>Yn}iv~S;ne#1enOg;~D>;o7cS})_2T8r})DaPRu=melX{dlwcgyF0z zz{e5`xa;$Sqd-tFr(VGsayN{{g^f1#xV+mf)F}-=R~)|M*nA72t@->fB#^ZaRZwq>>k-nIwV#FTgZBL(me^R|LHkRyj| zC-4OVPtbZFe~)5XJ{plUoEdZs=T_YJk_3LJe~1`*1}G-^Of;%bfoSeqqxy{kd~X5% zWC1><0IwLv&AjvLRkHq$q968V-o4=cnAsHlsCJE4_Y^M(w%E=eSBj3x!9&W_qs_qs z_vm9QWxc6Vmy6}1`Nbc5iyT^+<0syt%*k`|<4VyX#r?2rC;=XKg8Q7BlBSDRXKi%ozf{Wc1)2qJH!q z>P$_jDQ!n1Xc<|=M&4D_Vy#8w&H1%?-ZPjZNhlAa5iB7+Qy9ySe-pF;G+HMP#e7St zjg`(UW~Fr_5|%P6E3=F?ZChfk9XH9&%4c@4a?AZ}LZ_N8ftDA1HUQ&_rGFC^FCRC3!b-udM-|b~9i9+Q& z(f3lnpc>?53y3l=nof0sh22cOrf57x0+`pg%-hoJuA3f;Mj3n7I$UXEJ2^ftsj7_K zV7*(_P&h)xGDRGW{lWr@5=d!&D$_AG-@jy`U&+gP%cjMn@eOK{>9#~N64SycBG|$L zja%9yaqQ%;v-Wn>(YgYWSRlDJg}QBfdsDk8D+b%m;u>TxVeKn|32j~^iux!sSi9OC z(b$hzdypecvY9Xnxv_`_rY`n0L;WYU)?{T5Lvjj*MO28o0;q{5A$0)r<6ZOxkFhFC zd)UM_o1E=|M50~Wn7~+9YJD)NnenM*can72tEnE(eclx-R-z_-wW=BB5nU9(@-??v zGuz#DEI=cd)IAyM(N~iKX3*;}$<%+Ftu?Bu2{zS+GUdH7GeRMMcTi1GX;Vb#Q(Bt^ z@lZOd%Dh0ioUzuT#c4IUu|-XyCfY^CWYjWa6we1}ZI?!Z>o|e2AJA-3v0mCqb;{-t zW4j8dA`w7trMYI2dugOg^`>BfN*HUfEzJi?urv0LSIS_D?F-#tPZcfEqRS#lEggv7 z8{2@gA-)O{O^L_Xr4#egu^@TvM^$|zW8b8ara%JaWB3Nw_!q*fJ?*}t{ZU7NUey0j zFPxe9l@CGwNdNk3=~(zp9KFBy`jWw$UY_??ul(13&81nf5)eRchBBVgercvN&YUpm9W&+@fbq2qrlCWE=71+eJS zS=5t@X5zk#PAa(jaWZMeH{wP3Lfj6xkj(>5*T2|)`MLQYgO#O55oH8xnfLYU?gUnh zX2YL_$OYF<`u_m(G<-vj;M?ZmvhLALmbY3!!s_6wo6UeV z=3I&geaTM(lH`k?lGaOgP6uB%8uk>s8G04qYxyjSRswM%kJqwJNW}k&SJ0k=7j@i@ zdSI8wQNpha=yw2}up|yk;*b;pB?Vpu5<`&bv*A(P7Ea%Y(!LAMH(+`duSID`11!jr(&*EOsKyL=C=Qw2= z`9pcS44RU|UeJn|8H(o_avNW~F-lpGg0BHY;^O+2*j^xp-_KQP{@(2AXLdi+_sqFx QE>o6;XvUZpAsA$=8Ib-g{P&;yXc!wetz{g0a_kG! zn>F?qrstNFR+*fY6&ouHHk%3y%F8QUrVS2LWp%l!wA^G}zRI+@qR4T1LPD%rZF=h` z51AG=eqI#{lh*vG>N>=iCj6*s9#8+GY7*e2H9xJI&!M+!0nh&_FE?q;y*xf?&E35G zq&4+wd8=xPxW{EE5}54cwgwp;Mi2l~1-e`Z9yBswxWEMP#AHt2s9@#IVCUkpwQGu%f&rj2! z4t*| zD*Q@?@%M@HKUd?ItMLUYyk3RHDs-!`QH8%$;m0Z-uc`45RQRn5C#m^U)cDgX4B`FS zka2vg#>YYaBL9k2Jg-rqTZO4AT&co~)q1Hz>PNN}(k;ZYHzK4@MlgsHrpGP(!LP!Q zi26nLVga}spvT2hX0A1Ban?$T;MukY`b%1won=Mbw#=4G>DJ{pEF(yN%PLH(Tm>$N zDYK-Ye51pZvTS+IrAS{@y`kLUGG!N(7i@HFc9fIsudJwcIVzENYpJWSglaQJ)yf@Z zCaI{vNtvscEnP;vuvgHL{OLAEygh|znU<1VT5 z#U~3Bi*IXM^5`35Kfdkt_jj#Zclo-~4FzRHN3}e<(5^rN{hEr zD~z>E77j!B+ZT!WV=7F|6!CbgfbJ{-yA}&rmn~qZ{^x2ucbP~p8Umead`gbU|8RwX z@+twhdMEpq=F4!RA9V$%wW!8kbv#wd}#y}#s%&ROMu_RW9m zu|m;*2)`*sBEF8TW0xb8vJI?&kyRo7Ob!cw2$gSCkRajCf;{7Ugx&IcYfNH>s;9SZd$jYw0vE4m7{W9 z#fF=UD;;^a!t}~3*j#k=ymjKYbVGR&w#{3xYubQDM)-4|i*`+NpNnxNxz8D0M)$c1 zt~hs3s{7ouEzxQIW6jj}u@3QvvXlHXT4i6tQ|E#~!&81)(YC6YhNoKH=d>+46gMn+ z21R~9tVoAtmF)S@B>R$%eh~~x&G9HJE3XHW7^2(#hPnqR!I$&^&(-qW=YvV_d``L5 z+C!S&TajBaIQMucH>&sh$hGlY!a`P1_m`pE=-v$EUc_^0$H;T*L%G`CsmMKtsMM?v z<=)J54G;LWI@xpFbYqAL9UaBBXYLFZE1{uxr1{-v-0pL3!%ly=5YpxcBN7@Uji=iX z&hBs*X!_jm81ySN18H)b=36&#^RCc`6J=$ffUHR2jL!@eQHH}-bBbL#^~80b8Sa$y zpGk%aeX&=X)nGNJ7F)fxlzQn(DwB?ETQYLy8lJM6EybPxo~LWeiVASGNXY!U+x1_lD+wZ*YG4(ERksd_|gLXm1VZXM#bQp*^DA zE5gr`pEZ1RekS3-1tuvmnUh_-@>vj0w ziPo=}6h^%RW0}+%hz=(h4olkNnRf<52@NZ>BepN(m-qx?TfZW@K64BPmxFP7GZTpb z`6nbzt?bQArtv#ZNKu-!=Zd|VCW_V+dzWhKm&O20kbO(dEpI~5kU7Kd&D3L@OO3&# z!+pV^tmK+YWS;^-U{bRMzg+ryKEuXQ)kCsRvV?FgZha2aZf)%z7*5C8)-kTQ)}Cme zq#^b@hDRzEA+a4DXd^w;MuTJyJ$5LPc_ti|1GJD#gXY;?VCvJaFony5mNO&y0`b|A zad*LNgZ~|^e#Pu?70AA+?}Rj+g;G$*G^ox7lfi#RtG|Ar8=AA}gUd`8Wu}K(>F8*X zrVqh&I&d8*fHp3QY{T%>ii;$!;afk@O0zp%i|A*%nDE8Q)F~>p~#&AZ=A^ovKF~<(+M$FS5I~0AanY39=m|oDxJg5%JfJOra+y!ZF_dC<{E7HQ>KSXp$)8G{J)Uk&+A|r?NU&gBF>bTIuUC&>gXHhlTJ_q{g3HT4gYtoa z@_yIYhIqZ_jG>_we6Q6<(h#jhMxleX_2bnc%iB*|l~a=8q)RU;&$I@n)JJ7+_q)bO ztsf+2)f=<7Kj#`-dxjZcj(Eqabg6(98_K+*0Hz&ap*w1u9${av!On! zwMR=$#wx8?2!l-WGHNxZwI@-k4>j3gIEhwk-=T#Njm33>1pd}{^l_brlgcw{zYVPe ztU&FLX=wEav;yH6_Zfp!AEHo4hkJS-sk+cGI~p{x&j6XQ0n_k|;i>1)TXo0+HZ~XBgGo7M0H>$gga0)7&!`{Uuzsy9w^$u*GF1N)b*#h^&||a6=7olc8*@~ zZx|)2!_#O=iW4-f7c1^~JyJK$;2r+ML-8derkpJ!1r&U~05}z-3C0 zRN`|XnMPuCNFx_hy;QFup^Ao#hMI)#YK7Q z+}?U7+VIi*43`1wLKamW3CKOkH&R5 zEtbith zXsz0X7G6_p{#=FSD!fT;p`5pHE{bV@Ydw0e#d8#M)ezJ&R|!EqbFC$4WGh^kQ@&*6F~`GhOx+ znN7@UIDE2YHo6OJyNMf@VdC4BH?zD;O(?q7<|zy?TU%x~SuHkSW{S<5*#&5#+DPxD z?I~{s8SR0&qNTMF{dw1!lh@oJtww7(*wGnf1s4A6vU1XnB8~T z+W@wV7hI;Z(v&D{$G1+@Rl=aQ8SOsZqaOr=#VtuJa18{PMv%(zlftsnp3Cw{(b#Ew zG2j-U<8QN?O=MODlRkcr%Jv6gi>Das?!_jaYNmrZ?B*J~55~FdwVP9ULxL)@(k>n0 zgNkPv3@tfhc}wws4F)~^8pC#*+DIz1`YciCb^^Ku5}}?`P~0+rLD>KXsfN1v0SuI- zDYDmUw#dMOqhgVcfO)a&#g-`op_UZj5lq^Fc6k5KQ5ofYY}^C-1jgVmsQ*1rdOk2` zf`~tgc*Eie^_eDFS&Y3D?L)?R`lDRec=|7J#WqVC=I^cmW4!)Iz9W=dOmx#6&<0fnuKvV*r{Gm?N~ z?*1OY5^i{|xFr@l9ZXK(p>Kl0G=Hy-jBdZIJZyFoHlE`-()zrc!ptLscYsxy8Th!& z$u~&3(hcM}u`6ee>_-na<(y6Fla&twdlsVlcC^%G_bogEPH|<*o_r&klvBU}aw>jy zm!Jw#b~{yAqe8a|*AldQ)7vb1pv8Kl(%z7_zfnD=`2rx>`$N0(qFs5`ro2L=pNRo% z_bYu{H^`pf^9JqS42UhVJ|0iK{GRtFQg#}ULi6o87j(r0lk#C|Af6hE=ZLsbCes9) z;9w4<b<9eLA&x? zcE*o)1B>eMc4eg=k^L0GUba^^+3)EhFH(%ntA|@lR$^@#O7bKsD=XLQ5iwC@P+48J zSKoWOIm*J^q291>q`Bd<0crNU6zns!Zq<9*g4G*o6rjoxL1v$BRu4w&)wpzlSD>>8 zQ|7jDgDJE4%wWn~E#e`{j1R8llzBS?lxZW9fQH9SAtDUr7vdF(v`m+*d?hPAP&9Yc zqC=l<3xSOG5f#m%_es&jvB0UXNO@d@smo5a(&U3&e+nku1&@g`Ms$w(NT>O*M4~}! zUD%2XMi(vW$9n#Rf}mh}FlqnW z#5EXQd)nz>QUhW_ma1d9eD?erC3$PI(n-ygpgGR`$-$&DQJ4B@4<@Za3@m%6l4(8? z3MhQGv!WB`*$NyTG|;i=4gKnvgoKv~J8gj3mAR*uWwwa_pn zP%utlSYzVrl!*cokMswFM`!^EjGxaJw2#3{f|jny zszS<7h79DJplDJ?Rx}08pj=SS=-)EtC=|TE_d?mb32dJQMtlLwrSd0I4v!`AIq2Vd zne4ee8LiD0x@HcP6?98TVnVv)*C3YCM!Muep-WOpm#Dm}Y!ft+)`so~dm~3OEt0xzT9zYR$4zXWmV52wJ-A z#o(5u(CV7QmsIJ933=Jd@s^!pVNXd$+={X}c5ji{6imAGEv!-*C#uK%9!U6m*Lk*? zbCRpev*yn;SC`n7)yY&?rq(U=lz#RZ4MkG_5RLNL$_rTZQ{L2MFq?NXreIlWwjt}q ztTewJ3z#VozZ$mdvr3n1Ji365ue=nP2m&a_dMC*Wl>UpDSt#fP%3#Rgl4tj=Gw0is z8^Ms9Qfx|9s!h2)-KM-N-6XA-)`cgFbPPDSj2l|HWN6ux09SdWUmCXY+3^q7$CA`rM*X#a}s4hN?0VaFVC0-s|m~b5>rc)N(5ZEe;bZA7v(CTBHNlfHz8R~L`v}q4kk#6|nnpbCYD}AQA;=&FSAo}w z)eJf=d8emSD`q!L0rkQrx3=6Pj@w0f8OY} z%#kdL^JVYt7%P^Nm&gkCZUR$}Ae|A)3s!u1J;8*NO+>hd1@V#FIO4kqjDmO(G<0#x zcoz8VGaxQwk^+u-jKS2&zyi?kg2d?b9P2#vkO!2;)aRLsy?Ha*mheG%^UU>mDZSCM z^0l1tHH3xaws*Yjy%l4DS^8pdCB||GrYWRfXk{6)QV$;oY&%jdIoKDTlekd_D-Uzg zfpcY`^ellBNn*o0y3<`lq7NW>&c%2^^iJ5@x~tIx~CB} zNrw>R>;1FD`QRY3_=>&TAjPlERJHC8?7qw?c5j;GNoTHk>ZXF!E~LhS9ZSkEGakNV zCUzFVq@!IVX2X=U9$1qGs59>Z3Tfb+WX{J*+h83)@;_kt4W}jO2%d+2{#5XDeGSp@ zvRVS8?5?t*`-)r6V}Un60cySn>!D^1N26%da4~Hf#_&x;?+>NsDL~CWKe|e1_eS?# zK;}R}4$y|hmtm|LC$~i#QGi<-y*YF^hfM?2TY^^Ye8`639*|&r91{flkl=vs2)6(>oR8T6Iq@yTKY+!5f{UurZoE5x3&gUx@ zSD}KpNR4)rJteoyQ}w8snx~ynFlp~=>PGK0bR86^#cbH#MrzOV98tjcw3#fOPk$qS zc?$an4Hiyr3LZd_BPNdJPF_M*THV($SFG%T>huIO)t^8TTrks4Nk>qO$+J7<*(YUR zT_UyTx*FTGTrh#`+c_R8;!gW)Ke>Mfjh-~%+}N9fU1pMjlUjn)Gi z>Bwztj2p*?C%`lHo5^?qrKr_llH*mxAc0;F?XC=U%Xx0nSx}c7j0;7+q3&vAfL(M= zl;c~KPEEMc1lXsJKLk5z(+-^u;y*~Wl+1J*(PUY%W2NwUXgeqSa?LrxPHc*|{}`3& zEG}|JK;aTTEkyksbzQ?#meSFZj;Wxf29mpu)-ygC=c6O+x~nHz=VK+Ros(b-4R;nxVZ{kE>KQIVq+(SgPltIm#j3 zU3r9tg6UMwvt#UQX<;dWq~yceQI^P@$adu!8jVF(UL*2i0|I{Y_v}VvA&y@Tju%rE zCai_%48I2T*|I?8braPQ@S!CpxcefFrcu^zZ=pR zLT~aF6vKi?Dj~lLnE7@XF_uqwQe?$bPX&P46bzT#EPT>b2BKNwTT7JV3>YRA9`ZShtR01;tjbF* zc9mpYGZQAQ~qQ zCJhFZXP1gZY>1VrZ~HDbqS2^qlrx--ay!XUro%!! zlaB)d+_`ZX#li$%9AA^55#ffQQ#PrYb_cXz@d5dC_knk85K0(*LC0R3e>jJxKH~se)@0;ZO`cZ zA>j>hBp5YCi3SJ#21QX81_S9-z+pOs0gJ4E=?V>PuCMILQ0o5o4sY6 z(C-ldp;LJ0GkE9FNc`x-!h3^%={j;JDLc6WQe7mSaFHaCCjAF!Dpq*Xh{2>PKB(m; zVvSi?IE{9tz8*|P=XjI&x(ri~wSAFm3a7$YZcouNA90EhR?OHhLpRvGH|lM0q1n7u{0xDP zqA&;v20=}-t&KYsNDJ7Bv1~@n&suN^EG^5~HjVT&FlO)$C?7y~y-B<3rc?vz08PP02QSzKN0y zJC94sD^jpgPm83I@Y>BZnYxztBlcQI!{|_2KHJ`tQ9+^$k=%1?t5Vn1(jE-$^+(bk_-=Q&=O?nx|jA zhOY!^Z{RpM>@zjgz07HnZ6ZE0VH|vMR`~^kwu5)XlCV?okmqAfu#sWpWVinWm208% z-MkU#ec2oTcC{YtFi*yf0T|yF&pF+li(K=-$d}Lo@cDFPdok^W7R!qE?KWn@`Btx0 zJZo6GzZb4R?b{v`>%BrGVukYeUWA)m+Y&wJ#{6)F=iJyIqSaW}6`pf(E)X#O?KaI{ zQAqY2)5zP;8n$0X`xbI!gpaE7^KM#?IQ56-+`VuT&~p@~xEE@Lo);1xPm7~hZX*cM zul-Prjimph5d{U_&$MACgXR|W!%^FBpiTP z^SFA22R9S5(6|do)22C==#25_RBs8XQsEhbUQsHXu5cC42~6AyQ5&*DQzMDHa4`OlYw{XOd!P8ID;I1 zwncVSJ{_KufDRm%hs_<=o=Dk)=YU&6v46+$e9F5Zmb0~=j zN?udMWr`E6R0>!qzMJC6gjx=%7qTJ{5!58|ftsekXX2E-e=D4#;PdBk{EI^@?jiN_ zHj>5mJ|8O;*3wPt7XNna72xp?^OLaME1&pxZ1+HpgeF>=SFtGoc4M9~f!gr8T%XQTrjyi*yt#S&&bD zAUoqF*wbJLu>1mId0Y%((jQ4)vol^a+|!K*+2?}R_Y~^sqK&*g=#pgLY8(PB#u3({ zew)|wxqbdMhI`&cQ6@You9cpz#~SL!fds;STq(D;596{ z{bF_MpXXCg6Fp?=(G8{^w<9q1Z9}L2Y%5L>)u|^HK};N)ctr*yfE&Oz>2MKP1y_I_ zR8GLjn$i_`98wD=_#QC0DcxS;t~ftf?X zHp_6&uTV-*oMcKBxIQ6pByA*RFXF-gY?1*ZT!;!CoTdw2FSH>CWSsn1A3{)p-zi!-s|q|NZ0>>Y3S+Vo0ha@res$%%6&&P_34 zi$VTnq@GBti3JT7-U3jOjARRGS>jvD#cZrIQ^*k-EaMn>G(dK3fBb-5%k_*==$Wzn z_S~nyDdgUUBvfJji*Rob+|hD!w+Zbx?pViZwS~YawA$b&TH&h3DLQ2#%`fb8Y}7*< z8y6_?n96@;QsFsak)%xaoo1XyvP6D?=tPFN6L=!m)Km_4N7;D`vXZnB|W zj;=EsDH}E6!N8nnQzJPx2>Y8_bAt=OC>Oh z6itD`=NCj2_*uBG&YsZ?H9{iWvj_tst|eT{!;(O5AX&0i{81 zVTS|k2AOQi5)-lIHt%C*H{z6C&(jy1yic0B<&UIB%A(Du4?9t%o(>Yom5@Wc>kKxg zpd*NUEcej$EGGoW#D%sVI#UG zR|F(TunqN0J7_LFXadfMwTHzH+6aFVzthW4(MkqR(MkmRAPbz+p)mv-Q-P}$j*5~j zm#$#5I%BzmqT% zD{!XZY!m4lSxcO>Rz22o4?otr<9A|e$KZ4{N9)pK=GV?5Zc!2*S`C0}E*4JOYdC4= z@reeKoVH^$(MeDbh_Z%a~3 zIRC5t&`6|Fbb7)-C=Z4dIcl z{uX&IQ|}B~pc`Zp=$X%o@ms{|?dEl$3Ek8sD+6O#0@e!q194F$ z*q9FXQpHihMK1;ZKoH$ZcK~w2jw)zl3-KPQW!$oWkV!`frIzE|waw+M@4qn_0nL|P zI3vY){lu6waAM3Um=#*nNv6P{M4|8^0H2cjo{8{FkzC_b13oG7o(gA3;b~gMT~avH>$Li=k5N5N$!Lrp@p z^gOQp6@xYwG+2x`4GeHK<33L+DePsKoZ4^5Q)RekVR)iKlcefAm3AT(i^t_MLf!QW9nr*VutPBB~$E5(hV!g1Mr$FRXV!} zFXCJ(xo?AhnqgC}hqBk5g6vt9c77p!KFkBEhe|V038O%7`Eb{Fwe}i|PN<6T(UXbHF-5q!t|}a`^~RJzqLPShiSIs*`E2QvKCK{P^8T6xUXl^4mf1 z=1pjhh0~ygUsS=qFYB4!0w{>?9Kk_zx#pU~HP^**dw@;=M2mvQudNnjN8W^nXyRarylMdE!H2w(UZ9EVu# z{Qk~?UqhC0+|7-T(B)ka#uOrqJd$q_NabYn-IiG8CJT%bzXPMj$BAILgIs~rp$0l{ zK4z9ZZByzq|6#)^I?iOZW^vPd`RBCLhWLJNIs2?eHe~e)#Nlw}hpg@5j*0RrWK$+` z(OL!O*k(>8TW|+vpFAF>fNv%U#20RK08t{RCQj&UFua_wM^5nk^$-6$Jl_TPQJCku zDe}*Gp3m<{e&UOXFSBB; zawXw}6wZVjK+oRFv|!q4%iOcQ7ul5Of=PLgl6-&*`h!U~^5>T<4Sm6+>+q~}p3R)3 zam~PMCU|ibw=E$?-uYx3;ia7yy)vAEnMQApzIO`BMY-rQ8gTVvuDa-cI}SG{dRI*$ zYnAgAyiSAg6*PbCp?`^_g~ zTPWgSV5<%!K1SFMv7?1d^2^2{=Esp#!t|wF7AFxHMHY)8V*EU+8eUpl@b)FlL8s*G zBbEhAw_tXnh;`j~094;&z4{=DF5Y8R`V4hvpxb>*rXbe>46z8{^&NF_RJwvme?CC8 zB&nvixt>N$&VZ8ZGSuxy#eww=$5rCwm)12!ta{QBBU(g4N{ik!0}}Hh@hGf^1R#*n zW~d|EM~PEQXS8pb>D|G%hPZb(sM~DB)KaqYww&>{p)Q|xO?W*ZsetlUY5xd^Aj|j@ zj_<$HxDtGef-n)})-yUe`V-l6!f4BwXWlX+m=q`4RbMdsVm}=k&NFlOL(iLNAzRU! zDf5NEU#DYHE4)!l`PtEL;2nhdw`-UAtZ(UYU(j&x{X{F@se<2&9gCgEfR&f4+TjHOz`j;$aCE+B%I{$SE93<>k44lofhdQpq- zBS~hUr^-TlD!e*(;I0<;Td3}3Y&w1{qjxe?VYk3t5ps~yU8#D?^66>`jKY{NV1BuQ z`X)T3vgd1~;jVWu9K1ISHp?eABlBz~%rnDw{syw=>nVo2$n_6`EzzH_nu}s!u362F z-gsq+mf~7qatj$&@DutHtkC)3^U+y+)vv-V9(v`rL^aE>d|IR<#4g+hqvce#!i-EI z{>^VOhBy`C0zRa6lov@7`hN-Ii!Khq_+qXkDlKO#9a!v$U4DLyCV{`#KxSDD?N-XC zqL13+ZgfSLup>6mLQh@(lYd7~HR6S*@Y=A8BL94C_zpcaA2y}XQ)}#`r%nOeVR}l` z$87|hBSGsMHClIXu}#r~5AGvtC_`U8&ZfwEIYZVDMkm838v`rQCbT&8y)WSxRNkv^ z6R$->J-!VMr{Q<5ywNi4`o8R(IS#J?fSOSj<{qRm@shDPq{Od^6MgFLs(k}WikFvz zGsh!sq_Sg2DEm=UMA?CL6tC6tZCSztS-{_?yO+QyEdOUP3g5%@{E;{&4#nR_Pn1?d!N=V-_&~$wbU=%CBXdvvRB7}Z=u3tQ{Th= zI6JIQ=nEM$2YVjs1EvC3ANWl{BGdrbanhIA0nqD%P_!?@Y}C{JVGG|11(SZghtQo& z@9jShwC_c$a>iSRIyy4r!`y~pYAd&p8G~WIh6(3h+P^-{hj|mqMjDalxb+gn~FW;ds0!D{sl(snKd-6K{ADr#7~c9F66? zVZ6b16K|OOOJt_fCXW0ak@t`Xc%4MM3pO{tYe6*sK;tTNZ5D-!)j|Uw-hj`J-UlPg6Lk6w#4XVJdvOTJdgrweW`Po@ zWJm(GsKi-GdDT+d~*vy6R6iL3E zD01Q;Ej7z0$76%njw?id#QAk1IC zuV_4)HlZmynnJ7RQ;&fgMROrv(NMor$X}&mHR$^&dUS*4z5!E&{8;UQxY61x;diS- zek^=WQpK%Jd}xI_fIn*t8hi{55^g9$)Gz_pDTp@*eoLhG94}Fo!9T-w&|R=j72D-# z91YT4u=c=BqQTJMp`pcEIO9kE&RW<{Tch#nw=;89nt8kTmm#^TC{{lU}(^oLucaa~y zawd)-K3T}|`GCME_?*Dx@HIM%@S&}z=ST|a9zz`t(!l3O$ghmpdm=_IZNy#^F^h<` zix{0-AdhxfsFyEdkBL~fi2Yo|0wUJPW5<&5W-C0TcTr4HUnRxhaNGMOT6APeJcLaC zE*h;wi0|ixUlnT@;=3u1!l{(GOO0UNS5LB#V+vf~MXo}nzXpp5cG1}A>&cPsM`^n< z+3ualw*~aFC%>LO_{v-O*q_!1?#DK%$C?r663zv{Lin;7hG6Q{0WqK|FBl2q%za&2eb`AIl3IDjzv(Yd~>-g!^LgB;y4N_#h08oRD z((AKKIGDt)!rk*TN>{Rz<&1MUD*QV}*He!q!{It}35sT=1&gJOHfz0ji>O+sNGUi5 z$BP`r-mDqq(v@pl$$MPvxi+d8yWodFF1j)SI~4bN;LN3MAH7H5urAjANy@PHr*C%jnZiuA=ily z4dMPjzwW{3ncx4X&p`+WRQ!3kviBPxYB(B0Td9qFEA`^HaHab!I$H)_zi357_GQlj zj)4Re$5F`o%jp^qwpiKo@M+kG`MD=t&_6||`LL^gdkx=zxm9QR4l}Vc_nJS9haMpm z*voA%rlyH#o+Cu8D8u$#q@dlHG{4%O1;nFT0J{Zf0zhL~bAU+Jo^7gBoF{k0^n$14DbZH48Wf8vTM0;)yu+Fk2n5@x#}fMtYmne<`;c} zo>s6mwtKqY{R}Fez9cnH_1-tb8nlDeh_7Vd4)fFK5f^R)9oS|OU)h!QMi?FV)PTF(SDU?)aYR^wU)9Zl4k-DnKAK9WJA2l2Qrl0as`?u=qvn3&2D!fqd=5VGh2Baz`@ua?r5;$fLX!<4BSo<`+<)eQ>v9 zlPvCb(8c3{&uefWsOgNSL&n z(9Hnt$BSd%8D~jOs*?x4HA3f9ccKo+{uVYH$utA%i$L_ckJ1i;IYD1cC0_{$2UW@N z=^%?L?2qmQ^=U6=3Vg$9iB&{BeE~5Cf1@OPF9jYRTJhlFdHiYsp}y%DRfhUj)i(FP zLv7P-xSydEKBc!4V6;!^Id>6-vEBGP?d)s6CYBB9erde@w_=O4+965UtA^7 zyNu7yC46?KgOnw9xMK3vDQbU^@H|gUP1+YUHFrcz%`zaRA>E9rQD4&%Z)k-lCMP^G zxE?@mAE3o&g|E+G8zCVi7#-oM*?@c1QpRzm>khmUy9`rLE_b`PIGKE^JH+)XxQa64 zFp%2gG&lU>eT1`!)?mK;4qOTNVjfBlEWLc^hRZ}G5+5K7Z!*3MFxs1pN8xs&EnDDP zaySg0utzX?$`Z3@qZtmHvlw7LXkFw=q=|80QfM_f8jmtGS|wqd+RY`L@Y6ZrvFYc# zytU}Yicg*3D~o8y6lnhx=lXmNI=F1{@lLj9i zH0LAe(C;EcW0(vH;1>E+<0q&c{{wkaKfLo^5TGkPyWWtcnpuA@}h)gCIHYc2{E-A+YoGj^i(WE@gt)SjV+w?DU4 zUwe+8s;A4_KXSzzp28J0!UnExI*N{?gmXM02EHkX4H=x|VU34E#Jtng^6wW$h)wA1 zDtylqD+ewSVv3>KUI&HzSE8=(vLggW`Hsy4Kc*?h2QL!&F&5v}!4b!=&X)0`DcAzR zMqvu*tKa7i7<@!R`BUI2?8vtJYZe<0XU-`0E{Q^i%hjF+tsz!rInL9<-?qGn_n6Gc zD&bP|NBF$NahE0HonX)L)R4=MkJCi&PKIv`y_QPU;kQhPP<+sT7V6=7%JS)s6Bxw| z-+ey~8XskthOS5ATN2c?kGS?z`L~QK*%aR$Mu;tJS)7@Y)rx|oGldWjaPM+VTE??5 zoVIO+FFk)93~XY+-c1eAG468~*I4(teA$ro4GuN9F_LlZ7QOU)&Vw-Ttma~}#{)i3x-iba%m6;>3V#o&<_aR|Ww#I*g^#lDBa-6#3>`R+A%U$vPv}ze zaV>!DtbVxy<^+8PM_4Dc_Xrs!13dPw?7Lx(`YolYbid^^S^R@Pp>*LG_#M}c@Cl{s zL8j0rlv*zr_|D@B;&QIKQw)cfz%*Z;8o`nGHL5o9WYtEVgE4$JM~ZKu7}cw=WU+}i z6d(gjsaM}rEAM<{i_L@%i(97P`-lAOB|AAQ?IYM8A+mhI$&y8DM}+9TD{M~jI0fSVWS(yJ=531tisoAfDA5)1$rP-p8nhLk;JQ^uw9A4q%W1QZxtWiJcAe< zQzcLNCyq=Hu#z2^#$JTqXHked72#IyPv&gF> zvf*$WOsO$$oEt1_gV!v4<`G)s>E#%1i7C2QXJej)0;KsDet&XMkUGR0P~v0JwAlL+ z&h7jsM~QmEg`bvpRZqot9TA1@!k6VaI=Iv;8I~RE$)qI++D1#Ai7Ux9*z7kFmi71Mi81kU2)_rZcQK-tcz#~W$6@67yXf3>Hb=c$A51#u zg$w^Tzz^s#hvDKs7)Nw%>ot|7dP_I zC}$+Rekpeex3(IKzr^Wtb<}6aH$a#-V+O%0;e#*I z`&`?buv_w=Rr%=LJ#Sz3Q{pWUX*}*h;62(eJ8}558M(csG6FA*Rpq^mb(8EJC21>^hEL* zDFW~t^KU0u$g=TW^%GtOcO$5OSy}kQS*QaQkdNgCS|FIj`^OkE)QJU0J@0N7Z*Fpdkxlx3)_!Vv#H6{i&ytJ?_9Ss1^9cX z-VT7{syZaL>3)F7SS`M&uFJ&}CpQ?W2IQTCcp5B1#DYnGyN9Tg(1_lGNv9DBCLPCb z3sr?O3MT!A3V7f*zbSN;CO9(+rN#6HXMTJ4!1GCz4W*xmc>Xlv`Qz}jI8)A2pm^~n z>vNLl-G1pT)6$oT&XW|2KD#hS`(>9s`jlMQZolZ1t?=0ON+OmbT*Pja1Vw{6mP!p?VvGepSc5aMQ zqt`NhrPdjpAC;l{7I-gC>O=6ShC9=t%iX4q8ftg%WnW<zvyB=$xojsz=@p%>+H0kUO4Dm}g)U?jPrm zcgE#w;T8%1+I%ibovv23Etz$O-rV-s@|*u;79Y~uY%elZ@5DRMkC z*2<{ViIkh`uxroKK7__B8l}^TK%0&&Uj?!3$I4*;H2&n_85}rskf< zrp}wdrk)lvs!5Jz$wfw%oNHjo7;p0ZDj$s3vhm=H@t?*zMVSdGGahA7_6fC&K|9a~ zr%xQ{13Kl71%32v;!4nm<27F3#h>}OR2(OuKJX+QIbUF`sGEqoi7UYue7=l>=ZlX8 zV<^HHfX`ImGxdpy-4}GCFOCEHE<)d!kBQ)~iF1=W2Y3$csysJw2JuPKFg|IddEt2% zv9W!Fd5H7gY>Y8?Hrl?FO~kxS=reW-PKua6qCa@V03I>ikFoJ_#|RnbIF1L7MzjNd z0?j8rk>nRVeIam49>mklz-NRx>Lw}HB*cJ!j4a`Pa0aA6jME4j8ZgdrEb$4H;q!|* z23?au*W|g0eN=bi)EGAI)4}7bL0^48!jJm56U0a!57~)GOUKyzjoLWiv) zv!dKpSy8sQw9LUCiC*a{TT@!;sxBy7U4Cn6c@bkDMIp^zQL(AoxwyK#&{bMdo`o7) z8GBWOoJ{=R^3|0NYQ<6GV&yLUhs69pC0@faD;@aX>#1Rhea#kEIvn=W4V49`Iw4AS z6gZ`_vWh~M%%3Tjv3(l+zrZ<_6@`wfDk{O)Pbg*B{}j9@R8A@?szh1*AHz}hih?4w zeF>LX*^u$Et664QMU_J?C@;cDzaN3!4eVZKEv+ipQ0B-j0nVXjG7HMes@Nk@E2$}O*{dv6^dt8S5`X8U7{fy=(ji`RUI0VJ1BM4 z)+(1{b8hKohg3ymq9;}qo$ILFTv`s&sl~B%9Nq?VMXxR|;e#p4sws3hX=2a_A;xl| zS2Cd8!-oysV|5v`)S_JS%`R%K-c#yF25Zgb*4m(C?L**$EPqr6C3nZrt* zC1uV6XQ>$96*_9n34Xnjo~j(KNWHUh!!DWP%YF*|3izNJN}!{TKfsu27eX0hMVoN; z3%DO2=hzARG=Q+p#8?S}9`@`jNKXe$oC1A^=R|rQ5@3Qc8G#|pLzprXGlh_kD|Oun zZiM{^p|}&zeF%*R%?P2m3w@K#T!&DEzz~cGeHSt|XBy5D5g6Ki1;K^Tj*xOOW1j;i z0yY6^0gb@HgkUiP4}>WQZiGgJd;}-L9E8U(#%6>R;Lw433lMfAe275tGBv&c`ETMm z2jLI`Q{&szIQE9D1~@h%uu83-#7HX_`C@F2pY z2!4c4gdT)%5R5-#Y!-qQ;rj@;Ab1cSL3j$G1K|w9IfOVILR^AyEkZ8B7KHl{{u|*% zg!d7?K}h&7#%3T~gb*rgd+$qBK!?Ow-@4z za4kX}!tDt6BRq<54B=e_O%r3&5UdC#2=xg6h42JICqe*W{65C!A}m92Al!}c2*Oc> zR}p#;f(R3T$=Ga!g$TI_B?xyS+=uWBgkK~05#B+F`W4t3VFAKw1kz|xEShPVj>WJs zOwY!$ScW|gOJL)e0j3)EKqT1A<}(!(I$w zPGOg_S!_0&!!E=B6MqFuWmmGfEDf6YDy**GW9e)@u3==b1#BU^mR-jzOk#^zCbP0E zwwNtpGPAMk*-~a_*=!kG&T`lawvw%4xokCC!){<}u`2(7-H88zem%R1<*|HLfd7fU zkQFfpD`p#62`gncvrVjwZD!@Hf;rhOtddpXf2yx$TiC6vhHYiHvD?`;b_e?*yOX(D zExU_(SRLEW?#B91&%CUGDQpL-B#rQ~F#8}bS%C$F?BuX3}KClqbY z+lc=tzN)Okl~-1gS5iu)kcj1HNFo(0aJee;DvFC~aYDIpQdQ|~j!?qpf}*?))y1J` zc}0;U&$ZR*$b(TAN_3Vx^RT3rqmPKhib{&Zp2#b%tN;olW<{jlLiSD`@1W9A7;0Oj zItyGSdBaoDMQM3)#qbt|7iXoTxU?pe z0rGA}W2J>K0zzp-&mu>0L8z0;65iexN2NL^#bua2qA7By@bDZ$yi!b8kz-4qKqMm3 zSy@rUDNW5pq~GE!ACiK>I`dpG;_|i>lnu#qRTen&%BwdINycOq4enBmXb2vZI1(O| zJ{S*584VB08vzeWj>My)koqcyK`MB-dPCLLs_!Z>N@Z#n6ZsvLhxJ^|37l6{TCfq* z8YV)8nqI-J+&pkGk&z^#w7k@nSMK0b9FEiYE6S_5=3l$H-`hSOw7r{T1bOOMzouM_TcV@Pbm zEmKOkU5*$@PDpz5HoK}vOhg^dxLnS{rNOp&6d7K{xe4m6v{(#%P!FM;kz1?`k!w&L zN*%d;6~~IJB(7&@?yz|*D9dw_c2nhQa6WNmMWrfS?ve4T@N=C?#T9Br1D6W(?3|C4Vjw_{(ge#>F z!<8pSXs8GjXpM=$l}bk7%6WulqtwY2c$jCnpp8u&Jrwj+*lgk0xvL1UKQ*r!5V{*p((4V7VJJKV@uh{V?#Q60275J ze_-syn4@!wVB66+g>wHXr-TcA2OA72GmH&*{!lic#9?zxiK9;?<&HF!l#WTtgXb&n zJExQvB$|&lrQrf2Oey7!Fr_@1Thiqfd4=%4!C^Lt(WtNT;6@NB31`7p+N|U`g>_$5hTdG{7^C*7)rb_e%_?{qVYBnv zm8BJxrLL{K;6QSiGF6)eo~i)XlQ*gRZ%)8ou;0bl{{-Oa&s7hv%f@Y(v{%LV)hVTXe9srbL% zaZNQ9=???`b}oE+cy0z3moWuYONLRqMdlaTeIMo!?PRk)MEI$@!SO%B?D)) zqX&G*hPIKu8}Rn)MLTtXM-l2!rX6t3Qk(`cJz<(HGtc);fDkcp95wu6Zkj* z6LUa2l%E3l?h4?A@*e_9tKct0x(ryA3tunNw*g+e8g=k&1vK6O|1-)Y1Fk}dNBTOz z2i51@fOq}??P0w2fN$LhpEI7{158>EeDOR5a18?CzYcH@f)>yF0iVbRM46NV@DG9= z&rbr*+aSii0Pq9?&1nbVFAL!Zr#6cigV%)99J>HE(E*&%W+Px50`X@jU``2q>L|Y! z&|iu%f%npHW~>5%%G3a!Q=j8EF?Iz4rKbbls6Nv<%%n2W7oE#&Mp&z09s!q>15eOA z2QaBZl$io}Cj!k;J>ZK7G~QPLFU0vGJ(~e6rOztS}upTAL7Yw<7HBDzL1KO1%iuc95{0r zi@KR8vl(Xq1t`B2*c2cJC4;|gz%u-!oC=%@(4P~h7@BY&R+Om1$pZE51P-Mr2c?0( z<%m1*yqKRoP<|mwn$W%j}atFZ7$+CFIW4a8KvGsb4oEWJKu%7jFTvhO3Nag=@4xr=XJEXxdGS^Jt{CiRU8+? zuy%&dz||`KujZq^60s|BekJAtO<;bBKi2^7%J3)#I1_psJdG1M6L|DygE>3b}DH94E!YT_wCb?*!8>)V&QKYhPtzq~(Z|Jwce`%Csa_t)%q z@2}tAxPRCFzWx3C*#Z3l%Pb{`BJ o>^sE1MMc?0h2i~u&%HOLytwV3eYVf< zGz-fs0c|N^?E9Kb-;nq+f?=g zqbut?Atn&?FAX~D8JE-N^YhFSjtPc+jK{}VXIC)wem6HsuTM5AqW`kWWqav_erO^; zo*5YWGRlu<)(m|}g07((B-k+Y6~MjsH4NPf_|l1#q3=rN8ijwQ0z(SCTY+H-&h@zJ zsE#XpV*h7g9LN?QZF%xoltHTjUnG3LK$8jRHR#C)XcSV7mfe zQsAQsyib8{1v(U1rN~Lxb&Eo$a~Y{=s8asBGN(=|Fr?JWfRG0i`jZOWtiV?kc%uUE zQJ_zOwF<0KV5I^}6*x(OLlu~!K(zvU6hCz-<%B&67{8v7Yc1Op>_?QI&Rkue*vse= zsY0HBU2j4%5Tu%1h5uF=_G@Zd(-difprkcTYgRQ4(Q0irTXS=>7Rx)k$1Y&W`oM1whT+0A+vz< zWI4EkWEsjyL^;|uv+%c>ks9AX^L1w&PpB@V{#V=64O$m{^;E1Skxyk3vf=i*2>L)J`e5B1mp2IoIG3l^!@0jfgBR>}Pl&&&I@D|Mf{GHGFI?{k`gy<0?+s;*V}cwX z4*DRJ3kLl`CM(lDE}5{LGXN*}U0|qndc2%l7I9_!D_hq>LXdNM84g;wpp*B&v9d^M zm@>1cWV@M(OlE~FZ!WhKqch*>4Rf|eMH92!!`CsHlExB$*z0EKJ!2`yGft+?>2rHI zI$Fv&LfcYwIk=*)7D6g93<>u$xU@LEu*$sx*A!`(65Sz6xjMl+l|c`cv4oSP;Zw}Q zEKr<)p1^J^=f(7qOuyQ2NU?H?@P8!dT70IQt2HVnRXC;;&7#Wvq5PhGTApwW8y5%k;+D~L4+>4K zS`zkn-Jzxs}gmW8X8fu@EoBg<*<*yRcU@j06W8 z*~?=OW+VF~5dBmYeor(U>XEE59fCcgVFyYq%yh8PcBc9}i}!TOiEXJe>gKbN(`Tpz zL0dT-zyzPHHXaAXbd-gS4mM^Oe*pr(Y-U@m#ypB;+e{rcQ!Bf#4UA3utj4TSxGkZA z5e<5%E7-%vZXJUAU6^A9yU}5d%}j!zF%X1il%L-J*cP5P&`1r!dWlmse0&ZaA1xXT zlG*y3s3T*B1%C&Kw$)gLR?#qm(x_a~uuI|o1dtW16W(u8tK_Y;by>ntBubC zzzGz634rzs=&K5fb~}Iqw924e2K!~u1;7e#u`%ua=u+BNjhc<_R@WBnscl(3b~@Kq!^@)KYw8$1ia(W#pG*E( zD~&8`>WnYP;mZn78c7mhq@7NI?HD z&8^{?(rBP3qTxoFk{HclDALiK4v6ttNX>{DEObUCM$;~tCXMD7f2Gy(GI$5mF}`hp z+{Z!6+Q^bRGe(jkvu&YluU)}5PGY-2bw=B>?@G-0&Q590(|MP%0299pgO7=S7cMW# z#BY@-?+IT}E125tYPD#{m3qXF5Dlv&^Ymx2@PsbW@Do&=I)YJ3+luihRHrr5xmDX1 z*HN5KjOf?udq=Xubr^=*FbhRP9W^PYFGJzfdN8m;g^rDOh*(R)`7gJy=MJ*bQz|z4 zm6}g(nmUo!Hcg!z-kG*@hDx?0+T-D;M|-?{L9{2xkCH_ng*&kBWYM2V1%g8NSg41x z=Rx5$tS~b@KKg5rQ{rm4r&}ApAbIIlbjsA)JybM|m%CSp?umw-C`9*T%+?wur(n*M zV6NC>Pa3J)%7nYp%Y+b9CY-2N+>2S0Cuo;sV0j4o1pxM#HXEQtdk20o$8VQ_!6*T} zo&q|NMS~i;TC_tbA=;jo7%e(XI<{q{G2PzcHfC7n(P@eOM3b2o(PUuu*u-?n6}Nuz7YrB7$f9LMq$b!S@pnQZgIe3oF6XeGI#G z`Zh#3h%_r&ie?SN`JnwH_Cct;0rUO|DlevA1rVPM-TSn^K_%TjY|SUb127A|uYC;^ zHd;gh!3bmgfW^O$u0VVmR-h&3P@6hL!#dFIg7yHNy@KIQ071VEzwu!>Kw2bvwCFhy zNLWYDfmh*&Xwi0wkRlphKvB>>i(>c07VXo3(V~@7VAchtG%nB-2I4J1ih3FYQbb61RPH<$^Mv6Z9P!!7-@fys# z*{n-H^dssP?OUL-N&6%DG-47yk+#*6Q8W;v$&V-+2ub0Crq<|ox(%l`49}1B0O?Kz zX$7C!Fd#n?M>*}WR%t_T(f-N+N6diJXkUK(B?Jq40YK1ppr$PPd(qx;^024q9 zOx7=v7=8hU7FGP0P#G;+M1wOpn_~JWP!jn!@lZvnr%K^`<$%2m z)YjHMEaivU0{tn3R>FhWwzl-zb8Vs+R-fI&wsoYR+6XDVt)zXJu?ZQh@Sa`hutZPv zSpG&dcHtm@okcL~%3Ti4xg++ZgRE=6eeA(9*Pa=|kaSikNf%fhn>17RH%n3)pOW@# zOH%as8FsxrX}{g{yE37J4#{1o!dT_Ky6Cu+3zj@q0K>s}1a;f!I_ld|%7dYen{mU83$&{!jHrz^R;k1Nw_&AVY0^@X zG>dq()RR7arspt@U}?Y59!;ea&27vk?5>=jeK7Hf@hBJU<5SpkFh|>t>h1;>x8`io zFp4V1Wftg2pjP2Xndzu#$moL)p)BCeOH>X7!Hzb0cHC*V!B`|L{IN&aZ6-yXz2F0T zn)L`agIUm4cJ3`;c4`)yndy2-9CDaXY>OS|Qsgwf2x1D{fE@z&&`+sdTHSS^;Z#^i zZ%VjA)Xe5KJrcngb*Wggl?fEAa1R7n>uGsUv774F9dL2<@Tueo*s z5?NesEB3>?CYl{{c_Zf0R|m0v?mh- zKY$qAz~>I?n#9QVm*YJAIgal*GKET;Y)Qs6l=~-<{?n2i>TL8}mY1Dq`z~apk47v7 znZf~7?-)W7^%2lsMOLAI5=_>RgIjno?*OPbnXvX3Cl5 z_ImMeD0Oedjbuo8X^`Uxc}6vz;}JkYh~rjQmsBfZ zMpv~QY=e%uq`@e(uj&eSde^3xhp-c zWfB2rCoLm7`?Z(w^SWN#{A5Zm?%0#oiyQwkIDt3)X#LOx&g|GZ zT*fuKI<9{(m*N{g?R@k4-EYk@GiRQ9Eiiub3vc|?WA0jfG4taQ56&Lm9W~$mqr;Dt zq-ta1UVO^@)vgWJPa}gLimcjh-c|U{ray>fqZ)U=W%lnqzV=j8$Mn~KxY@F!dD*<1 z^JC`%$5&3<@&1FyzWV8f?)#pbQ?l_-4JSIk95;D&A`LiD{ zE_!2ef$-F$i^Rp7&TM2_E1y|=yYt{hVUchob8*X8N0&bLSkKv;cl~Vt;`f_;_Ku%C z&ip_& zXnE)VoGhMfzw5*D_MPh<`%_2pqwN(1Ve5e*JAeFQ@v)_^ugkZ+aO2B=SXXQuyZ56V zuNO>l3ilK{=iS+M-^&#%2mhwJ`0!_wWBqnM+HAOBEjAWEb1MItoBw?O=39#IeSYJ~ z9qyH-UGLAk!}Yy>ontGPN5w}XMUR}HI{Y42X33P+=L<)-9L!ra{qb$&;^BfnKCn=C z{mbzaYc?en>|Je}*8kT}gl|7OX{zb&k%z_}&H1eC;YInIzQ40|!F$iF+Vr#6az3hi zWze&8pSIq)HGM*6*~y8w<<73y^5(M4hwjw;qV?GBdoRTI-uTof4=h^nNZ~F=bWFyq zncR|#(T85_x9;9SCnp|z`prZ8HScdaf`~QJa|h;jjoVrJ(B6-42!!`YpFtA++n@iJ z9$1f<;UkEI>N4>@2=F;X?9Uhpc?mFzw?J{=YXLt(7%l@m^gY59e9IMro(K3R8j=N0 zzw0!3lVAl--zFSEZURo}${wV<0S;YaD<$7q3ig5+0n8`Si;Hzczd+RNy{78#nA~z*(deI8DFReEzlk1+OL{ zdeGuwnWOaXVa34c`KrO6ijw=JO={Yr0&hNoc>m!eOCimJ_aYo5`$#QbhY+$#;>jkt zc&wI>oQwA(=m*}L3_#lw@YLgFi4*)QV2u+cd{7{j1JbuAw8l>wC0ZlNLYssRD>y=U zaYA(mV1oyIK8f!izfPfc1qq|rNbUB(CaT8`DlOAjF-}Rl{CcI3r1sG}A9{n}xuh=# zFJ-c&oDE2^9Q4&hrobAxE%gU|X%hsbk}vg?YQ0?NB=9vNV@;7;pw|w3snm*kD*!*y z9xug9JRUq((|(Cwj#1BjSkmLa+1YSENx*^jl^>?{^a8eNdQom7FU_ z;5Nnow@JO8g3=U{C*gFa(f-hRIv<*Xy}h6#BF|7ljF{Vak%C$}3qt6t6P~yun-50gH-AD;2dstIXO*My^U8kQAO8D$m4D~gKUNO%>${bo@av(<|K`_s0{8FpYkB=9C)J@~ zY%?>qJW9oWvi3oX5_gJ4&z=~iX6#NvT#kLyj;ju^m+nn+IGV9I+$lgQ>?i<4`lu}m z^fJZr8!AVI`BlE?b}}}Uz*y%t#*@~5 zgrEukZ)|lmHY4MH^oe-^M7(z=MEa0)7vu0(=NyD60cp56A;l0a^if0UiSE2Uw6V2e1%u zH9!Rz!eBXsYfqq^*1+`y;=O?VfTsYF{**CJB3upFJsmKSvBgCzi_InGaw>!H(ke5q zT}ko9E2>NBTH36y5NZU6zQ|eAwArDbSGvOXEyP#2Ha0l~y|t#PX0v09 zqlv~=d9zD!v?6U=y|Bs27!_%9H0l@E*0fMUW$Cg~s&?fR6DtPf0gM1WKn-AkVV#Mc z1NZ>%0x0iI+a8}geq&ejJ9}^Y+s`j)((69@^pWXSmGwEDSm?NO@zK|x-V|SV+ha?f zJURa3+y3(5gG+C++e&TrB}%;jsI)6lqj5zM<7|spT1FBA<%<@N#EjI5xW&a>E`s}kn4mU! zhLQUsxj@i8Z;LXmU3=r&`i(V>U+((P8GyF{*FssYzsVS28Kv8Yqz^4CN=uilSb1*3Z;Rym?=Lg4 z|F(ffxF_%fZZ@0PuS(GF60~PAu|MBzVlm6Ud7GXuuA60cmKSf`x^dyH{%^0lzDjQM zR$TuR5M6Czc7Wk$=Ba7(bC$R@r`BD2*Td2OJgpt~%!Hbv?vwwq?B}ZE&pkQ$=*b=X ze^LCR!}sm{(~GWt^RriH3TFmyeyaMlbw%x`p6#By=lR0lFE0Mi)47`;_~1k5-apLQ z+qUL~B9Gqi%W-GRv{${gPV-m$hf98Ybbix2p6vI&DWAV-{-;;T{rnfM-?Z%bU++u$ z^yS949&)^BZnfS1Z|^N~UG5qai(O-4e_Cr|Yw_YYM4MG$0n_p^F51vgMf0} zQ+gAtWi70gHM4etwREKJF}wq0fL4#pU~rgg8V*b(~UxjQ@5F|De8vzlr5o7Ix4Yb_?Huq>)}$`TvM z(h`=XN=knU6I->^p>EJPY2K^$|Ji*|)8%!k(!7Jt29=X0!qLcmQG*qTq?s`&GJ3F} zA=XJ#Wl&_y;I|s$oHTa^MXCnl8`MskOqEFO1720EeSIkEl386h_YQw(BnXt;xv=xX ziJ4=GAz9T1Z)dh&d%rid2P=XqkuGL|o_hSd+RfskqbQF0D?x-X^(Yb=jlTG`kp`4i8fGWoMElRqm`e#x{@d#KN+p6mB( znf6dv`#q*_kB!qwm1^xFUZC%k+T9V)JU1^Cql9sjFb+G#C@D&NNQ~++Yr1bjSzRb~ z(Es>8-&k&Q`HGJyUsjD&nb>WK>aps&Ezv#J_-;!K_+V2@l}TU$u^~>hY3ig(V)azX zn%vK#z+atciC_1MKBRRGghyT75Vu+y{`~RI?J?015S^WBS?^Nyn_{9mEis*A#M7+q zF~wYNz)<4V*O@i*&6>b?3LXJHz}>9&o{mvlt_~X;AjH*uzL>GZCGk;(4_egIV^UoP z4UoFWs;9nnmq5kUi6yG;k~mzZO6DZ7BwjKnizSJYd7fC3R418Jx=Y8Q-$>Wv&M1~* zbjIWEIn$+rHusQ!|uUHv7|AZV%1pYR|QPJ_wsTDYqpLzz|i3ac1{ zo`c#+2_dxFJyy4%B{hu-?X!J-r^0*v#5f@i)G!Uk;Ft2%N#SG1?ac}!SC1(#X);@(NyI{P!ke># zz>6u~B4k^HnHGy{DI%`52n7~lyxSM&%GftyFl*nqL8Bb43q-5;sRw88iyNG^FLrRo z*d*0p+`gE>=zY&I!_PtVRv05H0DGm<{X8Xb0QVGF48R5W4VswE>G~?mWa)zN8&ng zPY*)_a}jhdj2`bP)r3BnP2G#rBTO3;lT`~rs%hNHOF))Z>!AWSDqlwo0POFFn!KqZxlA^35Qo<4?Sf&sW z_bDJ0yc}?MXc(A81}8XvC~7dqed-b!pAl<#(-&#{X!loYB(?Ui`^+7XN>DrI*u+=0 zhl!VCP?TVDYP5%UoyOh~>pP>qTs9yJgE|z`b!YdZ>d|#4dQ661L`w&zl7N(eQT>I=RGF@WlUINk-RVS8(_e`k~#@U;`P{cW+7Bv=6 ze}bUI9IkVpiK?6UX3s}GuLHS=RbBlxdKGRDOQxjAMw?_wGFUY-)e#UaNnsR0S)%#O zFw{|&_{*8*_mAs4Gm|qdVio!bYy&4CUl>aes>KnlF6fxJFu4TLphuB3X3m2O*bSF} zN;|0UIn;SN>Jmxyo4(zL(TH|`v06nVTt!lhh4xtKGXaS(xsnk?!YQ@5IGh`j2@2=R zg%ZcKgyGaDz1jDXZZ397*~C|*H+P*ufc2f$O?(p~fZ9m*UxA9?8c|0IgOn4$Q!DBt znv_Sdo+6dawPBArvT_2^BonX+9^i6Zon%=^0;qlV)xOg)+Fx5Wp5iDh1%fHG?(84O z4zTBx^$9Ze znx>i9**MHny(tvBNO5`!eMTW2g?toZ6ndXRLsJntj?kiLJ!%fC_Cd;YoHG5ILS70z zPNAnL^dkyANTGWvw3|YABlKJ}wl;zBVj(L(nnf{TJ}%KrNWw+Mgvq$5nX9n31qB9I z_pYT&d9Q^o%X*D;Del$NB_BEUOi0BAZudF3bTT0!A5C-$<9ZV*%hkOqx=ic+0M4pG zP47{<#P+^`OAuSCq;m#u*D>i;L6a3Hs7h{j4R;-L#r9USah+ZM^#Z(JJ7X8TDryh7 zZ(s*TRbI6;Oyz{w+)7Qml3kdB+!MfN-=kn3B>x6x4`plv^Sg9@SA4?)zf0qHsePX; zp*1H2C0M!|SVOeOoMdZIdCZBnF(qE)CFZSrB{67`%Kb-i{t9WG>}E0et3B3>W#CAW+SZ`<6eZf6kFi|lZ%-0CF8UKX(JMGe#|;#z=R*qJ z9T!y)M2tA^;5w;DojV}RFdTLdPto4rw{+L(bAEr}uD&UdR;h0=zadKcqc4vr7FXqm zE6A*qEV^px&CWAX)fe8Wu6Di~mCI}cEC$uhJod%0IY-N@%lz$n-$Q1`?CaMY4&frv zU?p%+C}mBFM5!y2U`l`lmVd+VaJIaN1&P{~OHK`xoXktE`c`nsimGbqFIO%;El~W8 zi+;c7cN@@Q?Xz)EX?nLW0rXrP1$s7PnrKK`ovm(-Cl~d&)t^5;xSHa0h-(`!s(WKW z*g*>t$&nh=pn(hQA_%Oi^@ci0+KBqOEg;oWONYzLWGRLSb17~H76vaU96E^IL-C^Z zVo<#i20}uhia1nJ0gw7l$VauoYqK@P;6KEhc`IICIZbwX5BnwSMR9SGDf_-c?dBdbf>)njxr@^SU7Yi!-`|kpF&}~E zAfK9amjzWy9UAGos}OKIhFRBP(fomQ*K*v7W*-K-Xg)8E1BSx2QdJ7nXkvwD-H{5Y zI(3=B8{dGeSWA%bM}xh7$>>0RSQt<>?sU`)e&yL1;d zjHS#psomIhLC|_`AMgaQ_{~F7} z00HMnr8aaZ$@7D?XtJE>X!E~GedDw{yN9sk{B}ErIIAm0INdd7aBsQXU(t1b zgn_Qv;NCpfOljQ*T_;`BCG(JYmkn)6=5yj*D-j@$t{I`kRZ_`HG_NI0cJ&idRTV^% z16-7epKC&>@HJR?b$|$KhKYq#rOd)rAuL=O#zMlf+Gz`BA@L}bgLm=zyuu`@eJwaf z9E1!;WEj?C~(&>58tyuA#2Oc2}I``P*R0^n;!U)&x)Y5NwIlc_JtBK->oC zA|ci{fJskg=tMqTVF>X=n*3a+tpnc>_VJ8fvpeH zBs3d+7icQ8(aV?-BF)i{+=f1s|ZHsf?Zt}Os zxh*OF!8obZBwA7=Y~NXo1EwsP!EC^kmxYcDm`wNMV(^~Tqk&PA^~waB6d+jTKmGoy z2EA|i2GXFbg^AK3-!mG>&$+>9!-cL3=r2xbc?Z8$O$2$JY}iGV43>3F;_O-ul1l@F zds$z>!khXsD)91-iXas_x&aC-(Y4%gEPJ}E1mXqbtsH;ce!K9FY`ncp zwwst0u-+P2nZHf%JDH2++kl{k)=){qgHK8^6@DA->`&&^&qwvW)R-MA?UneiZG!0n z6qG(E3hI0}UO&po&-FnR4CiDzC699QDY=7}0-efAk< z8_WsE0|tYqXr5D#js|YYJaC(EPc|JB-Nguc%tvhv$y9{w7`4r5lsPjpjcB_Pkj>q3 zG+Eml3g!Xq881fNMMY<#rswr0A79WD|M&a8l_tvfW*Cs$Io>nNr~>mkYP6(w#} zS&xVknNc(=u9B85#8egUseGb**8ysiEGXXKn(B7TVRE4O`w)+85AgX(d(Up%_LNNR zE=rV2)MAlND$!uYClg_{BDchZ#X=_Xl@%Y zsn9Be@)fcFjzBlJM!QKhh}~Q(bQ1~1Zkn;hZjyoEdCb{~2;Z6OM#s{FG%bzMiVL9? zb)*&f;{IWpOh<>)AQ>23F~%#3F?bCON+)8@nB{gwOV+s^HA$I_c|=Rve)7A^T7wy+X(@s z@#4qCV;e2PZRAXV9>C0LQkOZsSRMOY0!tc$_pstb;76jMmiWB)5(!pr{}`VIZ$a@{ ztU(Lptmx3eJVOhp9IJmxU5!yy7LOrLo(obZe0C1-Qr$4q~w-{8CmY@NJ9#rC+K z*dG(HD@NyG_t3)2PQ|>uG8Mo24{p2oF{j}$iubG+@>^9MtT|yWN+pTVIoC)y_aGic z2C`i}mB(SKYsggRrc|v|qQhcKDv1}(=K{*-yt1x>N}XrJz;)h38c3ldtuH6?bycj? zF(j|5N)u5zjd7`yJcDR#PV1ET~f~ zj;RwmS}mLdhIBp8SM#EIm|P(?ooC&-3hABBzeQa*>3m_-6@o6$Jo&|2b52N6u#B<3 zv&GwdcN?LaPVD-U=3AGIb`XS{fiZk99hzu3U*I=M^zYFjHHZgmI5R6bGgn~jI+NoS zUMl9uewe_6hM+Q|0ubFg6J;ub3Mn3>xhS+AhIgog#yyl*Ww?;-(rI`90%W9fc7ChQ z?KsFtxIJWl9xU^3OB5l3OVs{78X$}Lpt%s*nVgoFSWqAj4CgjvflJO;^;1tNO~ zr^~}bCmYnkqXZHYym4^lMJ%V9;&=POS?C--P^a^OdII=hppLpCR*dmPl<4^2-!pdb zk6Fj^DyQ)(=|B~#3^*f3SKG9tEYVFYi64kFVfB~|AbA@^l!lmEMBxE2F>o6Q!(no zTaTRvFTd=R86p?EhC0Cu z&?Me{iDrpBnkD8ywkixK-Ir9=+Phytx~HQv_j5QG#9LJ2y7=}2S}l21o)sEt)kSyB zFcVjwcW3#vcfW+ZQlhx}qO?jStyC@-PQdGJKA}Qsq(uuVxAjCcLMk4DH(M=P)D@meZ@IrEQFI*@JC6A8r5z|> zD4G-7$5s1&IfI-6=EMaDgj$Pmi`%CWO5MKk!cFoxDK{K$(A^^Se{n{;r|*luOgvQP zy9Fhr;oL7*YrANXE03W0sFA8$L)FPnz3H4LgSy_FXbiaVy;qZGm?omJODgw}>9T5R z59DWLXCwVXzAF&#{&IxkwDPE`mXSOVgyb`cgrwm@?xodOe@rLl&@Nl<`^1b^mFUuh zG6Redw+?e&&<0v0UTCCs3Om+mTK|X>;Ja}s(#6@~Iw-o1i)NV}Q1@U5tmptcl1A8( zD4BFsV25aaSLO}p%eqm%R7)p(iG=!r1^ugqOpCCH7&4m}k{~l=Znf_VbmyyROgj_7 zBgZ|oVT`PP3M@xtbq{Vl9eZ;>Cn?QJ#-LCvFxH`Q%U6- zaSDVJB1-c47A8lC+9}z|47rp0c1m0~B)Yg(zZX-YEP3%HDk;K0Z@N4y&-d@8ywDvH)jlH^xa zXz!*`?Rj`7&32V)_eK>Q;qk1?-@Y`UYL}^{7O2{bOc~yPCBCGHsZxtdYS)NcRZ>dV zg$~sh?;|QDsx?UIcEqrv5soK){|S@kCKv3?wf zxDsh#VELlcz8X`Rq&+;%s)to=J>{{!K;|zw=&?@kIKtKVyZ${~8T(bzx%P1tzH8!X zMD)`h$rW_*K#-byeHuCge&`bg>`0xWe}a62D?ZtQjj6epfQAG`~ylcP05< ziCB|ijoT0nqrm4x1M5OWD01c~%A6aPdDl=pm^SLPB6R^LAl|`lc?wV*YiXyj zE<5DUHtQe?B-UrAMKhLdj#q}Air*&U_B^Hj6K!UjlW+_7llSb0xb3A=SJ`Gg;=Z7` zV?0YTZingi0B);aR@1Mi>pT!sY+!4==t&|u!_K~kWOeyc&N8O}J5zjSInu?4ZTxp7 zenX6Hb@q!?OqejxMQ!noFhY(aOC9h(L-7ilj}dx2CzN;=w)HSwO%rB}Z13&BfB^Mp zdSi+DgAEvyU=?mjLk_Ny#dW1*Jug)bOE1=%G%{EIM85Z!0E<)ZikY|=N78?G^0zQm$`3Z;CZS-xSBT<&)1@p zD;lN-+#fH!2gKTG@$KK!iyo7`dR~!=Qfwh$OZvQ6Im}VUswjz9a}~!|S6)`ppzN$` zfEq0U;N!;WlJhWKx&7Z6!TzTs3Up5;I!|cRy88uG<*w5K;%UDnc)N1fkeAvS@h4w#p~#7?Y_UK3uHXOiJ`~1SA;D6p zLTA4$LMo2sFS>+O;fBElz@x{M=ze@Hvave%;~VJO-L(prfi7Y_-vRKDNFuApLnEln z4-(YS6*VW6;zGOaB($@vc#^T~xABXLPt5 zH$`#dE(~4aU15nDN9~6!QRy*Hc}*F%f;=`G!-dwL!J9wwTJ-HwoSvB9Vp0L(r7vO9OWgC2Z)CCl&LYx$dP$6xbpt2|B%61jp zk0BhPLOP-0DbcyJ*g#x7dLdi`4lYHimcenVy9v3njj zQdc8xXhUWapJCn*7F<}2CE?u89oSg8)+V!EUPqk&fXd$J~S#!GGLNi-_^PU$AT@n^_kgX7+E zKJB6M0P2$hs1N*Ep3{CTn(tA_s0xtL*$qN_csDpDi$TeL^ZmMh^8=Wq0=ou8%R7-z zrlrcY!~G(sBM$9(<)8|W5nJZ&ff|UNKtrP;XgHCW3VVgx~j1 zRH&%W8g(S~2|@lEKBq>E9y&RCnZu!|tE7?47&`oqIM`!?dpxIFwo@sjGta zcEV1jUb%9`*qcL@(=#7*e1{0#4SUS0lXk#srgQhlSD_;VQM4f;*@mToEg5dy+eR6B^??BW=pdl?)VyvR}F46oHTg5&Dz21iPQ;d z>=sHV$}7Z593D~O21mb;TH#M%lu~(7`)uje7y6=bw)J)$?0$G-#2MJk!J`FQsY)l_ ztf832>cLqT`l5yLQo9a*^l0f)k5wz$bz*gV_lg-E|PtIy}BaHM}qKX;tMb2v{_}%i!?~BRXQbWTgA6#) zfir%tiyrd_)eSnEQ?0)aybqjsgwk)3#^stXwoj|B!m$}%1)UEQUBl9)+)KhEJS!ct zla{EYLxy7uUTx5<7qm--@tzj{trrsA{Tg>aL{KNO+;iVyXKVn7)uDE5!MMA^m zwe4pEX4>!cR?GMXat@xv=C-Z8!dEk$@uu*2-!U;2hMPDoyISQbKA^h&7SD?Dl0hu~ zT%~xN|;?*>dLJfVAC01S|NSPPi`2bj^Tmg&(%bn!2~2mGifV%6eR;i>kus& zVh}134L7;_rsP(ucK*WcfH}#jXt;28Bed42jp!aK9<{kcd{S9!?J{x1PR8eYzrT8#F{)yxN0g=IReB=xk8R;B$UzB5UmTEdm)R5>gzrcS#usL<$ zKMv~B69VA)aeo3aH@oAwFg?5DgfP|bIz^eW(0bQ*?|WD?#d4hO>#xjJ{F@IWjn$ET)X=y84J^CE?p%I&uc8+pwI+@{N_XiyA^ zOw>F$2j@RV-88CJxM>tyU^>!$e;BPOC|@k)Q`|Y9;Q21hj3zu;5YV;q8+_o9B+qM` zMX17aTo%`|AYvABl(IQW23f1c4_+ITVbsP_DQZvg`rN_WGY#HIo>VwT)jrEqzKc7n zA<9R`l<*23=DQyO$^dnx_T7LCqm~HST4ZhBh`uS-=I5{)#w^y6Wgw2EQx?<`e&$Mk zpa{nYg5-zl`Zmfdb^ScNt}9u0e6%a)Y<>!>XJjT(Y`MggC*##j_)EXQzLr25WOwJT0rS-R-m_7VTe)uIN)dg;X7h^zm@I=??YiQ;-RyV z9qr3MfR;A}I`bg*wS#>OC=GXFQshV0Ch!tt9$Ax1T*i-a0<(tuAW8%uVnYddJx4H% zv}3=Qm%}sH$w8V!_7ULNH+%%ANNs$TkDvrk{lUJaqX_Cyw_C8oU_|7u_{`nfD;qB~ z4ywT^WgPq`#@CoJb)|7|6x|uIZ61Pii!Y?&|938=coWLRn#r5^-?xzBl`G@;oqmo# z7Jb-F_hj38w9lJiw)%^zazy1!4@>fXD^uZ_KajN&KVq%C6%C8YoR}}Wh`hLS zcO++Z7twYTIn>VnmNFD+vdtPihl2-vy=V6#2EK%(x~{|h<{vZHME($Yomez=5bu;) zHnmPFno19{d(1zu1)L!s?t~zpNL4I3X>4_(^(Vfc!dGfIoZBb+q&k-7Pe6m(E)%u5 zM8_XUdw{%E`@j=&&#Xnz=KB#0$T5p z@PLAKPtP)4xAmcZ^N-f{SY~%yf7oNucUvEnopk}n*&^=WyMa5-X3Ln7kBdLeD9fBS z;4*s>_)5=^j6!cOi!XCN2(tDUB}PfN>pUKEPnAh;;GzBte{mjyucB%oj&rz3J@fDv zZ_argo)b7NxbMUlkB;KhJa=MtOrkz;#p&LJ^&pQ7Z2pfNL4R&MS+pSA4Ao1vl7|B-3Nmf}=;_XoY8n-+x%zu4_=s58g^uSdXjSujtC9w`Ivc3cA?jCpx64 za*#tlff|bu!`ENI#`LZsQc^)WgI=a4pM8bSz~(IN6^UK6d&F^lp0BqL2+17bYq)`% zD05ITA)o346?nN_9r5?0c11f(LWgrpM^pptebHvt1GG<7?s;H2UAR0FW$3IvJRkCw z7evlOB0=)zAt@$7^U!ebG3mRzYADH!lhAyZtGkPqZJw@8__g_Bz*ES%Oy~nbs7sWO z2tSuL|1&F*U+#%woc;sP5%`}W^rRI!Ki?eoJixP&q8)jV<_PGmXqpm5)8J7WE^bgY z_o&ewe%hWoCCecBBnCzV&opkV19hw}=ckdu5xFCg9&_iTUfJH9=LOE|bi2L`roWpL zJf8MAyTDT-U-!c_qJ$|Go-PLh)i&Q-Kt-XK+{|3g>T9`d&hS4&#VTYvDI1;l;lV0; zgxL9GwBX@k5AGX(Kcbw1xj(Qu*#-wabS)IF59XtZLsUkQzL6lm`dk)H-oRv@&SE z#U)@4mw;LPx%3q({!luXIps;?cU~kZfQAWLt-wUwHMB1RPhm+(|N1&?RGtK`LhaI$ zD=@}z!NYgFZ8)1=RaGT&Mu=QoiB01`$=rTfigr$RzexOFD_q6*6GzD18$;w)v(6Vu zrI>^%a$tjiLu!NL8gzrvI3X)>;mcLGYq`-QJwTdCoG|!Ypj4g26_ETIpo(ZAIZ%l! z#l0z!iES#+chnxfcL#HzZXjkldweiK;+t$@kqQyf*zQ0cm;qeo2Ol1FL0wtol?R6B zzrc@zsyn}fi3oli9UhW$X?$B?h46QwU@C-21ZF7G5F{1Y8yb;{zgKzMk^&D&1>SJL z#it(QZ{!UJ8u<+e6%-fxhJ%|RO=Rl?VY`Tn&<$LKX4gr}uHj;|Og}0{GXn=o?t6Z$ z7|Ac)2%fddzSguHurGWwQRB!!2=w~D0=*6%3xB8Ch(GBtT*l{Mm_}c7lAaYn{`pTK z3twxJJ`s76-az&&=8BzovOZ+42%EV{DPI1>c@`dJ3|-&zLkYCX3_g^AS8?JErIx8U zc-|dy@Vu%a%lF+OdC^W!+G6yD?cYozALz)D3~Z|4q1rKfDqNN0VH(;}%rE`W^t|h4 zc%g^kw_6Lp8qN&djUD2N(I6EKA9UU`pP(-;V!*k?M*nrA>WEb!^R4b3_LAkRD^d z`ccy(aJx;)qu+!q`kiH@-}Dsv&0mP$x_c5<@yn#;{4#4Ezr-f<%UnIbEK0)V$KQ*! z;Ubywt_+pCPbE%TiQ7jl_!)@Z17xG{4@uI}i;phHtvv7f#Yx3HhY3;RRm{6Y%r8a| zKKVzeySS43vjtsoCc$UwPiUkdqoGhG4-xBe ziJPL*i)5A40PLE_19+`4KS4>Vw+`=Dx-j@yU`nj^okFd2X7W97#>3B5!dfQF*7tj! z;`&qFYw#d{ZdyJ4?Kuz0rsJMoH0R|@Dk?|rC~boyeJ@m z)iOU~#v^EQ#1pkw%SnWqoao_x2)h4W)L@po!yBdDeGbSWq`Do?_;}-PFY@r2R`Cn? z$Y<*wE^^(l`=oaF@4=I_lL423kGcgN(~B;CZ!e$b@B^BE>h5?Uu6@Ew5QWaO{HzqG zCw4mmL^Gdgz!CBzFbKA}VV}fpactStvIC!)iYgU~9qsj-97~&vnwx~y z=EjoxMhE+paa0Up@IF8&faa z!HQZP_(l*lu$Wn+N?IL`<@Fm|Yg%`(Vpi1H+~%;@d@>z10PO^%{w7tsRFdbGTuFW&IY%;x-}-h1d~RT;Eo+5qG?b0MbP@ zjg4(AQ?=63x~0AegbY-LuSkX0&n&9SCZG>;)}kYh7NNem2@Md^>{?ZYL%3}6GIM#U zdAT8tGo77_UfGK8=U73+n$4hlCAGz)%uR%1Nn_3CHdZwzcDbY7(a5StVwX2>-t1@% zu$Q@`%N?8R(eYMZ(GpjEZQEk@WprgzYgjzKIaOS>*tS%`XE)o_XeaS)Y-O9ksK)w@ z9MS)#wwBkl3Fg+;X7sU&;eKfgKGI``Sno(<>^Q>*d72yB9U-yf8RQ=C$d)2D)# zHq|<|uWViptXKina+9NNG}R0ZvD9W+ZUas?7Rz4(3e^zG_q49LlfmR+>^}mKwUe>? zZ)faWCv?qvd>d>7V}HfhOLG{TI+wBUqyB#b9zZw`unq7g?$_d)4jin2e_Y2{Eb{Ni)rzYFFbj|l zcpY)Cp^v8lQ-Q}s$MqkeV>I%uLAj6g7zwlK8;tk{8Q^CKKXeUaTX9VQ zjwf*)M|l8C5Wfg<$8r51z@VZ*9vF^Jcs)GL8lG^;Qaug7I8m8+Ht_M0OH4P)VCS%Ljdt+ z8hFqFo-Mc&i+_L>unN!!=mPu%@HF5M;2pqcfC+BQmw?59TL5iX2RIMV-ObnyfMUQ#zz)E70gnQn2fPh9517)8@c~!{pmU~o0DcPa0^SCE0#J*L z%>m>BDgn0wb_0F{_#NN~;J*QWz>FU76<`6Z0ki?`0sI{Bd%&B3vw&E*yJrAW0p);N zz;?j*0lx+u1$+vaaSvmKfI5H(cmyy2_$%N`K#~VC09XUK1Mm|-FW@-f6TsB(VlxP^ z60j5SD4-wkKHvgi`uEU(fF0ll{1os4;1u8@;MzSHJAf4cC!h=P5MUqRRlsS$mjE4( z;UohLfTe&Yz`cNH0B-{R0U(VQ&0?4eZ!(Bu}oa>-{_phu3^_QJ)6z0V{=$Co6D|eH?Vo^TWmgCz!tJa_-@s=Sqe*K zY0SWkES+VrOqRv6Sq>hG$-~P|3s@mDvBmfZW)bWKGb>?Bn1wB6H?w7IIkU1-wu0GM z87pTMY$dD23{cHhvDNGrwuY@`>)3i`XB$`zzLvF#)iMXGW1I2j#(JE7xs^4tE%=63 zGrpm<6<^V6!`HT4_>$H(e1micKFRnUwv%%!+g?`GXhWIcG* z!#&KyzRSMH_ON@|_t|~ye)a(S0b9DUv=) z0-CFDs%wsz2+I#2;0vp)t$kB)_WD}h&}Pi{O8){;ZT#z%_Lf#hUH$f;H1K2#Dz4uI zb0a947*Ok|tD!bi1JrMI^2%Xg1o&Ck*t|_)+GeCup|PoQS?X4+%TU(mKBSDZBy8si@-d@ufo-DN1wAhQwyC*A8Q5c^<>5I}RZyWuJBg>=+1#RtXmD8?)C#h4A?kWV zbA3~wM!ENaBAj*{i^$QJEfb7HfDEIpam^ACpDmk}aT?rNiVSO;MysMj>{|qv(v7k4 zD5;DKKEg-JU{u)YCSt*sTcM5X>qdwZQmc}9*{WNE$Qn|T5-wY+O~I4vOfGH2cPxi% z8tpBlghvN*XciLp=GM{CjIWGi{m-Rl6D?~4T?~p2k>H^C&5kBVYyGD10;8i}7R;bB z0#<^J3Z;VMLv>GZ3MQZMCPD{oAUpA0?t?6ILD7LegvBd;2um34Ls(Lv58?4jAHq|p z4`H!NALIlsYsAC)mfAqx5%J^0vT>ur^O1OrR7l>IdWC5t30#wKVK8N+@X2%}X;l1! zqk;r&Bw7*O0QJM-Bb5n~=8^nk1bHN$*hM|yy=$!*nHNVhjCA-`#||!`8*88hWua=J zI84QLu14+=fa+6HNc{PbAP)h)Hf;~aY9x_py{WODY?xLDJP%=|DTA|lD<9i5Yevk_ zSl@K3y%ryPrnx_$)B*^Gw8ZnZwKyEL_L@yXeft$^lXE$^e?_rL_>dkQ=G3U1t<9wv zhhwsAQ3mFicv(X@ci0`Rt#-i~xq&S$8zc4>!eif56S>7ENBw5!MyQq`2@4?sSl!;( z7O~!D*g*I|>lJb0{b+GwK9^^KNL1!|E}z^6gLFiuLU1Gg;=|+-k!x?fY@d;Y)*yhv zs&ka!TpJdxr_EF#5+us%|T0P7ZnwV&Vqs4*rvRm*81kwdSOQZqo9=FA=I`-?m&cA11Xf-r3`G!K4Qok1GrOM zn_ba0JWH7Q+P1A;*yIc-?nXyVtIWhuZUkqdF33C!VOVe$H0mHf_$H`b5`&aLS3#|s z){S;YBj3c}#AsI(!^ZWll z3PjPiB79zO6NUWI(Fgc!BgEq%1AJJFZP>?{DJG;4zcIxaajk+!%Z^_&_iZhOpQ{x2 zOYvKRF#Wx#`s9+l+Yrs#CQ?isvz2U}UWkwypu#_r)8N0KAR-Di3-EZ=!~3-vwbjED z=0MF&@QyVj^KCp&3Nr%xY=9BoH}nHOy74Gmggjf|Gpj-V9l)jrcMjf!6<7KOJ*70m zi$;Gf@cKoKv{8&4ZScfV+bzJM9{HMhz7@E4AY8(Ib(FpdIrXUDfqdJLuN59UJ$!)m z2sw}{;Le6{GkkXS+_#4fAovU?2ens)oaEUXZ6^&L#}p1JxKbz$eKf!W3I8?xnNge$ z0x#DDTpELVSPw5^6IyBsw0a}L)L;7B4v%IOJdzGjlu8kmT0kG_33(i`sTv{f7wAJ8 z$0G}&vGmi!qe(q=a(XtR#?7dWD5ytE0$OL_4B@;Y&<=5wT32{VcQSXbgu4Zv+o%97 zYvBVW+6MFWa=fVqHbjqR@CTc%z(pHY&sYv*1n8f^d!35ARF=iVBw_IT1EL`HHRzGj zi&4phuNpE9uQ*ak%G%J^8eluh-|?z_l;Qv0KOOcFqOkXsx*@eT)tTCu+L9`yZcp8r z+L^jLwL5i#q1NCuG#XkAf?>O1r=ioZ+t6*e$FRq6zu{>^ui*uQ*Kp8q$Z*7P)NtHz z!tk!)l)+>yHd>6!jW**-<0|7?;|61`(P?ZnwipHDcH>TCr*XHj+jx(0ukkVC6UL{E z`;1Q;dyOv`y~cyaL&hVhbA`=}GDO^yKt; z=|4{2oBmk(6X{Q-?@NC=y*K@ZbZ`2>^h4=K(vPMePd}0VZu+V857K?-@j`|-<6y?2j3XJRGCs)gWejDU%Q&AgoN*CuepRMAQ5)@3=fI_4BM3vfj+PDIe_B zV~-1$MLC%{i*uIcROZ-ocISLI=fRwx<~)`2Ud{(OgE^n&{3|CZcTVp7+|=Bx-06A7 zytR2d@@~(&H}AQ;{QR5pEAzjXZz?bstSY#z;Esat7d%(+Qo)-A^9#RSSY3E$;oXG~ z75=*LVBwdA5Ks0r+A^g$QW{b^QhtC9i-y$Z)H_pOPgSSQOIws?N?V=Qm$t}|W-uG> zHQ0Dbgap_m5x2Jc4+J=m*jAa?mgBovTY|Z?4W^`6U*4B0+>JSq34 zJdpCkl!sFuNqIcwg_ORO<0=10aio5h`bFx5v{h*jr$reSKsMhqRHUy;zcu}}%ww5< z&pe#pXMIRbL2PV@5vv?Ka&4e{ATYJ zN`Erlm{FGTos7pbewp!N#%CGo%t?@p6`79AhRlx4bD7(+Udj4v*7)r7>_@VHmHm9S zKIi(JqMQ{u^*PNsV$Q=kzs%W}vp?r`@Lrdj0KQvuD{}A16?1==dnEV$Ty0)<-XHSb z&l}GBcb+c4Gyksq-{e1^|3&`4^J5BT7GxDT(T}?dJOw{0*jwkYt!`xWSNN$TbuhmKs(X z)*3cLv$Y!B;NOFW#|-BotZ#lFYv&H K#?k*5{{APaEufJA literal 0 HcmV?d00001 diff --git a/branches/winverbs/ulp/nd/user/objfre_svr-08_x86/i386/ndinstall.exe b/branches/winverbs/ulp/nd/user/objfre_svr-08_x86/i386/ndinstall.exe new file mode 100644 index 0000000000000000000000000000000000000000..16d497f5b37776915023f2c211b3b7ac66542bf8 GIT binary patch literal 9216 zcmeHMeQ;Ywx0du$dY(^ z&v7~g>{Z-}P+YQYwv+Av#WlNScQZTL?NS2e!y#-!W&)cHS!x!BdZz)DI+V0z6Q>wn zf9L5bR=$|&AIx?-T+O-no^#JV=iGD8efR0S#s~MXT*jCdAj^y$L`$VcslU3Kz;Myx z7mL_0XTQ1Rpr+x?CCz*=W{3)rZo%Jc==6uf5z)}W83ZY82!;)H8{CH8NPt^8XHG$h zD!TpYTW+6!=+XKNte$mC{XF2SXXU7^wqB=~>Lcg`XP-%13@mWgS6zt?l#?=

7x5lp7}>0Y{ndifH@0)*8t_AEL2>f&N-mxi%(m`f(ysO*`0)t-R=_>GuBUnz3^ zBHDd=RAd*e6?SJA17llO2K=HQ#NA|@5r8$b+W=XmhOx1gLQLocO<4m7ima;)2DxUX zz=a}UEF?KDmCt}StJE;ICJX(h`$zOZ&mD~Q10DlB0r(lf3!riw?Kt2x-~xazWvm%s z1C#)Y0AH419B=~g3&38$Za^R4VT@CWF*bl$9B2Z~0ayy4avQaPvv)G~G++#H0I(bI z2tWYz0K9-^z@GtZ03%>2U=d(0;4MI=yfL5QG^I=UcBc7<1v7Yto_QAJfY1gQ$JrXk zo8XUtrJuOj!Alc5c4_oskpbtM6TDnRuTj*EUt}&MIjP$ zbE4PH34Os%F6M0whGUV=tzMTO4_i*;gqXK^t;-Mz8{D<`vkg%$ynce83KT8@braVc>Eje8Rnm=k9bsr+edUx`lW@%~QZfL0I{82b2(z*0* zLzh1o;sV#o!_ZX{)05$Ja-T2bGMfRGs6K@;iW^m-SOEm1G{e;yegiH@e+ZTZw&O}v zWRZHdHX?-r2D&f1Inm%Z@cwWh#L;f5`wVls;WA-+W)+MzaovcB7Jo?Moc*d&!?vKv ztBPS+9j7X}k!_Gs$Au6+ikNGKs7rIZY=#HI{tlQVMhsoSZ~!)9pNP{ z+BNK)GF@=Bv?Qdd)Ezf77N3%(MT0x0@4bsLQ7=o=eEy$6f}Ui0JZE-6v%6K6rjk`9 zp!DR+|KgJM26U6-_79{Y{yeEob5D9z1ul0>>In9NWZz0656&=oCYRZd^xZWaD@i!d z53h5%WHZ12@-XZkJR;&5DL0;%es8+qZc+3tfIi=*&BqSUtY`4-0N6&z{ia_foYU?W z(|hjWp`BmIvdhgc#w&%}?dJEv8$9?1WPLtl42?;i9vzQ&4oy$4n;$1%Jx5byy*52u zl)>@-qM2MI>zAT+jT}TBAFJVCf^V??mk52mFvC0dy1cut;B7rRv)lcrH zdia1+pP>4=QwsgGQm2&qtWuv->M7KwF<y+ZO5)SB7l9ci>rx<&oOBM|s6LaL> z#E)q0QlW5mb9?$Ajx-n%91c`+MLp_K>hjB}GE%h^m8?IYqr)_QT?Q`w^IOVO>HxJ*neUm?P_(F*Nx* z5Jv{U4TgfA#i<<_@=V$l63kwnsGvdqGQg4Z^Ni{;VzKH`8+kLM=?&~3*0?9 zZdp$>xq{pwSCBvC3UWw_jy)b8qcuRR3(HYUmDuK|b$F&a&nEMevlERIu8pZZ2u6?R zaFR9_HNJHb-acieQ$x@!vE$)b3_Sk(6t;VIz7 z-#?T*)T0@t1*GP~NwDd4SP`Ei(z@ew`l<1FWc_YXlXIHEm&lj(dlby2U?aN-VWSU| z_j;f=US+@llM4Bri!fWi7vr9k1tX@h^lDO{06Fs_e5l+7_BnI|NI9P!KC=Zj%-j!2 zXMU0mS~_FB6VkGHzb;oQP1ekLzXvwOS3pUx9-!*7Wd%68^veuD%NcHchrej@${qt_t6z_%0)U%T{ zJ$*rw!S#;ZH;SLHo$GEXe*U4GOecpfv<}zh&+A)=CsDG#(0)mnoovg`eb@fEP#Av= zH_}r3JHkhJmT%sqitWkg&(YD^fgOr-)9gjGnx@o)6#0nEDri~unyV?5$Q*@b$(oWw zsl^vh#3wQVG-;qXWJzW^R+Wc;NQ&7MhiZVC#!{USoAJmKmpvkv?z`OQ@zSCR$Q58@ z$+{9M7)P zOE9Ye4l#{k6Y;34jbD;Q#}enXvZ2zl78gExs4wLeFa@yJtcU9%03wa12 zXTG`u_4Q4sI4#aiDf`;`>ir+p{ImV2sQWSH(8SpnPmIpR>L=ZJ%bUVcEQhY7G;NZm zWW5D_(>Yo1P}>@%g`2@wM|{rnCg(ZRIr~K^@0AUZo2Vd9UVJA}K{7Y{)QdrVl^i^~ z6i02etqccxPBYbd7u5?JsII7|x~dL!*KU1OY32l!W?_xe13l`rPYpZK|Acs9}N*cG=YokXdb<=Yp|6gG&mXnb~KM(Pi2k&8pr2tz@8<Rwo2i~ zKVy;Do&B|}mMB^OPHWkNM4^O4hNG^f*0tWWYGojVw9Aij=%4*Mf#Vqad!?`Z%V&R@ zb1x_24=}POG2&qCV+|+`{+Q^bWC3IMIh*cxHe}>{ri}{0u-L`e-)Xzl?t2p`wCpgu3+qXzox0$$_G>aO4B4w|b03UtBI-4H2L|dYU&h0)<*X0G6r^Cbasg+5 zCv#;%J&^wFg+G7;SDHr`z4CeJ8al3RC1ocIQ0#C>S*XY0_X^0M~RFX_#YK0w8g3!|@Yu5}N6yBa;D444+b zt7Gv;IW)tK9J+vfdYE-XS`hhn4wA#H3)yzYzN_#USvm41RzNxO@kY>BA%&u0Yr)ft zOuZlc+hL6#7#^A^jc6&CPa_f5uh9D0Qq0Ndr~^j~&p^^0g$+UQg%!T9&i0dC0yC&K zlHEbrM0x_C5>8(k{EGJU*8m4A%F^2cy#jb<^sT~|2crT;wA5RLwVGHNtVz!$f6$i; z0WC?MSt)6~Qs+wW^#kx-h}{Fd4d{0(v&dUf_=#5BjTd=bb&3bCr2QDHSf$uuLGNlm z8t^rRzI*VnBmzq!kQ4+Z2Hplp#N!rUM`*npR6S%J*_`n^Vd-7^D%N)S`TNR`D{rwZuq?M&Ewz?=EgLK?mJZ8S%MMH2@|5KT z%VEng%dagftxeXb_0O%3T8FI1tZ!LQS>LsOVEs31%K9JH3)WoQBHLZIHMRzu&la?W zZ86*TZ9lM$+McyNXFFm$ZhPDIvF#JvXSPeWo9u;ly}jJ-usiMd+1+-Z{UQ4!_MP@O z?eE(E)&7zFQ~Mw6U)XaU1&&)B3mhemdmN39eU4`xKXbg~*jyE=daLSpRUcO6tZrWI zS>3<-@zqnSE2~?p1J%*${_271-PNPjKdnAeeZ2Z)^?3CM)u*c`tN*L|5{={;t~ zS;iux-nh_MYFuHo8rK-_H8vVsja!UcjoXcn7UH~x+B8RG%tOU5^hr;RD2wk*G_ Sq--f2)Njf+9{3}8;C}!Lcm4$c literal 0 HcmV?d00001 -- 2.46.0