<Component Id="cCHK_user"\r
Guid="A4AC788B-6517-47b2-A898-0493ADC794D5">\r
<CreateFolder />\r
- <File Id="file83" Name="DAPL2-_1.DLL"\r
- LongName="dapl2-scmd.dll"\r
+ <File Id="file82" Name="d2-cmad.dll"\r
+ LongName="dapl2-ofa-cmad.dll"\r
DiskId="1" Compressed="yes"\r
- Source="..\bin\DAPL2\$(var.ARCH)\dapl2-scmd.dll" />\r
+ Source="..\bin\DAPL2\$(var.ARCH)\dapl2-ofa-cmad.dll" />\r
+ <File Id="file83" Name="d2-scmd.dll"\r
+ LongName="dapl2-ofa-scmd.dll"\r
+ DiskId="1" Compressed="yes"\r
+ Source="..\bin\DAPL2\$(var.ARCH)\dapl2-ofa-scmd.dll" />\r
<File Id="file84" Name="dapl2d.dll"\r
DiskId="1" Compressed="yes"\r
Source="..\bin\DAPL2\$(var.ARCH)\dapl2d.dll" />\r
<File Id="_dapl2d.dll" Compressed="yes" DiskId="1"\r
LongName="dapl2d.dll" Name="DAPL2D.DLL"\r
Source="..\bin\DAPL2\$(var.ARCH)\dapl2d.dll" />\r
- <File Id="_dapl2_scm.dll" Compressed="yes" DiskId="1"\r
- LongName="dapl2-scm.dll" Name="D2-scm.DLL"\r
- Source="..\bin\DAPL2\$(var.ARCH)\dapl2-scm.dll" />\r
- <CopyFile Id="dupDAPL2_scm" FileId="_dapl2_scm.dll"\r
+ <File Id="_dapl2_ofa_scm.dll" Compressed="yes" DiskId="1"\r
+ LongName="dapl2-ofa-scm.dll" Name="D2-scm.DLL"\r
+ Source="..\bin\DAPL2\$(var.ARCH)\dapl2-ofa-scm.dll" />\r
+ <CopyFile Id="dupDAPL2_ofa-scm" FileId="_dapl2_ofa_scm.dll"\r
DestinationName="d2-scm.dll"\r
- DestinationLongName="dapl2-scm.dll"\r
+ DestinationLongName="dapl2-ofa-scm.dll"\r
DestinationProperty="WindowsFolder" />\r
- <File Id="_dapl2_scmd.dll" Compressed="yes" DiskId="1"\r
- LongName="dapl2-scmd.dll" Name="D2-scmd.DLL"\r
- Source="..\bin\DAPL2\$(var.ARCH)\dapl2-scmd.dll" />\r
+ <File Id="_dapl2_ofa_scmd.dll" Compressed="yes" DiskId="1"\r
+ LongName="dapl2-ofa-scmd.dll" Name="D2-scmd.DLL"\r
+ Source="..\bin\DAPL2\$(var.ARCH)\dapl2-ofa-scmd.dll" />\r
+ <File Id="_dapl2_ofa_cma.dll" Compressed="yes" DiskId="1"\r
+ LongName="dapl2-ofa-cma.dll" Name="D2-cma.DLL"\r
+ Source="..\bin\DAPL2\$(var.ARCH)\dapl2-ofa-cma.dll" />\r
+ <CopyFile Id="dupDAPL2_ofa-cma" FileId="_dapl2_ofa_cma.dll"\r
+ DestinationName="d2-cma.dll"\r
+ DestinationLongName="dapl2-ofa-cma.dll"\r
+ DestinationProperty="WindowsFolder" />\r
+ <File Id="_dapl2_ofa_cmad.dll" Compressed="yes" DiskId="1"\r
+ LongName="dapl2-ofa-cmad.dll" Name="D2-cmad.DLL"\r
+ Source="..\bin\DAPL2\$(var.ARCH)\dapl2-ofa-cmad.dll" />\r
<File Id="_dat2.dll" Compressed="yes" DiskId="1"\r
LongName="dat2.dll" Name="DAT2.DLL"\r
Source="..\bin\DAPL2\$(var.ARCH)\dat2.dll" />\r
Source="..\bin\net\$(var.ARCH)\ibwsd.dll" />\r
<?endif?>\r
\r
- <File Id="ibndprov.dll" Compressed="yes" DiskId="1"\r
- LongName="ibndprov.dll" Name="IBNDPROV.DLL"\r
- Source="..\bin\net\$(var.ARCH)\ibndprov.dll" />\r
- <File Id="ndinstall.exe" Compressed="yes" DiskId="1"\r
- LongName="ndinstall.exe" Name="NDINSTAL.EXE"\r
- Source="..\bin\net\$(var.ARCH)\ndinstall.exe" />\r
+ <?if $(var.ARCHP) != "ia64" ?>\r
+ <File Id="ibndprov.dll" Compressed="yes" DiskId="1"\r
+ LongName="ibndprov.dll" Name="IBNDPROV.DLL"\r
+ Source="..\bin\net\$(var.ARCH)\ibndprov.dll" />\r
+ <File Id="ndinstall.exe" Compressed="yes" DiskId="1"\r
+ LongName="ndinstall.exe" Name="NDINSTAL.EXE"\r
+ Source="..\bin\net\$(var.ARCH)\ndinstall.exe" />\r
+ <?endif?>\r
\r
<?if $(var.ARCHP) = "x64" Or $(var.ARCHP) = "ia64" ?>\r
<File Id="ibwsd32.dll" Compressed="yes" DiskId="1"\r
#\r
# DAT 2.0 [socket-cm] InfiniBand QPs setup by passing QP info over a socket\r
# connection; supports DAT Windows <==> Linux over IB connections.\r
-ibnic0v2-scm u2.0 nonthreadsafe default C:\Windows\dapl2-scm.dll ri.2.0 "IbalHca0 1" ""\r
+ibnic0v2-scm u2.0 nonthreadsafe default C:\Windows\dapl2-ofa-scm.dll ri.2.0 "ibv_device0 1" ""\r
#\r
# Socket-CM (debug)\r
-ibnic0v2-scmd u2.0 nonthreadsafe default "C:\\Program Files (x86)\\WinOF\\dapl2-scmd.dll" ri.2.0 "IbalHca0 1" ""\r
+ibnic0v2-scmd u2.0 nonthreadsafe default "C:\\Program Files (x86)\\WinOF\\dapl2-scmd.dll" ri.2.0 "ibv_device0 1" ""\r
context = p_cep->context;\r
pfn_destroy_cb = p_cep->pfn_destroy_cb;\r
\r
+ /* Cancel any queued IRP */\r
+ __cep_complete_irp( p_cep, STATUS_CANCELLED, IO_NO_INCREMENT );\r
+\r
__unbind_cep( p_cep );\r
ref_cnt = __cleanup_cep( p_cep );\r
if( reusable )\r
al_dev_open_context_t *p_context;\r
ual_cep_poll_ioctl_t *p_ioctl;\r
ib_mad_element_t *p_mad = NULL;\r
+ void* dummy;\r
\r
AL_ENTER( AL_DBG_CM );\r
\r
\r
p_ioctl->status = al_cep_poll( p_context->h_al,\r
*(net32_t*)cl_ioctl_in_buf( h_ioctl ),\r
- &(void*)(ULONG_PTR)p_ioctl->context,\r
+ &dummy,\r
&p_ioctl->new_cid, &p_mad );\r
\r
if( p_ioctl->status == IB_SUCCESS )\r
\r
cl_memcpy( p_mad->p_mad_buf, ioctl.mad_buf, MAD_BLOCK_SIZE );\r
\r
- *p_context = (void*)(ULONG_PTR)ioctl.context;\r
+ *p_context = p_cep->destroy_context;\r
*p_new_cid = ioctl.new_cid;\r
*pp_mad = p_mad;\r
}\r
status = ib_dereg_pnp( bus_globals.h_pnp_iou, NULL );\r
bus_globals.h_pnp_iou = NULL;\r
CL_ASSERT( status == IB_SUCCESS );\r
+ BUS_TRACE(BUS_DBG_PNP, ("%s deregister IOU PNP\n", p_bfi->whoami) );\r
}\r
cl_obj_deref( p_bfi->p_iou_mgr_obj );\r
\r
p_bfi->whoami, p_ext->cl_ext.vfptr_pnp_po->identity,\r
p_ext->cl_ext.p_self_do, p_ext ) );\r
\r
- BUS_TRACE( BUS_DBG_PNP,("%s p_ext->h_ca->obj.state %d ref_cnt %d\n",\r
- p_bfi->whoami,\r
- p_ext->h_ca->obj.state,\r
- p_ext->h_ca->obj.ref_cnt));\r
-\r
IoDeleteDevice( p_ext->cl_ext.p_self_do );\r
}\r
\r
p_ext->cl_ext.vfptr_pnp_po->identity, p_ext->cl_ext.p_self_do,\r
p_ext, p_ext->b_present, \r
p_ext->b_reported_missing, p_ext->b_hibernating ) );\r
- goto hca_deref;\r
+ deref_al_obj( &p_ext->h_ca->obj );\r
+ goto xit;\r
}\r
\r
p_ext->b_present = FALSE;\r
/* free PNP context */\r
cl_free( p_ctx );\r
p_pnp_rec->pnp_rec.context = NULL;\r
-\r
-hca_deref:\r
deref_al_obj( &p_ext->h_ca->obj );\r
+ p_ext->h_ca = NULL; // for free_iou_mgr()\r
+\r
+xit:\r
cl_mutex_release( &gp_iou_mgr->pdo_mutex );\r
\r
BUS_EXIT( BUS_DBG_PNP );\r
xcopy %bin_dir%\qlgcvnic.pdb %2\net\x86\ /yq\r
xcopy %bin_dir%\ibsrp.pdb %2\storage\x86\ /yq\r
\r
-xcopy %bin_dir%\netipoib.inf %2\net\x86\ /yq\r
+rem Use netipoib.inf without WSD support for XP32\r
+if /I "%OSE%" == "wxp" (\r
+ copy /A /Y %1\ulp\ipoib\kernel\netipoib-xp32.inf %2\net\x86\netipoib.inf\r
+) else (\r
+ xcopy %bin_dir%\netipoib.inf %2\net\x86\ /yq\r
+)\r
+\r
+rem allow XP SRP build & sign, WIX skips SRP for XP\r
+rem otherwise there is too much special casing for SRP on XP.\r
+xcopy %bin_dir%\ib_srp.inf %2\storage\x86\ /yq\r
xcopy %bin_dir%\netvnic.inf %2\net\x86\ /yq\r
xcopy %bin_dir%\ib_srp.inf %2\storage\x86\ /yq\r
\r
)\r
)\r
\r
-set F=dapl2d.dll dapl2d.pdb dapl2-scmd.dll dapl2-scmd.pdb dat2d.dll dat2d.lib dat2d.pdb\r
+set F=dapl2d.dll dapl2d.pdb dapl2-ofa-scmd.dll dapl2-ofa-scmd.pdb dapl2-ofa-cmad.dll dapl2-ofa-cmad.pdb dat2d.dll dat2d.lib dat2d.pdb\r
\r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %2\DAPL2\amd64\ /yq 1>nul\r
)\r
)\r
\r
-set F=dapl2d.dll dapl2d.pdb dapl2-scmd.dll dapl2-scmd.pdb dat2d.dll dat2d.lib dat2d.pdb\r
+set F=dapl2d.dll dapl2d.pdb dapl2-ofa-scmd.dll dapl2-ofa-scmd.pdb dapl2-ofa-cmad.dll dapl2-ofa-cmad.pdb dat2d.dll dat2d.lib dat2d.pdb\r
\r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %2\DAPL2\ia64\ /yq 1>nul\r
)\r
)\r
\r
-set F=dapl2d.dll dapl2d.pdb dapl2-scmd.dll dapl2-scmd.pdb dat2d.dll dat2d.lib dat2d.pdb\r
+set F=dapl2d.dll dapl2d.pdb dapl2-ofa-scmd.dll dapl2-ofa-scmd.pdb dapl2-ofa-cmad.dll dapl2-ofa-cmad.pdb dat2d.dll dat2d.lib dat2d.pdb\r
\r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %2\DAPL2\x86\ /yq 1>nul\r
xcopy %bin_dir%\ibwsd.dll %2\net\amd64\ /yq\r
xcopy %bin_dir%\installsp.exe %2\net\amd64\ /yq \r
xcopy %bin_dir%\installsp.exe %2\tools\amd64\release\ /yq\r
-xcopy %bin_dir%\ndinstall.exe %2\net\amd64\ /yq \r
-xcopy %bin_dir%\ndinstall.exe %2\tools\amd64\release\ /yq\r
-xcopy %bin_dir%\ibndprov.dll %2\net\amd64\ /yq\r
+\r
+if exist "%bin_dir%\ndinstall.exe" (\r
+ xcopy %bin_dir%\ndinstall.exe %2\net\amd64\ /yq \r
+ xcopy %bin_dir%\ndinstall.exe %2\tools\amd64\release\ /yq\r
+ xcopy %bin_dir%\ibndprov.dll %2\net\amd64\ /yq\r
+) else (\r
+ echo %0 - missing x64 Network Direct components [ibndprov.dll,ndinstall.exe]\r
+)\r
\r
echo xcopy amd64 Free *.exe tools\amd64\release\ \r
xcopy %bin_dir%\*.exe %2\tools\amd64\release\ /yq 1>nul\r
)\r
)\r
\r
-set F=dapl2.dll dapl2.pdb dapl2-scm.dll dapl2-scm.pdb dat2.dll dat2.lib dat2.pdb\r
+set F=dapl2.dll dapl2.pdb dapl2-ofa-scm.dll dapl2-ofa-scm.pdb dapl2-ofa-cma.dll dapl2-ofa-cma.pdb dat2.dll dat2.lib dat2.pdb\r
\r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %2\DAPL2\amd64\ /yq 1>nul\r
\r
xcopy %bin_dir%\ibwsd.dll %2\net\ia64\ /yq\r
xcopy %bin_dir%\installsp.exe %2\net\ia64\ /yq\r
-xcopy %bin_dir%\ndinstall.exe %2\net\ia64\ /yq\r
-xcopy %bin_dir%\ibndprov.dll %2\net\ia64\ /yq\r
+rem xcopy %bin_dir%\ndinstall.exe %2\net\ia64\ /yq\r
+rem xcopy %bin_dir%\ibndprov.dll %2\net\ia64\ /yq\r
\r
echo xcopy IA64 Free *.exe tools\ia64\release\ \r
xcopy %bin_dir%\*.exe %2\tools\ia64\release\ /yq 1>nul\r
)\r
)\r
\r
-set F=dapl2.dll dapl2.pdb dapl2-scm.dll dapl2-scm.pdb dat2.dll dat2.lib dat2.pdb\r
+set F=dapl2.dll dapl2.pdb dapl2-ofa-scm.dll dapl2-ofa-scm.pdb dapl2-ofa-cma.dll dapl2-ofa-cma.pdb dat2.dll dat2.lib dat2.pdb\r
\r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %2\DAPL2\ia64\ /yq 1>nul\r
)\r
)\r
\r
-set F=dapl2.dll dapl2.pdb dapl2-scm.dll dapl2-scm.pdb dat2.dll dat2.lib dat2.pdb\r
+set F=dapl2.dll dapl2.pdb dapl2-ofa-scm.dll dapl2-ofa-scm.pdb dapl2-ofa-cma.dll dapl2-ofa-cma.pdb dat2.dll dat2.lib dat2.pdb\r
\r
for %%i in ( %F% ) do (\r
xcopy %bin_dir%\%%i %2\DAPL2\x86\ /yq 1>nul\r
)\r
)\r
\r
-copy %bin_dir%\ndinstall.exe %2\net\x86\ /y\r
-copy %bin_dir%\ndinstall.exe %2\tools\x86\release\ /y\r
-copy %bin_dir%\ibndprov.dll %2\net\x86\ /y\r
-\r
-if /I "%OSE%" == "wxp" goto inf_files\r
+if exist "%bin_dir%\ndinstall.exe" (\r
+ copy %bin_dir%\ndinstall.exe %2\net\x86\ /y\r
+ copy %bin_dir%\ndinstall.exe %2\tools\x86\release\ /y\r
+ copy %bin_dir%\ibndprov.dll %2\net\x86\ /y\r
+) else (\r
+ echo %0 - missing x86 Network Direct components [ibndprov.dll,ndinstall.exe]\r
+)\r
\r
+if /I "%OSE%" == "wxp" goto mk_sym_bin\r
rem free x86 items\r
\r
if "%DBG%" == "TRUE" echo DBG: copy x86 Free WSD\r
\r
if "%DBG%" == "TRUE" echo DBG: x86 Free dlls to WOW64\r
\r
-copy %bin_dir%\ibndprov.dll %2\net\amd64\ibndprov32.dll /y\r
-copy %bin_dir%\ibndprov.dll %2\net\ia64\ibndprov32.dll /y\r
+if exist "%bin_dir%\ibndprov.dll" (\r
+ copy %bin_dir%\ibndprov.dll %2\net\amd64\ibndprov32.dll /y\r
+ copy %bin_dir%\ibndprov.dll %2\net\ia64\ibndprov32.dll /y\r
+)\r
copy /B %bin_dir%\ibwsd.dll %2\net\amd64\ibwsd32.dll /y\r
copy /B %bin_dir%\ibwsd.dll %2\net\ia64\ibwsd32.dll /y\r
\r
copy /B %bin_dir%\dapl2.dll %2\DAPL2\ia64\dapl232.dll /y\r
copy /B %bin_dir%\dat2.dll %2\DAPL2\ia64\dat232.dll /y\r
\r
-\r
-rem Copy x86 .cdf files to arch specific dirs for Driver signing\r
-rem .inf files now resident alongside .sys files\r
-\r
-:inf_files\r
-\r
-if "%DBG%" == "TRUE" echo DBG: cpy %OSE% x86 INF files\r
-\r
-copy /A/Y %1\hw\mthca\kernel\mthca32.cdf %2\HCA\x86\mthca.cdf\r
-copy /A/Y %1\hw\mlx4\kernel\hca\mlx4_hca32.cdf %2\HCA\x86\mlx4_hca.cdf\r
-copy /A/Y %1\hw\mlx4\kernel\bus\drv\mlx4_bus32.cdf %2\HCA\x86\mlx4_bus.cdf\r
-copy /A/Y %1\core\iou\kernel\ib_iou.cdf %2\HCA\x86\ib_iou.cdf\r
-rem [future?]copy /A/Y %1\core\winverbs\kernel\winverbs32.cdf %2\HCA\x86\winverbs.cdf\r
-copy /A/Y %1\core\winverbs\kernel\winverbs.cdf %2\HCA\x86\winverbs.cdf\r
-\r
-rem No WSD support for XP32\r
-if /I "%OSE%" == "wxp" (\r
- copy /A /Y %1\ulp\ipoib\kernel\netipoib-xp32.inf %2\net\x86\netipoib.inf\r
- copy /A /Y %1\ulp\ipoib\kernel\ipoib32-xp.cdf %2\net\x86\ipoib.cdf\r
-) else (\r
- copy /A/Y %1\ulp\ipoib\kernel\ipoib32.cdf %2\net\x86\ipoib.cdf\r
-)\r
-xcopy %1\ulp\qlgcvnic\kernel\netvnic.cdf %2\net\x86\ /yq\r
-xcopy %1\ulp\srp\kernel\ib_srp.cdf %2\storage\x86\ /yq\r
-\r
-if /I "%OSE%" == "wxp" goto cpy_bin_files\r
-\r
-if "%DBG%" == "TRUE" echo DBG: copy 64bit INF files\r
-\r
-xcopy %1\hw\mthca\kernel\mthca.cdf %2\HCA\amd64 /yq\r
-\r
-xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.cdf %2\HCA\amd64\ /yq\r
-xcopy %1\hw\mlx4\kernel\hca\mlx4_hca32.cdf %2\HCA\amd64\ /yq\r
-xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.cdf %2\HCA\amd64\ /yq\r
-\r
-xcopy %1\core\iou\kernel\ib_iou.cdf %2\HCA\amd64\ /yq\r
-xcopy %1\core\winverbs\kernel\winverbs.cdf %2\HCA\amd64\ /yq\r
-\r
-xcopy %1\ulp\ipoib\kernel\ipoib.cdf %2\net\amd64\ /yq\r
-xcopy %1\ulp\qlgcvnic\kernel\netvnic.cdf %2\net\amd64\ /yq\r
-xcopy %1\ulp\srp\kernel\ib_srp.cdf %2\storage\amd64\ /yq\r
-\r
-\r
-rem When ConnectX works for ia64\r
-rem xcopy %1\hw\mlx4\kernel\hca\mlx4_hca.cdf %2\HCA\ia64\ /yq\r
-rem xcopy %1\hw\mlx4\kernel\bus\drv\mlx4_bus.cdf %2\HCA\ia64\ /yq\r
-\r
-xcopy %1\core\iou\kernel\ib_iou.cdf %2\HCA\ia64\ /yq\r
-xcopy %1\core\winverbs\kernel\winverbs.cdf %2\HCA\ia64\ /yq\r
-\r
-xcopy %1\hw\mthca\kernel\mthca.cdf %2\HCA\ia64\ /yq\r
-\r
-xcopy %1\ulp\ipoib\kernel\ipoib.cdf %2\net\ia64\ /yq\r
-xcopy %1\ulp\qlgcvnic\kernel\netvnic.cdf %2\net\ia64\ /yq\r
-xcopy %1\ulp\srp\kernel\ib_srp.cdf %2\storage\ia64\ /yq\r
-\r
-\r
-:cpy_bin_files\r
+:mk_sym_bin\r
\r
rem bin\bin used to generate a web based symbol store in build-ofa-dist.bat.\r
\r
)\r
\r
rem Copy DAT v2.0 header files\r
-if "%DBG%" == "TRUE" echo DBG: DAT v2.0 header files\r
-if exist %1\ulp\dapl2\dat\include\dat (\r
- pushd %1\ulp\dapl2\dat\include\dat\r
-\r
- xcopy dat.h %2\DAPL2 /Y/Q \r
- xcopy dat_error.h %2\DAPL2 /Y/Q\r
- xcopy dat_platform_specific.h %2\DAPL2 /Y/Q\r
- xcopy dat_redirection.h %2\DAPL2 /Y/Q\r
- xcopy dat_registry.h %2\DAPL2 /Y/Q\r
- xcopy dat_vendor_specific.h %2\DAPL2 /Y/Q\r
- xcopy udat.h %2\DAPL2 /Y/Q\r
- xcopy udat_config.h %2\DAPL2 /Y/Q\r
- xcopy udat_redirection.h %2\DAPL2 /Y/Q\r
- xcopy udat_vendor_specific.h %2\DAPL2 /Y/Q\r
- popd\r
- pushd %1\ulp\dapl2\test\dapltest\scripts\r
- xcopy dt-svr.bat %2\DAPL2 /Y/Q\r
- xcopy dt-cli.bat %2\DAPL2 /Y/Q\r
- popd\r
-)\r
+\r
+if "%DBG%" == "TRUE" echo DBG: [%OSE%] DAT v2.0 header files\r
+\r
+if exist %1\ulp\dapl2\dat\include\dat (set DATINC=dat) else (set DATINC=dat2)\r
+pushd %1\ulp\dapl2\dat\include\%DATINC%\r
+if ERRORLEVEL 1 (\r
+ echo %0: ERR - missing DAT files @ %1\ulp\dapl2\dat\include\%DATINC%\r
+ exit /B 1\r
+)\r
+xcopy dat.h %2\DAPL2 /Y/Q \r
+xcopy dat_error.h %2\DAPL2 /Y/Q\r
+xcopy dat_platform_specific.h %2\DAPL2 /Y/Q\r
+xcopy dat_redirection.h %2\DAPL2 /Y/Q\r
+xcopy dat_registry.h %2\DAPL2 /Y/Q\r
+xcopy dat_vendor_specific.h %2\DAPL2 /Y/Q\r
+xcopy udat.h %2\DAPL2 /Y/Q\r
+xcopy udat_config.h %2\DAPL2 /Y/Q\r
+xcopy udat_redirection.h %2\DAPL2 /Y/Q\r
+xcopy udat_vendor_specific.h %2\DAPL2 /Y/Q\r
+popd\r
+\r
+pushd %1\ulp\dapl2\test\dapltest\scripts\r
+xcopy dt-svr.bat %2\DAPL2 /Y/Q\r
+xcopy dt-cli.bat %2\DAPL2 /Y/Q\r
+popd\r
\r
rem Copy IBAL header files\r
if "%DBG%" == "TRUE" echo DBG: IBAL header files\r
typedef struct _ual_cep_poll_ioctl\r
{\r
ib_api_status_t status;\r
- uint64_t context;\r
net32_t new_cid;\r
ib_mad_element_t element;\r
ib_grh_t grh;\r
#define IB_PORT_LINK_SPEED_ENABLED_MASK 0x0F\r
#define IB_PORT_PHYS_STATE_MASK 0xF0\r
#define IB_PORT_PHYS_STATE_SHIFT 4\r
-#define IB_PORT_PHYS_STATE_NO_CHANGE 0\r
-#define IB_PORT_PHYS_STATE_SLEEP 1\r
-#define IB_PORT_PHYS_STATE_POLLING 2\r
-#define IB_PORT_PHYS_STATE_DISABLED 3\r
-#define IB_PORT_PHYS_STATE_PORTCONFTRAIN 4\r
-#define IB_PORT_PHYS_STATE_LINKUP 5\r
-#define IB_PORT_PHYS_STATE_LINKERRRECOVER 6\r
-#define IB_PORT_PHYS_STATE_PHYTEST 7\r
#define IB_PORT_LNKDWNDFTSTATE_MASK 0x0F\r
\r
#define IB_PORT_CAP_RESV0 (CL_NTOH32(0x00000001))\r
ibat \\r
limits \\r
wherebu \\r
- wvtests \\r
perftest\r
perftests \\r
part_man \\r
infiniband-diags \\r
- qlgcvnic_config\r
+ qlgcvnic_config \\r
+ ndinstall\r
--- /dev/null
+DIRS=\\r
+ user\r
--- /dev/null
+TARGETNAME=ndinstall\r
+TARGETPATH=..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE=PROGRAM\r
+UMTYPE=console\r
+USE_MSVCRT=1\r
+\r
+INCLUDES=..\..\..\inc;\\r
+ ..\..\..\inc\user;\r
+\r
+SOURCES= \\r
+ installsp.rc \\r
+ installsp.c\r
+\r
+USER_C_FLAGS=$(USER_C_FLAGS) \r
+#-DPERFMON_ENABLED\r
+\r
+TARGETLIBS=\\r
+ $(SDK_LIB_PATH)\ws2_32.lib \\r
+ $(SDK_LIB_PATH)\LoadPerf.lib \r
+\r
+MSC_WARNING_LEVEL= /W3\r
+\r
+LINKER_FLAGS=$(LINKER_FLAGS)\r
--- /dev/null
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id$\r
+ */\r
+\r
+/*\r
+ * Module Name: installsp.c\r
+ * Description: This module installs/removes the NetworkDirect provider for infiniband.\r
+ * execute:\r
+ * To install the service provider\r
+ * installsp -i\r
+ * To remove the service provider\r
+ * installsp -r\r
+ */\r
+\r
+#include <winsock2.h>\r
+#include <ws2spi.h>\r
+#include <stdio.h>\r
+\r
+#ifndef PFL_NETWORKDIRECT_PROVIDER\r
+#define PFL_NETWORKDIRECT_PROVIDER 0x00000010\r
+#endif\r
+\r
+/* Initialize the LSP's provider path for Infiband Service Provider dll */\r
+static const WCHAR provider_path[] = L"%SYSTEMROOT%\\system32\\ibndprov.dll";\r
+static const WCHAR provider_name[] = L"OpenFabrics Network Direct Provider";\r
+static const char openib_key_name[] = "OpenFabrics Alliance";\r
+\r
+/* Unique provider GUID generated with "uuidgen -s" */\r
+static GUID provider_guid = {\r
+ /* {52CDAA00-29D0-46be-8FC6-E51D7075C338} */\r
+ 0x52CDAA00, 0x29D0, 0x46be,\r
+ { 0x8f, 0xc6, 0xe5, 0x1d, 0x70, 0x75, 0xc3, 0x38 }\r
+};\r
+\r
+#ifdef _WIN64\r
+#define WSCInstallProvider WSCInstallProvider64_32\r
+#endif /* _WIN64 */\r
+\r
+/*\r
+ * Function: usage\r
+ * Description: Prints usage information.\r
+ */\r
+static void\r
+usage (char *progname)\r
+{\r
+ printf ("usage: %s [-i/-r [-p]]\n", progname);\r
+ printf (" -i Install the OpenFabrics NetworkDirect service provider\n"\r
+ " -r Remove the OpenFabrics NetworkDirect service provider\n"\r
+ " -r <name> Remove the specified service provider\n"\r
+ " -l List service providers\n");\r
+}\r
+\r
+\r
+/* Function: print_providers\r
+ * Description: \r
+ * This function prints out each entry in the Winsock catalog.\r
+*/\r
+static void print_providers(void)\r
+{\r
+ WSAPROTOCOL_INFOW *protocol_info;\r
+ unsigned int protocol_count;\r
+ unsigned int i;\r
+ DWORD protocol_size;\r
+ INT err_no;\r
+ int rc;\r
+\r
+ /* Find the size of the buffer */\r
+ protocol_size = 0;\r
+ rc = WSCEnumProtocols (NULL, NULL, &protocol_size, &err_no);\r
+ if (rc == SOCKET_ERROR && err_no != WSAENOBUFS) {\r
+ printf("WSCEnumProtocols() returned error (%d)\n", err_no);\r
+ return;\r
+ }\r
+\r
+ /* Allocate the buffer */\r
+ protocol_info = HeapAlloc (GetProcessHeap (), HEAP_ZERO_MEMORY, protocol_size);\r
+ if (protocol_info == NULL) {\r
+ printf("HeapAlloc() failed\n");\r
+ return;\r
+ }\r
+\r
+ /* Enumerate the catalog for real */\r
+ rc = WSCEnumProtocols (NULL, protocol_info, &protocol_size, &err_no);\r
+ if (rc == SOCKET_ERROR) {\r
+ printf("WSCEnumProtocols returned error for real enumeration (%d)\n",\r
+ err_no);\r
+ HeapFree (GetProcessHeap (), 0, protocol_info);\r
+ return;\r
+ }\r
+\r
+ protocol_count = rc;\r
+\r
+ for (i = 0; i < protocol_count; i++) {\r
+ printf ("%010d - %S\n", protocol_info[i].dwCatalogEntryId,\r
+ protocol_info[i].szProtocol);\r
+ }\r
+\r
+ HeapFree (GetProcessHeap (), 0, protocol_info);\r
+\r
+ return;\r
+}\r
+\r
+/*\r
+ * Function: install_provider\r
+ * Description: installs the service provider\r
+ *\r
+ * Note: most of the information setup here comes from "MSDN Home >\r
+ * MSDN Library > Windows Development > Network Devices and\r
+ * Protocols > Design Guide > System Area Networks > Windows Sockets\r
+ * Direct > Windows Sockets Direct Component Operation > Installing\r
+ * Windows Sockets Direct Components".\r
+ * The direct link is http://msdn.microsoft.com/library/default.asp?url=/library/en-us/network/hh/network/wsdp_2xrb.asp\r
+ */\r
+static void install_provider(void)\r
+{\r
+ int rc;\r
+ INT err_no;\r
+ LONG reg_error;\r
+ WSAPROTOCOL_INFOW provider;\r
+ HKEY hkey;\r
+\r
+ /* Setup the values in PROTOCOL_INFO */\r
+ provider.dwServiceFlags1 = \r
+ XP1_GUARANTEED_DELIVERY | \r
+ XP1_GUARANTEED_ORDER | \r
+ XP1_MESSAGE_ORIENTED |\r
+ XP1_CONNECT_DATA; /*XP1_GRACEFUL_CLOSE;*/\r
+ provider.dwServiceFlags2 = 0; /* Reserved */\r
+ provider.dwServiceFlags3 = 0; /* Reserved */\r
+ provider.dwServiceFlags4 = 0; /* Reserved */\r
+ provider.dwProviderFlags = PFL_HIDDEN | PFL_NETWORKDIRECT_PROVIDER;\r
+ provider.ProviderId = provider_guid; /* Service Provider ID provided by vendor. Need to be changed later */\r
+ provider.dwCatalogEntryId = 0;\r
+ provider.ProtocolChain.ChainLen = 1; /* Base Protocol Service Provider */\r
+ provider.iVersion = 1;\r
+ provider.iAddressFamily = AF_INET;\r
+ provider.iMaxSockAddr = 16;\r
+ provider.iMinSockAddr = 16;\r
+ provider.iSocketType = -1;\r
+ provider.iProtocol = 0;\r
+ provider.iProtocolMaxOffset = 0;\r
+ provider.iNetworkByteOrder = BIGENDIAN;\r
+ provider.iSecurityScheme = SECURITY_PROTOCOL_NONE;\r
+ provider.dwMessageSize = 0xFFFFFFFF; /* IB supports 32-bit lengths for data transfers on RC */\r
+ provider.dwProviderReserved = 0;\r
+ wcscpy( provider.szProtocol, provider_name );\r
+\r
+ rc = WSCInstallProvider(\r
+ &provider_guid, provider_path, &provider, 1, &err_no );\r
+ if( rc == SOCKET_ERROR )\r
+ {\r
+ if( err_no == WSANO_RECOVERY )\r
+ printf("The provider is already installed\n");\r
+ else\r
+ printf("install_provider: WSCInstallProvider failed: %d\n", err_no);\r
+ }\r
+}\r
+\r
+/*\r
+ * Function: remove_provider\r
+ * Description: removes our provider.\r
+ */\r
+static void remove_provider( const char* const provider_name )\r
+{\r
+ int rc;\r
+ int err_no;\r
+ LONG reg_error;\r
+ HKEY hkey;\r
+\r
+ /* Remove from the catalog */\r
+ rc = WSCDeinstallProvider(&provider_guid, &err_no);\r
+ if (rc == SOCKET_ERROR) {\r
+ printf ("WSCDeinstallProvider failed: %d\n", err_no);\r
+ }\r
+\r
+#ifdef _WIN64\r
+ /* Remove from the 32-bit catalog too! */\r
+ rc = WSCDeinstallProvider32(&provider_guid, &err_no);\r
+ if (rc == SOCKET_ERROR) {\r
+ printf ("WSCDeinstallProvider32 failed: %d\n", err_no);\r
+ }\r
+#endif /* _WIN64 */\r
+}\r
+\r
+/* Function: main\r
+ *\r
+ * Description:\r
+ * Parse the command line arguments and call either the install or remove\r
+ * routine.\r
+ */\r
+int __cdecl main (int argc, char *argv[])\r
+{\r
+ WSADATA wsd;\r
+\r
+ /* Load Winsock */\r
+ if (WSAStartup (MAKEWORD (2, 2), &wsd) != 0) {\r
+ printf ("InstallSP: Unable to load Winsock: %d\n", GetLastError ());\r
+ return -1;\r
+ }\r
+\r
+ /* Confirm that the WinSock DLL supports 2.2. Note that if the\r
+ * DLL supports versions greater than 2.2 in addition to 2.2, it\r
+ * will still return 2.2 in wVersion since that is the version we\r
+ * requested. */\r
+ if (LOBYTE (wsd.wVersion) != 2 || HIBYTE (wsd.wVersion) != 2) {\r
+\r
+ /* Tell the user that we could not find a usable WinSock DLL. */\r
+ WSACleanup ();\r
+ printf\r
+ ("InstallSP: Unable to find a usable version of Winsock DLL\n");\r
+ return -1;\r
+ }\r
+ if (argc < 2) {\r
+ usage (argv[0]);\r
+ return -1;\r
+ }\r
+ if ((strlen (argv[1]) != 2) && (argv[1][0] != '-')\r
+ && (argv[1][0] != '/')) {\r
+ usage (argv[0]);\r
+ return -1;\r
+ }\r
+ switch (tolower (argv[1][1])) {\r
+\r
+ case 'i':\r
+ /* Install the Infiniband Service Provider */\r
+ install_provider ();\r
+#ifdef PERFMON_ENABLED\r
+ _IBSPPerfmonIniFilesGenerate();\r
+ if ( _IBSPPerfmonRegisterKeys() == ERROR_SUCCESS )\r
+ _IBSPPerfmonRegisterCounters();\r
+#endif\r
+ break;\r
+\r
+ case 'r':\r
+ /* Remove the service provider */\r
+ if( argc == 2 )\r
+ remove_provider( openib_key_name );\r
+ else\r
+ remove_provider( argv[2] );\r
+#ifdef PERFMON_ENABLED\r
+ _IBSPPerfmonIniFilesRemove();\r
+ if ( _IBSPPerfmonDeregisterCounters() == ERROR_SUCCESS )\r
+ _IBSPPerfmonDeregisterKeys();\r
+#endif\r
+ break;\r
+\r
+ case 'l':\r
+ /* List existing providers */\r
+ print_providers();\r
+ break;\r
+ \r
+ default:\r
+ usage (argv[0]);\r
+ break;\r
+ }\r
+\r
+ WSACleanup ();\r
+\r
+ return 0;\r
+}\r
--- /dev/null
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.\r
+ * Portions Copyright (c) 2008 Microsoft Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id$\r
+ */\r
+\r
+\r
+#include <oib_ver.h>\r
+\r
+#define VER_FILETYPE VFT_APP\r
+#define VER_FILESUBTYPE VFT2_UNKNOWN\r
+\r
+#ifdef _DEBUG_\r
+#define VER_FILEDESCRIPTION_STR "NetworkDirect for InfiniBand installer (Debug)"\r
+#else\r
+#define VER_FILEDESCRIPTION_STR "NetworkDirect for InfiniBand installer"\r
+#endif\r
+\r
+#define VER_INTERNALNAME_STR "ndinstall.exe"\r
+#define VER_ORIGINALFILENAME_STR "ndinstall.exe"\r
+\r
+#include <common.ver>\r
--- /dev/null
+#\r
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source\r
+# file to this component. This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+# If Network Direct SDK installed then disable only IA64 builds\r
+# Otherwise disable all builds.\r
+!IFDEF ND_INC\r
+DDK_BLOCK_ON_IA64=1\r
+!ELSE\r
+!MESSAGE Skipping ndinstall build: HPC SDK not installed.\r
+DDK_BLOCK_ON_IA64=1\r
+DDK_BLOCK_ON_X86=1\r
+DDK_BLOCK_ON_AMD64=1\r
+!ENDIF\r
+\r
+!INCLUDE ..\..\..\inc\openib.def\r
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\r
+ <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">\r
+ <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">\r
+ <security>\r
+ <requestedPrivileges>\r
+ <requestedExecutionLevel level="requireAdministrator" uiAccess="false"/>\r
+ </requestedPrivileges>\r
+ </security>\r
+ </trustInfo>\r
+</assembly>
\ No newline at end of file
+commit 29bf0a24f54c45d2742026756f31f1a1f26fb6f3
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Thu Apr 16 14:35:18 2009 -0700
+
+ dapltest: reset server listen ports to avoid collisions during long runs
+
+ If server is running continuously the port number increments
+ from base without reseting between tests. This will
+ eventually cause collisions in port space.
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit c27af8de0501d132b8152ec8546023cdba212de5
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Thu Apr 16 10:21:51 2009 -0700
+
+ To avoid duplicating port numbers between different tests, the next port
+ number to use must increment based on the number of endpoints per thread *
+ the number of threads.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 3084310197c20aaa50abe82260fc835786f591f5
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Thu Apr 16 10:21:45 2009 -0700
+
+ dapltest assumes that events across multiple endpoints occur in a specific
+ order. Since this is a false assumption, avoid this by directing events to
+ per endpoint EVDs, rather than using shared EVDs.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit ef87a0a462f4fa07ac252e28d3aeb44af73cc202
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Thu Apr 16 10:21:41 2009 -0700
+
+ Synchronization is missing between removing items from an EVD and queuing
+ them. Since the removal thread is the user's, but the queuing thread is
+ not, the synchronization must be provided by DAPL. Hold the evd lock
+ around any calls to dapls_rbuf_*.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit f5e86d28f803162ffdf94b41ec7435dec92f728d
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Thu Apr 16 10:21:26 2009 -0700
+
+ Communication to the CR thread is done using an internal socket. When a
+ new connection request is ready for processing, an object is placed on
+ the CR list, and data is written to the internal socket. The write causes
+ the CR thread to wake-up and process anything on its cr list.
+
+ If multiple objects are placed on the CR list around the same time, then
+ the CR thread will read in a single character, but process the entire list.
+ This results in additional data being left on the internal socket. When
+ the CR does a select(), it will find more data to read, read the data, but
+ not have any real work to do. The result is that the thread spins in a
+ loop checking for changes when none have occurred until all data on the
+ internal socket has been read.
+
+ Avoid this overhead by reading all data off the internal socket before
+ processing the CR list.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 2ab52e9b1ab37c6eb44206c135e0568a8c2d01fa
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Thu Apr 16 10:21:13 2009 -0700
+
+ The dapl connect call takes as input an address (sockaddr) and a port number
+ as separate input parameters. It modifies the sockaddr address to set the
+ port number before trying to connect. This leads to a situation in
+ dapltest with multiple threads that reference the same buffer for their
+ address, but specify different port numbers, where the different threads
+ end up trying to connect to the same remote port.
+
+ To solve this, do not modify the caller's address buffer and instead use
+ a local buffer. This fixes an issue seen running multithreaded tests with
+ dapltest.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 7947026ede478f08b4a7b8cb607f457765bf2afa
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Thu Apr 16 10:21:03 2009 -0700
+
+ Windows socket calls should check return values against SOCKET_ERROR to
+ determine if an error occurred.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit e66e42fc44c50c8202f7c98f76d799a69aa3f1b6
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Fri Apr 10 08:33:41 2009 -0700
+
+ Build: add new file dapl/openib_cma/linux/openib_osd.h to EXTRA_DIST
+
+ Fix rpmbuild problem with new cma osd include file.
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit acb213adb3268e9bf6999e2bf040d4a71212b701
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Fri Apr 10 08:32:24 2009 -0700
+
+ dapl scm: reduce wait time for thread startup.
+
+ thread startup wait reduce to 2ms to reduce open times.
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit 55459699fa9c0e5fb7e2b17822f0916412c64b35
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Fri Apr 10 08:31:22 2009 -0700
+
+ dapl-scm: getsockopt optlen needs initialized to size of optval
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit d710c5327e05a40796341d16b45a2b098b03f588
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:17:32 2009 -0700
+
+ The connection request thread adds sockets to a select list unless
+ the cr->socket is invalid and the cr request state is set to destroy. If the
+ cr->socket is invalid, but the cr->state is not destroy, then the cr->socket
+ is added to an FD set for select/poll. This results in select/poll
+ returning an error when select is called. As a result, the cr thread never
+ actually blocks during this state.
+
+ Fix this by only destroying a cr based on its state being set to destroy
+ and skip adding cr->sockets to the FD set when they are invalid.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 0be961c432f897d4f92d9a24dcb7c42ad30ea160
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:08:16 2009 -0700
+
+ Make sure all locks are initialized properly and don't zero their memory
+ once they are.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 9abdc26cd6154aa55588759ba54c9ca69e3fe2b5
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:08:13 2009 -0700
+
+ The lock functions are defined just a few lines beneath the prototypes
+ as inline. Remove the duplicate prototypes.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 9578c4aeb9878d98374e4b7abc02db182aef82c6
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:08:07 2009 -0700
+
+ Make sure all locks are initialized and don't zero out their memory once
+ they are.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 97edcbb662b489303ef68c0da02831efaddeed91
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:08:03 2009 -0700
+
+ The IBAL library allocates a small number of threads for callbacks to the
+ user. If the user blocks all of the callback threads, no additional
+ callbacks can be invoked. The DAPL IBAL provider cancels listen requests
+ from within an IBAL callback, then waits for a second callback to confirm
+ that the listen has been canceled. If there is a single IBAL callback
+ thread, or multiple listens are canceled simultaneously, then the provider
+ can deadlock waiting for a cancel callback that never occurs.
+
+ This problem is seen when running dapltest with multiple threads.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 3e56e63bcb68de352edadafdcfcc4cb222c08c7b
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:57 2009 -0700
+
+ We need to check the return value from select for errors before checking
+ the FD sets. An item may be in an FD set but select could have returned
+ an error.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit a8a977becaeefe0d7f8e01e01631a11988d2d54e
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:53 2009 -0700
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit ecc79cc0a1ae2bdbb3dfd19e15b3b562ac9a2957
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:49 2009 -0700
+
+ Enable building with CQ_WAIT_OBJECTS support to directly wait on CQ
+ completion channels in the Windows version of the openib_scm provider.
+ Also minor fixup to use DAPL_DBG_TYPE_UTIL for debug log messages
+ instead of DAPL_DBG_TYPE_CM.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 73728763666a46df5789af93b50db53cdf64afd6
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:44 2009 -0700
+
+ The IBAL-SCM provider will run into an inifinite loop if the check for
+ cr->socket > SCM_MAX_CONN - 1 fails. The code continues back to the start
+ of the while loop without moving to the next connection request entry
+ in the list.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 9b1b396539926d36ffacfff04fbe7c081e436b45
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:40 2009 -0700
+
+ next_cr is set just before and inside the check
+ if ((cr->socket == DAPL_INVALID_SOCKET && cr->state == SCM_DESTROY)
+ Remove setting it inside the if statement.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 7b49a9850f62276bb7bfccb2d85a1e94e311813c
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:35 2009 -0700
+
+ Some errors on windows are more easily interpretted in hex than decimal.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 08ee072a1396ac2c28983878dbc6b02feb035787
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:32 2009 -0700
+
+ The WinOF HCA driver cannot handle transitioning from RTS -> RESET ->
+ INIT -> ERROR. Simply delete the QP and re-create it to reinitialize
+ the endpoint until the bug is fixed.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit 51ef5d96ce67d6141ec02b2a318f1b6e12be1bcf
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:23 2009 -0700
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit a8582be0e1fc89e856f1d0b43a3c1b271295a352
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:07:18 2009 -0700
+
+ Convert the openib_cma provider to common code between linux and windows.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit cc2a71dfe0c35a70f6b1ba66070a3a06059a8bb5
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: Fri Apr 10 08:06:53 2009 -0700
+
+ Move from using pipes to sockets for internal communication. This
+ avoids issues with windows only supporting select() on sockets.
+
+ Remove windows specific definition of dapl_dbg_log.
+
+ Update to latest windows libibverbs implementation using completion
+ channel abstraction to improve windows scalability and simplify
+ porting where FD's are accessed directly in Linux.
+
+ Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+commit b3ad2ed97399a24a869841e17d1314e11c379aae
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Tue Mar 31 05:41:50 2009 -0800
+
+ Release 2.0.17
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit 5d732929f8a90a490994e8e35a3666c3647ad4fe
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Tue Mar 31 05:22:11 2009 -0800
+
+ dapl: ia64 build problem on SuSE 11, atomic.h no longer exists.
+
+ Add autotools check for SuSE 11 and include intrinsics.h
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit d7b8654db3a1f4ead16cb2e6d15f0902a322a188
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Mon Mar 16 13:23:50 2009 -0800
+
+ Release 2.0.16
+
+ Fix changelog year in spec file.
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
+commit 08d9e26a85911f99d47cbb92ec8bccfc7f073be0
+Author: Arlin Davis <arlin.r.davis@intel.com>
+Date: Mon Mar 16 13:15:22 2009 -0800
+
+ Release 2.0.16
+
+ Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>
+
commit 42c97b2a11d63ac6ba8a15fe8c82061e7da6e136
Author: Arlin Davis <arlin.r.davis@intel.com>
Date: Fri Mar 13 12:39:12 2009 -0800
OSFLAGS += -DREDHAT_EL5
endif
+if OS_SUSE11
+OSFLAGS += -DSUSE_11
+endif
+
if EXT_TYPE_IB
XFLAGS = -DDAT_EXTENSIONS
XPROGRAMS_CMA = dapl/openib_cma/dapl_ib_extensions.c
dapl/include/dapl_vendor.h \
dapl/openib_cma/dapl_ib_dto.h \
dapl/openib_cma/dapl_ib_util.h \
+ dapl/openib_cma/linux/openib_osd.h \
dapl/openib_scm/dapl_ib_dto.h \
dapl/openib_scm/dapl_ib_util.h \
dapl/openib_scm/linux/openib_osd.h \
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.57)
-AC_INIT(dapl, 2.0.16, general@lists.openfabrics.org)
+AC_INIT(dapl, 2.0.18, general@lists.openfabrics.org)
AC_CONFIG_SRCDIR([dat/udat/udat.c])
AC_CONFIG_AUX_DIR(config)
AM_CONFIG_HEADER(config.h)
-AM_INIT_AUTOMAKE(dapl, 2.0.16)
+AM_INIT_AUTOMAKE(dapl, 2.0.18)
AM_PROG_LIBTOOL
fi)
AM_CONDITIONAL(OS_RHEL5, test "$ac_cv_rhel5" = "yes")
+dnl Check for SuSE release 11
+AC_CACHE_CHECK(Check for SUSE_11 system, ac_cv_suse11,
+ if test -f /etc/SuSE-release &&
+ test -n "`grep -e "VERSION = 11" /etc/SuSE-release`"; then
+ ac_cv_suse11=yes
+ else
+ ac_cv_suse11=no
+ fi)
+AM_CONDITIONAL(OS_SUSE11, test "$ac_cv_suse11" = "yes")
+
AC_CONFIG_FILES([Makefile test/dtest/Makefile test/dapltest/Makefile dapl.spec])
AC_OUTPUT
%{_mandir}/man5/*.5*
%changelog
+* Fri Apr 17 2009 Arlin Davis <ardavis@ichips.intel.com> - 2.0.18
+- DAT/DAPL Version 2.0.18 Release 1, OFED 1.4.1 GA
+
+* Tue Mar 31 2009 Arlin Davis <ardavis@ichips.intel.com> - 2.0.17
+- DAT/DAPL Version 2.0.17 Release 1, OFED 1.4.1 GA
+
* Mon Mar 16 2009 Arlin Davis <ardavis@ichips.intel.com> - 2.0.16
- DAT/DAPL Version 2.0.16 Release 1, OFED 1.4.1
-DIRS = ibal ibal-scm openib_scm openib_cma
+DIRS = ibal openib_scm openib_cma
+++ /dev/null
-!if $(FREEBUILD)
-TARGETNAME = dapl2-scm
-!else
-TARGETNAME = dapl2-scmd
-!endif
-
-TARGETPATH = ..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)
-TARGETTYPE = DYNLINK
-DLLENTRY = _DllMainCRTStartup
-
-!if $(_NT_TOOLS_VERSION) == 0x700
-DLLDEF=$O\udapl_scm_exports.def
-!else
-DLLDEF=$(OBJ_PATH)\$O\udapl_scm_exports.def
-!endif
-
-USE_MSVCRT = 1
-
-SOURCES = udapl.rc \
- udapl_sources.c \
- ..\dapl_common_src.c \
- ..\dapl_udapl_src.c \
- dapl_ibal-scm_cm.c \
- dapl_ibal-scm_util.c
-
-INCLUDES = ..\include;..\common;..\ibal;..\..\dat\include;\
- ..\..\dat\udat\windows;..\udapl\windows;\
- ..\..\..\..\inc;..\..\..\..\inc\user;
-
-DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -D_VENDOR_IBAL_ -DDAPL_MERGE_CM_DTO\
- -DDAT_EXTENSIONS -DSOCK_CM
-
-USER_C_FLAGS = $(USER_C_FLAGS) $(DAPL_OPTS)
-!if !$(FREEBUILD)
-USER_C_FLAGS = $(USER_C_FLAGS) -DDAPL_DBG #-DDAPL_COUNTERS
-!endif
-
-TARGETLIBS = \
- $(SDK_LIB_PATH)\kernel32.lib \
- $(SDK_LIB_PATH)\ws2_32.lib \
-!if $(FREEBUILD)
- $(TARGETPATH)\*\dat2.lib \
- $(TARGETPATH)\*\complib.lib \
- $(TARGETPATH)\*\ibal.lib
-!else
- $(TARGETPATH)\*\dat2d.lib \
- $(TARGETPATH)\*\complibd.lib \
- $(TARGETPATH)\*\ibald.lib
-!endif
-
-# FIX ME ASAP
-#MSC_WARNING_LEVEL= /W3
-MSC_WARNING_LEVEL= /W1 /wd4113 /Wp64
+++ /dev/null
-/*
- * This Software is licensed under one of the following licenses:
- *
- * 1) under the terms of the "Common Public License 1.0" a copy of which is
- * available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/cpl.php.
- *
- * 2) under the terms of the "The BSD License" a copy of which is
- * available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/bsd-license.php.
- *
- * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
- * copy of which is available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/gpl-license.php.
- *
- * Licensee has the right to choose one of the above licenses.
- *
- * Redistributions of source code must retain the above copyright
- * notice and one of the license notices.
- *
- * Redistributions in binary form must reproduce both the above copyright
- * notice, one of the license notices in the documentation
- * and/or other materials provided with the distribution.
- */
-
-/***************************************************************************
- *
- * Module: uDAPL
- *
- * Filename: dapl_sock_cm.c
- *
- * Authors: Arlin Davis / Stan C. Smith
- *
- * Created: 3/10/2005
- *
- * Description:
- *
- * The uDAPL socket CM provider - connection management
- *
- ****************************************************************************
- * Source Control System Information
- *
- * $Id: $
- *
- * Copyright (c) 2007 Intel Corporation. All rights reserved.
- *
- **************************************************************************/
-
-#include "dapl.h"
-#include "dapl_adapter_util.h"
-#include "dapl_evd_util.h"
-#include "dapl_cr_util.h"
-#include "dapl_name_service.h"
-#include "dapl_ibal_name_service.h"
-#include "dapl_ibal_util.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <io.h>
-
-extern int g_scm_pipe[2];
-
-extern DAT_RETURN
-dapls_ib_query_gid( IN DAPL_HCA *hca_ptr,
- IN GID *gid );
-
-
-static struct ib_cm_handle * dapli_cm_create(void)
-{
- struct ib_cm_handle *cm_ptr;
-
- /* Allocate CM, init lock, and initialize */
- if ((cm_ptr = dapl_os_alloc(sizeof(*cm_ptr))) == NULL)
- return NULL;
-
- (void)dapl_os_memzero(cm_ptr, sizeof(*cm_ptr));
- if (dapl_os_lock_init(&cm_ptr->lock))
- goto bail;
-
- cm_ptr->dst.ver = htons(DSCM_VER);
- cm_ptr->socket = -1;
- cm_ptr->l_socket = -1;
- return cm_ptr;
-bail:
- dapl_os_free(cm_ptr, sizeof(*cm_ptr));
- return NULL;
-}
-
-
-/* mark for destroy, remove all references, schedule cleanup */
-
-static void dapli_cm_destroy(struct ib_cm_handle *cm_ptr)
-{
- dapl_dbg_log(DAPL_DBG_TYPE_CM,
- " cm_destroy: cm %p ep %p\n", cm_ptr,cm_ptr->ep);
-
- /* cleanup, never made it to work queue */
- if (cm_ptr->state == SCM_INIT) {
- if (cm_ptr->socket >= 0)
- closesocket(cm_ptr->socket);
- if (cm_ptr->l_socket >= 0)
- closesocket(cm_ptr->l_socket);
- dapl_os_free(cm_ptr, sizeof(*cm_ptr));
- return;
- }
-
- dapl_os_lock(&cm_ptr->lock);
- cm_ptr->state = SCM_DESTROY;
- if (cm_ptr->ep)
- cm_ptr->ep->cm_handle = IB_INVALID_HANDLE;
-
- /* close socket if still active */
- if (cm_ptr->socket >= 0) {
- closesocket(cm_ptr->socket);
- cm_ptr->socket = -1;
- }
- if (cm_ptr->l_socket >= 0) {
- closesocket(cm_ptr->l_socket);
- cm_ptr->l_socket = -1;
- }
- dapl_os_unlock(&cm_ptr->lock);
-
- /* wakeup work thread */
- _write(g_scm_pipe[1], "w", sizeof "w");
-}
-
-
-/* queue socket for processing CM work */
-static void dapli_cm_queue(struct ib_cm_handle *cm_ptr)
-{
- /* add to work queue for cr thread processing */
- dapl_llist_init_entry((DAPL_LLIST_ENTRY*)&cm_ptr->entry);
- dapl_os_lock( &cm_ptr->hca->ib_trans.lock );
- dapl_llist_add_tail((DAPL_LLIST_HEAD*)&cm_ptr->hca->ib_trans.list,
- (DAPL_LLIST_ENTRY*)&cm_ptr->entry, (void*)cm_ptr);
- dapl_os_unlock(&cm_ptr->hca->ib_trans.lock);
-
- /* wakeup CM work thread */
- _write(g_scm_pipe[1], "w", sizeof "w");
-}
-
-
-
-static uint16_t
-dapli_get_lid(IN DAPL_HCA *hca, IN int port)
-{
- dapl_ibal_ca_t *p_ca = (dapl_ibal_ca_t *) hca->ib_hca_handle;
- dapl_ibal_port_t *p_active_port;
-
- p_ca = (dapl_ibal_ca_t *) hca->ib_hca_handle;
-
- if (NULL == p_ca)
- {
- dapl_dbg_log ( DAPL_DBG_TYPE_ERR,
- "--> %s() There is no HCA @ line #%d\n",
- __FUNCTION__, __LINE__);
- return 0;
- }
-
- /*
- * We are using the first active port in the list for
- * communication. We have to get back here when we decide to support
- * fail-over and high-availability.
- */
- p_active_port = dapli_ibal_get_port ( p_ca, (uint8_t)port );
-
- if (NULL == p_active_port)
- {
- dapl_dbg_log ( DAPL_DBG_TYPE_ERR,
- "--> %s() Port %d is not available @ line #%d\n",
- __FUNCTION__, port, __LINE__);
- return 0;
- }
-
- if (p_active_port->p_attr->link_state != IB_LINK_ACTIVE)
- {
- /*
- * Port is DOWN; can not send or recv messages
- */
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,"--> %s() Port %d is DOWN; "
- "can not send/recv to/from fabric\n",
- __FUNCTION__, p_active_port->p_attr->port_num);
- return 0;
- }
-
- if (p_active_port->p_attr->lid == 0)
- {
- dapl_dbg_log ( DAPL_DBG_TYPE_ERR, "--> %s() Port %d has no LID "
- "assigned; can not operate\n",
- __FUNCTION__, p_active_port->p_attr->port_num);
- }
-#if 0 && defined(DAPL_DBG)
- dapl_dbg_log(DAPL_DBG_TYPE_CM, "--> %s() Port_num %d LID 0x%x\n",
- __FUNCTION__, p_active_port->p_attr->port_num,
- p_active_port->p_attr->lid);
-#endif
-
- return p_active_port->p_attr->lid;
-}
-
-
-/*
- * ACTIVE/PASSIVE: called from CR thread or consumer via ep_disconnect
- */
-static DAT_RETURN
-dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr)
-{
- DAPL_EP *ep_ptr = cm_ptr->ep;
- DAT_UINT32 disc_data = htonl(0xdead);
-
- if (ep_ptr == NULL)
- return DAT_SUCCESS;
-
- dapl_os_lock(&cm_ptr->lock);
- if ((cm_ptr->state == SCM_INIT) ||
- (cm_ptr->state == SCM_DISCONNECTED)) {
- dapl_os_unlock(&cm_ptr->lock);
- return DAT_SUCCESS;
- } else {
- /* send disc date, close socket, schedule destroy */
- if (cm_ptr->socket >= 0) {
- send(cm_ptr->socket, (const char *)&disc_data,
- sizeof(disc_data), 0);
- closesocket(cm_ptr->socket);
- cm_ptr->socket = -1;
- }
- cm_ptr->state = SCM_DISCONNECTED;
- _write(g_scm_pipe[1], "w", sizeof "w");
- }
- dapl_os_unlock(&cm_ptr->lock);
-
-
- if (ep_ptr->cr_ptr) {
- dapls_cr_callback(cm_ptr,
- IB_CME_DISCONNECTED,
- NULL,
- ((DAPL_CR *)ep_ptr->cr_ptr)->sp_ptr);
- } else {
- dapl_evd_connection_callback(ep_ptr->cm_handle,
- IB_CME_DISCONNECTED,
- NULL,
- ep_ptr);
- }
-
- /* remove reference from endpoint */
- ep_ptr->cm_handle = NULL;
-
- /* schedule destroy */
-
-
- return DAT_SUCCESS;
-}
-
-
-
-/*
- * PASSIVE: consumer accept, send local QP information, private data,
- * queue on work thread to receive RTU information to avoid blocking
- * user thread.
- */
-static DAT_RETURN
-dapli_socket_accept_usr( DAPL_EP *ep_ptr,
- DAPL_CR *cr_ptr,
- DAT_COUNT p_size,
- DAT_PVOID p_data )
-{
- DAPL_IA *ia_ptr = ep_ptr->header.owner_ia;
- dp_ib_cm_handle_t cm_ptr = cr_ptr->ib_cm_handle;
- WSABUF iovec[2];
- int len, rc;
- short rtu_data = 0;
- ib_api_status_t ibs;
- ib_qp_attr_t qpa;
- dapl_ibal_port_t *p_port;
- dapl_ibal_ca_t *p_ca;
-
- dapl_dbg_log (DAPL_DBG_TYPE_EP, "%s() p_sz %d sock %d port 0x%x\n",
- __FUNCTION__,p_size,cm_ptr->socket,
- ia_ptr->hca_ptr->port_num);
-
- if (p_size > IB_MAX_REP_PDATA_SIZE)
- return DAT_LENGTH_ERROR;
-
- /* must have a accepted socket */
- if ( cm_ptr->socket < 0 ) {
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- "%s() Not accepted socket? remote port=0x%x lid=0x%x"
- " qpn=0x%x psize=%d\n",
- cm_ptr->dst.port, cm_ptr->dst.lid,
- ntohs(cm_ptr->dst.qpn), cm_ptr->dst.p_size);
- return DAT_INTERNAL_ERROR;
- }
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " accept_usr: remote port=0x%x lid=0x%x"
- " qpn=0x%x psize=%d\n",
- cm_ptr->dst.port, cm_ptr->dst.lid,
- ntohs(cm_ptr->dst.qpn), cm_ptr->dst.p_size);
-
- /* modify QP to RTR and then to RTS with remote info already read */
-
- p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle;
- p_port = dapli_ibal_get_port (p_ca, (uint8_t)ia_ptr->hca_ptr->port_num);
- if (!p_port)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() dapli_ibal_get_port() failed @ line #%d\n",
- __FUNCTION__,__LINE__);
- goto bail;
- }
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- "%s() DST: qpn 0x%x port 0x%x lid %x psize %d\n",
- __FUNCTION__,
- cl_ntoh32(cm_ptr->dst.qpn),
- cm_ptr->dst.port,
- cl_ntoh16(cm_ptr->dst.lid), cm_ptr->dst.p_size);
-
- /* modify QP to RTR and then to RTS with remote info */
-
- ibs = dapls_modify_qp_state_to_rtr( ep_ptr->qp_handle,
- cm_ptr->dst.qpn,
- cm_ptr->dst.lid,
- p_port );
- if (ibs != IB_SUCCESS)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() QP --> RTR failed @ line #%d\n",
- __FUNCTION__,__LINE__);
- goto bail;
- }
-
- if ( dapls_modify_qp_state_to_rts( ep_ptr->qp_handle ) )
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() QP --> RTS failed @ line #%d\n",
- __FUNCTION__,__LINE__);
- goto bail;
- }
-
- ep_ptr->qp_state = IB_QP_STATE_RTS;
-
- /* save remote address information */
- dapl_os_memcpy( &ep_ptr->remote_ia_address,
- &cm_ptr->dst.ia_address,
- sizeof(ep_ptr->remote_ia_address));
-
- /* determine QP & port numbers */
- ibs = ib_query_qp(ep_ptr->qp_handle, &qpa);
- if (ibs != IB_SUCCESS)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " ib_query_qp() ERR %s\n", ib_get_err_str(ibs));
- goto bail;
- }
-
- /* Send our QP info, IA address, and private data */
- cm_ptr->dst.qpn = qpa.num; /* ib_net32_t */
- cm_ptr->dst.port = ia_ptr->hca_ptr->port_num;
- cm_ptr->dst.lid = dapli_get_lid(ia_ptr->hca_ptr, ia_ptr->hca_ptr->port_num);
- /* set gid in network order */
- ibs = dapls_ib_query_gid( ia_ptr->hca_ptr, &cm_ptr->dst.gid );
- if ( ibs != IB_SUCCESS )
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() dapls_ib_query_gid() returns '%s'\n",
- __FUNCTION__,ib_get_err_str(ibs));
- goto bail;
- }
-
- cm_ptr->dst.ia_address = ia_ptr->hca_ptr->hca_address;
- cm_ptr->dst.p_size = p_size;
-
- dapl_dbg_log(DAPL_DBG_TYPE_CM,
- "%s()\n Tx QP info: qpn %x port 0x%x lid 0x%x p_sz %d IP %s\n",
- __FUNCTION__, cl_ntoh32(cm_ptr->dst.qpn), cm_ptr->dst.port,
- cl_ntoh16(cm_ptr->dst.lid), cm_ptr->dst.p_size,
- dapli_get_ip_addr_str(&cm_ptr->dst.ia_address,NULL));
-
- /* network byte-ordering - QPN & LID already are */
- cm_ptr->dst.p_size = cl_hton32(cm_ptr->dst.p_size);
- cm_ptr->dst.port = cl_hton16(cm_ptr->dst.port);
-
- iovec[0].buf = (char*)&cm_ptr->dst;
- iovec[0].len = sizeof(ib_qp_cm_t);
- if (p_size) {
- iovec[1].buf = p_data;
- iovec[1].len = p_size;
- }
- rc = WSASend( cm_ptr->socket, iovec, (p_size ? 2:1), &len, 0, 0, 0 );
- if (rc || len != (p_size + sizeof(ib_qp_cm_t))) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " accept_usr: ERR %d, wcnt=%d\n",
- WSAGetLastError(), len);
- goto bail;
- }
- dapl_dbg_log(DAPL_DBG_TYPE_CM,
- " accept_usr: local port=0x%x lid=0x%x"
- " qpn=0x%x psize=%d\n",
- ntohs(cm_ptr->dst.port), ntohs(cm_ptr->dst.lid),
- ntohl(cm_ptr->dst.qpn), ntohl(cm_ptr->dst.p_size));
-
- /* save state and reference to EP, queue for RTU data */
- cm_ptr->ep = ep_ptr;
- cm_ptr->hca = ia_ptr->hca_ptr;
- cm_ptr->state = SCM_ACCEPTED;
-
- /* restore remote address information for query */
- dapl_os_memcpy( &cm_ptr->dst.ia_address,
- &ep_ptr->remote_ia_address,
- sizeof(cm_ptr->dst.ia_address));
-
- dapl_dbg_log( DAPL_DBG_TYPE_EP," PASSIVE: accepted!\n" );
- dapli_cm_queue(cm_ptr);
-
- return DAT_SUCCESS;
-
-bail:
- dapl_dbg_log( DAPL_DBG_TYPE_ERR," accept_usr: ERR !QP_RTR_RTS \n");
- dapli_cm_destroy(cm_ptr);
- dapls_ib_reinit_ep( ep_ptr ); /* reset QP state */
-
- return DAT_INTERNAL_ERROR;
-}
-
-
-/*
- * PASSIVE: read RTU from active peer, post CONN event
- */
-void
-dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr)
-{
- int len;
- short rtu_data = 0;
-
- /* complete handshake after final QP state change */
- len = recv(cm_ptr->socket, (char*)&rtu_data, sizeof(rtu_data), 0);
- if ( len != sizeof(rtu_data) || ntohs(rtu_data) != 0x0e0f ) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " accept_rtu: ERR %d, rcnt=%d rdata=%x\n",
- WSAGetLastError(), len, ntohs(rtu_data) );
- goto bail;
- }
-
- /* save state and reference to EP, queue for disc event */
- cm_ptr->state = SCM_CONNECTED;
-
- /* final data exchange if remote QP state is good to go */
- dapl_dbg_log( DAPL_DBG_TYPE_EP," PASSIVE: connected!\n" );
- dapls_cr_callback(cm_ptr, IB_CME_CONNECTED, NULL, cm_ptr->sp);
- return;
-bail:
- dapls_ib_reinit_ep(cm_ptr->ep); /* reset QP state */
- dapli_cm_destroy(cm_ptr);
- dapls_cr_callback(cm_ptr, IB_CME_DESTINATION_REJECT, NULL, cm_ptr->sp);
-}
-
-
-/*
- * ACTIVE: Create socket, connect, and exchange QP information
- */
-static DAT_RETURN
-dapli_socket_connect ( DAPL_EP *ep_ptr,
- DAT_IA_ADDRESS_PTR r_addr,
- DAT_CONN_QUAL r_qual,
- DAT_COUNT p_size,
- DAT_PVOID p_data )
-{
- dp_ib_cm_handle_t cm_ptr;
- DAPL_IA *ia_ptr = ep_ptr->header.owner_ia;
- int len, rc, opt = 1;
- WSABUF iovec[2];
- DWORD ioflags;
- short rtu_data = htons(0x0E0F);
- ib_qp_attr_t qpa;
- ib_api_status_t ibs;
- dapl_ibal_port_t *p_port;
- dapl_ibal_ca_t *p_ca;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect: r_qual %d psize %d\n",
- r_qual, p_size);
-
- cm_ptr = dapli_cm_create();
- if (cm_ptr == NULL)
- return DAT_INSUFFICIENT_RESOURCES;
-
- /* create, connect, sockopt, and exchange QP information */
- if ((cm_ptr->socket = socket(AF_INET,SOCK_STREAM,0)) < 0 ) {
- dapli_cm_destroy(cm_ptr);
- return DAT_INSUFFICIENT_RESOURCES;
- }
-
- ((struct sockaddr_in*)r_addr)->sin_port = htons(r_qual);
-
- if (connect(cm_ptr->socket, r_addr, sizeof(*r_addr)) == SOCKET_ERROR) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " connect: %d on r_qual %d\n",
- WSAGetLastError(), (unsigned int)r_qual);
- dapli_cm_destroy(cm_ptr);
- return DAT_INVALID_ADDRESS;
- }
-
- (void)setsockopt( cm_ptr->socket,
- IPPROTO_TCP,TCP_NODELAY,
- (const char*)&opt,
- sizeof(opt) );
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket connected!\n");
-
- /* determine QP & port numbers */
- ibs = ib_query_qp(ep_ptr->qp_handle, &qpa);
- if (ibs != IB_SUCCESS)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " ib_query_qp() ERR %s\n", ib_get_err_str(ibs));
- goto bail;
- }
-
- /* Send QP info, IA address and private data */
- cm_ptr->dst.qpn = qpa.num; /* ib_net32_t */
- cm_ptr->dst.port = cl_hton16(ia_ptr->hca_ptr->port_num);
- cm_ptr->dst.lid = dapli_get_lid( ia_ptr->hca_ptr,
- ia_ptr->hca_ptr->port_num );
- if (cm_ptr->dst.lid == 0)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() ERR lid == 0? @ line #%d\n",
- __FUNCTION__, __LINE__);
- goto bail;
- }
-
- /* set gid in network order */
- ibs = dapls_ib_query_gid( ia_ptr->hca_ptr, &cm_ptr->dst.gid );
- if ( ibs != IB_SUCCESS )
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() dapls_ib_query_gid() returns '%s'\n",
- __FUNCTION__,ib_get_err_str(ibs));
- goto bail;
- }
-
- cm_ptr->dst.ia_address = ia_ptr->hca_ptr->hca_address;
- cm_ptr->dst.p_size = cl_hton32(p_size);
-
- dapl_dbg_log(DAPL_DBG_TYPE_CM, "%s()\n Tx QP info: qpn %x "
- "port %d lid 0x%x p_sz %d IP %s\n",__FUNCTION__,
- cl_ntoh32(cm_ptr->dst.qpn), cl_ntoh16(cm_ptr->dst.port),
- cl_ntoh16(cm_ptr->dst.lid),
- cl_ntoh32(cm_ptr->dst.p_size),
- dapli_get_ip_addr_str(&cm_ptr->dst.ia_address,NULL));
-
- iovec[0].buf = (char*)&cm_ptr->dst;
- iovec[0].len = sizeof(ib_qp_cm_t);
- if ( p_size ) {
- iovec[1].buf = p_data;
- iovec[1].len = p_size;
- }
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP," socket connected, write QP and private data\n");
- rc = WSASend (cm_ptr->socket, iovec, (p_size ? 2:1), &len, 0, 0, NULL);
- if ( rc || len != (p_size + sizeof(ib_qp_cm_t)) ) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " connect write: ERR %d, wcnt=%d\n",
- WSAGetLastError(), len);
- goto bail;
- }
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " connect: SRC port=0x%x lid=0x%x, qpn=0x%x, psize=%d\n",
- cm_ptr->dst.port, cm_ptr->dst.lid,
- cm_ptr->dst.qpn, cm_ptr->dst.p_size );
-
- /* queue up to work thread to avoid blocking consumer */
- cm_ptr->state = SCM_CONN_PENDING;
- cm_ptr->hca = ia_ptr->hca_ptr;
- cm_ptr->ep = ep_ptr;
- dapli_cm_queue(cm_ptr);
- return DAT_SUCCESS;
-bail:
- /* close socket, free cm structure */
- dapli_cm_destroy(cm_ptr);
- return DAT_INTERNAL_ERROR;
-}
-
-
-/*
- * ACTIVE: exchange QP information, called from CR thread
- */
-void
-dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
-{
- DAPL_EP *ep_ptr = cm_ptr->ep;
- DAPL_IA *ia_ptr = cm_ptr->ep->header.owner_ia;
- int len, rc;
- DWORD ioflags;
- WSABUF iovec[1];
- short rtu_data = htons(0x0E0F);
- ib_cm_events_t event = IB_CME_DESTINATION_REJECT;
- ib_api_status_t ibs;
- dapl_ibal_port_t *p_port;
- dapl_ibal_ca_t *p_ca;
-
- /* read DST information into cm_ptr, overwrite SRC info */
- dapl_dbg_log(DAPL_DBG_TYPE_EP," connect_rtu: recv peer QP data\n");
-
- iovec[0].buf = (char*)&cm_ptr->dst;
- iovec[0].len = sizeof(ib_qp_cm_t);
- ioflags = len = 0;
- rc = WSARecv (cm_ptr->socket, iovec, 1, &len, &ioflags, 0, 0);
- if ( rc == SOCKET_ERROR || len != sizeof(ib_qp_cm_t) ||
- ntohs(cm_ptr->dst.ver) != DSCM_VER )
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "connect_rtu read: ERR %d rcnt=%d ver=%d\n",
- WSAGetLastError(), len, cm_ptr->dst.ver);
- goto bail;
- }
-
- /* check for consumer reject */
- if (cm_ptr->dst.rej) {
- dapl_dbg_log(DAPL_DBG_TYPE_CM,
- " connect_rtu read: PEER REJ reason=0x%x\n",
- ntohs(cm_ptr->dst.rej));
- event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA;
- goto bail;
- }
-
- /* convert peer response values to host order */
- cm_ptr->dst.port = cl_ntoh16(cm_ptr->dst.port);
- cm_ptr->dst.lid = ntohs(cm_ptr->dst.lid);
- cm_ptr->dst.qpn = cm_ptr->dst.qpn;
- cm_ptr->dst.p_size = cl_ntoh32(cm_ptr->dst.p_size);
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect: Rx DST: qpn %x port %d "
- "lid %x psize %d IP %s\n",
- cl_ntoh32(cm_ptr->dst.qpn), cm_ptr->dst.port,
- cl_ntoh16(cm_ptr->dst.lid),
- cm_ptr->dst.p_size,
- dapli_get_ip_addr_str(&cm_ptr->dst.ia_address,NULL));
-
- /* save remote address information */
- dapl_os_memcpy( &ep_ptr->remote_ia_address,
- &cm_ptr->dst.ia_address,
- sizeof(ep_ptr->remote_ia_address));
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " connect_rtu: DST %s port=0x%x lid=0x%x, qpn=0x%x, psize=%d\n",
- inet_ntoa(((struct sockaddr_in *)&cm_ptr->dst.ia_address)->sin_addr),
- cm_ptr->dst.port, cm_ptr->dst.lid,
- cm_ptr->dst.qpn, cm_ptr->dst.p_size);
-
- /* validate private data size before reading */
- if (cm_ptr->dst.p_size > IB_MAX_REP_PDATA_SIZE) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " connect_rtu read: psize (%d) wrong\n",
- cm_ptr->dst.p_size );
- goto bail;
- }
-
- /* read private data into cm_handle if any present */
- dapl_dbg_log(DAPL_DBG_TYPE_EP," socket connected, read private data\n");
- if ( cm_ptr->dst.p_size ) {
- iovec[0].buf = cm_ptr->p_data;
- iovec[0].len = cm_ptr->dst.p_size;
- ioflags = 0;
- rc = WSARecv (cm_ptr->socket, iovec, 1, &len, &ioflags, 0, 0 );
- if (rc || len != cm_ptr->dst.p_size, &len, 0, 0, 0 ) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " connect read pdata: ERR %d, rcnt=%d\n",
- WSAGetLastError(), len);
- goto bail;
- }
- }
-
- p_ca = (dapl_ibal_ca_t *) ia_ptr->hca_ptr->ib_hca_handle;
- p_port = dapli_ibal_get_port (p_ca, (uint8_t)ia_ptr->hca_ptr->port_num);
- if (!p_port)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() dapli_ibal_get_port() failed @ line #%d\n",
- __FUNCTION__,__LINE__);
- goto bail;
- }
-
- /* modify QP to RTR and then to RTS with remote info */
- ibs = dapls_modify_qp_state_to_rtr( ep_ptr->qp_handle,
- cm_ptr->dst.qpn,
- cm_ptr->dst.lid,
- p_port );
- if (ibs != IB_SUCCESS)
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() QP --> RTR failed @ line #%d\n",
- __FUNCTION__,__LINE__);
- goto bail;
- }
-
- if ( dapls_modify_qp_state_to_rts( ep_ptr->qp_handle ) )
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "%s() QP --> RTS failed @ line #%d\n",
- __FUNCTION__,__LINE__);
- goto bail;
- }
-
- ep_ptr->qp_state = IB_QP_STATE_RTS;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP," connect_rtu: send RTU\n");
-
- /* complete handshake after final QP state change */
- send(cm_ptr->socket, (const char *)&rtu_data, sizeof(rtu_data), 0);
-
- /* init cm_handle and post the event with private data */
- ep_ptr->cm_handle = cm_ptr;
- cm_ptr->state = SCM_CONNECTED;
- dapl_dbg_log( DAPL_DBG_TYPE_EP," ACTIVE: connected!\n" );
-
- dapl_evd_connection_callback( ep_ptr->cm_handle,
- IB_CME_CONNECTED,
- cm_ptr->p_data,
- ep_ptr );
- return;
-bail:
- /* close socket, free cm structure and post error event */
- dapli_cm_destroy(cm_ptr);
- dapls_ib_reinit_ep(ep_ptr); /* reset QP state */
- dapl_evd_connection_callback( NULL /*ep_ptr->cm_handle*/,
- event,
- NULL,
- ep_ptr );
-}
-
-
-/*
- * PASSIVE: Create socket, listen, accept, exchange QP information
- */
-static DAT_RETURN
-dapli_socket_listen ( DAPL_IA *ia_ptr,
- DAT_CONN_QUAL serviceID,
- DAPL_SP *sp_ptr )
-{
- struct sockaddr_in addr;
- ib_cm_srvc_handle_t cm_ptr = NULL;
- int opt = 1, rc;
- DAT_RETURN dat_status = DAT_SUCCESS;
-
- dapl_dbg_log ( DAPL_DBG_TYPE_EP,
- " listen(ia_ptr %p ServiceID %d sp_ptr %p)\n",
- ia_ptr, serviceID, sp_ptr);
-
- /* Allocate CM and initialize */
- cm_ptr = dapli_cm_create();
- if (cm_ptr == NULL)
- return DAT_INSUFFICIENT_RESOURCES;
-
- cm_ptr->sp = sp_ptr;
- cm_ptr->hca = ia_ptr->hca_ptr;
-
- /* bind, listen, set sockopt, accept, exchange data */
- if ((cm_ptr->l_socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- "socket for listen returned %d\n", errno);
- dat_status = DAT_INSUFFICIENT_RESOURCES;
- goto bail;
- }
-
- setsockopt(cm_ptr->l_socket,
- SOL_SOCKET,
- SO_REUSEADDR,
- (const char *)&opt,
- sizeof(opt));
- addr.sin_port = htons(serviceID);
- addr.sin_family = AF_INET;
- addr.sin_addr.s_addr = INADDR_ANY;
-
- rc = bind (cm_ptr->l_socket, (struct sockaddr*)&addr, sizeof(addr));
- if (rc == SOCKET_ERROR)
- {
- rc = WSAGetLastError();
- dapl_dbg_log( DAPL_DBG_TYPE_CM,
- " bind: ERROR %d on conn_qual 0x%x\n",
- rc,serviceID);
-
- if ( rc == WSAEADDRINUSE )
- dat_status = DAT_CONN_QUAL_IN_USE;
- else
- dat_status = DAT_CONN_QUAL_UNAVAILABLE;
- goto bail;
- }
-
- rc = listen (cm_ptr->l_socket, 128);
- if (rc == SOCKET_ERROR)
- {
- rc = WSAGetLastError();
- dapl_dbg_log( DAPL_DBG_TYPE_CM,
- " listen: ERROR %d on conn_qual 0x%x\n",
- rc,serviceID);
-
- if ( rc == WSAEADDRINUSE )
- dat_status = DAT_CONN_QUAL_IN_USE;
- else
- dat_status = DAT_CONN_QUAL_UNAVAILABLE;
- goto bail;
- }
-
- /* set cm_handle for this service point, save listen socket */
- sp_ptr->cm_srvc_handle = cm_ptr;
-
- /* queue up listen socket to process inbound CR's */
- cm_ptr->state = SCM_LISTEN;
- dapli_cm_queue(cm_ptr);
-
- dapl_dbg_log( DAPL_DBG_TYPE_CM,
- " listen: qual 0x%x cr %p s_fd %d\n",
- ntohs(serviceID), cm_ptr, cm_ptr->l_socket );
-
- return dat_status;
-bail:
- dapl_dbg_log( DAPL_DBG_TYPE_CM,
- " listen: ERROR on conn_qual 0x%x\n",serviceID);
- dapli_cm_destroy(cm_ptr);
- return dat_status;
-}
-
-
-/*
- * PASSIVE: send local QP information, private data, and wait for
- * active side to respond with QP RTS/RTR status
- */
-static DAT_RETURN
-dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)
-{
- dp_ib_cm_handle_t acm_ptr;
- void *p_data = NULL;
- int len;
- DAT_RETURN dat_status = DAT_SUCCESS;
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP," socket_accept\n");
-
- /* Allocate accept CM and initialize */
- if ((acm_ptr = dapli_cm_create()) == NULL)
- return DAT_INSUFFICIENT_RESOURCES;
-
- acm_ptr->l_socket = -1;
- acm_ptr->sp = cm_ptr->sp;
- acm_ptr->hca = cm_ptr->hca;
-
- len = sizeof(acm_ptr->dst.ia_address);
- acm_ptr->socket = accept(cm_ptr->l_socket,
- (struct sockaddr*)&acm_ptr->dst.ia_address,
- &len );
-
- if ( acm_ptr->socket < 0 ) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " accept: ERR %d on FD %d l_cr %p\n",
- WSAGetLastError(),cm_ptr->l_socket,cm_ptr);
- dat_status = DAT_INTERNAL_ERROR;
- goto bail;
- }
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP," socket accepted, read QP data\n");
-
- /* read in DST QP info, IA address. check for private data */
- len = recv(acm_ptr->socket,(char*)&acm_ptr->dst,sizeof(ib_qp_cm_t),0);
- if ( len != sizeof(ib_qp_cm_t) || ntohs(acm_ptr->dst.ver) != DSCM_VER )
- {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " accept read: ERR %d, rcnt=%d ver=%d\n",
- WSAGetLastError(), len, acm_ptr->dst.ver);
- dat_status = DAT_INTERNAL_ERROR;
- goto bail;
-
- }
- /* convert accepted values to host byte ordering */
- acm_ptr->dst.port = cl_ntoh16(acm_ptr->dst.port);
- acm_ptr->dst.lid = ntohs(acm_ptr->dst.lid);
- acm_ptr->dst.qpn = acm_ptr->dst.qpn;
- acm_ptr->dst.p_size = cl_ntoh32(acm_ptr->dst.p_size);
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP, " accept: DST %s port 0x%x "
- "lid 0x%x qpn 0x%x psize %d\n",
- dapli_get_ip_addr_str(&acm_ptr->dst.ia_address,NULL),
- acm_ptr->dst.port,
- cl_ntoh16(acm_ptr->dst.lid),
- cl_ntoh32(acm_ptr->dst.qpn),
- acm_ptr->dst.p_size);
-
- /* validate private data size before reading */
- if ( acm_ptr->dst.p_size > IB_MAX_REQ_PDATA_SIZE ) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " accept read: psize (%d) wrong\n",
- acm_ptr->dst.p_size );
- dat_status = DAT_INTERNAL_ERROR;
- goto bail;
- }
-
- dapl_dbg_log(DAPL_DBG_TYPE_EP," socket accepted, read private data\n");
-
- /* read private data into cm_handle if any present */
- if ( acm_ptr->dst.p_size ) {
- len = recv( acm_ptr->socket,
- acm_ptr->p_data,
- acm_ptr->dst.p_size,
- 0 );
- if ( len != acm_ptr->dst.p_size ) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- " accept read pdata: ERR %d, rcnt=%d\n",
- WSAGetLastError(), len );
- dat_status = DAT_INTERNAL_ERROR;
- goto bail;
- }
- dapl_dbg_log(DAPL_DBG_TYPE_EP,
- " accept: psize=%d read\n",
- acm_ptr->dst.p_size);
- p_data = acm_ptr->p_data;
- }
-
- acm_ptr->state = SCM_ACCEPTING;
-
- /* trigger CR event and return SUCCESS */
- dapls_cr_callback( acm_ptr,
- IB_CME_CONNECTION_REQUEST_PENDING,
- p_data,
- acm_ptr->sp );
-
- return DAT_SUCCESS;
-bail:
- dapli_cm_destroy(acm_ptr);
- return DAT_INTERNAL_ERROR;
-}
-
-
-/*
- * dapls_ib_connect
- *
- * Initiate a connection with the passive listener on another node
- *
- * Input:
- * ep_handle,
- * remote_ia_address,
- * remote_conn_qual,
- * prd_size size of private data and structure
- * prd_prt pointer to private data structure
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_SUCCESS
- * DAT_INSUFFICIENT_RESOURCES
- * DAT_INVALID_PARAMETER
- *
- */
-DAT_RETURN
-dapls_ib_connect (
- IN DAT_EP_HANDLE ep_handle,
- IN DAT_IA_ADDRESS_PTR remote_ia_address,
- IN DAT_CONN_QUAL remote_conn_qual,
- IN DAT_COUNT private_data_size,
- IN void *private_data )
-{
- DAPL_EP *ep_ptr;
- ib_qp_handle_t qp_ptr;
-
- dapl_dbg_log ( DAPL_DBG_TYPE_EP,
- " connect(ep_handle %p ....)\n", ep_handle);
- /*
- * Sanity check
- */
- if ( NULL == ep_handle )
- return DAT_SUCCESS;
-
- ep_ptr = (DAPL_EP*)ep_handle;
- qp_ptr = ep_ptr->qp_handle;
-
- return (dapli_socket_connect( ep_ptr, remote_ia_address,
- remote_conn_qual,
- private_data_size, private_data ));
-}
-
-/*
- * dapls_ib_disconnect
- *
- * Disconnect an EP
- *
- * Input:
- * ep_handle,
- * disconnect_flags
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_SUCCESS
- *
- */
-DAT_RETURN
-dapls_ib_disconnect (
- IN DAPL_EP *ep_ptr,
- IN DAT_CLOSE_FLAGS close_flags )
-{
- dp_ib_cm_handle_t cm_ptr = ep_ptr->cm_handle;
- ib_api_status_t ib_status;
-
- dapl_dbg_log (DAPL_DBG_TYPE_EP,
- "dapls_ib_disconnect(ep_handle %p ....)\n", ep_ptr);
-
-#if 0 // XXX
- /* disconnect QP ala transition to RESET state */
- ib_status = dapls_modify_qp_state_to_reset (ep_ptr->qp_handle);
-
- if ( ib_status != IB_SUCCESS )
- {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- "--> %s() failed to move qp to RESET status = %s\n",
- __FUNCTION__, ib_get_err_str(ib_status));
- return DAT_INTERNAL_ERROR;
- }
- ep_ptr->qp_state = IB_QPS_RESET;
-
- /* inform DAPL of disconnect */
- if ( ep_ptr->cr_ptr ) {
- dapls_cr_callback ( ep_ptr->cm_handle,
- IB_CME_DISCONNECTED,
- NULL,
- ((DAPL_CR *)ep_ptr->cr_ptr)->sp_ptr );
- } else {
- dapl_evd_connection_callback ( ep_ptr->cm_handle,
- IB_CME_DISCONNECTED,
- NULL,
- ep_ptr );
- ep_ptr->cm_handle = NULL;
- }
-#endif
- /* modify QP state --> INIT */
- dapls_ib_reinit_ep(ep_ptr);
-
- if (cm_ptr == NULL)
- return DAT_SUCCESS;
- else
- return dapli_socket_disconnect(cm_ptr);
-}
-
-
-/*
- * dapls_ib_disconnect_clean
- *
- * Clean up outstanding connection data. This routine is invoked
- * after the final disconnect callback has occurred. Only on the
- * ACTIVE side of a connection.
- *
- * Input:
- * ep_ptr DAPL_EP
- * active Indicates active side of connection
- *
- * Output:
- * none
- *
- * Returns:
- * void
- *
- */
-void
-dapls_ib_disconnect_clean (
- IN DAPL_EP *ep_ptr,
- IN DAT_BOOLEAN active,
- IN const ib_cm_events_t ib_cm_event )
-{
- return;
-}
-
-/*
- * dapl_ib_setup_conn_listener
- *
- * Have the CM set up a connection listener.
- *
- * Input:
- * ibm_hca_handle HCA handle
- * qp_handle QP handle
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_SUCCESS
- * DAT_INSUFFICIENT_RESOURCES
- * DAT_INTERNAL_ERROR
- * DAT_CONN_QUAL_UNAVAILBLE
- * DAT_CONN_QUAL_IN_USE
- *
- */
-DAT_RETURN
-dapls_ib_setup_conn_listener (
- IN DAPL_IA *ia_ptr,
- IN DAT_UINT64 ServiceID,
- IN DAPL_SP *sp_ptr )
-{
- return (dapli_socket_listen( ia_ptr, ServiceID, sp_ptr ));
-}
-
-
-/*
- * dapl_ib_remove_conn_listener
- *
- * Have the CM remove a connection listener.
- *
- * Input:
- * ia_handle IA handle
- * ServiceID IB Channel Service ID
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_SUCCESS
- * DAT_INVALID_STATE
- *
- */
-DAT_RETURN
-dapls_ib_remove_conn_listener (
- IN DAPL_IA *ia_ptr,
- IN DAPL_SP *sp_ptr )
-{
- ib_cm_srvc_handle_t cm_ptr = sp_ptr->cm_srvc_handle;
-
- dapl_dbg_log (DAPL_DBG_TYPE_EP,
- "dapls_ib_remove_conn_listener(ia_ptr %p sp_ptr %p cm_ptr %p)\n",
- ia_ptr, sp_ptr, cm_ptr );
-
- /* close accepted socket, free cm_srvc_handle and return */
- if ( cm_ptr != NULL ) {
- if ( cm_ptr->l_socket >= 0 ) {
- closesocket( cm_ptr->l_socket );
- cm_ptr->l_socket = -1;
- }
- if ( cm_ptr->socket >= 0 ) {
- closesocket( cm_ptr->socket );
- cm_ptr->socket = -1;
- }
- /* cr_thread will free */
- sp_ptr->cm_srvc_handle = NULL;
- _write(g_scm_pipe[1], "w", sizeof "w");
- }
- return DAT_SUCCESS;
-}
-
-
-/*
- * dapls_ib_accept_connection
- *
- * Perform necessary steps to accept a connection
- *
- * Input:
- * cr_handle
- * ep_handle
- * private_data_size
- * private_data
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_SUCCESS
- * DAT_INSUFFICIENT_RESOURCES
- * DAT_INTERNAL_ERROR
- *
- */
-DAT_RETURN
-dapls_ib_accept_connection (
- IN DAT_CR_HANDLE cr_handle,
- IN DAT_EP_HANDLE ep_handle,
- IN DAT_COUNT p_size,
- IN const DAT_PVOID p_data )
-{
- DAPL_CR *cr_ptr;
- DAPL_EP *ep_ptr;
-
- dapl_dbg_log (DAPL_DBG_TYPE_EP,
- "dapls_ib_accept_connection(cr %llx ep %llx p_sz %d)\n",
- cr_handle, ep_handle, p_size);
-
- cr_ptr = (DAPL_CR *) cr_handle;
- ep_ptr = (DAPL_EP *) ep_handle;
-
- /* allocate and attach a QP if necessary */
- if ( ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED ) {
- DAT_RETURN status;
- status = dapls_ib_qp_alloc( ep_ptr->header.owner_ia,
- ep_ptr, ep_ptr );
- if ( status != DAT_SUCCESS )
- return status;
- }
-
- return ( dapli_socket_accept_usr(ep_ptr, cr_ptr, p_size, p_data) );
-}
-
-
-/*
- * dapls_ib_reject_connection
- *
- * Reject a connection
- *
- * Input:
- * cr_handle
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_SUCCESS
- * DAT_INTERNAL_ERROR
- *
- */
-
-DAT_RETURN
-dapls_ib_reject_connection (
- IN dp_ib_cm_handle_t cm_ptr,
- IN int reject_reason,
- IN DAT_COUNT psize,
- IN const DAT_PVOID pdata)
-{
- WSABUF iovec[1];
- int len;
-
- dapl_dbg_log (DAPL_DBG_TYPE_EP,
- " reject(cm %p reason %x pdata %p psize %d)\n",
- cm_ptr, reject_reason, pdata, psize );
-
- /* write reject data to indicate reject */
- if (cm_ptr->socket >= 0) {
- cm_ptr->dst.rej = (uint16_t)reject_reason;
- cm_ptr->dst.rej = cl_hton16(cm_ptr->dst.rej);
- iovec[0].buf = (char*)&cm_ptr->dst;
- iovec[0].len = sizeof(ib_qp_cm_t);
- (void) WSASend (cm_ptr->socket, iovec, 1, &len, 0, 0, NULL);
- closesocket(cm_ptr->socket);
- cm_ptr->socket = -1;
- }
-
- /* cr_thread will destroy CR */
- cm_ptr->state = SCM_REJECTED;
- _write(g_scm_pipe[1], "w", sizeof "w");
- return DAT_SUCCESS;
-}
-
-
-/*
- * dapls_ib_cm_remote_addr
- *
- * Obtain the remote IP address given a connection
- *
- * Input:
- * cr_handle
- *
- * Output:
- * remote_ia_address: where to place the remote address
- *
- * Returns:
- * DAT_SUCCESS
- * DAT_INVALID_HANDLE
- *
- */
-DAT_RETURN
-dapls_ib_cm_remote_addr (
- IN DAT_HANDLE dat_handle,
- OUT DAT_SOCK_ADDR6 *remote_ia_address )
-{
- DAPL_HEADER *header;
- dp_ib_cm_handle_t ib_cm_handle;
-
- dapl_dbg_log (DAPL_DBG_TYPE_EP,
- "dapls_ib_cm_remote_addr(dat_handle %p, ....)\n",
- dat_handle );
-
- header = (DAPL_HEADER *)dat_handle;
-
- if (header->magic == DAPL_MAGIC_EP)
- ib_cm_handle = ((DAPL_EP *) dat_handle)->cm_handle;
- else if (header->magic == DAPL_MAGIC_CR)
- ib_cm_handle = ((DAPL_CR *) dat_handle)->ib_cm_handle;
- else
- return DAT_INVALID_HANDLE;
-
- dapl_os_memcpy( remote_ia_address,
- &ib_cm_handle->dst.ia_address,
- sizeof(DAT_SOCK_ADDR6) );
-
- return DAT_SUCCESS;
-}
-
-/*
- * dapls_ib_private_data_size
- *
- * Return the size of private data given a connection op type
- *
- * Input:
- * prd_ptr private data pointer
- * conn_op connection operation type
- * hca_ptr hca pointer, needed for transport type
- *
- * If prd_ptr is NULL, this is a query for the max size supported by
- * the provider, otherwise it is the actual size of the private data
- * contained in prd_ptr.
- *
- *
- * Output:
- * None
- *
- * Returns:
- * length of private data
- *
- */
-int dapls_ib_private_data_size (
- IN DAPL_PRIVATE *prd_ptr,
- IN DAPL_PDATA_OP conn_op,
- IN DAPL_HCA *hca_ptr)
-{
- int size;
-
- switch (conn_op)
- {
- case DAPL_PDATA_CONN_REQ:
- {
- size = IB_MAX_REQ_PDATA_SIZE;
- break;
- }
- case DAPL_PDATA_CONN_REP:
- {
- size = IB_MAX_REP_PDATA_SIZE;
- break;
- }
- case DAPL_PDATA_CONN_REJ:
- {
- size = IB_MAX_REJ_PDATA_SIZE;
- break;
- }
- case DAPL_PDATA_CONN_DREQ:
- {
- size = IB_MAX_DREQ_PDATA_SIZE;
- break;
- }
- case DAPL_PDATA_CONN_DREP:
- {
- size = IB_MAX_DREP_PDATA_SIZE;
- break;
- }
- default:
- {
- size = 0;
- }
-
- } /* end case */
-
- return size;
-}
-
-/*
- * Map all socket CM event codes to the DAT equivelent.
- */
-#define DAPL_IB_EVENT_CNT 11
-
-static struct ib_cm_event_map
-{
- const ib_cm_events_t ib_cm_event;
- DAT_EVENT_NUMBER dat_event_num;
- } ib_cm_event_map[DAPL_IB_EVENT_CNT] = {
- /* 00 */ { IB_CME_CONNECTED,
- DAT_CONNECTION_EVENT_ESTABLISHED},
- /* 01 */ { IB_CME_DISCONNECTED,
- DAT_CONNECTION_EVENT_DISCONNECTED},
- /* 02 */ { IB_CME_DISCONNECTED_ON_LINK_DOWN,
- DAT_CONNECTION_EVENT_DISCONNECTED},
- /* 03 */ { IB_CME_CONNECTION_REQUEST_PENDING,
- DAT_CONNECTION_REQUEST_EVENT},
- /* 04 */ { IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA,
- DAT_CONNECTION_REQUEST_EVENT},
- /* 05 */ { IB_CME_DESTINATION_REJECT,
- DAT_CONNECTION_EVENT_NON_PEER_REJECTED},
- /* 06 */ { IB_CME_DESTINATION_REJECT_PRIVATE_DATA,
- DAT_CONNECTION_EVENT_PEER_REJECTED},
- /* 07 */ { IB_CME_DESTINATION_UNREACHABLE,
- DAT_CONNECTION_EVENT_UNREACHABLE},
- /* 08 */ { IB_CME_TOO_MANY_CONNECTION_REQUESTS,
- DAT_CONNECTION_EVENT_NON_PEER_REJECTED},
- /* 09 */ { IB_CME_LOCAL_FAILURE,
- DAT_CONNECTION_EVENT_BROKEN},
- /* 10 */ { IB_CM_LOCAL_FAILURE,
- DAT_CONNECTION_EVENT_BROKEN}
-};
-
-#ifdef NOT_USED
-/*
- * dapls_ib_get_cm_event
- *
- * Return a DAT connection event given a provider CM event.
- *
- * Input:
- * dat_event_num DAT event we need an equivelent CM event for
- *
- * Output:
- * none
- *
- * Returns:
- * ib_cm_event of translated DAPL value
- */
-DAT_EVENT_NUMBER
-dapls_ib_get_dat_event (
- IN const ib_cm_events_t ib_cm_event,
- IN DAT_BOOLEAN active)
-{
- DAT_EVENT_NUMBER dat_event_num;
- int i;
-
- active = active;
-
- if (ib_cm_event > IB_CM_LOCAL_FAILURE)
- return (DAT_EVENT_NUMBER) 0;
-
- dat_event_num = 0;
- for (i = 0; i < DAPL_IB_EVENT_CNT; i++) {
- if (ib_cm_event == ib_cm_event_map[i].ib_cm_event) {
- dat_event_num = ib_cm_event_map[i].dat_event_num;
- break;
- }
- }
- dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK,
- "dapls_ib_get_dat_event: event translate(%s) ib=0x%x dat=0x%x\n",
- active ? "active" : "passive", ib_cm_event, dat_event_num);
-
- return dat_event_num;
-}
-
-
-/*
- * dapls_ib_get_cm_event
- *
- * Return a DAT connection event given a provider CM event.
- *
- * Input:
- * ib_cm_event event provided to the dapl callback routine
- * active switch indicating active or passive connection
- *
- * Output:
- * none
- *
- * Returns:
- * DAT_EVENT_NUMBER of translated provider value
- */
-ib_cm_events_t
-dapls_ib_get_cm_event (
- IN DAT_EVENT_NUMBER dat_event_num)
-{
- ib_cm_events_t ib_cm_event;
- int i;
-
- ib_cm_event = 0;
- for (i = 0; i < DAPL_IB_EVENT_CNT; i++) {
- if ( dat_event_num == ib_cm_event_map[i].dat_event_num ) {
- ib_cm_event = ib_cm_event_map[i].ib_cm_event;
- break;
- }
- }
- return ib_cm_event;
-}
-#endif /* NOT_USED */
-
-/* outbound/inbound CR processing thread to avoid blocking applications */
-
-#define SCM_MAX_CONN (8 * sizeof(fd_set))
-
-void cr_thread(void *arg)
-{
- struct dapl_hca *hca_ptr = arg;
- ib_cm_srvc_handle_t cr, next_cr;
- int max_fd, rc;
- char rbuf[2];
- fd_set rfd, rfds;
- struct timeval to;
-
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cr_thread: ENTER hca %p\n",hca_ptr);
-
- dapl_os_lock( &hca_ptr->ib_trans.lock );
- hca_ptr->ib_trans.cr_state = IB_THREAD_RUN;
-
- while (hca_ptr->ib_trans.cr_state == IB_THREAD_RUN) {
-
- FD_ZERO( &rfds );
- FD_SET(g_scm_pipe[0], &rfds);
- max_fd = g_scm_pipe[0];
-
- if (!dapl_llist_is_empty((DAPL_LLIST_HEAD*)&hca_ptr->ib_trans.list))
- next_cr = dapl_llist_peek_head((DAPL_LLIST_HEAD*)
- &hca_ptr->ib_trans.list);
- else
- next_cr = NULL;
-
- while (next_cr) {
- cr = next_cr;
- next_cr = dapl_llist_next_entry((DAPL_LLIST_HEAD*)
- &hca_ptr->ib_trans.list,
- (DAPL_LLIST_ENTRY*)&cr->entry);
- //dapl_dbg_log (DAPL_DBG_TYPE_CM," CR_thread: cm_ptr %p\n", cr );
- if (cr->l_socket == -1 ||
- hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) {
-
- dapl_dbg_log(DAPL_DBG_TYPE_CM," thread: Freeing %p\n", cr);
- dapl_llist_remove_entry((DAPL_LLIST_HEAD*)
- &hca_ptr->ib_trans.list,
- (DAPL_LLIST_ENTRY*)&cr->entry);
- dapl_os_free( cr, sizeof(*cr) );
- continue;
- }
-
- if (cr->socket == -1) {
- continue;
- }
- if (cr->socket > SCM_MAX_CONN-1) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,
- "SCM ERR: cr->socket(%d) exceeded FD_SETSIZE %d\n",
- cr->socket,SCM_MAX_CONN-1);
- continue;
- }
- FD_SET( cr->socket, &rfds ); /* add to select SET */
- if ( cr->socket > max_fd )
- max_fd = cr->l_socket;
-
- /* individual select poll to check for work */
- FD_ZERO(&rfd);
- FD_SET(cr->socket, &rfd);
- dapl_os_unlock(&hca_ptr->ib_trans.lock);
-
- to.tv_sec = 0;
- to.tv_usec = 0; /* wakeup and check destroy */
-
- /* block waiting for Rx data */
- if (select(cr->socket+1,&rfd,NULL,NULL,&to) == SOCKET_ERROR) {
- rc = WSAGetLastError();
- if ( rc != SOCKET_ERROR /*WSAENOTSOCK*/ )
- {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR/*CM*/,
- " thread: select(sock %d) ERR %d on cr %p\n",
- cr->socket, rc, cr);
- }
- closesocket(cr->socket);
- cr->socket = -1;
- } else if (FD_ISSET(cr->socket,&rfd)) {
- if (cr->socket > 0) {
- if (cr->state == SCM_LISTEN)
- dapli_socket_accept(cr);
- else if (cr->state == SCM_ACCEPTED)
- dapli_socket_accept_rtu(cr);
- else if (cr->state == SCM_CONN_PENDING)
- dapli_socket_connect_rtu(cr);
- else if (cr->state == SCM_CONNECTED)
- dapli_socket_disconnect(cr);
- }
- }
- dapl_os_lock( &hca_ptr->ib_trans.lock );
- }
- dapl_os_unlock( &hca_ptr->ib_trans.lock );
-
- to.tv_sec = 0;
- to.tv_usec = 100000; /* wakeup and check destroy */
-
- (void) select(max_fd+1, &rfds, NULL, NULL, &to);
-
- /* if pipe data consume - used to wake this thread up */
- if (FD_ISSET(g_scm_pipe[0],&rfds)) {
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cr_thread() read pipe data\n");
-printf(" cr_thread() read pipe data\n");
- _read(g_scm_pipe[0], rbuf, 2);
-printf(" cr_thread() Finished read pipe data\n");
- }
- dapl_os_lock( &hca_ptr->ib_trans.lock );
- }
- dapl_os_unlock( &hca_ptr->ib_trans.lock );
- hca_ptr->ib_trans.cr_state = IB_THREAD_EXIT;
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cr_thread(hca %p) exit\n",hca_ptr);
-}
-
-/*
- * Local variables:
- * c-indent-level: 4
- * c-basic-offset: 4
- * tab-width: 8
- * End:
- */
+++ /dev/null
-/*
- * This Software is licensed under one of the following licenses:
- *
- * 1) under the terms of the "Common Public License 1.0" a copy of which is
- * available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/cpl.php.
- *
- * 2) under the terms of the "The BSD License" a copy of which is
- * available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/bsd-license.php.
- *
- * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
- * copy of which is available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/gpl-license.php.
- *
- * Licensee has the right to choose one of the above licenses.
- *
- * Redistributions of source code must retain the above copyright
- * notice and one of the license notices.
- *
- * Redistributions in binary form must reproduce both the above copyright
- * notice, one of the license notices in the documentation
- * and/or other materials provided with the distribution.
- */
-
-/***************************************************************************
- *
- * Module: uDAPL
- *
- * Filename: dapl_ib_util.c
- *
- * Author: Arlin Davis
- *
- * Created: 3/10/2005
- *
- * Description:
- *
- * The uDAPL openib provider - init, open, close, utilities
- *
- ****************************************************************************
- * Source Control System Information
- *
- * $Id: $
- *
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
- *
- **************************************************************************/
-#ifdef RCSID
-static const char rcsid[] = "$Id: $";
-#endif
-
-#include "dapl.h"
-#include "dapl_adapter_util.h"
-#include "dapl_ibal_util.h"
-#include "dapl_ibal_name_service.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <io.h>
-#include <fcntl.h>
-
-extern void cr_thread(void *arg);
-
-int g_dapl_loopback_connection = 0;
-int g_scm_pipe[2];
-
-#ifdef NOT_USED
-
-/* just get IP address for hostname */
-static DAT_RETURN getipaddr( char *addr, int addr_len)
-{
- struct sockaddr_in *ipv4_addr = (struct sockaddr_in*)addr;
- struct hostent *h_ptr;
- struct utsname ourname;
-
- if ( uname( &ourname ) < 0 )
- return DAT_INTERNAL_ERROR;
-
- h_ptr = gethostbyname( ourname.nodename );
- if ( h_ptr == NULL )
- return DAT_INTERNAL_ERROR;
-
- if ( h_ptr->h_addrtype == AF_INET ) {
- int i;
- struct in_addr **alist =
- (struct in_addr **)h_ptr->h_addr_list;
-
- *(uint32_t*)&ipv4_addr->sin_addr = 0;
- ipv4_addr->sin_family = AF_INET;
-
- /* Walk the list of addresses for host */
- for (i=0; alist[i] != NULL; i++) {
-
- /* first non-loopback address */
- if ( *(uint32_t*)alist[i] != htonl(0x7f000001) ) {
- dapl_os_memcpy( &ipv4_addr->sin_addr,
- h_ptr->h_addr_list[i],
- 4 );
- break;
- }
- }
- /* if no acceptable address found */
- if (*(uint32_t*)&ipv4_addr->sin_addr == 0)
- return DAT_INVALID_ADDRESS;
- } else
- return DAT_INVALID_ADDRESS;
-
- return DAT_SUCCESS;
-}
-#endif
-
-/*
- * dapli_init_sock_cm
- *
- * Input:
- * *hca_p pointer to provide HCA handle
- *
- * Output:
- * none
- *
- * Return:
- * DAT_SUCCESS
- * dapl_convert_errno
- *
- */
-DAT_RETURN dapli_init_sock_cm ( IN DAPL_HCA *hca_ptr )
-{
- int opts;
- int i;
- DAT_RETURN dat_status = DAT_SUCCESS;
-
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL, " %s(): %p\n",__FUNCTION__,hca_ptr );
-
- /* set RC tunables via enviroment or default */
- hca_ptr->ib_trans.max_inline_send =
- dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
-#if 0
- hca_ptr->ib_trans.ack_retry =
- dapl_os_get_env_val("DAPL_ACK_RETRY", SCM_ACK_RETRY);
- hca_ptr->ib_trans.ack_timer =
- dapl_os_get_env_val("DAPL_ACK_TIMER", SCM_ACK_TIMER);
- hca_ptr->ib_trans.rnr_retry =
- dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY);
- hca_ptr->ib_trans.rnr_timer =
- dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER);
- hca_ptr->ib_trans.global =
- dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL);
- hca_ptr->ib_trans.hop_limit =
- dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT);
- hca_ptr->ib_trans.tclass =
- dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS);
-#endif
-
- /* initialize cr_list lock */
- dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.lock);
- if (dat_status != DAT_SUCCESS)
- {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- "%s() failed to init cr_list lock\n", __FUNCTION__);
- return DAT_INTERNAL_ERROR;
- }
-
-#if 0
- /* initialize cq_lock */
- dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);
- if (dat_status != DAT_SUCCESS) {
- dapl_log(DAPL_DBG_TYPE_ERR,
- "%s() failed to init cq_lock\n", __FUNCTION__);
- return DAT_INTERNAL_ERROR;
- }
-#endif
-
- /* initialize CM list for listens on this HCA */
- dapl_llist_init_head((DAPL_LLIST_HEAD*)&hca_ptr->ib_trans.list);
-
- /* create pipe communication endpoints */
- if (_pipe(g_scm_pipe, 256, O_TEXT)) {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- "%s() failed to create thread\n", __FUNCTION__);
- return DAT_INTERNAL_ERROR;
- }
-
- /* create thread to process inbound connect request */
- hca_ptr->ib_trans.cr_state = IB_THREAD_INIT;
- dat_status = dapl_os_thread_create(cr_thread,
- (void*)hca_ptr,
- &hca_ptr->ib_trans.thread );
- if (dat_status != DAT_SUCCESS)
- {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- " open_hca: failed to create thread\n");
- return DAT_INTERNAL_ERROR;
- }
-
- /* wait for thread */
- while (hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) {
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
- " open_hca: waiting for cr_thread\n");
- Sleep (250); /* 1/4 sec */
- }
-
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
- "IB hca: port %d, %s %d.%d.%d.%d\n",
- hca_ptr->port_num,
- ((struct sockaddr_in *)&hca_ptr->hca_address)->sin_family == AF_INET ? "AF_INET":"AF_INET6",
- dapli_get_ip_addr_str(&hca_ptr->hca_address,NULL) );
-
- return dat_status;
-}
-
-
-/*
- * dapli_close_sock_cm
- *
- * Open HCA
- *
- * Input:
- * DAPL_HCA provide CA handle
- *
- * Output:
- * none
- *
- * Return:
- * DAT_SUCCESS
- * dapl_convert_errno
- *
- */
-DAT_RETURN dapli_close_sock_cm ( IN DAPL_HCA *hca_ptr )
-{
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL," %s(): %p\n", __FUNCTION__, hca_ptr);
-
- /* destroy cr_thread and lock */
- hca_ptr->ib_trans.cr_state = IB_THREAD_CANCEL;
-
- while (hca_ptr->ib_trans.cr_state != IB_THREAD_EXIT) {
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
- " close_hca: waiting for cr_thread\n");
- Sleep (500); /* 1/2 seconds */
- }
- dapl_os_lock_destroy(&hca_ptr->ib_trans.lock);
-
- return (DAT_SUCCESS);
-}
-
+++ /dev/null
-#
-# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source
-# file to this component. This file merely indirects to the real make file
-# that is shared by all the driver components of the OpenIB Windows project.
-#
-
-!INCLUDE ..\..\..\..\inc\openib.def
+++ /dev/null
-/*
- * Copyright (c) 2007 Intel Corporation. All rights reserved.
- *
- * This software is available to you under the OpenIB.org BSD license
- * below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * $Id$
- */
-
-
-#include <oib_ver.h>
-
-#define VER_FILETYPE VFT_DLL
-#define VER_FILESUBTYPE VFT2_UNKNOWN
-
-#if DBG
-#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0 (socket-cm) (Debug)"
-#define VER_INTERNALNAME_STR "dapl2-scmd.dll"
-#define VER_ORIGINALFILENAME_STR "dapl2-scmd.dll"
-#else
-#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0 (socket-cm)"
-#define VER_INTERNALNAME_STR "dapl2-scm.dll"
-#define VER_ORIGINALFILENAME_STR "dapl2-scm.dll"
-#endif
-
-#include <common.ver>
+++ /dev/null
-#if DBG
-LIBRARY dapl2-scmd.dll
-#else
-LIBRARY dapl2-scm.dll
-#endif
-
-
-EXPORTS
-dat_provider_init
-dat_provider_fini
-#ifdef DAT_EXTENSIONS
-dapl_extensions
-#endif
-
+++ /dev/null
-#include "..\ibal\dapl_ibal_qp.c"
-#include "..\ibal\dapl_ibal_cq.c"
-#include "..\ibal\dapl_ibal_util.c"
-#include "..\ibal\dapl_ibal_name_service.c"
-#include "..\ibal\dapl_ibal_extensions.c"
IN DAPL_EP *ep_ptr)
{
/* work around bug in low level driver - 3/24/09 */
- /* RTS -> RESET -> INIT -> ERROR QP transition crashes system */\r
+ /* RTS -> RESET -> INIT -> ERROR QP transition crashes system */
if (ep_ptr->qp_handle != IB_INVALID_HANDLE) {
dapls_ib_qp_free(ep_ptr->header.owner_ia, ep_ptr);
dapls_ib_qp_alloc(ep_ptr->header.owner_ia, ep_ptr, ep_ptr);
/* wait for thread */
while (hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) {
- dapl_os_sleep_usec(20000);
+ dapl_os_sleep_usec(2000);
}
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
dapl_log(DAPL_DBG_TYPE_UTIL,
" thread_destroy: thread wakeup err = %s\n",
strerror(errno));
- dapl_os_sleep_usec(20000);
+ dapl_os_sleep_usec(2000);
}
dapl_os_lock_destroy(&hca_ptr->ib_trans.lock);
--- /dev/null
+DAPL_SCM_1.2 {\r
+ global:\r
+ dat_provider_fini;\r
+ dat_provider_init;\r
+ local: *;\r
+};\r
#include <sys/utsname.h>
#include <fcntl.h>
-#if !defined(REDHAT_EL5) && (defined(__ia64__))
+#if !defined(SUSE_11) && !defined(REDHAT_EL5) && defined(__ia64__)
#include <asm/atomic.h>
#endif
+#if defined(SUSE_11) && defined(__ia64__)
+#include <asm/types.h>
+#include <asm/intrinsics.h>
+#endif
+
/* Useful debug definitions */
#ifndef STATIC
#define STATIC static
DT_Tdep_PT_Debug (1,(phead,"%s: Send Server_Info\n", module));
pt_ptr->Server_Info.dapltest_version = DAPLTEST_VERSION;
pt_ptr->Server_Info.is_little_endian = DT_local_is_little_endian;
+ /* reset port, don't eat up port space on long runs */
+ if (ps_ptr->NextPortNumber >= SERVER_PORT_NUMBER + 1000)
+ ps_ptr->NextPortNumber = SERVER_PORT_NUMBER + 1;
pt_ptr->Server_Info.first_port_number = ps_ptr->NextPortNumber;
ps_ptr->NextPortNumber += pt_ptr->Client_Info.total_threads;
case TRANSACTION_TEST:
{
/* create a thread to handle this pt_ptr; */
- ps_ptr->NextPortNumber +=\r
- (pt_ptr->Params.u.Transaction_Cmd.eps_per_thread - 1) *\r
- pt_ptr->Client_Info.total_threads;\r
+ ps_ptr->NextPortNumber +=
+ (pt_ptr->Params.u.Transaction_Cmd.eps_per_thread - 1) *
+ pt_ptr->Client_Info.total_threads;
DT_Tdep_PT_Debug (1,(phead,"%s: Creating Transaction Test Thread\n", module));
pt_ptr->thread = DT_Thread_Create (pt_ptr,
DT_Transaction_Test_Server,
#include "..\..\..\..\etc\user\getopt.c"
#define getpid() ((int)GetCurrentProcessId())
-#define F64x "%I64x"
+#define F64x "%I64x"
#ifdef DBG
#define DAPL_PROVIDER "ibnic0v2d"
#define ntohll(x) bswap_64(x)
#endif
-#endif // _WIN32 || _WIN64
+#endif // _WIN32 || _WIN64
/* Debug: 1 == connect & close only, otherwise full-meal deal */
#define CONNECT_ONLY 0
#define RDMA_BUFFER_SIZE (64)
/* Global DAT vars */
-static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL;
-static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL;
-static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL;
-static DAT_PSP_HANDLE h_psp = DAT_HANDLE_NULL;
-static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL;
-
-static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL;
-static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL;
+static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL;
+static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL;
+static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL;
+static DAT_PSP_HANDLE h_psp = DAT_HANDLE_NULL;
+static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL;
+
+static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL;
+static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL;
/* RDMA buffers */
-static DAT_LMR_HANDLE h_lmr_send = DAT_HANDLE_NULL;
-static DAT_LMR_HANDLE h_lmr_recv = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT lmr_context_send;
-static DAT_LMR_CONTEXT lmr_context_recv;
-static DAT_RMR_CONTEXT rmr_context_send;
-static DAT_RMR_CONTEXT rmr_context_recv;
-static DAT_VLEN registered_size_send;
-static DAT_VLEN registered_size_recv;
-static DAT_VADDR registered_addr_send;
-static DAT_VADDR registered_addr_recv;
+static DAT_LMR_HANDLE h_lmr_send = DAT_HANDLE_NULL;
+static DAT_LMR_HANDLE h_lmr_recv = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_send;
+static DAT_LMR_CONTEXT lmr_context_recv;
+static DAT_RMR_CONTEXT rmr_context_send;
+static DAT_RMR_CONTEXT rmr_context_recv;
+static DAT_VLEN registered_size_send;
+static DAT_VLEN registered_size_recv;
+static DAT_VADDR registered_addr_send;
+static DAT_VADDR registered_addr_recv;
/* Initial msg receive buf, RMR exchange, and Rdma-write notification */
#define MSG_BUF_COUNT 3
#define MSG_IOV_COUNT 2
-static DAT_RMR_TRIPLET rmr_recv_msg[MSG_BUF_COUNT];
-static DAT_LMR_HANDLE h_lmr_recv_msg = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT lmr_context_recv_msg;
-static DAT_RMR_CONTEXT rmr_context_recv_msg;
-static DAT_VLEN registered_size_recv_msg;
-static DAT_VADDR registered_addr_recv_msg;
+static DAT_RMR_TRIPLET rmr_recv_msg[MSG_BUF_COUNT];
+static DAT_LMR_HANDLE h_lmr_recv_msg = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_recv_msg;
+static DAT_RMR_CONTEXT rmr_context_recv_msg;
+static DAT_VLEN registered_size_recv_msg;
+static DAT_VADDR registered_addr_recv_msg;
/* message send buffer */
-static DAT_RMR_TRIPLET rmr_send_msg;
-static DAT_LMR_HANDLE h_lmr_send_msg = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT lmr_context_send_msg;
-static DAT_RMR_CONTEXT rmr_context_send_msg;
-static DAT_VLEN registered_size_send_msg;
-static DAT_VADDR registered_addr_send_msg;
-static DAT_EP_ATTR ep_attr;
-char hostname[256] = {0};
-char provider[64] = DAPL_PROVIDER;
-char addr_str[INET_ADDRSTRLEN];
+static DAT_RMR_TRIPLET rmr_send_msg;
+static DAT_LMR_HANDLE h_lmr_send_msg = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_send_msg;
+static DAT_RMR_CONTEXT rmr_context_send_msg;
+static DAT_VLEN registered_size_send_msg;
+static DAT_VADDR registered_addr_send_msg;
+static DAT_EP_ATTR ep_attr;
+char hostname[256] = { 0 };
+char provider[64] = DAPL_PROVIDER;
+char addr_str[INET_ADDRSTRLEN];
/* rdma pointers */
-char *rbuf = NULL;
-char *sbuf = NULL;
-int status;
+char *rbuf = NULL;
+char *sbuf = NULL;
+int status;
/* timers */
-double start,stop,total_us,total_sec;
+double start, stop, total_us, total_sec;
struct dt_time {
- double total;
- double open;
- double reg;
- double unreg;
- double pzc;
- double pzf;
- double evdc;
- double evdf;
- double cnoc;
- double cnof;
- double epc;
- double epf;
- double rdma_wr;
- double rdma_rd[MAX_RDMA_RD];
- double rdma_rd_total;
- double rtt;
- double close;
+ double total;
+ double open;
+ double reg;
+ double unreg;
+ double pzc;
+ double pzf;
+ double evdc;
+ double evdf;
+ double cnoc;
+ double cnof;
+ double epc;
+ double epf;
+ double rdma_wr;
+ double rdma_rd[MAX_RDMA_RD];
+ double rdma_rd_total;
+ double rtt;
+ double close;
};
struct dt_time time;
/* defaults */
-static int failed=0;
-static int performance_times=0;
-static int connected=0;
-static int burst=10;
-static int server=1;
-static int verbose=0;
-static int polling=0;
-static int poll_count=0;
-static int rdma_wr_poll_count=0;
-static int rdma_rd_poll_count[MAX_RDMA_RD]={0};
-static int delay=0;
-static int buf_len=RDMA_BUFFER_SIZE;
-static int use_cno=0;
-static int recv_msg_index=0;
-static int burst_msg_posted=0;
-static int burst_msg_index=0;
+static int failed = 0;
+static int performance_times = 0;
+static int connected = 0;
+static int burst = 10;
+static int server = 1;
+static int verbose = 0;
+static int polling = 0;
+static int poll_count = 0;
+static int rdma_wr_poll_count = 0;
+static int rdma_rd_poll_count[MAX_RDMA_RD] = { 0 };
+static int delay = 0;
+static int buf_len = RDMA_BUFFER_SIZE;
+static int use_cno = 0;
+static int recv_msg_index = 0;
+static int burst_msg_posted = 0;
+static int burst_msg_index = 0;
/* forward prototypes */
-const char * DT_RetToString (DAT_RETURN ret_value);
-const char * DT_EventToSTr (DAT_EVENT_NUMBER event_code);
-void print_usage(void);
-double get_time(void);
-void init_data(void);
-
-DAT_RETURN send_msg( void *data,
- DAT_COUNT size,
- DAT_LMR_CONTEXT context,
- DAT_DTO_COOKIE cookie,
- DAT_COMPLETION_FLAGS flags );
-
-DAT_RETURN connect_ep( char *hostname, DAT_CONN_QUAL conn_id );
-void disconnect_ep( void );
-DAT_RETURN register_rdma_memory( void );
-DAT_RETURN unregister_rdma_memory( void );
-DAT_RETURN create_events( void );
-DAT_RETURN destroy_events(void);
-DAT_RETURN do_rdma_write_with_msg( void );
-DAT_RETURN do_rdma_read_with_msg( void );
-DAT_RETURN do_ping_pong_msg( void );
+const char *DT_RetToString(DAT_RETURN ret_value);
+const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code);
+void print_usage(void);
+double get_time(void);
+void init_data(void);
+
+DAT_RETURN send_msg(void *data,
+ DAT_COUNT size,
+ DAT_LMR_CONTEXT context,
+ DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags);
+
+DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id);
+void disconnect_ep(void);
+DAT_RETURN register_rdma_memory(void);
+DAT_RETURN unregister_rdma_memory(void);
+DAT_RETURN create_events(void);
+DAT_RETURN destroy_events(void);
+DAT_RETURN do_rdma_write_with_msg(void);
+DAT_RETURN do_rdma_read_with_msg(void);
+DAT_RETURN do_ping_pong_msg(void);
#define LOGPRINTF if (verbose) printf
-int
-main(int argc, char **argv)
+void flush_evds(void)
{
- int i,c;
- DAT_RETURN ret;
- DAT_EP_PARAM ep_param;
-
- /* parse arguments */
- while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1)
- {
- switch(c)
- {
- case 't':
- performance_times = 1;
- fflush(stdout);
- break;
- case 's':
- server = 1;
- fflush(stdout);
- break;
- case 'c':
- use_cno = 1;
- printf("%d Creating CNO for DTO EVD's\n",getpid());
- fflush(stdout);
- break;
- case 'v':
- verbose = 1;
- printf("%d Verbose\n",getpid());
- fflush(stdout);
- break;
- case 'p':
- polling = 1;
- printf("%d Polling\n",getpid());
- fflush(stdout);
- break;
- case 'B':
- burst = atoi(optarg);
- break;
- case 'd':
- delay = atoi(optarg);
- break;
- case 'b':
- buf_len = atoi(optarg);
- break;
- case 'h':
- server = 0;
- strcpy (hostname, optarg);
- break;
- case 'P':
- strcpy (provider, optarg);
- break;
- default:
- print_usage();
- exit(-12);
- }
- }
+ DAT_EVENT event;
+
+ /* Flush async error queue */
+ printf("%d ERR: Checking ASYNC EVD...\n", getpid());
+ while (dat_evd_dequeue(h_async_evd, &event) == DAT_SUCCESS) {
+ printf(" ASYNC EVD ENTRY: handle=%p reason=%d\n",
+ event.event_data.asynch_error_event_data.dat_handle,
+ event.event_data.asynch_error_event_data.reason);
+ }
+ /* Flush receive queue */
+ printf("%d ERR: Checking RECEIVE EVD...\n", getpid());
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_SUCCESS) {
+ printf(" RCV EVD ENTRY: op=%d stat=%d ln=%d ck="F64x"\n",
+ event.event_data.dto_completion_event_data.operation,
+ event.event_data.dto_completion_event_data.status,
+ event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64);
+ }
+ /* Flush request queue */
+ printf("%d ERR: Checking REQUEST EVD...\n", getpid());
+ while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_SUCCESS) {
+ printf(" REQ EVD ENTRY: op=%d stat=%d ln=%d ck="F64x"\n",
+ event.event_data.dto_completion_event_data.operation,
+ event.event_data.dto_completion_event_data.status,
+ event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64);
+ }
+}
+
+int main(int argc, char **argv)
+{
+ int i, c;
+ DAT_RETURN ret;
+ DAT_EP_PARAM ep_param;
+
+ /* parse arguments */
+ while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1) {
+ switch (c) {
+ case 't':
+ performance_times = 1;
+ fflush(stdout);
+ break;
+ case 's':
+ server = 1;
+ fflush(stdout);
+ break;
+ case 'c':
+ use_cno = 1;
+ printf("%d Creating CNO for DTO EVD's\n", getpid());
+ fflush(stdout);
+ break;
+ case 'v':
+ verbose = 1;
+ printf("%d Verbose\n", getpid());
+ fflush(stdout);
+ break;
+ case 'p':
+ polling = 1;
+ printf("%d Polling\n", getpid());
+ fflush(stdout);
+ break;
+ case 'B':
+ burst = atoi(optarg);
+ break;
+ case 'd':
+ delay = atoi(optarg);
+ break;
+ case 'b':
+ buf_len = atoi(optarg);
+ break;
+ case 'h':
+ server = 0;
+ strcpy(hostname, optarg);
+ break;
+ case 'P':
+ strcpy(provider, optarg);
+ break;
+ default:
+ print_usage();
+ exit(-12);
+ }
+ }
#if defined(_WIN32) || defined(_WIN64)
- {
- WSADATA wsaData;
-
- i = WSAStartup ( MAKEWORD(2,2), &wsaData );
- if ( i != 0 ) {
- printf("%s WSAStartup(2.2) failed? (0x%x)\n",argv[0],i);
- fflush(stdout);
- exit(1);
- }
- }
+ {
+ WSADATA wsaData;
+
+ i = WSAStartup(MAKEWORD(2, 2), &wsaData);
+ if (i != 0) {
+ printf("%s WSAStartup(2.2) failed? (0x%x)\n", argv[0],
+ i);
+ fflush(stdout);
+ exit(1);
+ }
+ }
#endif
- if (!server) {
- printf("%d Running as client - %s\n",getpid(),provider);
- } else {
- printf("%d Running as server - %s\n",getpid(),provider);
- }
- fflush(stdout);
-
- /* allocate send and receive buffers */
- if (((rbuf = malloc(buf_len*burst)) == NULL) ||
- ((sbuf = malloc(buf_len*burst)) == NULL)) {
- perror("malloc");
- exit(1);
- }
- memset( &time, 0, sizeof(struct dt_time) );
- LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",
- getpid(), rbuf, sbuf, buf_len);
-
- /* dat_ia_open, dat_pz_create */
- h_async_evd = DAT_HANDLE_NULL;
- start = get_time();
- ret = dat_ia_open( provider, 8, &h_async_evd, &h_ia );
- stop = get_time();
- time.open += ((stop - start)*1.0e6);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: Error Adaptor open: %s\n",
- getpid(),DT_RetToString(ret));
- exit(1);
- } else
- LOGPRINTF("%d Opened Interface Adaptor\n",getpid());
-
- /* Create Protection Zone */
- start = get_time();
- LOGPRINTF("%d Create Protection Zone\n",getpid());
- ret = dat_pz_create(h_ia, &h_pz);
- stop = get_time();
- time.pzc += ((stop - start)*1.0e6);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error creating Protection Zone: %s\n",
- getpid(),DT_RetToString(ret));
- exit(1);
- } else
- LOGPRINTF("%d Created Protection Zone\n",getpid());
-
- /* Register memory */
- LOGPRINTF("%d Register RDMA memory\n", getpid());
- ret = register_rdma_memory();
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering RDMA memory: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d Register RDMA memory done\n", getpid());
-
- LOGPRINTF("%d Create events\n", getpid());
- ret = create_events();
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error creating events: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else {
- LOGPRINTF("%d Create events done\n", getpid());
- }
-
- /* create EP */
- memset( &ep_attr, 0, sizeof(ep_attr) );
- ep_attr.service_type = DAT_SERVICE_TYPE_RC;
- ep_attr.max_rdma_size = 0x10000;
- ep_attr.qos = 0;
- ep_attr.recv_completion_flags = 0;
- ep_attr.max_recv_dtos = MSG_BUF_COUNT + (burst*3);
- ep_attr.max_request_dtos = MSG_BUF_COUNT + (burst*3) + MAX_RDMA_RD;
- ep_attr.max_recv_iov = MSG_IOV_COUNT;
- ep_attr.max_request_iov = MSG_IOV_COUNT;
- ep_attr.max_rdma_read_in = MAX_RDMA_RD;
- ep_attr.max_rdma_read_out = MAX_RDMA_RD;
- ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
- ep_attr.ep_transport_specific_count = 0;
- ep_attr.ep_transport_specific = NULL;
- ep_attr.ep_provider_specific_count = 0;
- ep_attr.ep_provider_specific = NULL;
-
- start = get_time();
- ret = dat_ep_create( h_ia, h_pz, h_dto_rcv_evd,
- h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep );
- stop = get_time();
- time.epc += ((stop - start)*1.0e6);
- time.total += time.epc;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_create: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
-
- /*
- * register message buffers, establish connection, and
- * exchange DMA RMR information info via messages
- */
- ret = connect_ep( hostname, SERVER_CONN_QUAL );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error connect_ep: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d connect_ep complete\n", getpid());
-
- /* Query EP for local and remote address information, print */
- ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_query: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
+ if (!server) {
+ printf("%d Running as client - %s\n", getpid(), provider);
+ } else {
+ printf("%d Running as server - %s\n", getpid(), provider);
+ }
+ fflush(stdout);
+
+ /* allocate send and receive buffers */
+ if (((rbuf = malloc(buf_len * (burst+1))) == NULL) ||
+ ((sbuf = malloc(buf_len * (burst+1))) == NULL)) {
+ perror("malloc");
+ exit(1);
+ }
+ memset(&time, 0, sizeof(struct dt_time));
+ LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",
+ getpid(), rbuf, sbuf, buf_len);
+
+ /* dat_ia_open, dat_pz_create */
+ h_async_evd = DAT_HANDLE_NULL;
+ start = get_time();
+ ret = dat_ia_open(provider, 8, &h_async_evd, &h_ia);
+ stop = get_time();
+ time.open += ((stop - start) * 1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error Adaptor open: %s\n",
+ getpid(), DT_RetToString(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
+
+ /* Create Protection Zone */
+ start = get_time();
+ LOGPRINTF("%d Create Protection Zone\n", getpid());
+ ret = dat_pz_create(h_ia, &h_pz);
+ stop = get_time();
+ time.pzc += ((stop - start) * 1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error creating Protection Zone: %s\n",
+ getpid(), DT_RetToString(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Created Protection Zone\n", getpid());
+
+ /* Register memory */
+ LOGPRINTF("%d Register RDMA memory\n", getpid());
+ ret = register_rdma_memory();
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering RDMA memory: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d Register RDMA memory done\n", getpid());
+
+ LOGPRINTF("%d Create events\n", getpid());
+ ret = create_events();
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error creating events: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else {
+ LOGPRINTF("%d Create events done\n", getpid());
+ }
+
+ /* create EP */
+ memset(&ep_attr, 0, sizeof(ep_attr));
+ ep_attr.service_type = DAT_SERVICE_TYPE_RC;
+ ep_attr.max_rdma_size = 0x10000;
+ ep_attr.qos = 0;
+ ep_attr.recv_completion_flags = 0;
+ ep_attr.max_recv_dtos = MSG_BUF_COUNT + (burst * 3);
+ ep_attr.max_request_dtos = MSG_BUF_COUNT + (burst * 3) + MAX_RDMA_RD;
+ ep_attr.max_recv_iov = MSG_IOV_COUNT;
+ ep_attr.max_request_iov = MSG_IOV_COUNT;
+ ep_attr.max_rdma_read_in = MAX_RDMA_RD;
+ ep_attr.max_rdma_read_out = MAX_RDMA_RD;
+ ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
+ ep_attr.ep_transport_specific_count = 0;
+ ep_attr.ep_transport_specific = NULL;
+ ep_attr.ep_provider_specific_count = 0;
+ ep_attr.ep_provider_specific = NULL;
+
+ start = get_time();
+ ret = dat_ep_create(h_ia, h_pz, h_dto_rcv_evd,
+ h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep);
+ stop = get_time();
+ time.epc += ((stop - start) * 1.0e6);
+ time.total += time.epc;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_create: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
+
+ /*
+ * register message buffers, establish connection, and
+ * exchange DMA RMR information info via messages
+ */
+ ret = connect_ep(hostname, SERVER_CONN_QUAL);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error connect_ep: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d connect_ep complete\n", getpid());
+
+ /* Query EP for local and remote address information, print */
+ ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_query: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
#if defined(_WIN32)
- printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),
- inet_ntoa(((struct sockaddr_in *)
- ep_param.local_ia_address_ptr)->sin_addr),
- (ep_param.local_port_qual));
+ printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),
+ inet_ntoa(((struct sockaddr_in *)
+ ep_param.local_ia_address_ptr)->sin_addr),
+ (ep_param.local_port_qual));
#else
- inet_ntop(AF_INET,
- &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->sin_addr,
- addr_str, sizeof(addr_str));
- printf("\n%d Query EP: LOCAL addr %s port "F64x"\n", getpid(),
- addr_str, (ep_param.local_port_qual));
+ inet_ntop(AF_INET,
+ &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->
+ sin_addr, addr_str, sizeof(addr_str));
+ printf("\n%d Query EP: LOCAL addr %s port " F64x "\n", getpid(),
+ addr_str, (ep_param.local_port_qual));
#endif
#if defined(_WIN32)
- printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),
- inet_ntoa(((struct sockaddr_in *)
- ep_param.local_ia_address_ptr)->sin_addr),
- (ep_param.remote_port_qual));
+ printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),
+ inet_ntoa(((struct sockaddr_in *)
+ ep_param.local_ia_address_ptr)->sin_addr),
+ (ep_param.remote_port_qual));
#else
- inet_ntop(AF_INET,
- &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->sin_addr,
- addr_str, sizeof(addr_str));
- printf("%d Query EP: REMOTE addr %s port "F64x"\n", getpid(),
- addr_str, (ep_param.remote_port_qual));
+ inet_ntop(AF_INET,
+ &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->
+ sin_addr, addr_str, sizeof(addr_str));
+ printf("%d Query EP: REMOTE addr %s port " F64x "\n", getpid(),
+ addr_str, (ep_param.remote_port_qual));
#endif
- fflush(stdout);
+ fflush(stdout);
#if CONNECT_ONLY
#if defined(_WIN32) || defined(_WIN64)
- Sleep(1*1000);
+ Sleep(1 * 1000);
#else
- sleep(1);
+ sleep(1);
#endif
- goto cleanup;
+ goto cleanup;
#endif
- /*********** RDMA write data *************/
- ret = do_rdma_write_with_msg();
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
-
- /*********** RDMA read data *************/
- ret = do_rdma_read_with_msg();
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
-
- /*********** PING PING messages ************/
- ret = do_ping_pong_msg();
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
- getpid(),DT_RetToString(ret));
- goto cleanup;
- } else {
- LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
- goto complete;
- }
+ /*********** RDMA write data *************/
+ ret = do_rdma_write_with_msg();
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
+
+ /*********** RDMA read data *************/
+ ret = do_rdma_read_with_msg();
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
+
+ /*********** PING PING messages ************/
+ ret = do_ping_pong_msg();
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ goto cleanup;
+ } else {
+ LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
+ goto complete;
+ }
cleanup:
- failed++;
+
+ fflush(stderr);
+ fflush(stdout);
+ flush_evds();
+ failed++;
+ fflush(stderr);
+ fflush(stdout);
+
complete:
- /* disconnect and free EP resources */
- if ( h_ep != DAT_HANDLE_NULL ) {
- /* unregister message buffers and tear down connection */
- LOGPRINTF("%d Disconnect and Free EP %p \n",getpid(),h_ep);
- disconnect_ep();
-
- /* free EP */
- LOGPRINTF("%d Free EP %p \n",getpid(),h_ep);
- start = get_time();
- ret = dat_ep_free( h_ep );
- stop = get_time();
- time.epf += ((stop - start)*1.0e6);
- time.total += time.epf;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing EP: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Freed EP\n",getpid());
- h_ep = DAT_HANDLE_NULL;
- }
- }
-
- /* free EVDs */
- LOGPRINTF("%d destroy events\n", getpid());
- ret = destroy_events();
- if(ret != DAT_SUCCESS)
- fprintf(stderr, "%d Error destroy_events: %s\n",
- getpid(),DT_RetToString(ret));
- else
- LOGPRINTF("%d destroy events done\n", getpid());
-
-
- ret = unregister_rdma_memory();
- LOGPRINTF("%d unregister_rdma_memory \n", getpid());
- if(ret != DAT_SUCCESS)
- fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
- getpid(),DT_RetToString(ret));
- else
- LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
-
- /* Free protection domain */
- LOGPRINTF("%d Freeing pz\n",getpid());
- start = get_time();
- ret = dat_pz_free( h_pz );
- stop = get_time();
- time.pzf += ((stop - start)*1.0e6);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing PZ: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Freed pz\n",getpid());
- h_pz = NULL;
- }
-
- /* close the device */
- LOGPRINTF("%d Closing Interface Adaptor\n",getpid());
- start = get_time();
- ret = dat_ia_close( h_ia, DAT_CLOSE_ABRUPT_FLAG );
- stop = get_time();
- time.close += ((stop - start)*1.0e6);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: Error Adaptor close: %s\n",
- getpid(),DT_RetToString(ret));
- exit(1);
- } else
- LOGPRINTF("%d Closed Interface Adaptor\n",getpid());
-
- /* free rdma buffers */
- free(rbuf);
- free(sbuf);
-
- printf("\n%d: DAPL Test Complete. %s\n\n",
- getpid(), failed?"FAILED":"PASSED");
-
- if (!performance_times)
- exit(0);
-
- printf( "\n%d: DAPL Test Complete.\n\n",getpid());
- printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
- " usec, pc=%d\n",
- getpid(), time.rtt, burst, time.rtt/burst, poll_count );
- printf( "%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
- " usec, pc=%d\n",
- getpid(), time.rdma_wr, burst,
- time.rdma_wr/burst, rdma_wr_poll_count );
- for(i=0;i<MAX_RDMA_RD;i++) {
- printf( "%d: RDMA read: Total=%10.2lf usec, %d bursts, "
- "itime=%10.2lf usec, pc=%d\n",
- getpid(),time.rdma_rd_total,MAX_RDMA_RD,
- time.rdma_rd[i],rdma_rd_poll_count[i] );
- }
- printf( "%d: open: %10.2lf usec\n", getpid(), time.open );
- printf( "%d: close: %10.2lf usec\n", getpid(), time.close );
- printf( "%d: PZ create: %10.2lf usec\n", getpid(), time.pzc );
- printf( "%d: PZ free: %10.2lf usec\n", getpid(), time.pzf );
- printf( "%d: LMR create:%10.2lf usec\n", getpid(), time.reg );
- printf( "%d: LMR free: %10.2lf usec\n", getpid(), time.unreg );
- printf( "%d: EVD create:%10.2lf usec\n", getpid(), time.evdc );
- printf( "%d: EVD free: %10.2lf usec\n", getpid(), time.evdf );
- if (use_cno) {
- printf( "%d: CNO create: %10.2lf usec\n", getpid(), time.cnoc );
- printf( "%d: CNO free: %10.2lf usec\n", getpid(), time.cnof );
- }
- printf( "%d: EP create: %10.2lf usec\n",getpid(), time.epc );
- printf( "%d: EP free: %10.2lf usec\n",getpid(), time.epf );
- printf( "%d: TOTAL: %10.2lf usec\n",getpid(), time.total );
+ /* disconnect and free EP resources */
+ if (h_ep != DAT_HANDLE_NULL) {
+ /* unregister message buffers and tear down connection */
+ LOGPRINTF("%d Disconnect and Free EP %p \n", getpid(), h_ep);
+ disconnect_ep();
+
+ /* free EP */
+ LOGPRINTF("%d Free EP %p \n", getpid(), h_ep);
+ start = get_time();
+ ret = dat_ep_free(h_ep);
+ stop = get_time();
+ time.epf += ((stop - start) * 1.0e6);
+ time.total += time.epf;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing EP: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Freed EP\n", getpid());
+ h_ep = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free EVDs */
+ LOGPRINTF("%d destroy events\n", getpid());
+ ret = destroy_events();
+ if (ret != DAT_SUCCESS)
+ fprintf(stderr, "%d Error destroy_events: %s\n",
+ getpid(), DT_RetToString(ret));
+ else
+ LOGPRINTF("%d destroy events done\n", getpid());
+
+ ret = unregister_rdma_memory();
+ LOGPRINTF("%d unregister_rdma_memory \n", getpid());
+ if (ret != DAT_SUCCESS)
+ fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
+ getpid(), DT_RetToString(ret));
+ else
+ LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
+
+ /* Free protection domain */
+ LOGPRINTF("%d Freeing pz\n", getpid());
+ start = get_time();
+ ret = dat_pz_free(h_pz);
+ stop = get_time();
+ time.pzf += ((stop - start) * 1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing PZ: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Freed pz\n", getpid());
+ h_pz = NULL;
+ }
+
+ /* close the device */
+ LOGPRINTF("%d Closing Interface Adaptor\n", getpid());
+ start = get_time();
+ ret = dat_ia_close(h_ia, DAT_CLOSE_ABRUPT_FLAG);
+ stop = get_time();
+ time.close += ((stop - start) * 1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error Adaptor close: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else
+ LOGPRINTF("%d Closed Interface Adaptor\n", getpid());
+
+ /* free rdma buffers */
+ free(rbuf);
+ free(sbuf);
+
+ printf("\n%d: DAPL Test Complete. %s\n\n",
+ getpid(), failed ? "FAILED" : "PASSED");
+
+ if (!performance_times)
+ exit(0);
+
+ printf("\n%d: DAPL Test Complete.\n\n", getpid());
+ printf("%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
+ " usec, pc=%d\n",
+ getpid(), time.rtt, burst, time.rtt / burst, poll_count);
+ printf("%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
+ " usec, pc=%d\n",
+ getpid(), time.rdma_wr, burst,
+ time.rdma_wr / burst, rdma_wr_poll_count);
+ for (i = 0; i < MAX_RDMA_RD; i++) {
+ printf("%d: RDMA read: Total=%10.2lf usec, %d bursts, "
+ "itime=%10.2lf usec, pc=%d\n",
+ getpid(), time.rdma_rd_total, MAX_RDMA_RD,
+ time.rdma_rd[i], rdma_rd_poll_count[i]);
+ }
+ printf("%d: open: %10.2lf usec\n", getpid(), time.open);
+ printf("%d: close: %10.2lf usec\n", getpid(), time.close);
+ printf("%d: PZ create: %10.2lf usec\n", getpid(), time.pzc);
+ printf("%d: PZ free: %10.2lf usec\n", getpid(), time.pzf);
+ printf("%d: LMR create:%10.2lf usec\n", getpid(), time.reg);
+ printf("%d: LMR free: %10.2lf usec\n", getpid(), time.unreg);
+ printf("%d: EVD create:%10.2lf usec\n", getpid(), time.evdc);
+ printf("%d: EVD free: %10.2lf usec\n", getpid(), time.evdf);
+ if (use_cno) {
+ printf("%d: CNO create: %10.2lf usec\n", getpid(), time.cnoc);
+ printf("%d: CNO free: %10.2lf usec\n", getpid(), time.cnof);
+ }
+ printf("%d: EP create: %10.2lf usec\n", getpid(), time.epc);
+ printf("%d: EP free: %10.2lf usec\n", getpid(), time.epf);
+ printf("%d: TOTAL: %10.2lf usec\n", getpid(), time.total);
#if defined(_WIN32) || defined(_WIN64)
- WSACleanup();
+ WSACleanup();
#endif
- return(0);
+ return (0);
}
#if defined(_WIN32) || defined(_WIN64)
-void gettimeofday( struct timeval *t, char *jnk)
+void gettimeofday(struct timeval *t, char *jnk)
{
SYSTEMTIME now;
GetLocalTime(&now);
double get_time(void)
{
- struct timeval tp;
+ struct timeval tp;
- gettimeofday(&tp, NULL);
- return ((double) tp.tv_sec + (double) tp.tv_usec * 1e-6);
+ gettimeofday(&tp, NULL);
+ return ((double)tp.tv_sec + (double)tp.tv_usec * 1e-6);
}
void init_data(void)
{
- memset(rbuf, 'a', buf_len);
- memset(sbuf, 'b', buf_len);
+ memset(rbuf, 'a', buf_len);
+ memset(sbuf, 'b', buf_len);
}
-
DAT_RETURN
-send_msg( void *data,
- DAT_COUNT size,
- DAT_LMR_CONTEXT context,
- DAT_DTO_COOKIE cookie,
- DAT_COMPLETION_FLAGS flags )
+send_msg(void *data,
+ DAT_COUNT size,
+ DAT_LMR_CONTEXT context,
+ DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags)
{
- DAT_LMR_TRIPLET iov;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_RETURN ret;
+ DAT_LMR_TRIPLET iov;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_RETURN ret;
- iov.lmr_context = context;
+ iov.lmr_context = context;
#if defined(_WIN32)
- iov.virtual_address = (DAT_VADDR)data;
+ iov.virtual_address = (DAT_VADDR) data;
#else
- iov.virtual_address = (DAT_VADDR)(unsigned long)data;
+ iov.virtual_address = (DAT_VADDR) (unsigned long)data;
#endif
- iov.segment_length = size;
-
- LOGPRINTF("%d calling post_send\n", getpid());
- cookie.as_64 = 0xaaaa;
- ret = dat_ep_post_send( h_ep,
- 1,
- &iov,
- cookie,
- flags );
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
- getpid(),DT_RetToString(ret));
- return ret;
- }
-
- if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
- if ( polling ) {
- printf("%d Polling post send completion...\n",getpid());
- while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY) ;
- }
- else {
- LOGPRINTF("%d waiting for post_send completion event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
- LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_req_evd ) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
- getpid());
- return( DAT_ABORT );
- }
- }
- /* use wait to dequeue */
- ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(),DT_RetToString(ret));
- return ret;
- }
- }
-
- /* validate event number, len, cookie, and status */
- if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
- fprintf(stderr, "%d: ERROR: DTO event number %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
-
- if ((event.event_data.dto_completion_event_data.transfered_length != size ) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0xaaaa )) {
- fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x" \n",
- getpid(),
- event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64 );
- return( DAT_ABORT );
-
- }
- if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO event status %s\n",
- getpid(),DT_RetToString(ret));
- return( DAT_ABORT );
- }
- }
-
- return DAT_SUCCESS;
+ iov.segment_length = size;
+
+ LOGPRINTF("%d calling post_send\n", getpid());
+ cookie.as_64 = 0xaaaa;
+ ret = dat_ep_post_send(h_ep, 1, &iov, cookie, flags);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
+ getpid(), DT_RetToString(ret));
+ return ret;
+ }
+
+ if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
+ if (polling) {
+ printf("%d Polling post send completion...\n",
+ getpid());
+ while (dat_evd_dequeue(h_dto_req_evd, &event) ==
+ DAT_QUEUE_EMPTY) ;
+ } else {
+ LOGPRINTF("%d waiting for post_send completion event\n",
+ getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret =
+ dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(), evd);
+ if (evd != h_dto_req_evd) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+ getpid());
+ return (DAT_ABORT);
+ }
+ }
+ /* use wait to dequeue */
+ ret =
+ dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
+ &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(), DT_RetToString(ret));
+ return ret;
+ }
+ }
+
+ /* validate event number, len, cookie, and status */
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+
+ if ((event.event_data.dto_completion_event_data.
+ transfered_length != size)
+ || (event.event_data.dto_completion_event_data.user_cookie.
+ as_64 != 0xaaaa)) {
+ fprintf(stderr,
+ "%d: ERROR: DTO len %d or cookie " F64x " \n",
+ getpid(),
+ event.event_data.dto_completion_event_data.
+ transfered_length,
+ event.event_data.dto_completion_event_data.
+ user_cookie.as_64);
+ return (DAT_ABORT);
+
+ }
+ if (event.event_data.dto_completion_event_data.status !=
+ DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+ getpid(), DT_RetToString(ret));
+ return (DAT_ABORT);
+ }
+ }
+
+ return DAT_SUCCESS;
}
-DAT_RETURN
-connect_ep( char *hostname, DAT_CONN_QUAL conn_id )
+DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
{
- DAT_SOCK_ADDR remote_addr;
- DAT_RETURN ret;
- DAT_REGION_DESCRIPTION region;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov;
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- int i;
- unsigned char *buf;
- DAT_CR_PARAM cr_param = { 0 };
- unsigned char pdata[48] = { 0 };
-
- /* Register send message buffer */
- LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",
- getpid(), &rmr_send_msg, (int)sizeof(DAT_RMR_TRIPLET) );
- region.for_va = &rmr_send_msg;
- ret = dat_lmr_create( h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- sizeof(DAT_RMR_TRIPLET),
- h_pz,
- DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_send_msg,
- &lmr_context_send_msg,
- &rmr_context_send_msg,
- ®istered_size_send_msg,
- ®istered_addr_send_msg );
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering send msg buffer: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d Registered send Message Buffer %p \n",
- getpid(),region.for_va );
-
- /* Register Receive buffers */
- LOGPRINTF("%d Registering Receive Message Buffer %p\n",
- getpid(), rmr_recv_msg );
- region.for_va = rmr_recv_msg;
- ret = dat_lmr_create( h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- sizeof(DAT_RMR_TRIPLET)*MSG_BUF_COUNT,
- h_pz,
- DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_recv_msg,
- &lmr_context_recv_msg,
- &rmr_context_recv_msg,
- ®istered_size_recv_msg,
- ®istered_addr_recv_msg );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d Registered Receive Message Buffer %p\n",
- getpid(),region.for_va);
-
- for ( i = 0; i < MSG_BUF_COUNT; i++ ) {
- cookie.as_64 = i;
- l_iov.lmr_context = lmr_context_recv_msg;
+ DAT_SOCK_ADDR remote_addr;
+ DAT_RETURN ret;
+ DAT_REGION_DESCRIPTION region;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ int i;
+ unsigned char *buf;
+ DAT_CR_PARAM cr_param = { 0 };
+ unsigned char pdata[48] = { 0 };
+
+ /* Register send message buffer */
+ LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",
+ getpid(), &rmr_send_msg, (int)sizeof(DAT_RMR_TRIPLET));
+ region.for_va = &rmr_send_msg;
+ ret = dat_lmr_create(h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ sizeof(DAT_RMR_TRIPLET),
+ h_pz,
+ DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_send_msg,
+ &lmr_context_send_msg,
+ &rmr_context_send_msg,
+ ®istered_size_send_msg,
+ ®istered_addr_send_msg);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering send msg buffer: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d Registered send Message Buffer %p \n",
+ getpid(), region.for_va);
+
+ /* Register Receive buffers */
+ LOGPRINTF("%d Registering Receive Message Buffer %p\n",
+ getpid(), rmr_recv_msg);
+ region.for_va = rmr_recv_msg;
+ ret = dat_lmr_create(h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ sizeof(DAT_RMR_TRIPLET) * MSG_BUF_COUNT,
+ h_pz,
+ DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_recv_msg,
+ &lmr_context_recv_msg,
+ &rmr_context_recv_msg,
+ ®istered_size_recv_msg,
+ ®istered_addr_recv_msg);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d Registered Receive Message Buffer %p\n",
+ getpid(), region.for_va);
+
+ for (i = 0; i < MSG_BUF_COUNT; i++) {
+ cookie.as_64 = i;
+ l_iov.lmr_context = lmr_context_recv_msg;
#if defined(_WIN32)
- l_iov.virtual_address = (DAT_VADDR)&rmr_recv_msg[ i ];
+ l_iov.virtual_address = (DAT_VADDR) & rmr_recv_msg[i];
#else
- l_iov.virtual_address = (DAT_VADDR)(unsigned long)&rmr_recv_msg[ i ];
+ l_iov.virtual_address =
+ (DAT_VADDR) (unsigned long)&rmr_recv_msg[i];
#endif
- l_iov.segment_length = sizeof(DAT_RMR_TRIPLET);
-
- LOGPRINTF("%d Posting Receive Message Buffer %p\n",
- getpid(), &rmr_recv_msg[ i ]);
- ret = dat_ep_post_recv( h_ep,
- 1,
- &l_iov,
- cookie,
- DAT_COMPLETION_DEFAULT_FLAG );
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d Registered Receive Message Buffer %p\n",
- getpid(),region.for_va);
-
- }
-
- /* setup receive rdma buffer to initial string to be overwritten */
- strcpy( (char*)rbuf, "blah, blah, blah\n" );
-
- if ( server ) { /* SERVER */
-
- /* create the service point for server listen */
- LOGPRINTF("%d Creating service point for listen\n",getpid());
- ret = dat_psp_create( h_ia,
- conn_id,
- h_cr_evd,
- DAT_PSP_CONSUMER_FLAG,
- &h_psp );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_psp_create: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d dat_psp_created for server listen\n", getpid());
-
- printf("%d Server waiting for connect request on port "F64x"\n",
- getpid(), conn_id);
-
- ret = dat_evd_wait( h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d dat_evd_wait for cr_evd completed\n", getpid());
-
- if ( event.event_number != DAT_CONNECTION_REQUEST_EVENT ) {
- fprintf(stderr, "%d Error unexpected cr event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
- if ( (event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
- (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != h_psp) ) {
- fprintf(stderr, "%d Error wrong cr event data : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
-
- /* use to test rdma_cma timeout logic */
+ l_iov.segment_length = sizeof(DAT_RMR_TRIPLET);
+
+ LOGPRINTF("%d Posting Receive Message Buffer %p\n",
+ getpid(), &rmr_recv_msg[i]);
+ ret = dat_ep_post_recv(h_ep,
+ 1,
+ &l_iov,
+ cookie, DAT_COMPLETION_DEFAULT_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error registering recv msg buffer: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d Registered Receive Message Buffer %p\n",
+ getpid(), region.for_va);
+
+ }
+
+ /* setup receive rdma buffer to initial string to be overwritten */
+ strcpy((char *)rbuf, "blah, blah, blah\n");
+
+ if (server) { /* SERVER */
+
+ /* create the service point for server listen */
+ LOGPRINTF("%d Creating service point for listen\n", getpid());
+ ret = dat_psp_create(h_ia,
+ conn_id,
+ h_cr_evd, DAT_PSP_CONSUMER_FLAG, &h_psp);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_psp_create: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d dat_psp_created for server listen\n",
+ getpid());
+
+ printf("%d Server waiting for connect request on port " F64x
+ "\n", getpid(), conn_id);
+
+ ret = dat_evd_wait(h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d dat_evd_wait for cr_evd completed\n",
+ getpid());
+
+ if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
+ fprintf(stderr, "%d Error unexpected cr event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+ if ((event.event_data.cr_arrival_event_data.conn_qual !=
+ SERVER_CONN_QUAL)
+ || (event.event_data.cr_arrival_event_data.sp_handle.
+ psp_handle != h_psp)) {
+ fprintf(stderr, "%d Error wrong cr event data : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+
+ /* use to test rdma_cma timeout logic */
#if defined(_WIN32) || defined(_WIN64)
- if (delay) Sleep(delay*1000);
+ if (delay)
+ Sleep(delay * 1000);
#else
- if (delay) sleep(delay);
+ if (delay)
+ sleep(delay);
#endif
- /* accept connect request from client */
- h_cr = event.event_data.cr_arrival_event_data.cr_handle;
- LOGPRINTF("%d Accepting connect request from client\n",getpid());
-
- /* private data - check and send it back */
- dat_cr_query( h_cr, DAT_CSP_FIELD_ALL, &cr_param);
-
- buf = (unsigned char*)cr_param.private_data;
- LOGPRINTF("%d CONN REQUEST Private Data %p[0]=%d [47]=%d\n",
- getpid(),buf,buf[0],buf[47]);
- for (i=0;i<48;i++) {
- if (buf[i] != i+1) {
- fprintf(stderr, "%d Error with CONNECT REQUEST"
- " private data: %p[%d]=%d s/be %d\n",
- getpid(), buf, i, buf[i], i+1);
- dat_cr_reject(h_cr, 0, NULL);
- return(DAT_ABORT);
- }
- buf[i]++; /* change for trip back */
- }
+ /* accept connect request from client */
+ h_cr = event.event_data.cr_arrival_event_data.cr_handle;
+ LOGPRINTF("%d Accepting connect request from client\n",
+ getpid());
+
+ /* private data - check and send it back */
+ dat_cr_query(h_cr, DAT_CSP_FIELD_ALL, &cr_param);
+
+ buf = (unsigned char *)cr_param.private_data;
+ LOGPRINTF("%d CONN REQUEST Private Data %p[0]=%d [47]=%d\n",
+ getpid(), buf, buf[0], buf[47]);
+ for (i = 0; i < 48; i++) {
+ if (buf[i] != i + 1) {
+ fprintf(stderr, "%d Error with CONNECT REQUEST"
+ " private data: %p[%d]=%d s/be %d\n",
+ getpid(), buf, i, buf[i], i + 1);
+ dat_cr_reject(h_cr, 0, NULL);
+ return (DAT_ABORT);
+ }
+ buf[i]++; /* change for trip back */
+ }
#ifdef TEST_REJECT_WITH_PRIVATE_DATA
- printf("%d REJECT request with 48 bytes of private data\n", getpid());
- ret = dat_cr_reject(h_cr, 48, cr_param.private_data);
- printf("\n%d: DAPL Test Complete. %s\n\n",
- getpid(), ret?"FAILED":"PASSED");
- exit(0);
+ printf("%d REJECT request with 48 bytes of private data\n",
+ getpid());
+ ret = dat_cr_reject(h_cr, 48, cr_param.private_data);
+ printf("\n%d: DAPL Test Complete. %s\n\n",
+ getpid(), ret ? "FAILED" : "PASSED");
+ exit(0);
#endif
- ret = dat_cr_accept(h_cr, h_ep, 48, cr_param.private_data);
+ ret = dat_cr_accept(h_cr, h_ep, 48, cr_param.private_data);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_cr_accept: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d dat_cr_accept completed\n", getpid());
- }
- else { /* CLIENT */
- struct addrinfo *target;
- int rval;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_cr_accept: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d dat_cr_accept completed\n", getpid());
+ } else { /* CLIENT */
+ struct addrinfo *target;
+ int rval;
#if defined(_WIN32) || defined(_WIN64)
- if ((rval=getaddrinfo (hostname, "ftp", NULL, &target)) != 0) {
- printf("\n remote name resolution failed! %s\n",gai_strerror(rval));
- exit ( 1 );
- }
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+ if ((rval = getaddrinfo(hostname, "ftp", NULL, &target)) != 0) {
+ printf("\n remote name resolution failed! %s\n",
+ gai_strerror(rval));
+ exit(1);
+ }
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
#else
- if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
- perror("\n remote name resolution failed!");
- exit ( 1 );
- }
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+ if (getaddrinfo(hostname, NULL, NULL, &target) != 0) {
+ perror("\n remote name resolution failed!");
+ exit(1);
+ }
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
#endif
- printf ("%d Server Name: %s \n", getpid(), hostname);
- printf ("%d Server Net Address: %d.%d.%d.%d port "F64x"\n", getpid(),
- (rval >> 0) & 0xff, (rval >> 8) & 0xff,
- (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);
-
- remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
- freeaddrinfo(target);
-
- for (i=0;i<48;i++) /* simple pattern in private data */
- pdata[i]=i+1;
-
- LOGPRINTF("%d Connecting to server\n",getpid());
- ret = dat_ep_connect( h_ep,
- &remote_addr,
- conn_id,
- CONN_TIMEOUT,
- 48,
- (DAT_PVOID)pdata,
- 0,
- DAT_CONNECT_DEFAULT_FLAG );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_connect: %s\n",
- getpid(), DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d dat_ep_connect completed\n", getpid());
- }
-
- printf("%d Waiting for connect response\n",getpid());
-
- ret = dat_evd_wait( h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
+ printf("%d Server Name: %s \n", getpid(), hostname);
+ printf("%d Server Net Address: %d.%d.%d.%d port " F64x "\n",
+ getpid(), (rval >> 0) & 0xff, (rval >> 8) & 0xff,
+ (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);
+
+ remote_addr = *((DAT_IA_ADDRESS_PTR) target->ai_addr);
+ freeaddrinfo(target);
+
+ for (i = 0; i < 48; i++) /* simple pattern in private data */
+ pdata[i] = i + 1;
+
+ LOGPRINTF("%d Connecting to server\n", getpid());
+ ret = dat_ep_connect(h_ep,
+ &remote_addr,
+ conn_id,
+ CONN_TIMEOUT,
+ 48,
+ (DAT_PVOID) pdata,
+ 0, DAT_CONNECT_DEFAULT_FLAG);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_connect: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d dat_ep_connect completed\n", getpid());
+ }
+
+ printf("%d Waiting for connect response\n", getpid());
+
+ ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
+ getpid());
#ifdef TEST_REJECT_WITH_PRIVATE_DATA
- if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
- fprintf(stderr, "%d expected conn reject event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
- /* get the reject private data and validate */
- buf = (unsigned char*)event.event_data.connect_event_data.private_data;
- printf("%d Received REJECT with private data %p[0]=%d [47]=%d\n",
- getpid(),buf,buf[0],buf[47]);
- for (i=0;i<48;i++) {
- if (buf[i] != i+2) {
- fprintf(stderr, "%d client: Error with REJECT event"
- " private data: %p[%d]=%d s/be %d\n",
- getpid(), buf, i, buf[i], i+2);
- dat_ep_disconnect( h_ep, DAT_CLOSE_ABRUPT_FLAG);
- return(DAT_ABORT);
- }
- }
- printf("\n%d: DAPL Test Complete. PASSED\n\n", getpid());
- exit(0);
+ if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
+ fprintf(stderr, "%d expected conn reject event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+ /* get the reject private data and validate */
+ buf = (unsigned char *)event.event_data.connect_event_data.private_data;
+ printf("%d Received REJECT with private data %p[0]=%d [47]=%d\n",
+ getpid(), buf, buf[0], buf[47]);
+ for (i = 0; i < 48; i++) {
+ if (buf[i] != i + 2) {
+ fprintf(stderr, "%d client: Error with REJECT event"
+ " private data: %p[%d]=%d s/be %d\n",
+ getpid(), buf, i, buf[i], i + 2);
+ dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
+ return (DAT_ABORT);
+ }
+ }
+ printf("\n%d: DAPL Test Complete. PASSED\n\n", getpid());
+ exit(0);
#endif
- if ( event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED ) {
- fprintf(stderr, "%d Error unexpected conn event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
-
- /* check private data back from server */
- if (!server) {
- buf = (unsigned char*)event.event_data.connect_event_data.private_data;
- LOGPRINTF("%d CONN Private Data %p[0]=%d [47]=%d\n",
- getpid(),buf,buf[0],buf[47]);
- for (i=0;i<48;i++) {
- if (buf[i] != i+2) {
- fprintf(stderr, "%d Error with CONNECT event"
- " private data: %p[%d]=%d s/be %d\n",
- getpid(), buf, i, buf[i], i+2);
- dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
- LOGPRINTF("%d waiting for disconnect event...\n", getpid());
- dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
- return(DAT_ABORT);
- }
- }
- }
-
- printf("\n%d CONNECTED!\n\n",getpid());
- connected = 1;
+ if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
+ fprintf(stderr, "%d Error unexpected conn event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+
+ /* check private data back from server */
+ if (!server) {
+ buf =
+ (unsigned char *)event.event_data.connect_event_data.
+ private_data;
+ LOGPRINTF("%d CONN Private Data %p[0]=%d [47]=%d\n", getpid(),
+ buf, buf[0], buf[47]);
+ for (i = 0; i < 48; i++) {
+ if (buf[i] != i + 2) {
+ fprintf(stderr, "%d Error with CONNECT event"
+ " private data: %p[%d]=%d s/be %d\n",
+ getpid(), buf, i, buf[i], i + 2);
+ dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
+ LOGPRINTF
+ ("%d waiting for disconnect event...\n",
+ getpid());
+ dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE,
+ 1, &event, &nmore);
+ return (DAT_ABORT);
+ }
+ }
+ }
+
+ printf("\n%d CONNECTED!\n\n", getpid());
+ connected = 1;
#if CONNECT_ONLY
- return 0;
+ return 0;
#endif
- /*
- * Setup our remote memory and tell the other side about it
- */
- rmr_send_msg.virtual_address = htonll((DAT_VADDR)(uintptr_t)rbuf);
- rmr_send_msg.segment_length = htonl(RDMA_BUFFER_SIZE);
- rmr_send_msg.rmr_context = htonl(rmr_context_recv);
-
- printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",
- getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE );
-
- ret = send_msg(&rmr_send_msg,
- sizeof( DAT_RMR_TRIPLET ),
- lmr_context_send_msg,
- cookie,
- DAT_COMPLETION_SUPPRESS_FLAG);
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else
- LOGPRINTF("%d send_msg completed\n", getpid());
-
- /*
- * Wait for remote RMR information for RDMA
- */
- if ( polling ) {
- printf("%d Polling for remote to send RMR data\n",getpid());
- while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
- }
- else {
- printf("%d Waiting for remote to send RMR data\n",getpid());
- if (use_cno)
- {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
- LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_rcv_evd ) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
- getpid());
- return( DAT_ABORT );
- }
- }
- /* use wait to dequeue */
- ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error waiting on h_dto_rcv_evd: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else {
- LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());
- }
- }
-
- printf("%d remote RMR data arrived!\n",getpid());
-
- if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
- if ((event.event_data.dto_completion_event_data.transfered_length !=
- sizeof( DAT_RMR_TRIPLET )) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
- recv_msg_index) ) {
- fprintf(stderr,"ERR recv event: len=%d cookie="F64x" expected %d/%d\n",
- (int)event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64,
- (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
- return( DAT_ABORT );
- }
-
- /* swap received RMR msg: network order to host order */
- r_iov = rmr_recv_msg[recv_msg_index];
- rmr_recv_msg[recv_msg_index].rmr_context =
- ntohl(r_iov.rmr_context);
- rmr_recv_msg[recv_msg_index].virtual_address =
+ /*
+ * Setup our remote memory and tell the other side about it
+ */
+ rmr_send_msg.virtual_address = htonll((DAT_VADDR) (uintptr_t) rbuf);
+ rmr_send_msg.segment_length = htonl(RDMA_BUFFER_SIZE);
+ rmr_send_msg.rmr_context = htonl(rmr_context_recv);
+
+ printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",
+ getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE);
+
+ ret = send_msg(&rmr_send_msg,
+ sizeof(DAT_RMR_TRIPLET),
+ lmr_context_send_msg,
+ cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else
+ LOGPRINTF("%d send_msg completed\n", getpid());
+
+ /*
+ * Wait for remote RMR information for RDMA
+ */
+ if (polling) {
+ printf("%d Polling for remote to send RMR data\n", getpid());
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+ DAT_QUEUE_EMPTY) ;
+ } else {
+ printf("%d Waiting for remote to send RMR data\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(), evd);
+ if (evd != h_dto_rcv_evd) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
+ return (DAT_ABORT);
+ }
+ }
+ /* use wait to dequeue */
+ ret =
+ dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_rcv_evd: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n",
+ getpid());
+ }
+ }
+
+ printf("%d remote RMR data arrived!\n", getpid());
+
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+ if ((event.event_data.dto_completion_event_data.transfered_length !=
+ sizeof(DAT_RMR_TRIPLET)) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+ recv_msg_index)) {
+ fprintf(stderr,
+ "ERR recv event: len=%d cookie=" F64x
+ " expected %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.
+ transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.
+ as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
+ return (DAT_ABORT);
+ }
+
+ /* swap received RMR msg: network order to host order */
+ r_iov = rmr_recv_msg[recv_msg_index];
+ rmr_recv_msg[recv_msg_index].rmr_context = ntohl(r_iov.rmr_context);
+ rmr_recv_msg[recv_msg_index].virtual_address =
ntohll(r_iov.virtual_address);
- rmr_recv_msg[recv_msg_index].segment_length =
+ rmr_recv_msg[recv_msg_index].segment_length =
ntohl(r_iov.segment_length);
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
- getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
- rmr_recv_msg[recv_msg_index].virtual_address,
- rmr_recv_msg[recv_msg_index].segment_length );
+ printf("%d Received RMR from remote: "
+ "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+ rmr_recv_msg[recv_msg_index].virtual_address,
+ rmr_recv_msg[recv_msg_index].segment_length);
- recv_msg_index++;
+ recv_msg_index++;
- return ( DAT_SUCCESS );
+ return (DAT_SUCCESS);
}
-void
-disconnect_ep( void )
+void disconnect_ep(void)
{
- DAT_RETURN ret;
- DAT_EVENT event;
- DAT_COUNT nmore;
-
- if (connected) {
-
- /*
- * Only the client needs to call disconnect. The server _should_ be able
- * to just wait on the EVD associated with connection events for a
- * disconnect request and then exit.
- */
- if ( !server ) {
- LOGPRINTF("%d dat_ep_disconnect\n", getpid());
- ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_disconnect: %s\n",
- getpid(),DT_RetToString(ret));
- }
- else {
- LOGPRINTF("%d dat_ep_disconnect completed\n", getpid());
- }
- }
- else {
- LOGPRINTF("%d Server waiting for disconnect...\n", getpid());
- }
-
- ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(),DT_RetToString(ret));
- }
- else {
- LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
- }
- }
-
- /* destroy service point */
- if (( server ) && ( h_psp != DAT_HANDLE_NULL )) {
- ret = dat_psp_free( h_psp );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_psp_free: %s\n",
- getpid(),DT_RetToString(ret));
- }
- else {
- LOGPRINTF("%d dat_psp_free completed\n", getpid());
- }
- }
-
- /* Unregister Send message Buffer */
- if ( h_lmr_send_msg != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Unregister send message h_lmr %p \n",getpid(),h_lmr_send_msg);
- ret = dat_lmr_free(h_lmr_send_msg);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error deregistering send msg mr: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Unregistered send message Buffer\n",getpid());
- h_lmr_send_msg = NULL;
- }
- }
-
- /* Unregister recv message Buffer */
- if ( h_lmr_recv_msg != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Unregister recv message h_lmr %p \n",getpid(),h_lmr_recv_msg);
- ret = dat_lmr_free(h_lmr_recv_msg);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error deregistering recv msg mr: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Unregistered recv message Buffer\n",getpid());
- h_lmr_recv_msg = NULL;
- }
- }
- return;
-}
+ DAT_RETURN ret;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+
+ if (connected) {
+
+ /*
+ * Only the client needs to call disconnect. The server _should_ be able
+ * to just wait on the EVD associated with connection events for a
+ * disconnect request and then exit.
+ */
+ if (!server) {
+ LOGPRINTF("%d dat_ep_disconnect\n", getpid());
+ ret = dat_ep_disconnect(h_ep, DAT_CLOSE_DEFAULT);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error dat_ep_disconnect: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d dat_ep_disconnect completed\n",
+ getpid());
+ }
+ } else {
+ LOGPRINTF("%d Server waiting for disconnect...\n",
+ getpid());
+ }
+ ret =
+ dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event,
+ &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
+ getpid());
+ }
+ }
-DAT_RETURN
-do_rdma_write_with_msg( void )
-{
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- DAT_RETURN ret;
- int i;
-
- printf("\n %d RDMA WRITE DATA with SEND MSG\n\n",getpid());
-
- cookie.as_64 = 0x5555;
-
- if ( recv_msg_index >= MSG_BUF_COUNT )
- return( DAT_ABORT );
-
- /* get RMR information from previously received message */
- r_iov = rmr_recv_msg[ recv_msg_index-1 ];
-
- if ( server )
- strcpy( (char*)sbuf, "server RDMA write data..." );
- else
- strcpy( (char*)sbuf, "client RDMA write data..." );
-
- for (i=0;i<MSG_IOV_COUNT;i++) {
- l_iov[i].lmr_context = lmr_context_send;
- l_iov[i].segment_length = buf_len/MSG_IOV_COUNT;
- l_iov[i].virtual_address = (DAT_VADDR)(uintptr_t)
- (&sbuf[l_iov[i].segment_length*i]);
-
- LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n",
- getpid(), i, &sbuf[l_iov[i].segment_length*i],
- l_iov[i].segment_length);
- }
-
- start = get_time();
- for (i=0;i<burst;i++) {
- cookie.as_64 = 0x9999;
- ret = dat_ep_post_rdma_write( h_ep, // ep_handle
- MSG_IOV_COUNT, // num_segments
- l_iov, // LMR
- cookie, // user_cookie
- &r_iov, // RMR
- DAT_COMPLETION_SUPPRESS_FLAG );
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_write() %s\n",
- getpid(),DT_RetToString(ret));
- return( DAT_ABORT );
- }
- LOGPRINTF("%d rdma_write # %d completed\n", getpid(),i+1);
- }
-
- /*
- * Send RMR information a 2nd time to indicate completion
- * NOTE: already swapped to network order in connect_ep
- */
- printf("%d Sending RDMA WRITE completion message\n",getpid());
-
- ret = send_msg( &rmr_send_msg,
- sizeof( DAT_RMR_TRIPLET ),
- lmr_context_send_msg,
- cookie,
- DAT_COMPLETION_SUPPRESS_FLAG );
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- } else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
-
- /*
- * Collect first event, write completion or the inbound recv
- */
- if (polling) {
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_QUEUE_EMPTY)
- rdma_wr_poll_count++;
- }
- else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(),evd);
- if ( evd != h_dto_rcv_evd ) {
- fprintf(stderr, "%d Error waiting on h_dto_cno: "
- "evd != h_dto_rcv_evd\n",
- getpid());
- return( ret );
- }
- }
- /* use wait to dequeue */
- ret = dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(),DT_RetToString(ret));
- return( ret );
- }
- }
- stop = get_time();
- time.rdma_wr = ((stop - start)*1.0e6);
-
- /* validate event number and status */
- printf("%d inbound rdma_write; send message arrived!\n",getpid());
- if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
-
- if ( (event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET )) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index) ) {
- fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",
- (int)event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64,
- (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
-
- return( DAT_ABORT );
- }
-
- /* swap received RMR msg: network order to host order */
- r_iov = rmr_recv_msg[recv_msg_index];
- rmr_recv_msg[recv_msg_index].virtual_address =
- ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
- rmr_recv_msg[recv_msg_index].segment_length =
- ntohl(rmr_recv_msg[recv_msg_index].segment_length);
- rmr_recv_msg[recv_msg_index].rmr_context =
- ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
-
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
- getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
- rmr_recv_msg[recv_msg_index].virtual_address,
- rmr_recv_msg[recv_msg_index].segment_length);
-
- LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
- getpid());
-
- printf("%d %s RDMA write buffer contains: %s\n",
- getpid(),
- server ? "SERVER:" : "CLIENT:",
- rbuf );
-
- recv_msg_index++;
-
- return ( DAT_SUCCESS );
+ /* destroy service point */
+ if ((server) && (h_psp != DAT_HANDLE_NULL)) {
+ ret = dat_psp_free(h_psp);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_psp_free: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d dat_psp_free completed\n", getpid());
+ }
+ }
+
+ /* Unregister Send message Buffer */
+ if (h_lmr_send_msg != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Unregister send message h_lmr %p \n", getpid(),
+ h_lmr_send_msg);
+ ret = dat_lmr_free(h_lmr_send_msg);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error deregistering send msg mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Unregistered send message Buffer\n",
+ getpid());
+ h_lmr_send_msg = NULL;
+ }
+ }
+
+ /* Unregister recv message Buffer */
+ if (h_lmr_recv_msg != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Unregister recv message h_lmr %p \n", getpid(),
+ h_lmr_recv_msg);
+ ret = dat_lmr_free(h_lmr_recv_msg);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error deregistering recv msg mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Unregistered recv message Buffer\n",
+ getpid());
+ h_lmr_recv_msg = NULL;
+ }
+ }
+ return;
}
-DAT_RETURN
-do_rdma_read_with_msg( void )
+DAT_RETURN do_rdma_write_with_msg(void)
{
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov;
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- DAT_RETURN ret;
- int i;
-
- printf("\n %d RDMA READ DATA with SEND MSG\n\n",getpid());
-
- if (recv_msg_index >= MSG_BUF_COUNT)
- return( DAT_ABORT );
-
- /* get RMR information from previously received message */
- r_iov = rmr_recv_msg[recv_msg_index-1];
-
- /* setup rdma read buffer to initial string to be overwritten */
- strcpy((char*)sbuf, "blah, blah, blah\n");
-
- if (server)
- strcpy((char*)rbuf, "server RDMA read data...");
- else
- strcpy((char*)rbuf, "client RDMA read data...");
-
- l_iov.lmr_context = lmr_context_send;
- l_iov.virtual_address = (DAT_VADDR)(uintptr_t)sbuf;
- l_iov.segment_length = buf_len;
-
- for (i=0;i<MAX_RDMA_RD;i++) {
- cookie.as_64 = 0x9999;
- start = get_time();
- ret = dat_ep_post_rdma_read(h_ep, // ep_handle
- 1, // num_segments
- &l_iov, // LMR
- cookie, // user_cookie
- &r_iov, // RMR
- DAT_COMPLETION_DEFAULT_FLAG );
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n",
- getpid(),DT_RetToString(ret));
- return(DAT_ABORT);
- }
-
- if (polling) {
- while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY)
- rdma_rd_poll_count[i]++;
- }
- else {
- LOGPRINTF("%d waiting for rdma_read completion event\n", getpid());
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN ret;
+ int i;
+
+ printf("\n %d RDMA WRITE DATA with SEND MSG\n\n", getpid());
+
+ cookie.as_64 = 0x5555;
+
+ if (recv_msg_index >= MSG_BUF_COUNT)
+ return (DAT_ABORT);
+
+ /* get RMR information from previously received message */
+ r_iov = rmr_recv_msg[recv_msg_index - 1];
+
+ if (server)
+ strcpy((char *)sbuf, "server RDMA write data...");
+ else
+ strcpy((char *)sbuf, "client RDMA write data...");
+
+ for (i = 0; i < MSG_IOV_COUNT; i++) {
+ l_iov[i].lmr_context = lmr_context_send;
+ l_iov[i].segment_length = buf_len / MSG_IOV_COUNT;
+ l_iov[i].virtual_address = (DAT_VADDR) (uintptr_t)
+ (&sbuf[l_iov[i].segment_length * i]);
+
+ LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n",
+ getpid(), i, &sbuf[l_iov[i].segment_length * i],
+ l_iov[i].segment_length);
+ }
+
+ start = get_time();
+ for (i = 0; i < burst; i++) {
+ cookie.as_64 = 0x9999;
+ ret = dat_ep_post_rdma_write(h_ep, // ep_handle
+ MSG_IOV_COUNT, // num_segments
+ l_iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ DAT_COMPLETION_SUPPRESS_FLAG);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d: ERROR: dat_ep_post_rdma_write() %s\n",
+ getpid(), DT_RetToString(ret));
+ return (DAT_ABORT);
+ }
+ LOGPRINTF("%d rdma_write # %d completed\n", getpid(), i + 1);
+ }
+
+ /*
+ * Send RMR information a 2nd time to indicate completion
+ * NOTE: already swapped to network order in connect_ep
+ */
+ printf("%d Sending RDMA WRITE completion message\n", getpid());
+
+ ret = send_msg(&rmr_send_msg,
+ sizeof(DAT_RMR_TRIPLET),
+ lmr_context_send_msg,
+ cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+
+ /*
+ * Collect first event, write completion or the inbound recv
+ */
+ if (polling) {
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+ DAT_QUEUE_EMPTY)
+ rdma_wr_poll_count++;
+ } else {
+ LOGPRINTF("%d waiting for message receive event\n", getpid());
if (use_cno) {
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
- LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_req_evd ) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
- getpid());
- return( DAT_ABORT );
+ ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(), evd);
+ if (evd != h_dto_rcv_evd) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: "
+ "evd != h_dto_rcv_evd\n", getpid());
+ return (ret);
}
}
/* use wait to dequeue */
- ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ ret =
+ dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(),DT_RetToString(ret));
- return ret;
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
}
- }
- /* validate event number, len, cookie, and status */
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d: ERROR: DTO event number %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
- if ((event.event_data.dto_completion_event_data.transfered_length != buf_len ) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0x9999 )) {
- fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x"\n",
- getpid(),
- event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64 );
- return( DAT_ABORT );
- }
- if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO event status %s\n",
- getpid(),DT_RetToString(ret));
- return( DAT_ABORT );
- }
- stop = get_time();
- time.rdma_rd[i] = ((stop - start)*1.0e6);
- time.rdma_rd_total += time.rdma_rd[i];
-
- LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
- }
-
- /*
- * Send RMR information a 3rd time to indicate completion
- * NOTE: already swapped to network order in connect_ep
- */
- printf("%d Sending RDMA read completion message\n",getpid());
-
- ret = send_msg( &rmr_send_msg,
- sizeof( DAT_RMR_TRIPLET ),
- lmr_context_send_msg,
- cookie,
- DAT_COMPLETION_SUPPRESS_FLAG );
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- } else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
-
- /*
- * Collect first event, write completion or the inbound recv with immed
- */
- printf("%d Waiting for inbound message....\n",getpid());
- if ( polling ) {
- while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
- }
- else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
- LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_rcv_evd ) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
- getpid());
- return( ret );
- }
- }
- /* use wait to dequeue */
- ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(),DT_RetToString(ret));
- return( ret );
- }
- }
-
- /* validate event number and status */
- printf("%d inbound rdma_read; send message arrived!\n",getpid());
- if (event.event_number != DAT_DTO_COMPLETION_EVENT ) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
-
- if ((event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET)) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index)) {
-
- fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",
- (int)event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64,
- (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
-
- return( DAT_ABORT );
- }
-
- /* swap received RMR msg: network order to host order */
- r_iov = rmr_recv_msg[recv_msg_index];
- rmr_recv_msg[recv_msg_index].virtual_address =
- ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
- rmr_recv_msg[recv_msg_index].segment_length =
- ntohl(rmr_recv_msg[ recv_msg_index].segment_length);
- rmr_recv_msg[recv_msg_index].rmr_context =
- ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
-
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
- getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
- rmr_recv_msg[recv_msg_index].virtual_address,
- rmr_recv_msg[recv_msg_index].segment_length);
-
- LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",getpid());
-
- printf("%d %s RCV RDMA read buffer contains: %s\n",
- getpid(),
- server ? "SERVER:" : "CLIENT:",
- sbuf );
-
- recv_msg_index++;
-
- return ( DAT_SUCCESS );
+ }
+ stop = get_time();
+ time.rdma_wr = ((stop - start) * 1.0e6);
+
+ /* validate event number and status */
+ printf("%d inbound rdma_write; send message arrived!\n", getpid());
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+
+ if ((event.event_data.dto_completion_event_data.transfered_length !=
+ sizeof(DAT_RMR_TRIPLET))
+ || (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+ recv_msg_index)) {
+ fprintf(stderr,
+ "unexpected event data for receive: len=%d cookie=" F64x
+ " exp %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.
+ transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.
+ as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
+
+ return (DAT_ABORT);
+ }
+
+ /* swap received RMR msg: network order to host order */
+ r_iov = rmr_recv_msg[recv_msg_index];
+ rmr_recv_msg[recv_msg_index].virtual_address =
+ ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
+ rmr_recv_msg[recv_msg_index].segment_length =
+ ntohl(rmr_recv_msg[recv_msg_index].segment_length);
+ rmr_recv_msg[recv_msg_index].rmr_context =
+ ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
+
+ printf("%d Received RMR from remote: "
+ "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+ rmr_recv_msg[recv_msg_index].virtual_address,
+ rmr_recv_msg[recv_msg_index].segment_length);
+
+ LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
+ getpid());
+
+ printf("%d %s RDMA write buffer contains: %s\n",
+ getpid(), server ? "SERVER:" : "CLIENT:", rbuf);
+
+ recv_msg_index++;
+
+ return (DAT_SUCCESS);
}
+DAT_RETURN do_rdma_read_with_msg(void)
+{
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN ret;
+ int i;
+
+ printf("\n %d RDMA READ DATA with SEND MSG\n\n", getpid());
+
+ if (recv_msg_index >= MSG_BUF_COUNT)
+ return (DAT_ABORT);
+
+ /* get RMR information from previously received message */
+ r_iov = rmr_recv_msg[recv_msg_index - 1];
+
+ /* setup rdma read buffer to initial string to be overwritten */
+ strcpy((char *)sbuf, "blah, blah, blah\n");
+
+ if (server)
+ strcpy((char *)rbuf, "server RDMA read data...");
+ else
+ strcpy((char *)rbuf, "client RDMA read data...");
+
+ l_iov.lmr_context = lmr_context_send;
+ l_iov.virtual_address = (DAT_VADDR) (uintptr_t) sbuf;
+ l_iov.segment_length = buf_len;
+
+ for (i = 0; i < MAX_RDMA_RD; i++) {
+ cookie.as_64 = 0x9999;
+ start = get_time();
+ ret = dat_ep_post_rdma_read(h_ep, // ep_handle
+ 1, // num_segments
+ &l_iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ DAT_COMPLETION_DEFAULT_FLAG);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d: ERROR: dat_ep_post_rdma_read() %s\n",
+ getpid(), DT_RetToString(ret));
+ return (DAT_ABORT);
+ }
-DAT_RETURN
-do_ping_pong_msg( )
+ if (polling) {
+ while (dat_evd_dequeue(h_dto_req_evd, &event) ==
+ DAT_QUEUE_EMPTY)
+ rdma_rd_poll_count[i]++;
+ } else {
+ LOGPRINTF("%d waiting for rdma_read completion event\n",
+ getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret =
+ dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(), evd);
+ if (evd != h_dto_req_evd) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+ getpid());
+ return (DAT_ABORT);
+ }
+ }
+ /* use wait to dequeue */
+ ret =
+ dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
+ &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(), DT_RetToString(ret));
+ return ret;
+ }
+ }
+ /* validate event number, len, cookie, and status */
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+ if ((event.event_data.dto_completion_event_data.
+ transfered_length != buf_len)
+ || (event.event_data.dto_completion_event_data.user_cookie.
+ as_64 != 0x9999)) {
+ fprintf(stderr,
+ "%d: ERROR: DTO len %d or cookie " F64x "\n",
+ getpid(),
+ event.event_data.dto_completion_event_data.
+ transfered_length,
+ event.event_data.dto_completion_event_data.
+ user_cookie.as_64);
+ return (DAT_ABORT);
+ }
+ if (event.event_data.dto_completion_event_data.status !=
+ DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+ getpid(), DT_RetToString(ret));
+ return (DAT_ABORT);
+ }
+ stop = get_time();
+ time.rdma_rd[i] = ((stop - start) * 1.0e6);
+ time.rdma_rd_total += time.rdma_rd[i];
+
+ LOGPRINTF("%d rdma_read # %d completed\n", getpid(), i + 1);
+ }
+
+ /*
+ * Send RMR information a 3rd time to indicate completion
+ * NOTE: already swapped to network order in connect_ep
+ */
+ printf("%d Sending RDMA read completion message\n", getpid());
+
+ ret = send_msg(&rmr_send_msg,
+ sizeof(DAT_RMR_TRIPLET),
+ lmr_context_send_msg,
+ cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+
+ /*
+ * Collect first event, write completion or the inbound recv with immed
+ */
+ printf("%d Waiting for inbound message....\n", getpid());
+ if (polling) {
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+ DAT_QUEUE_EMPTY) ;
+ } else {
+ LOGPRINTF("%d waiting for message receive event\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(), evd);
+ if (evd != h_dto_rcv_evd) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
+ return (ret);
+ }
+ }
+ /* use wait to dequeue */
+ ret =
+ dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ }
+
+ /* validate event number and status */
+ printf("%d inbound rdma_read; send message arrived!\n", getpid());
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+
+ if ((event.event_data.dto_completion_event_data.transfered_length !=
+ sizeof(DAT_RMR_TRIPLET))
+ || (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+ recv_msg_index)) {
+
+ fprintf(stderr,
+ "unexpected event data for receive: len=%d cookie=" F64x
+ " exp %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.
+ transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.
+ as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
+
+ return (DAT_ABORT);
+ }
+
+ /* swap received RMR msg: network order to host order */
+ r_iov = rmr_recv_msg[recv_msg_index];
+ rmr_recv_msg[recv_msg_index].virtual_address =
+ ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
+ rmr_recv_msg[recv_msg_index].segment_length =
+ ntohl(rmr_recv_msg[recv_msg_index].segment_length);
+ rmr_recv_msg[recv_msg_index].rmr_context =
+ ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
+
+ printf("%d Received RMR from remote: "
+ "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+ rmr_recv_msg[recv_msg_index].virtual_address,
+ rmr_recv_msg[recv_msg_index].segment_length);
+
+ LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
+ getpid());
+
+ printf("%d %s RCV RDMA read buffer contains: %s\n",
+ getpid(), server ? "SERVER:" : "CLIENT:", sbuf);
+
+ recv_msg_index++;
+
+ return (DAT_SUCCESS);
+}
+
+DAT_RETURN do_ping_pong_msg()
{
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_DTO_COOKIE cookie;
- DAT_LMR_TRIPLET l_iov;
- DAT_RETURN ret;
- int i;
- char *snd_buf;
- char *rcv_buf;
-
- printf("\n %d PING DATA with SEND MSG\n\n",getpid());
-
- snd_buf = sbuf;
- rcv_buf = rbuf;
-
- /* pre-post all buffers */
- for ( i=0; i < burst; i++ ) {
- burst_msg_posted++;
- cookie.as_64 = i;
- l_iov.lmr_context = lmr_context_recv;
- l_iov.virtual_address = (DAT_VADDR)(uintptr_t)rcv_buf;
- l_iov.segment_length = buf_len;
-
- LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",
- getpid(), rcv_buf );
-
- ret = dat_ep_post_recv( h_ep,
- 1,
- &l_iov,
- cookie,
- DAT_COMPLETION_DEFAULT_FLAG );
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error posting recv msg buffer: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else {
- LOGPRINTF("%d Posted Receive Message Buffer %p\n",getpid(),rcv_buf);
- }
-
- /* next buffer */
- rcv_buf += buf_len;
- }
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_DTO_COOKIE cookie;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RETURN ret;
+ int i;
+ char *snd_buf;
+ char *rcv_buf;
+
+ printf("\n %d PING DATA with SEND MSG\n\n", getpid());
+
+ snd_buf = sbuf;
+ rcv_buf = rbuf;
+
+ /* pre-post all buffers */
+ for (i = 0; i < burst; i++) {
+ burst_msg_posted++;
+ cookie.as_64 = i;
+ l_iov.lmr_context = lmr_context_recv;
+ l_iov.virtual_address = (DAT_VADDR) (uintptr_t) rcv_buf;
+ l_iov.segment_length = buf_len;
+
+ LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",
+ getpid(), rcv_buf);
+
+ ret = dat_ep_post_recv(h_ep,
+ 1,
+ &l_iov,
+ cookie, DAT_COMPLETION_DEFAULT_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error posting recv msg buffer: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Posted Receive Message Buffer %p\n",
+ getpid(), rcv_buf);
+ }
+
+ /* next buffer */
+ rcv_buf += buf_len;
+ }
#if defined(_WIN32) || defined(_WIN64)
- Sleep(1000);
+ Sleep(1000);
#else
- sleep(1);
+ sleep(1);
#endif
- /* Initialize recv_buf and index to beginning */
- rcv_buf = rbuf;
- burst_msg_index=0;
-
- /* client ping 0x55, server pong 0xAA in first byte */
- start = get_time();
- for ( i=0;i<burst;i++ ) {
- /* walk the send and recv buffers */
- if ( !server ) {
- *snd_buf = 0x55;
-
- LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
- getpid(), server ? "SERVER:" : "CLIENT:",
- snd_buf, *snd_buf, buf_len );
-
- ret = send_msg( snd_buf,
- buf_len,
- lmr_context_send,
- cookie,
- DAT_COMPLETION_SUPPRESS_FLAG );
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
- }
-
- /* Wait for recv message */
- if ( polling ) {
- poll_count=0;
- LOGPRINTF("%d Polling for message receive event\n", getpid());
- while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
- poll_count++;
- }
- else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
- LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_rcv_evd )
- {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
- getpid());
- return( ret );
- }
- }
- /* use wait to dequeue */
- ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(),DT_RetToString(ret));
- return( ret );
- }
- }
- /* start timer after first message arrives on server */
- if ( i == 0) {
- start = get_time();
- }
- /* validate event number and status */
- LOGPRINTF("%d inbound message; message arrived!\n",getpid());
- if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(),DT_EventToSTr(event.event_number));
- return( DAT_ABORT );
- }
- if ((event.event_data.dto_completion_event_data.transfered_length
- != buf_len) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64
- != burst_msg_index) ) {
- fprintf(stderr,"ERR: recv event: len=%d cookie="F64x" exp %d/%d\n",
- (int)event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64,
- (int)buf_len, (int)burst_msg_index );
-
- return( DAT_ABORT );
- }
-
- LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",
- getpid(), server ? "SERVER:" : "CLIENT:",
- rcv_buf, *rcv_buf, buf_len );
-
- burst_msg_index++;
-
- /* If server, change data and send it back to client */
- if ( server ) {
- *snd_buf = 0xaa;
-
- LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
- getpid(), server ? "SERVER:" : "CLIENT:",
- snd_buf, *snd_buf, buf_len );
-
- ret = send_msg( snd_buf,
- buf_len,
- lmr_context_send,
- cookie,
- DAT_COMPLETION_SUPPRESS_FLAG );
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(),DT_RetToString(ret));
- return(ret);
- }
- else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
- }
-
- /* next buffers */
- rcv_buf += buf_len;
- snd_buf += buf_len;
- }
- stop = get_time();
- time.rtt = ((stop - start)*1.0e6);
-
- return ( DAT_SUCCESS );
+ /* Initialize recv_buf and index to beginning */
+ rcv_buf = rbuf;
+ burst_msg_index = 0;
+
+ /* client ping 0x55, server pong 0xAA in first byte */
+ start = get_time();
+ for (i = 0; i < burst; i++) {
+ /* walk the send and recv buffers */
+ if (!server) {
+ *snd_buf = 0x55;
+
+ LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
+ getpid(), server ? "SERVER:" : "CLIENT:",
+ snd_buf, *snd_buf, buf_len);
+
+ ret = send_msg(snd_buf,
+ buf_len,
+ lmr_context_send,
+ cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+ }
+
+ /* Wait for recv message */
+ if (polling) {
+ poll_count = 0;
+ LOGPRINTF("%d Polling for message receive event\n",
+ getpid());
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+ DAT_QUEUE_EMPTY)
+ poll_count++;
+ } else {
+ LOGPRINTF("%d waiting for message receive event\n",
+ getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret =
+ dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(), evd);
+ if (evd != h_dto_rcv_evd) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
+ return (ret);
+ }
+ }
+ /* use wait to dequeue */
+ ret =
+ dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event,
+ &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ }
+ /* start timer after first message arrives on server */
+ if (i == 0) {
+ start = get_time();
+ }
+ /* validate event number and status */
+ LOGPRINTF("%d inbound message; message arrived!\n", getpid());
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(), DT_EventToSTr(event.event_number));
+ return (DAT_ABORT);
+ }
+ if ((event.event_data.dto_completion_event_data.
+ transfered_length != buf_len)
+ || (event.event_data.dto_completion_event_data.user_cookie.
+ as_64 != burst_msg_index)) {
+ fprintf(stderr,
+ "ERR: recv event: len=%d cookie=" F64x
+ " exp %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.
+ transfered_length,
+ event.event_data.dto_completion_event_data.
+ user_cookie.as_64, (int)buf_len,
+ (int)burst_msg_index);
+
+ return (DAT_ABORT);
+ }
+
+ LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",
+ getpid(), server ? "SERVER:" : "CLIENT:",
+ rcv_buf, *rcv_buf, buf_len);
+
+ burst_msg_index++;
+
+ /* If server, change data and send it back to client */
+ if (server) {
+ *snd_buf = 0xaa;
+
+ LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
+ getpid(), server ? "SERVER:" : "CLIENT:",
+ snd_buf, *snd_buf, buf_len);
+
+ ret = send_msg(snd_buf,
+ buf_len,
+ lmr_context_send,
+ cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+ }
+
+ /* next buffers */
+ rcv_buf += buf_len;
+ snd_buf += buf_len;
+ }
+ stop = get_time();
+ time.rtt = ((stop - start) * 1.0e6);
+
+ return (DAT_SUCCESS);
}
/* Register RDMA Receive buffer */
-DAT_RETURN
-register_rdma_memory(void)
+DAT_RETURN register_rdma_memory(void)
{
- DAT_RETURN ret;
- DAT_REGION_DESCRIPTION region;
-
- region.for_va = rbuf;
- start = get_time();
- ret = dat_lmr_create( h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- buf_len*burst,
- h_pz,
- DAT_MEM_PRIV_ALL_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_recv,
- &lmr_context_recv,
- &rmr_context_recv,
- ®istered_size_recv,
- ®istered_addr_recv );
- stop = get_time();
- time.reg += ((stop - start)*1.0e6);
- time.total += time.reg;
-
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering Receive RDMA buffer: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
- getpid(),region.for_va);
- }
-
- /* Register RDMA Send buffer */
- region.for_va = sbuf;
- ret = dat_lmr_create( h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- buf_len*burst,
- h_pz,
- DAT_MEM_PRIV_ALL_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_send,
- &lmr_context_send,
- &rmr_context_send,
- ®istered_size_send,
- ®istered_addr_send );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
- getpid(),region.for_va);
- }
-
- return DAT_SUCCESS;
+ DAT_RETURN ret;
+ DAT_REGION_DESCRIPTION region;
+
+ region.for_va = rbuf;
+ start = get_time();
+ ret = dat_lmr_create(h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ buf_len * (burst+1),
+ h_pz,
+ DAT_MEM_PRIV_ALL_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_recv,
+ &lmr_context_recv,
+ &rmr_context_recv,
+ ®istered_size_recv, ®istered_addr_recv);
+ stop = get_time();
+ time.reg += ((stop - start) * 1.0e6);
+ time.total += time.reg;
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr,
+ "%d Error registering Receive RDMA buffer: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
+ getpid(), region.for_va);
+ }
+
+ /* Register RDMA Send buffer */
+ region.for_va = sbuf;
+ ret = dat_lmr_create(h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ buf_len * (burst + 1),
+ h_pz,
+ DAT_MEM_PRIV_ALL_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_send,
+ &lmr_context_send,
+ &rmr_context_send,
+ ®istered_size_send, ®istered_addr_send);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
+ getpid(), region.for_va);
+ }
+
+ return DAT_SUCCESS;
}
/*
* Unregister RDMA memory
*/
-DAT_RETURN
-unregister_rdma_memory(void)
+DAT_RETURN unregister_rdma_memory(void)
{
- DAT_RETURN ret;
-
- /* Unregister Recv Buffer */
- if ( h_lmr_recv != DAT_HANDLE_NULL )
- {
- LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_recv);
- start = get_time();
- ret = dat_lmr_free(h_lmr_recv);
- stop = get_time();
- time.unreg += ((stop - start)*1.0e6);
- time.total += time.unreg;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error deregistering recv mr: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d Unregistered Recv Buffer\n",getpid());
- h_lmr_recv = NULL;
- }
- }
-
- /* Unregister Send Buffer */
- if ( h_lmr_send != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_send);
- ret = dat_lmr_free(h_lmr_send);
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error deregistering send mr: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d Unregistered send Buffer\n",getpid());
- h_lmr_send = NULL;
- }
- }
- return DAT_SUCCESS;
+ DAT_RETURN ret;
+
+ /* Unregister Recv Buffer */
+ if (h_lmr_recv != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Unregister h_lmr %p \n", getpid(), h_lmr_recv);
+ start = get_time();
+ ret = dat_lmr_free(h_lmr_recv);
+ stop = get_time();
+ time.unreg += ((stop - start) * 1.0e6);
+ time.total += time.unreg;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error deregistering recv mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Unregistered Recv Buffer\n", getpid());
+ h_lmr_recv = NULL;
+ }
+ }
+
+ /* Unregister Send Buffer */
+ if (h_lmr_send != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Unregister h_lmr %p \n", getpid(), h_lmr_send);
+ ret = dat_lmr_free(h_lmr_send);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error deregistering send mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Unregistered send Buffer\n", getpid());
+ h_lmr_send = NULL;
+ }
+ }
+ return DAT_SUCCESS;
}
/*
* Create CNO, CR, CONN, and DTO events
*/
-DAT_RETURN
-create_events(void)
+DAT_RETURN create_events(void)
{
- DAT_RETURN ret;
- DAT_EVD_PARAM param;
+ DAT_RETURN ret;
+ DAT_EVD_PARAM param;
- /* create CNO */
- if (use_cno) {
- start = get_time();
+ /* create CNO */
+ if (use_cno) {
+ start = get_time();
#if defined(_WIN32) || defined(_WIN64)
- { DAT_OS_WAIT_PROXY_AGENT pa = {NULL, NULL};
- ret = dat_cno_create(h_ia, pa, &h_dto_cno);
- }
+ {
+ DAT_OS_WAIT_PROXY_AGENT pa = { NULL, NULL };
+ ret = dat_cno_create(h_ia, pa, &h_dto_cno);
+ }
#else
- ret = dat_cno_create(h_ia, DAT_OS_WAIT_PROXY_AGENT_NULL, &h_dto_cno);
+ ret =
+ dat_cno_create(h_ia, DAT_OS_WAIT_PROXY_AGENT_NULL,
+ &h_dto_cno);
#endif
- stop = get_time();
- time.cnoc += ((stop - start)*1.0e6);
- time.total += time.cnoc;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_cno_create: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d cr_evd created, %p\n", getpid(), h_dto_cno);
- }
- }
-
- /* create cr EVD */
- start = get_time();
- ret = dat_evd_create(h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &h_cr_evd);
- stop = get_time();
- time.evdc += ((stop - start)*1.0e6);
- time.total += time.evdc;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d cr_evd created %p\n", getpid(),h_cr_evd);
- }
-
- /* create conn EVD */
- ret = dat_evd_create( h_ia,
- 10,
- DAT_HANDLE_NULL,
- DAT_EVD_CONNECTION_FLAG,
- &h_conn_evd );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d con_evd created %p\n", getpid(),h_conn_evd);
- }
-
- /* create dto SND EVD, with CNO if use_cno was set */
- ret = dat_evd_create( h_ia,
- (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,
- h_dto_cno,
- DAT_EVD_DTO_FLAG,
- &h_dto_req_evd );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );
- }
-
- /* create dto RCV EVD, with CNO if use_cno was set */
- ret = dat_evd_create( h_ia,
- MSG_BUF_COUNT,
- h_dto_cno,
- DAT_EVD_DTO_FLAG,
- &h_dto_rcv_evd );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );
- }
-
- /* query DTO req EVD and check size */
- ret = dat_evd_query( h_dto_req_evd,
- DAT_EVD_FIELD_EVD_QLEN,
- ¶m );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
- getpid(),DT_RetToString(ret));
- return (ret);
- }
- else if (param.evd_qlen < (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2) {
- fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
- getpid(), param.evd_qlen,
- (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2);
- return (ret);
- }
-
- LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n",
- getpid(), (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,
- param.evd_qlen);
-
- return DAT_SUCCESS;
+ stop = get_time();
+ time.cnoc += ((stop - start) * 1.0e6);
+ time.total += time.cnoc;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_cno_create: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d cr_evd created, %p\n", getpid(),
+ h_dto_cno);
+ }
+ }
+
+ /* create cr EVD */
+ start = get_time();
+ ret =
+ dat_evd_create(h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
+ &h_cr_evd);
+ stop = get_time();
+ time.evdc += ((stop - start) * 1.0e6);
+ time.total += time.evdc;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d cr_evd created %p\n", getpid(), h_cr_evd);
+ }
+
+ /* create conn EVD */
+ ret = dat_evd_create(h_ia,
+ 10,
+ DAT_HANDLE_NULL,
+ DAT_EVD_CONNECTION_FLAG, &h_conn_evd);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d con_evd created %p\n", getpid(), h_conn_evd);
+ }
+
+ /* create dto SND EVD, with CNO if use_cno was set */
+ ret = dat_evd_create(h_ia,
+ (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2,
+ h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_req_evd);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d dto_req_evd created %p\n", getpid(),
+ h_dto_req_evd);
+ }
+
+ /* create dto RCV EVD, with CNO if use_cno was set */
+ ret = dat_evd_create(h_ia,
+ MSG_BUF_COUNT + burst,
+ h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_rcv_evd);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(),
+ h_dto_rcv_evd);
+ }
+
+ /* query DTO req EVD and check size */
+ ret = dat_evd_query(h_dto_req_evd, DAT_EVD_FIELD_EVD_QLEN, ¶m);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else if (param.evd_qlen < (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2) {
+ fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
+ getpid(), param.evd_qlen,
+ (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2);
+ return (ret);
+ }
+
+ LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n",
+ getpid(), (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2,
+ param.evd_qlen);
+
+ return DAT_SUCCESS;
}
/*
* Destroy CR, CONN, CNO, and DTO events
*/
-DAT_RETURN
-destroy_events(void)
+DAT_RETURN destroy_events(void)
{
- DAT_RETURN ret;
-
- /* free cr EVD */
- if ( h_cr_evd != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Free cr EVD %p \n",getpid(),h_cr_evd);
- ret = dat_evd_free( h_cr_evd );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing cr EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Freed cr EVD\n",getpid());
- h_cr_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free conn EVD */
- if ( h_conn_evd != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Free conn EVD %p \n",getpid(),h_conn_evd);
- ret = dat_evd_free( h_conn_evd );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing conn EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d Freed conn EVD\n",getpid());
- h_conn_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free RCV dto EVD */
- if ( h_dto_rcv_evd != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);
- start = get_time();
- ret = dat_evd_free( h_dto_rcv_evd );
- stop = get_time();
- time.evdf += ((stop - start)*1.0e6);
- time.total += time.evdf;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing dto EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d Freed dto EVD\n",getpid());
- h_dto_rcv_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free REQ dto EVD */
- if ( h_dto_req_evd != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);
- ret = dat_evd_free( h_dto_req_evd );
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing dto EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d Freed dto EVD\n",getpid());
- h_dto_req_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free CNO */
- if ( h_dto_cno != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Free dto CNO %p \n",getpid(),h_dto_cno);
- start = get_time();
- ret = dat_cno_free( h_dto_cno );
- stop = get_time();
- time.cnof += ((stop - start)*1.0e6);
- time.total += time.cnof;
- if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing dto CNO: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- else {
- LOGPRINTF("%d Freed dto CNO\n",getpid());
- h_dto_cno = DAT_HANDLE_NULL;
- }
- }
- return DAT_SUCCESS;
+ DAT_RETURN ret;
+
+ /* free cr EVD */
+ if (h_cr_evd != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Free cr EVD %p \n", getpid(), h_cr_evd);
+ ret = dat_evd_free(h_cr_evd);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing cr EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Freed cr EVD\n", getpid());
+ h_cr_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free conn EVD */
+ if (h_conn_evd != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Free conn EVD %p \n", getpid(), h_conn_evd);
+ ret = dat_evd_free(h_conn_evd);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing conn EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Freed conn EVD\n", getpid());
+ h_conn_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free RCV dto EVD */
+ if (h_dto_rcv_evd != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Free RCV dto EVD %p \n", getpid(), h_dto_rcv_evd);
+ start = get_time();
+ ret = dat_evd_free(h_dto_rcv_evd);
+ stop = get_time();
+ time.evdf += ((stop - start) * 1.0e6);
+ time.total += time.evdf;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Freed dto EVD\n", getpid());
+ h_dto_rcv_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free REQ dto EVD */
+ if (h_dto_req_evd != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Free REQ dto EVD %p \n", getpid(), h_dto_req_evd);
+ ret = dat_evd_free(h_dto_req_evd);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Freed dto EVD\n", getpid());
+ h_dto_req_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free CNO */
+ if (h_dto_cno != DAT_HANDLE_NULL) {
+ LOGPRINTF("%d Free dto CNO %p \n", getpid(), h_dto_cno);
+ start = get_time();
+ ret = dat_cno_free(h_dto_cno);
+ stop = get_time();
+ time.cnof += ((stop - start) * 1.0e6);
+ time.total += time.cnof;
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto CNO: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Freed dto CNO\n", getpid());
+ h_dto_cno = DAT_HANDLE_NULL;
+ }
+ }
+ return DAT_SUCCESS;
}
/*
* Map DAT_RETURN values to readable strings,
* but don't assume the values are zero-based or contiguous.
*/
-char errmsg[512] = {0};
-const char *
-DT_RetToString (DAT_RETURN ret_value)
+char errmsg[512] = { 0 };
+const char *DT_RetToString(DAT_RETURN ret_value)
{
- const char *major_msg, *minor_msg;
+ const char *major_msg, *minor_msg;
- dat_strerror (ret_value, &major_msg, &minor_msg);
+ dat_strerror(ret_value, &major_msg, &minor_msg);
- strcpy(errmsg, major_msg);
- strcat(errmsg, " ");
- strcat(errmsg, minor_msg);
+ strcpy(errmsg, major_msg);
+ strcat(errmsg, " ");
+ strcat(errmsg, minor_msg);
- return errmsg;
+ return errmsg;
}
/*
* Map DAT_EVENT_CODE values to readable strings
*/
-const char *
-DT_EventToSTr (DAT_EVENT_NUMBER event_code)
+const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code)
{
- unsigned int i;
- static struct {
- const char *name;
- DAT_RETURN value;
- }
- dat_events[] =
- {
- # define DATxx(x) { # x, x }
- DATxx (DAT_DTO_COMPLETION_EVENT),
- DATxx (DAT_RMR_BIND_COMPLETION_EVENT),
- DATxx (DAT_CONNECTION_REQUEST_EVENT),
- DATxx (DAT_CONNECTION_EVENT_ESTABLISHED),
- DATxx (DAT_CONNECTION_EVENT_PEER_REJECTED),
- DATxx (DAT_CONNECTION_EVENT_NON_PEER_REJECTED),
- DATxx (DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR),
- DATxx (DAT_CONNECTION_EVENT_DISCONNECTED),
- DATxx (DAT_CONNECTION_EVENT_BROKEN),
- DATxx (DAT_CONNECTION_EVENT_TIMED_OUT),
- DATxx (DAT_CONNECTION_EVENT_UNREACHABLE),
- DATxx (DAT_ASYNC_ERROR_EVD_OVERFLOW),
- DATxx (DAT_ASYNC_ERROR_IA_CATASTROPHIC),
- DATxx (DAT_ASYNC_ERROR_EP_BROKEN),
- DATxx (DAT_ASYNC_ERROR_TIMED_OUT),
- DATxx (DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR),
- DATxx (DAT_SOFTWARE_EVENT)
- # undef DATxx
- };
- # define NUM_EVENTS (sizeof(dat_events)/sizeof(dat_events[0]))
-
- for (i = 0; i < NUM_EVENTS; i++) {
- if (dat_events[i].value == event_code)
- {
- return ( dat_events[i].name );
- }
- }
-
- return ( "Invalid_DAT_EVENT_NUMBER" );
-}
+ unsigned int i;
+ static struct {
+ const char *name;
+ DAT_RETURN value;
+ } dat_events[] = {
+# define DATxx(x) { # x, x }
+ DATxx(DAT_DTO_COMPLETION_EVENT),
+ DATxx(DAT_RMR_BIND_COMPLETION_EVENT),
+ DATxx(DAT_CONNECTION_REQUEST_EVENT),
+ DATxx(DAT_CONNECTION_EVENT_ESTABLISHED),
+ DATxx(DAT_CONNECTION_EVENT_PEER_REJECTED),
+ DATxx(DAT_CONNECTION_EVENT_NON_PEER_REJECTED),
+ DATxx(DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR),
+ DATxx(DAT_CONNECTION_EVENT_DISCONNECTED),
+ DATxx(DAT_CONNECTION_EVENT_BROKEN),
+ DATxx(DAT_CONNECTION_EVENT_TIMED_OUT),
+ DATxx(DAT_CONNECTION_EVENT_UNREACHABLE),
+ DATxx(DAT_ASYNC_ERROR_EVD_OVERFLOW),
+ DATxx(DAT_ASYNC_ERROR_IA_CATASTROPHIC),
+ DATxx(DAT_ASYNC_ERROR_EP_BROKEN),
+ DATxx(DAT_ASYNC_ERROR_TIMED_OUT),
+ DATxx(DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR),
+ DATxx(DAT_SOFTWARE_EVENT)
+# undef DATxx
+ };
+# define NUM_EVENTS (sizeof(dat_events)/sizeof(dat_events[0]))
+
+ for (i = 0; i < NUM_EVENTS; i++) {
+ if (dat_events[i].value == event_code) {
+ return (dat_events[i].name);
+ }
+ }
+ return ("Invalid_DAT_EVENT_NUMBER");
+}
void print_usage(void)
{
- printf("\n DAPL USAGE \n\n");
- printf("s: server\n");
- printf("t: performance times\n");
- printf("c: use cno\n");
- printf("v: verbose\n");
- printf("p: polling\n");
- printf("d: delay before accept\n");
- printf("b: buf length to allocate\n");
- printf("B: burst count, rdma and msgs \n");
- printf("h: hostname/address of server, specified on client\n");
- printf("P: provider name (default = OpenIB-cma)\n");
- printf("\n");
+ printf("\n DAPL USAGE \n\n");
+ printf("s: server\n");
+ printf("t: performance times\n");
+ printf("c: use cno\n");
+ printf("v: verbose\n");
+ printf("p: polling\n");
+ printf("d: delay before accept\n");
+ printf("b: buf length to allocate\n");
+ printf("B: burst count, rdma and msgs \n");
+ printf("h: hostname/address of server, specified on client\n");
+ printf("P: provider name (default = OpenIB-cma)\n");
+ printf("\n");
}
--- /dev/null
+[1-06-09]\r
+\r
+These 'shared' OFED files (shared between Linux and Windows) are mirrored here for the convience\r
+of Windows developers.\r
+\r
+The 'offical' OFED source files are located in the OFED git repository.\r
+\r
+The policy is 'no source file changes' in non-build files (.h & .c) are accepted here.\r
+Any source file changes here will likely be overwritten when refreshed from the OFED git repository.\r
+\r
+The plan here is to have a single Linux and Windows source.\r
+\r
+If you require source file changes, push the changes to the OFED maintainers.\r
+\r
+For continuing policy discussion, please contact sean.hefty@intel.com\r
+\r
+thank you,\r
+\r
+Stan (stan.smith@intel.com)
\ No newline at end of file
--- /dev/null
+DIRS = \\r
+ src
\ No newline at end of file
--- /dev/null
+/*
+ * Copyright (c) 2008 Lawrence Livermore National Lab. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef _IBNETDISC_H_
+#define _IBNETDISC_H_
+
+#include <stdio.h>
+#include <infiniband/mad.h>
+#include <iba/ib_types.h>
+#include <infiniband/mad_osd.h>
+
+struct ib_fabric; /* forward declare */
+struct chassis; /* forward declare */
+struct port; /* forward declare */
+
+/** =========================================================================
+ * Node
+ */
+typedef struct node {
+ struct node *next; /* all node list in fabric */
+ struct ib_fabric *fabric; /* the fabric node belongs to */
+
+ ib_portid_t path_portid; /* path from "from_node" */
+ int dist; /* num of hops from "from_node" */
+ int smalid;
+ int smalmc;
+
+ /* quick cache of switchinfo below */
+ int smaenhsp0;
+ /* use libibmad decoder functions for switchinfo */
+ //WHY does this not work???
+ //uint8_t switchinfo[sizeof (ib_switch_info_t)];
+ uint8_t switchinfo[64];
+
+ /* quick cache of info below */
+ uint64_t guid;
+ int type;
+ int numports;
+ /* use libibmad decoder functions for info */
+ uint8_t info[sizeof(ib_node_info_t)];
+
+ char nodedesc[IB_NODE_DESCRIPTION_SIZE];
+
+ struct port **ports; /* in order array of port pointers */
+ /* the size of this array is info.numports + 1 */
+ /* items MAY BE NULL! (ie 0 == switches only) */
+
+ /* chassis info */
+ struct node *next_chassis_node; /* next node in ibnd_chassis_t->nodes */
+ struct chassis *chassis; /* if != NULL the chassis this node belongs to */
+ unsigned char ch_type;
+ unsigned char ch_anafanum;
+ unsigned char ch_slotnum;
+ unsigned char ch_slot;
+} ibnd_node_t;
+
+/** =========================================================================
+ * Port
+ */
+typedef struct port {
+ uint64_t guid;
+ int portnum;
+ int ext_portnum; /* optional if != 0 external port num */
+ ibnd_node_t *node; /* node this port belongs to */
+ struct port *remoteport; /* null if SMA, or does not exist */
+ /* quick cache of info below */
+ uint16_t base_lid;
+ uint8_t lmc;
+ /* use libibmad decoder functions for info */
+ uint8_t info[sizeof(ib_port_info_t)];
+} ibnd_port_t;
+
+
+/** =========================================================================
+ * Chassis
+ */
+typedef struct chassis {
+ struct chassis *next;
+ uint64_t chassisguid;
+ int chassisnum;
+
+ /* generic grouping by SystemImageGUID */
+ int nodecount;
+ ibnd_node_t *nodes;
+
+ /* specific to voltaire type nodes */
+#define SPINES_MAX_NUM 12
+#define LINES_MAX_NUM 36
+ ibnd_node_t *spinenode[SPINES_MAX_NUM + 1];
+ ibnd_node_t *linenode[LINES_MAX_NUM + 1];
+} ibnd_chassis_t;
+
+/** =========================================================================
+ * Fabric
+ * Main fabric object which is returned and represents the data discovered
+ */
+typedef struct ib_fabric {
+ /* the node the discover was initiated from
+ * "from" parameter in ibnd_discover_fabric
+ * or by default the node you ar running on
+ */
+ ibnd_node_t *from_node;
+ /* NULL term list of all nodes in the fabric */
+ ibnd_node_t *nodes;
+ /* NULL terminated list of all chassis found in the fabric */
+ ibnd_chassis_t *chassis;
+ int maxhops_discovered;
+} ibnd_fabric_t;
+
+
+/** =========================================================================
+ * Initialization (fabric operations)
+ */
+MAD_EXPORT void ibnd_debug(int i);
+MAD_EXPORT void ibnd_show_progress(int i);
+
+MAD_EXPORT ibnd_fabric_t *ibnd_discover_fabric(char *dev_name, int dev_port,
+ int timeout_ms,
+ ib_portid_t *from, int hops);
+ /**
+ * dev_name: (required) local device name to use to access the fabric
+ * dev_port: (required) local device port to use to access the fabric
+ * timeout_ms: (required) gives the timeout for a _SINGLE_ query on
+ * the fabric. So if there are multiple nodes not
+ * responding this may result in a lengthy delay.
+ * from: (optional) specify the node to start scanning from.
+ * If NULL start from the node we are running on.
+ * hops: (optional) Specify how much of the fabric to traverse.
+ * negative value == scan entire fabric
+ */
+MAD_EXPORT void ibnd_destroy_fabric(ibnd_fabric_t *fabric);
+
+/** =========================================================================
+ * Node operations
+ */
+MAD_EXPORT ibnd_node_t *ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid);
+MAD_EXPORT ibnd_node_t *ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str);
+MAD_EXPORT ibnd_node_t *ibnd_update_node(ibnd_node_t *node);
+
+typedef void (*ibnd_iter_node_func_t)(ibnd_node_t *node, void *user_data);
+MAD_EXPORT void ibnd_iter_nodes(ibnd_fabric_t *fabric,
+ ibnd_iter_node_func_t func,
+ void *user_data);
+MAD_EXPORT void ibnd_iter_nodes_type(ibnd_fabric_t *fabric,
+ ibnd_iter_node_func_t func,
+ int node_type,
+ void *user_data);
+
+/** =========================================================================
+ * Chassis queries
+ */
+MAD_EXPORT uint64_t ibnd_get_chassis_guid(ibnd_fabric_t *fabric,
+ unsigned char chassisnum);
+MAD_EXPORT char *ibnd_get_chassis_type(ibnd_node_t *node);
+MAD_EXPORT char *ibnd_get_chassis_slot_str(ibnd_node_t *node,
+ char *str, size_t size);
+
+MAD_EXPORT int ibnd_is_xsigo_guid(uint64_t guid);
+MAD_EXPORT int ibnd_is_xsigo_tca(uint64_t guid);
+MAD_EXPORT int ibnd_is_xsigo_hca(uint64_t guid);
+
+#endif /* _IBNETDISC_H_ */
--- /dev/null
+!if $(FREEBUILD)\r
+TARGETNAME = libibnetdisc\r
+!else\r
+TARGETNAME = libibnetdiscd\r
+!endif\r
+\r
+TARGETPATH = ..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE = DYNLINK\r
+\r
+DLLDEF = $(OBJ_PATH)\$O\ibnetdisc_exports.def\r
+\r
+DLLENTRY = DllMain\r
+USE_MSVCRT = 1\r
+\r
+SOURCES = \\r
+ ibnetdisc_main.cpp \\r
+ ibnetdisc.c \\r
+ chassis.c\r
+ \r
+INCLUDES = ..\include\infiniband;\\r
+ ..\include;\\r
+ ..\..\libibmad\include;\\r
+ ..\..\..\tools\infiniband-diags\include;\\r
+ ..\..\libibverbs\include;\\r
+ ..\..\libibumad\include;\\r
+ ..\..\..\inc;\\r
+ ..\..\..\inc\user;\\r
+ ..\..\..\inc\user\linux;\r
+\r
+USER_C_FLAGS = $(USER_C_FLAGS)\r
+\r
+TARGETLIBS = \\r
+ $(SDK_LIB_PATH)\kernel32.lib \\r
+ $(SDK_LIB_PATH)\uuid.lib \\r
+ $(SDK_LIB_PATH)\ws2_32.lib \\r
+ $(SDK_LIB_PATH)\advapi32.lib \\r
+ $(SDK_LIB_PATH)\user32.lib \\r
+ $(SDK_LIB_PATH)\ole32.lib \\r
+!if $(FREEBUILD)\r
+ $(TARGETPATH)\*\libibumad.lib \\r
+ $(TARGETPATH)\*\libibmad.lib \r
+!else\r
+ $(TARGETPATH)\*\libibumadd.lib \\r
+ $(TARGETPATH)\*\libibmadd.lib \r
+!endif\r
--- /dev/null
+/*
+ * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2007 Xsigo Systems Inc. All rights reserved.
+ * Copyright (c) 2008 Lawrence Livermore National Lab. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+/*========================================================*/
+/* FABRIC SCANNER SPECIFIC DATA */
+/*========================================================*/
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include <infiniband/mad.h>
+
+#include "internal.h"
+#include "chassis.h"
+
+static char *ChassisTypeStr[5] = { "", "ISR9288", "ISR9096", "ISR2012", "ISR2004" };
+static char *ChassisSlotTypeStr[4] = { "", "Line", "Spine", "SRBD" };
+
+char *ibnd_get_chassis_type(ibnd_node_t *node)
+{
+ /* Currently, only if Voltaire chassis */
+ if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID)
+ return (NULL);
+ if (!node->chassis)
+ return (NULL);
+ if (node->ch_type == UNRESOLVED_CT
+ || node->ch_type > ISR2004_CT)
+ return (NULL);
+ return ChassisTypeStr[node->ch_type];
+}
+
+char *ibnd_get_chassis_slot_str(ibnd_node_t *node, char *str, size_t size)
+{
+ /* Currently, only if Voltaire chassis */
+ if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID)
+ return (NULL);
+ if (!node->chassis)
+ return (NULL);
+ if (node->ch_slot == UNRESOLVED_CS
+ || node->ch_slot > SRBD_CS)
+ return (NULL);
+ if (!str)
+ return (NULL);
+ snprintf(str, size, "%s %d Chip %d",
+ ChassisSlotTypeStr[node->ch_slot],
+ node->ch_slotnum,
+ node->ch_anafanum);
+ return (str);
+}
+
+static ibnd_chassis_t *find_chassisnum(struct ibnd_fabric *fabric, unsigned char chassisnum)
+{
+ ibnd_chassis_t *current;
+
+ for (current = fabric->first_chassis; current; current = current->next) {
+ if (current->chassisnum == chassisnum)
+ return current;
+ }
+
+ return NULL;
+}
+
+static uint64_t topspin_chassisguid(uint64_t guid)
+{
+ /* Byte 3 in system image GUID is chassis type, and */
+ /* Byte 4 is location ID (slot) so just mask off byte 4 */
+ return guid & 0xffffffff00ffffffULL;
+}
+
+int ibnd_is_xsigo_guid(uint64_t guid)
+{
+ if ((guid & 0xffffff0000000000ULL) == 0x0013970000000000ULL)
+ return 1;
+ else
+ return 0;
+}
+
+static int is_xsigo_leafone(uint64_t guid)
+{
+ if ((guid & 0xffffffffff000000ULL) == 0x0013970102000000ULL)
+ return 1;
+ else
+ return 0;
+}
+
+int ibnd_is_xsigo_hca(uint64_t guid)
+{
+ /* NodeType 2 is HCA */
+ if ((guid & 0xffffffff00000000ULL) == 0x0013970200000000ULL)
+ return 1;
+ else
+ return 0;
+}
+
+int ibnd_is_xsigo_tca(uint64_t guid)
+{
+ /* NodeType 3 is TCA */
+ if ((guid & 0xffffffff00000000ULL) == 0x0013970300000000ULL)
+ return 1;
+ else
+ return 0;
+}
+
+static int is_xsigo_ca(uint64_t guid)
+{
+ if (ibnd_is_xsigo_hca(guid) || ibnd_is_xsigo_tca(guid))
+ return 1;
+ else
+ return 0;
+}
+
+static int is_xsigo_switch(uint64_t guid)
+{
+ if ((guid & 0xffffffff00000000ULL) == 0x0013970100000000ULL)
+ return 1;
+ else
+ return 0;
+}
+
+static uint64_t xsigo_chassisguid(ibnd_node_t *node)
+{
+ uint64_t sysimgguid = mad_get_field64(node->info, 0, IB_NODE_SYSTEM_GUID_F);
+ uint64_t remote_sysimgguid;
+
+ if (!is_xsigo_ca(sysimgguid)) {
+ /* Byte 3 is NodeType and byte 4 is PortType */
+ /* If NodeType is 1 (switch), PortType is masked */
+ if (is_xsigo_switch(sysimgguid))
+ return sysimgguid & 0xffffffff00ffffffULL;
+ else
+ return sysimgguid;
+ } else {
+ if (!node->ports || !node->ports[1])
+ return (0);
+
+ /* Is there a peer port ? */
+ if (!node->ports[1]->remoteport)
+ return sysimgguid;
+
+ /* If peer port is Leaf 1, use its chassis GUID */
+ remote_sysimgguid = mad_get_field64(
+ node->ports[1]->remoteport->node->info,
+ 0, IB_NODE_SYSTEM_GUID_F);
+ if (is_xsigo_leafone(remote_sysimgguid))
+ return remote_sysimgguid & 0xffffffff00ffffffULL;
+ else
+ return sysimgguid;
+ }
+}
+
+static uint64_t get_chassisguid(ibnd_node_t *node)
+{
+ uint32_t vendid = mad_get_field(node->info, 0, IB_NODE_VENDORID_F);
+ uint64_t sysimgguid = mad_get_field64(node->info, 0, IB_NODE_SYSTEM_GUID_F);
+
+ if (vendid == TS_VENDOR_ID || vendid == SS_VENDOR_ID)
+ return topspin_chassisguid(sysimgguid);
+ else if (vendid == XS_VENDOR_ID || ibnd_is_xsigo_guid(sysimgguid))
+ return xsigo_chassisguid(node);
+ else
+ return sysimgguid;
+}
+
+static ibnd_chassis_t *find_chassisguid(ibnd_node_t *node)
+{
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(node->fabric);
+ ibnd_chassis_t *current;
+ uint64_t chguid;
+
+ chguid = get_chassisguid(node);
+ for (current = f->first_chassis; current; current = current->next) {
+ if (current->chassisguid == chguid)
+ return current;
+ }
+
+ return NULL;
+}
+
+uint64_t ibnd_get_chassis_guid(ibnd_fabric_t *fabric, unsigned char chassisnum)
+{
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric);
+ ibnd_chassis_t *chassis;
+
+ chassis = find_chassisnum(f, chassisnum);
+ if (chassis)
+ return chassis->chassisguid;
+ else
+ return 0;
+}
+
+static int is_router(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_IB_FC_ROUTER ||
+ devid == VTR_DEVID_IB_IP_ROUTER);
+}
+
+static int is_spine_9096(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SFB4 ||
+ devid == VTR_DEVID_SFB4_DDR);
+}
+
+static int is_spine_9288(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SFB12 ||
+ devid == VTR_DEVID_SFB12_DDR);
+}
+
+static int is_spine_2004(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SFB2004);
+}
+
+static int is_spine_2012(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SFB2012);
+}
+
+static int is_spine(struct ibnd_node *n)
+{
+ return (is_spine_9096(n) || is_spine_9288(n) ||
+ is_spine_2004(n) || is_spine_2012(n));
+}
+
+static int is_line_24(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SLB24 ||
+ devid == VTR_DEVID_SLB24_DDR ||
+ devid == VTR_DEVID_SRB2004);
+}
+
+static int is_line_8(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SLB8);
+}
+
+static int is_line_2024(struct ibnd_node *n)
+{
+ uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F);
+ return (devid == VTR_DEVID_SLB2024);
+}
+
+static int is_line(struct ibnd_node *n)
+{
+ return (is_line_24(n) || is_line_8(n) || is_line_2024(n));
+}
+
+int is_chassis_switch(struct ibnd_node *n)
+{
+ return (is_spine(n) || is_line(n));
+}
+
+/* these structs help find Line (Anafa) slot number while using spine portnum */
+char line_slot_2_sfb4[25] = { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4 };
+char anafa_line_slot_2_sfb4[25] = { 0, 1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2 };
+char line_slot_2_sfb12[25] = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9,10, 10, 11, 11, 12, 12 };
+char anafa_line_slot_2_sfb12[25] = { 0, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 };
+
+/* IPR FCR modules connectivity while using sFB4 port as reference */
+char ipr_slot_2_sfb4_port[25] = { 0, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1 };
+
+/* these structs help find Spine (Anafa) slot number while using spine portnum */
+char spine12_slot_2_slb[25] = { 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+char anafa_spine12_slot_2_slb[25]= { 0, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+char spine4_slot_2_slb[25] = { 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+char anafa_spine4_slot_2_slb[25] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+/* reference { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 }; */
+
+static void get_sfb_slot(struct ibnd_node *node, ibnd_port_t *lineport)
+{
+ ibnd_node_t *n = (ibnd_node_t *)node;
+
+ n->ch_slot = SPINE_CS;
+ if (is_spine_9096(node)) {
+ n->ch_type = ISR9096_CT;
+ n->ch_slotnum = spine4_slot_2_slb[lineport->portnum];
+ n->ch_anafanum = anafa_spine4_slot_2_slb[lineport->portnum];
+ } else if (is_spine_9288(node)) {
+ n->ch_type = ISR9288_CT;
+ n->ch_slotnum = spine12_slot_2_slb[lineport->portnum];
+ n->ch_anafanum = anafa_spine12_slot_2_slb[lineport->portnum];
+ } else if (is_spine_2012(node)) {
+ n->ch_type = ISR2012_CT;
+ n->ch_slotnum = spine12_slot_2_slb[lineport->portnum];
+ n->ch_anafanum = anafa_spine12_slot_2_slb[lineport->portnum];
+ } else if (is_spine_2004(node)) {
+ n->ch_type = ISR2004_CT;
+ n->ch_slotnum = spine4_slot_2_slb[lineport->portnum];
+ n->ch_anafanum = anafa_spine4_slot_2_slb[lineport->portnum];
+ } else {
+ IBPANIC("Unexpected node found: guid 0x%016" PRIx64,
+ node->node.guid);
+ }
+}
+
+static void get_router_slot(struct ibnd_node *node, ibnd_port_t *spineport)
+{
+ ibnd_node_t *n = (ibnd_node_t *)node;
+ uint64_t guessnum = 0;
+
+ node->ch_found = 1;
+
+ n->ch_slot = SRBD_CS;
+ if (is_spine_9096(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR9096_CT;
+ n->ch_slotnum = line_slot_2_sfb4[spineport->portnum];
+ n->ch_anafanum = ipr_slot_2_sfb4_port[spineport->portnum];
+ } else if (is_spine_9288(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR9288_CT;
+ n->ch_slotnum = line_slot_2_sfb12[spineport->portnum];
+ /* this is a smart guess based on nodeguids order on sFB-12 module */
+ guessnum = spineport->node->guid % 4;
+ /* module 1 <--> remote anafa 3 */
+ /* module 2 <--> remote anafa 2 */
+ /* module 3 <--> remote anafa 1 */
+ n->ch_anafanum = (guessnum == 3 ? 1 : (guessnum == 1 ? 3 : 2));
+ } else if (is_spine_2012(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR2012_CT;
+ n->ch_slotnum = line_slot_2_sfb12[spineport->portnum];
+ /* this is a smart guess based on nodeguids order on sFB-12 module */
+ guessnum = spineport->node->guid % 4;
+ // module 1 <--> remote anafa 3
+ // module 2 <--> remote anafa 2
+ // module 3 <--> remote anafa 1
+ n->ch_anafanum = (guessnum == 3? 1 : (guessnum == 1 ? 3 : 2));
+ } else if (is_spine_2004(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR2004_CT;
+ n->ch_slotnum = line_slot_2_sfb4[spineport->portnum];
+ n->ch_anafanum = ipr_slot_2_sfb4_port[spineport->portnum];
+ } else {
+ IBPANIC("Unexpected node found: guid 0x%016" PRIx64,
+ spineport->node->guid);
+ }
+}
+
+static void get_slb_slot(ibnd_node_t *n, ibnd_port_t *spineport)
+{
+ n->ch_slot = LINE_CS;
+ if (is_spine_9096(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR9096_CT;
+ n->ch_slotnum = line_slot_2_sfb4[spineport->portnum];
+ n->ch_anafanum = anafa_line_slot_2_sfb4[spineport->portnum];
+ } else if (is_spine_9288(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR9288_CT;
+ n->ch_slotnum = line_slot_2_sfb12[spineport->portnum];
+ n->ch_anafanum = anafa_line_slot_2_sfb12[spineport->portnum];
+ } else if (is_spine_2012(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR2012_CT;
+ n->ch_slotnum = line_slot_2_sfb12[spineport->portnum];
+ n->ch_anafanum = anafa_line_slot_2_sfb12[spineport->portnum];
+ } else if (is_spine_2004(CONV_NODE_INTERNAL(spineport->node))) {
+ n->ch_type = ISR2004_CT;
+ n->ch_slotnum = line_slot_2_sfb4[spineport->portnum];
+ n->ch_anafanum = anafa_line_slot_2_sfb4[spineport->portnum];
+ } else {
+ IBPANIC("Unexpected node found: guid 0x%016" PRIx64,
+ spineport->node->guid);
+ }
+}
+
+/* forward declare this */
+static void voltaire_portmap(ibnd_port_t *port);
+/*
+ This function called for every Voltaire node in fabric
+ It could be optimized so, but time overhead is very small
+ and its only diag.util
+*/
+static void fill_voltaire_chassis_record(struct ibnd_node *node)
+{
+ ibnd_node_t *n = (ibnd_node_t *)node;
+ int p = 0;
+ ibnd_port_t *port;
+ struct ibnd_node *remnode = 0;
+
+ if (node->ch_found) /* somehow this node has already been passed */
+ return;
+ node->ch_found = 1;
+
+ /* node is router only in case of using unique lid */
+ /* (which is lid of chassis router port) */
+ /* in such case node->ports is actually a requested port... */
+ if (is_router(node)) {
+ /* find the remote node */
+ for (p = 1; p <= node->node.numports; p++) {
+ port = node->node.ports[p];
+ if (port && is_spine(CONV_NODE_INTERNAL(port->remoteport->node)))
+ get_router_slot(node, port->remoteport);
+ }
+ } else if (is_spine(node)) {
+ for (p = 1; p <= node->node.numports; p++) {
+ port = node->node.ports[p];
+ if (!port || !port->remoteport)
+ continue;
+ remnode = CONV_NODE_INTERNAL(port->remoteport->node);
+ if (remnode->node.type != IB_NODE_SWITCH) {
+ if (!remnode->ch_found)
+ get_router_slot(remnode, port);
+ continue;
+ }
+ if (!n->ch_type)
+ /* we assume here that remoteport belongs to line */
+ get_sfb_slot(node, port->remoteport);
+
+ /* we could break here, but need to find if more routers connected */
+ }
+
+ } else if (is_line(node)) {
+ for (p = 1; p <= node->node.numports; p++) {
+ port = node->node.ports[p];
+ if (!port || port->portnum > 12 || !port->remoteport)
+ continue;
+ /* we assume here that remoteport belongs to spine */
+ get_slb_slot(n, port->remoteport);
+ break;
+ }
+ }
+
+ /* for each port of this node, map external ports */
+ for (p = 1; p <= node->node.numports; p++) {
+ port = node->node.ports[p];
+ if (!port)
+ continue;
+ voltaire_portmap(port);
+ }
+
+ return;
+}
+
+static int get_line_index(ibnd_node_t *node)
+{
+ int retval = 3 * (node->ch_slotnum - 1) + node->ch_anafanum;
+
+ if (retval > LINES_MAX_NUM || retval < 1)
+ IBPANIC("Internal error");
+ return retval;
+}
+
+static int get_spine_index(ibnd_node_t *node)
+{
+ int retval;
+
+ if (is_spine_9288(CONV_NODE_INTERNAL(node)) || is_spine_2012(CONV_NODE_INTERNAL(node)))
+ retval = 3 * (node->ch_slotnum - 1) + node->ch_anafanum;
+ else
+ retval = node->ch_slotnum;
+
+ if (retval > SPINES_MAX_NUM || retval < 1)
+ IBPANIC("Internal error");
+ return retval;
+}
+
+static void insert_line_router(ibnd_node_t *node, ibnd_chassis_t *chassis)
+{
+ int i = get_line_index(node);
+
+ if (chassis->linenode[i])
+ return; /* already filled slot */
+
+ chassis->linenode[i] = node;
+ node->chassis = chassis;
+}
+
+static void insert_spine(ibnd_node_t *node, ibnd_chassis_t *chassis)
+{
+ int i = get_spine_index(node);
+
+ if (chassis->spinenode[i])
+ return; /* already filled slot */
+
+ chassis->spinenode[i] = node;
+ node->chassis = chassis;
+}
+
+static void pass_on_lines_catch_spines(ibnd_chassis_t *chassis)
+{
+ ibnd_node_t *node, *remnode;
+ ibnd_port_t *port;
+ int i, p;
+
+ for (i = 1; i <= LINES_MAX_NUM; i++) {
+ node = chassis->linenode[i];
+
+ if (!(node && is_line(CONV_NODE_INTERNAL(node))))
+ continue; /* empty slot or router */
+
+ for (p = 1; p <= node->numports; p++) {
+ port = node->ports[p];
+ if (!port || port->portnum > 12 || !port->remoteport)
+ continue;
+
+ remnode = port->remoteport->node;
+
+ if (!CONV_NODE_INTERNAL(remnode)->ch_found)
+ continue; /* some error - spine not initialized ? FIXME */
+ insert_spine(remnode, chassis);
+ }
+ }
+}
+
+static void pass_on_spines_catch_lines(ibnd_chassis_t *chassis)
+{
+ ibnd_node_t *node, *remnode;
+ ibnd_port_t *port;
+ int i, p;
+
+ for (i = 1; i <= SPINES_MAX_NUM; i++) {
+ node = chassis->spinenode[i];
+ if (!node)
+ continue; /* empty slot */
+ for (p = 1; p <= node->numports; p++) {
+ port = node->ports[p];
+ if (!port || !port->remoteport)
+ continue;
+ remnode = port->remoteport->node;
+
+ if (!CONV_NODE_INTERNAL(remnode)->ch_found)
+ continue; /* some error - line/router not initialized ? FIXME */
+ insert_line_router(remnode, chassis);
+ }
+ }
+}
+
+/*
+ Stupid interpolation algorithm...
+ But nothing to do - have to be compliant with VoltaireSM/NMS
+*/
+static void pass_on_spines_interpolate_chguid(ibnd_chassis_t *chassis)
+{
+ ibnd_node_t *node;
+ int i;
+
+ for (i = 1; i <= SPINES_MAX_NUM; i++) {
+ node = chassis->spinenode[i];
+ if (!node)
+ continue; /* skip the empty slots */
+
+ /* take first guid minus one to be consistent with SM */
+ chassis->chassisguid = node->guid - 1;
+ break;
+ }
+}
+
+/*
+ This function fills chassis structure with all nodes
+ in that chassis
+ chassis structure = structure of one standalone chassis
+*/
+static void build_chassis(struct ibnd_node *node, ibnd_chassis_t *chassis)
+{
+ int p = 0;
+ struct ibnd_node *remnode = 0;
+ ibnd_port_t *port = 0;
+
+ /* we get here with node = chassis_spine */
+ insert_spine((ibnd_node_t *)node, chassis);
+
+ /* loop: pass on all ports of node */
+ for (p = 1; p <= node->node.numports; p++ ) {
+ port = node->node.ports[p];
+ if (!port || !port->remoteport)
+ continue;
+ remnode = CONV_NODE_INTERNAL(port->remoteport->node);
+
+ if (!remnode->ch_found)
+ continue; /* some error - line or router not initialized ? FIXME */
+
+ insert_line_router(&(remnode->node), chassis);
+ }
+
+ pass_on_lines_catch_spines(chassis);
+ /* this pass needed for to catch routers, since routers connected only */
+ /* to spines in slot 1 or 4 and we could miss them first time */
+ pass_on_spines_catch_lines(chassis);
+
+ /* additional 2 passes needed for to overcome a problem of pure "in-chassis" */
+ /* connectivity - extra pass to ensure that all related chips/modules */
+ /* inserted into the chassis */
+ pass_on_lines_catch_spines(chassis);
+ pass_on_spines_catch_lines(chassis);
+ pass_on_spines_interpolate_chguid(chassis);
+}
+
+/*========================================================*/
+/* INTERNAL TO EXTERNAL PORT MAPPING */
+/*========================================================*/
+
+/*
+Description : On ISR9288/9096 external ports indexing
+ is not matching the internal ( anafa ) port
+ indexes. Use this MAP to translate the data you get from
+ the OpenIB diagnostics (smpquery, ibroute, ibtracert, etc.)
+
+
+Module : sLB-24
+ anafa 1 anafa 2
+ext port | 13 14 15 16 17 18 | 19 20 21 22 23 24
+int port | 22 23 24 18 17 16 | 22 23 24 18 17 16
+ext port | 1 2 3 4 5 6 | 7 8 9 10 11 12
+int port | 19 20 21 15 14 13 | 19 20 21 15 14 13
+------------------------------------------------
+
+Module : sLB-8
+ anafa 1 anafa 2
+ext port | 13 14 15 16 17 18 | 19 20 21 22 23 24
+int port | 24 23 22 18 17 16 | 24 23 22 18 17 16
+ext port | 1 2 3 4 5 6 | 7 8 9 10 11 12
+int port | 21 20 19 15 14 13 | 21 20 19 15 14 13
+
+----------->
+ anafa 1 anafa 2
+ext port | - - 5 - - 6 | - - 7 - - 8
+int port | 24 23 22 18 17 16 | 24 23 22 18 17 16
+ext port | - - 1 - - 2 | - - 3 - - 4
+int port | 21 20 19 15 14 13 | 21 20 19 15 14 13
+------------------------------------------------
+
+Module : sLB-2024
+
+ext port | 13 14 15 16 17 18 19 20 21 22 23 24
+A1 int port| 13 14 15 16 17 18 19 20 21 22 23 24
+ext port | 1 2 3 4 5 6 7 8 9 10 11 12
+A2 int port| 13 14 15 16 17 18 19 20 21 22 23 24
+---------------------------------------------------
+
+*/
+
+int int2ext_map_slb24[2][25] = {
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 4, 18, 17, 16, 1, 2, 3, 13, 14, 15 },
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 11, 10, 24, 23, 22, 7, 8, 9, 19, 20, 21 }
+ };
+int int2ext_map_slb8[2][25] = {
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 6, 6, 6, 1, 1, 1, 5, 5, 5 },
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 8, 8, 8, 3, 3, 3, 7, 7, 7 }
+ };
+int int2ext_map_slb2024[2][25] = {
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 },
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
+ };
+/* reference { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 }; */
+
+/* map internal ports to external ports if appropriate */
+static void
+voltaire_portmap(ibnd_port_t *port)
+{
+ struct ibnd_node *n = CONV_NODE_INTERNAL(port->node);
+ int portnum = port->portnum;
+ int chipnum = 0;
+ ibnd_node_t *node = port->node;
+
+ if (!n->ch_found || !is_line(CONV_NODE_INTERNAL(node)) || (portnum < 13 || portnum > 24)) {
+ port->ext_portnum = 0;
+ return;
+ }
+
+ if (port->node->ch_anafanum < 1 || port->node->ch_anafanum > 2) {
+ port->ext_portnum = 0;
+ return;
+ }
+
+ chipnum = port->node->ch_anafanum - 1;
+
+ if (is_line_24(CONV_NODE_INTERNAL(node)))
+ port->ext_portnum = int2ext_map_slb24[chipnum][portnum];
+ else if (is_line_2024(CONV_NODE_INTERNAL(node)))
+ port->ext_portnum = int2ext_map_slb2024[chipnum][portnum];
+ else
+ port->ext_portnum = int2ext_map_slb8[chipnum][portnum];
+}
+
+static void add_chassis(struct ibnd_fabric *fabric)
+{
+ if (!(fabric->current_chassis = calloc(1, sizeof(ibnd_chassis_t))))
+ IBPANIC("out of mem");
+
+ if (fabric->first_chassis == NULL) {
+ fabric->first_chassis = fabric->current_chassis;
+ fabric->last_chassis = fabric->current_chassis;
+ } else {
+ fabric->last_chassis->next = fabric->current_chassis;
+ fabric->last_chassis = fabric->current_chassis;
+ }
+}
+
+static void
+add_node_to_chassis(ibnd_chassis_t *chassis, ibnd_node_t *node)
+{
+ node->chassis = chassis;
+ node->next_chassis_node = chassis->nodes;
+ chassis->nodes = node;
+}
+
+/*
+ Main grouping function
+ Algorithm:
+ 1. pass on every Voltaire node
+ 2. catch spine chip for every Voltaire node
+ 2.1 build/interpolate chassis around this chip
+ 2.2 go to 1.
+ 3. pass on non Voltaire nodes (SystemImageGUID based grouping)
+ 4. now group non Voltaire nodes by SystemImageGUID
+ Returns:
+ Pointer to the first chassis in a NULL terminated list of chassis in
+ the fabric specified.
+*/
+ibnd_chassis_t *group_nodes(struct ibnd_fabric *fabric)
+{
+ struct ibnd_node *node;
+ int dist;
+ int chassisnum = 0;
+ ibnd_chassis_t *chassis;
+
+ fabric->first_chassis = NULL;
+ fabric->current_chassis = NULL;
+
+ /* first pass on switches and build for every Voltaire node */
+ /* an appropriate chassis record (slotnum and position) */
+ /* according to internal connectivity */
+ /* not very efficient but clear code so... */
+ for (dist = 0; dist <= fabric->fabric.maxhops_discovered; dist++) {
+ for (node = fabric->nodesdist[dist]; node; node = node->dnext) {
+ if (mad_get_field(node->node.info, 0, IB_NODE_VENDORID_F) == VTR_VENDOR_ID)
+ fill_voltaire_chassis_record(node);
+ }
+ }
+
+ /* separate every Voltaire chassis from each other and build linked list of them */
+ /* algorithm: catch spine and find all surrounding nodes */
+ for (dist = 0; dist <= fabric->fabric.maxhops_discovered; dist++) {
+ for (node = fabric->nodesdist[dist]; node; node = node->dnext) {
+ if (mad_get_field(node->node.info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID)
+ continue;
+ //if (!node->node.chrecord || node->node.chrecord->chassisnum || !is_spine(node))
+ if (!node->ch_found
+ || (node->node.chassis && node->node.chassis->chassisnum)
+ || !is_spine(node))
+ continue;
+ add_chassis(fabric);
+ fabric->current_chassis->chassisnum = ++chassisnum;
+ build_chassis(node, fabric->current_chassis);
+ }
+ }
+
+ /* now make pass on nodes for chassis which are not Voltaire */
+ /* grouped by common SystemImageGUID */
+ for (dist = 0; dist <= fabric->fabric.maxhops_discovered; dist++) {
+ for (node = fabric->nodesdist[dist]; node; node = node->dnext) {
+ if (mad_get_field(node->node.info, 0, IB_NODE_VENDORID_F) == VTR_VENDOR_ID)
+ continue;
+ if (mad_get_field64(node->node.info, 0, IB_NODE_SYSTEM_GUID_F)) {
+ chassis = find_chassisguid((ibnd_node_t *)node);
+ if (chassis)
+ chassis->nodecount++;
+ else {
+ /* Possible new chassis */
+ add_chassis(fabric);
+ fabric->current_chassis->chassisguid =
+ get_chassisguid((ibnd_node_t *)node);
+ fabric->current_chassis->nodecount = 1;
+ }
+ }
+ }
+ }
+
+ /* now, make another pass to see which nodes are part of chassis */
+ /* (defined as chassis->nodecount > 1) */
+ for (dist = 0; dist <= MAXHOPS; ) {
+ for (node = fabric->nodesdist[dist]; node; node = node->dnext) {
+ if (mad_get_field(node->node.info, 0, IB_NODE_VENDORID_F) == VTR_VENDOR_ID)
+ continue;
+ if (mad_get_field64(node->node.info, 0, IB_NODE_SYSTEM_GUID_F)) {
+ chassis = find_chassisguid((ibnd_node_t *)node);
+ if (chassis && chassis->nodecount > 1) {
+ if (!chassis->chassisnum)
+ chassis->chassisnum = ++chassisnum;
+ if (!node->ch_found) {
+ node->ch_found = 1;
+ add_node_to_chassis(chassis, (ibnd_node_t *)node);
+ }
+ }
+ }
+ }
+ if (dist == fabric->fabric.maxhops_discovered)
+ dist = MAXHOPS; /* skip to CAs */
+ else
+ dist++;
+ }
+
+ return (fabric->first_chassis);
+}
--- /dev/null
+/*
+ * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2007 Xsigo Systems Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef _CHASSIS_H_
+#define _CHASSIS_H_
+
+#include <infiniband/ibnetdisc.h>
+
+#include "internal.h"
+
+/*========================================================*/
+/* CHASSIS RECOGNITION SPECIFIC DATA */
+/*========================================================*/
+
+/* Device IDs */
+#define VTR_DEVID_IB_FC_ROUTER 0x5a00
+#define VTR_DEVID_IB_IP_ROUTER 0x5a01
+#define VTR_DEVID_ISR9600_SPINE 0x5a02
+#define VTR_DEVID_ISR9600_LEAF 0x5a03
+#define VTR_DEVID_HCA1 0x5a04
+#define VTR_DEVID_HCA2 0x5a44
+#define VTR_DEVID_HCA3 0x6278
+#define VTR_DEVID_SW_6IB4 0x5a05
+#define VTR_DEVID_ISR9024 0x5a06
+#define VTR_DEVID_ISR9288 0x5a07
+#define VTR_DEVID_SLB24 0x5a09
+#define VTR_DEVID_SFB12 0x5a08
+#define VTR_DEVID_SFB4 0x5a0b
+#define VTR_DEVID_ISR9024_12 0x5a0c
+#define VTR_DEVID_SLB8 0x5a0d
+#define VTR_DEVID_RLX_SWITCH_BLADE 0x5a20
+#define VTR_DEVID_ISR9024_DDR 0x5a31
+#define VTR_DEVID_SFB12_DDR 0x5a32
+#define VTR_DEVID_SFB4_DDR 0x5a33
+#define VTR_DEVID_SLB24_DDR 0x5a34
+#define VTR_DEVID_SFB2012 0x5a37
+#define VTR_DEVID_SLB2024 0x5a38
+#define VTR_DEVID_ISR2012 0x5a39
+#define VTR_DEVID_SFB2004 0x5a40
+#define VTR_DEVID_ISR2004 0x5a41
+#define VTR_DEVID_SRB2004 0x5a42
+
+/* Vendor IDs (for chassis based systems) */
+#define VTR_VENDOR_ID 0x8f1 /* Voltaire */
+#define TS_VENDOR_ID 0x5ad /* Cisco */
+#define SS_VENDOR_ID 0x66a /* InfiniCon */
+#define XS_VENDOR_ID 0x1397 /* Xsigo */
+
+enum ibnd_chassis_type { UNRESOLVED_CT, ISR9288_CT, ISR9096_CT, ISR2012_CT, ISR2004_CT };
+enum ibnd_chassis_slot_type { UNRESOLVED_CS, LINE_CS, SPINE_CS, SRBD_CS };
+
+ibnd_chassis_t *group_nodes(struct ibnd_fabric *fabric);
+
+#endif /* _CHASSIS_H_ */
--- /dev/null
+/*
+ * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2007 Xsigo Systems Inc. All rights reserved.
+ * Copyright (c) 2008 Lawrence Livermore National Laboratory
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <time.h>
+#include <string.h>
+#include <getopt.h>
+#include <errno.h>
+#include <inttypes.h>
+
+#include <infiniband/umad.h>
+#include <infiniband/mad.h>
+
+#include <infiniband/ibnetdisc.h>
+#include <complib/cl_nodenamemap.h>
+
+#include "internal.h"
+#include "chassis.h"
+
+static int timeout_ms = 2000;
+static int show_progress = 0;
+int ibdebug;
+
+void
+decode_port_info(ibnd_port_t *port)
+{
+ port->base_lid = (uint16_t) mad_get_field(port->info, 0, IB_PORT_LID_F);
+ port->lmc = (uint8_t) mad_get_field(port->info, 0, IB_PORT_LMC_F);
+}
+
+static int
+get_port_info(struct ibnd_fabric *fabric, struct ibnd_port *port,
+ int portnum, ib_portid_t *portid)
+{
+ char width[64], speed[64];
+ int iwidth;
+ int ispeed;
+
+ port->port.portnum = portnum;
+ iwidth = mad_get_field(port->port.info, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
+ ispeed = mad_get_field(port->port.info, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
+
+ if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, timeout_ms,
+ fabric->ibmad_port))
+ return -1;
+
+ decode_port_info(&(port->port));
+
+ IBND_DEBUG("portid %s portnum %d: base lid %d state %d physstate %d %s %s\n",
+ portid2str(portid), portnum, port->port.base_lid,
+ mad_get_field(port->port.info, 0, IB_PORT_STATE_F),
+ mad_get_field(port->port.info, 0, IB_PORT_PHYS_STATE_F),
+ mad_dump_val(IB_PORT_LINK_WIDTH_ACTIVE_F, width, 64, &iwidth),
+ mad_dump_val(IB_PORT_LINK_SPEED_ACTIVE_F, speed, 64, &ispeed));
+ return 0;
+}
+
+/*
+ * Returns -1 if error.
+ */
+static int
+query_node_info(struct ibnd_fabric *fabric, struct ibnd_node *node, ib_portid_t *portid)
+{
+ if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, timeout_ms,
+ fabric->ibmad_port))
+ return -1;
+
+ /* decode just a couple of fields for quicker reference. */
+ mad_decode_field(node->node.info, IB_NODE_GUID_F, &(node->node.guid));
+ mad_decode_field(node->node.info, IB_NODE_TYPE_F, &(node->node.type));
+ mad_decode_field(node->node.info, IB_NODE_NPORTS_F,
+ &(node->node.numports));
+
+ return (0);
+}
+
+/*
+ * Returns 0 if non switch node is found, 1 if switch is found, -1 if error.
+ */
+static int
+query_node(struct ibnd_fabric *fabric, struct ibnd_node *inode,
+ struct ibnd_port *iport, ib_portid_t *portid)
+{
+ ibnd_node_t *node = &(inode->node);
+ ibnd_port_t *port = &(iport->port);
+ void *nd = inode->node.nodedesc;
+
+ if (query_node_info(fabric, inode, portid))
+ return -1;
+
+ port->portnum = mad_get_field(node->info, 0, IB_NODE_LOCAL_PORT_F);
+ port->guid = mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F);
+
+ if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, timeout_ms,
+ fabric->ibmad_port))
+ return -1;
+
+ if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, timeout_ms,
+ fabric->ibmad_port))
+ return -1;
+ decode_port_info(port);
+
+ if (node->type != IB_NODE_SWITCH)
+ return 0;
+
+ node->smalid = port->base_lid;
+ node->smalmc = port->lmc;
+
+ /* after we have the sma information find out the real PortInfo for this port */
+ if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, timeout_ms,
+ fabric->ibmad_port))
+ return -1;
+ decode_port_info(port);
+
+ port->base_lid = (uint16_t) node->smalid; /* LID is still defined by port 0 */
+ port->lmc = (uint8_t) node->smalmc;
+
+ if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, timeout_ms,
+ fabric->ibmad_port))
+ node->smaenhsp0 = 0; /* assume base SP0 */
+ else
+ mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F, &node->smaenhsp0);
+
+ IBND_DEBUG("portid %s: got switch node %" PRIx64 " '%s'\n",
+ portid2str(portid), node->guid, node->nodedesc);
+ return 0;
+}
+
+static int
+add_port_to_dpath(ib_dr_path_t *path, int nextport)
+{
+ if (path->cnt+2 >= sizeof(path->p))
+ return -1;
+ ++path->cnt;
+ path->p[path->cnt] = (uint8_t) nextport;
+ return path->cnt;
+}
+
+static int
+extend_dpath(struct ibnd_fabric *f, ib_dr_path_t *path, int nextport)
+{
+ int rc = add_port_to_dpath(path, nextport);
+ if ((rc != -1) && (path->cnt > f->fabric.maxhops_discovered))
+ f->fabric.maxhops_discovered = path->cnt;
+ return (rc);
+}
+
+static void
+dump_endnode(ib_portid_t *path, char *prompt,
+ struct ibnd_node *node, struct ibnd_port *port)
+{
+ char type[64];
+ if (!show_progress)
+ return;
+
+ mad_dump_node_type(type, 64, &(node->node.type), sizeof(int)),
+
+ printf("%s -> %s %s {%016" PRIx64 "} portnum %d base lid %d-%d\"%s\"\n",
+ portid2str(path), prompt, type,
+ node->node.guid,
+ node->node.type == IB_NODE_SWITCH ? 0 : port->port.portnum,
+ port->port.base_lid, port->port.base_lid + (1 << port->port.lmc) - 1,
+ node->node.nodedesc);
+}
+
+static struct ibnd_node *
+find_existing_node(struct ibnd_fabric *fabric, struct ibnd_node *new)
+{
+ int hash = HASHGUID(new->node.guid) % HTSZ;
+ struct ibnd_node *node;
+
+ for (node = fabric->nodestbl[hash]; node; node = node->htnext)
+ if (node->node.guid == new->node.guid)
+ return node;
+
+ return NULL;
+}
+
+ibnd_node_t *
+ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid)
+{
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric);
+ int hash = HASHGUID(guid) % HTSZ;
+ struct ibnd_node *node;
+
+ for (node = f->nodestbl[hash]; node; node = node->htnext)
+ if (node->node.guid == guid)
+ return (ibnd_node_t *)node;
+
+ return NULL;
+}
+
+ibnd_node_t *
+ibnd_update_node(ibnd_node_t *node)
+{
+ char portinfo_port0[sizeof (ib_port_info_t)];
+ void *nd = node->nodedesc;
+ int p = 0;
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(node->fabric);
+ struct ibnd_node *n = CONV_NODE_INTERNAL(node);
+
+ if (query_node_info(f, n, &(n->node.path_portid)))
+ return (NULL);
+
+ if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, timeout_ms,
+ f->ibmad_port))
+ return (NULL);
+
+ /* update all the port info's */
+ for (p = 1; p >= n->node.numports; p++) {
+ get_port_info(f, CONV_PORT_INTERNAL(n->node.ports[p]), p, &(n->node.path_portid));
+ }
+
+ if (n->node.type != IB_NODE_SWITCH)
+ goto done;
+
+ if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, timeout_ms,
+ f->ibmad_port))
+ return (NULL);
+
+ n->node.smalid = mad_get_field(portinfo_port0, 0, IB_PORT_LID_F);
+ n->node.smalmc = mad_get_field(portinfo_port0, 0, IB_PORT_LMC_F);
+
+ if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, timeout_ms,
+ f->ibmad_port))
+ node->smaenhsp0 = 0; /* assume base SP0 */
+ else
+ mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F, &n->node.smaenhsp0);
+
+done:
+ return (node);
+}
+
+ibnd_node_t *
+ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str)
+{
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric);
+ int i = 0;
+ ibnd_node_t *rc = f->fabric.from_node;
+ ib_dr_path_t path;
+
+ if (str2drpath(&path, dr_str, 0, 0) == -1) {
+ return (NULL);
+ }
+
+ for (i = 0; i <= path.cnt; i++) {
+ ibnd_port_t *remote_port = NULL;
+ if (path.p[i] == 0)
+ continue;
+ if (!rc->ports)
+ return (NULL);
+
+ remote_port = rc->ports[path.p[i]]->remoteport;
+ if (!remote_port)
+ return (NULL);
+
+ rc = remote_port->node;
+ }
+
+ return (rc);
+}
+
+static void
+add_to_nodeguid_hash(struct ibnd_node *node, struct ibnd_node *hash[])
+{
+ int hash_idx = HASHGUID(node->node.guid) % HTSZ;
+
+ node->htnext = hash[hash_idx];
+ hash[hash_idx] = node;
+}
+
+static void
+add_to_portguid_hash(struct ibnd_port *port, struct ibnd_port *hash[])
+{
+ int hash_idx = HASHGUID(port->port.guid) % HTSZ;
+
+ port->htnext = hash[hash_idx];
+ hash[hash_idx] = port;
+}
+
+static void
+add_to_type_list(struct ibnd_node*node, struct ibnd_fabric *fabric)
+{
+ switch (node->node.type) {
+ case IB_NODE_CA:
+ node->type_next = fabric->ch_adapters;
+ fabric->ch_adapters = node;
+ break;
+ case IB_NODE_SWITCH:
+ node->type_next = fabric->switches;
+ fabric->switches = node;
+ break;
+ case IB_NODE_ROUTER:
+ node->type_next = fabric->routers;
+ fabric->routers = node;
+ break;
+ }
+}
+
+static void
+add_to_nodedist(struct ibnd_node *node, struct ibnd_fabric *fabric)
+{
+ int dist = node->node.dist;
+ if (node->node.type != IB_NODE_SWITCH)
+ dist = MAXHOPS; /* special Ca list */
+
+ node->dnext = fabric->nodesdist[dist];
+ fabric->nodesdist[dist] = node;
+}
+
+
+static struct ibnd_node *
+create_node(struct ibnd_fabric *fabric, struct ibnd_node *temp, ib_portid_t *path, int dist)
+{
+ struct ibnd_node *node;
+
+ node = malloc(sizeof(*node));
+ if (!node) {
+ IBPANIC("OOM: node creation failed\n");
+ return NULL;
+ }
+
+ memcpy(node, temp, sizeof(*node));
+ node->node.dist = dist;
+ node->node.path_portid = *path;
+ node->node.fabric = (ibnd_fabric_t *)fabric;
+
+ add_to_nodeguid_hash(node, fabric->nodestbl);
+
+ /* add this to the all nodes list */
+ node->node.next = fabric->fabric.nodes;
+ fabric->fabric.nodes = (ibnd_node_t *)node;
+
+ add_to_type_list(node, fabric);
+ add_to_nodedist(node, fabric);
+
+ return node;
+}
+
+static struct ibnd_port *
+find_existing_port_node(struct ibnd_node *node, struct ibnd_port *port)
+{
+ if (port->port.portnum > node->node.numports || node->node.ports == NULL )
+ return (NULL);
+
+ return (CONV_PORT_INTERNAL(node->node.ports[port->port.portnum]));
+}
+
+static struct ibnd_port *
+add_port_to_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_port *temp)
+{
+ struct ibnd_port *port;
+
+ port = malloc(sizeof(*port));
+ if (!port)
+ return NULL;
+
+ memcpy(port, temp, sizeof(*port));
+ port->port.node = (ibnd_node_t *)node;
+ port->port.ext_portnum = 0;
+
+ if (node->node.ports == NULL) {
+ node->node.ports = calloc(sizeof(*node->node.ports), node->node.numports + 1);
+ if (!node->node.ports) {
+ IBND_ERROR("Failed to allocate the ports array\n");
+ return (NULL);
+ }
+ }
+
+ node->node.ports[temp->port.portnum] = (ibnd_port_t *)port;
+
+ add_to_portguid_hash(port, fabric->portstbl);
+ return port;
+}
+
+static void
+link_ports(struct ibnd_node *node, struct ibnd_port *port,
+ struct ibnd_node *remotenode, struct ibnd_port *remoteport)
+{
+ IBND_DEBUG("linking: 0x%" PRIx64 " %p->%p:%u and 0x%" PRIx64 " %p->%p:%u\n",
+ node->node.guid, node, port, port->port.portnum,
+ remotenode->node.guid, remotenode,
+ remoteport, remoteport->port.portnum);
+ if (port->port.remoteport)
+ port->port.remoteport->remoteport = NULL;
+ if (remoteport->port.remoteport)
+ remoteport->port.remoteport->remoteport = NULL;
+ port->port.remoteport = (ibnd_port_t *)remoteport;
+ remoteport->port.remoteport = (ibnd_port_t *)port;
+}
+
+static int
+get_remote_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_port *port, ib_portid_t *path,
+ int portnum, int dist)
+{
+ struct ibnd_node node_buf;
+ struct ibnd_port port_buf;
+ struct ibnd_node *remotenode, *oldnode;
+ struct ibnd_port *remoteport, *oldport;
+
+ memset(&node_buf, 0, sizeof(node_buf));
+ memset(&port_buf, 0, sizeof(port_buf));
+
+ IBND_DEBUG("handle node %p port %p:%d dist %d\n", node, port, portnum, dist);
+
+ if (mad_get_field(port->port.info, 0, IB_PORT_PHYS_STATE_F)
+ != IB_PORT_PHYS_STATE_LINKUP)
+ return -1;
+
+ if (extend_dpath(fabric, &path->drpath, portnum) < 0)
+ return -1;
+
+ if (query_node(fabric, &node_buf, &port_buf, path)) {
+ IBWARN("NodeInfo on %s failed, skipping port",
+ portid2str(path));
+ path->drpath.cnt--; /* restore path */
+ return -1;
+ }
+
+ oldnode = find_existing_node(fabric, &node_buf);
+ if (oldnode)
+ remotenode = oldnode;
+ else if (!(remotenode = create_node(fabric, &node_buf, path, dist + 1)))
+ IBPANIC("no memory");
+
+ oldport = find_existing_port_node(remotenode, &port_buf);
+ if (oldport) {
+ remoteport = oldport;
+ } else if (!(remoteport = add_port_to_node(fabric, remotenode, &port_buf)))
+ IBPANIC("no memory");
+
+ dump_endnode(path, oldnode ? "known remote" : "new remote",
+ remotenode, remoteport);
+
+ link_ports(node, port, remotenode, remoteport);
+
+ path->drpath.cnt--; /* restore path */
+ return 0;
+}
+
+static void *
+ibnd_init_port(char *dev_name, int dev_port)
+{
+ int mgmt_classes[2] = {IB_SMI_CLASS, IB_SMI_DIRECT_CLASS};
+
+ /* Crank up the mad lib */
+ return (mad_rpc_open_port(dev_name, dev_port, mgmt_classes, 2));
+}
+
+ibnd_fabric_t *
+ibnd_discover_fabric(char *dev_name, int dev_port, int timeout_ms,
+ ib_portid_t *from, int hops)
+{
+ struct ibnd_fabric *fabric = NULL;
+ ib_portid_t my_portid = {0};
+ struct ibnd_node node_buf;
+ struct ibnd_port port_buf;
+ struct ibnd_node *node;
+ struct ibnd_port *port;
+ int i;
+ int dist = 0;
+ ib_portid_t *path;
+ int max_hops = MAXHOPS-1; /* default find everything */
+
+ /* if not everything how much? */
+ if (hops >= 0) {
+ max_hops = hops;
+ }
+
+ /* If not specified start from "my" port */
+ if (!from) {
+ from = &my_portid;
+ }
+
+ fabric = malloc(sizeof(*fabric));
+
+ if (!fabric) {
+ IBPANIC("OOM: failed to malloc ibnd_fabric_t\n");
+ return (NULL);
+ }
+
+ memset(fabric, 0, sizeof(*fabric));
+
+ fabric->ibmad_port = ibnd_init_port(dev_name, dev_port);
+ if (!fabric->ibmad_port) {
+ IBPANIC("OOM: failed to open \"%s\" port %d\n",
+ dev_name, dev_port);
+ goto error;
+ }
+
+ IBND_DEBUG("from %s\n", portid2str(from));
+
+ memset(&node_buf, 0, sizeof(node_buf));
+ memset(&port_buf, 0, sizeof(port_buf));
+
+ if (query_node(fabric, &node_buf, &port_buf, from)) {
+ IBWARN("can't reach node %s\n", portid2str(from));
+ goto error;
+ }
+
+ node = create_node(fabric, &node_buf, from, 0);
+ if (!node)
+ goto error;
+
+ fabric->fabric.from_node = (ibnd_node_t *)node;
+
+ port = add_port_to_node(fabric, node, &port_buf);
+ if (!port)
+ IBPANIC("out of memory");
+
+ if (node->node.type != IB_NODE_SWITCH &&
+ get_remote_node(fabric, node, port, from,
+ mad_get_field(node->node.info, 0, IB_NODE_LOCAL_PORT_F),
+ 0) < 0)
+ return ((ibnd_fabric_t *)fabric);
+
+ for (dist = 0; dist <= max_hops; dist++) {
+
+ for (node = fabric->nodesdist[dist]; node; node = node->dnext) {
+
+ path = &node->node.path_portid;
+
+ IBND_DEBUG("dist %d node %p\n", dist, node);
+ dump_endnode(path, "processing", node, port);
+
+ for (i = 1; i <= node->node.numports; i++) {
+ if (i == mad_get_field(node->node.info, 0,
+ IB_NODE_LOCAL_PORT_F))
+ continue;
+
+ if (get_port_info(fabric, &port_buf, i, path)) {
+ IBWARN("can't reach node %s port %d", portid2str(path), i);
+ continue;
+ }
+
+ port = find_existing_port_node(node, &port_buf);
+ if (port)
+ continue;
+
+ port = add_port_to_node(fabric, node, &port_buf);
+ if (!port)
+ IBPANIC("out of memory");
+
+ /* If switch, set port GUID to node port GUID */
+ if (node->node.type == IB_NODE_SWITCH) {
+ port->port.guid = mad_get_field64(node->node.info,
+ 0, IB_NODE_PORT_GUID_F);
+ }
+
+ get_remote_node(fabric, node, port, path, i, dist);
+ }
+ }
+ }
+
+ fabric->fabric.chassis = group_nodes(fabric);
+
+ return ((ibnd_fabric_t *)fabric);
+error:
+ free(fabric);
+ return (NULL);
+}
+
+static void
+destroy_node(struct ibnd_node *node)
+{
+ int p = 0;
+
+ for (p = 0; p <= node->node.numports; p++) {
+ free(node->node.ports[p]);
+ }
+ free(node->node.ports);
+ free(node);
+}
+
+void
+ibnd_destroy_fabric(ibnd_fabric_t *fabric)
+{
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric);
+ int dist = 0;
+ struct ibnd_node *node = NULL;
+ struct ibnd_node *next = NULL;
+ ibnd_chassis_t *ch, *ch_next;
+
+ ch = f->first_chassis;
+ while (ch) {
+ ch_next = ch->next;
+ free(ch);
+ ch = ch_next;
+ }
+ for (dist = 0; dist <= MAXHOPS; dist++) {
+ node = f->nodesdist[dist];
+ while (node) {
+ next = node->dnext;
+ destroy_node(node);
+ node = next;
+ }
+ }
+ if (f->ibmad_port)
+ mad_rpc_close_port(f->ibmad_port);
+ free(f);
+}
+
+void
+ibnd_debug(int i)
+{
+ if (i) {
+ ibdebug++;
+ madrpc_show_errors(1);
+ umad_debug(i);
+ } else {
+ ibdebug = 0;
+ madrpc_show_errors(0);
+ umad_debug(0);
+ }
+}
+
+void
+ibnd_show_progress(int i)
+{
+ show_progress = i;
+}
+
+void
+ibnd_iter_nodes(ibnd_fabric_t *fabric,
+ ibnd_iter_node_func_t func,
+ void *user_data)
+{
+ ibnd_node_t *cur = NULL;
+
+ for (cur = fabric->nodes; cur; cur = cur->next) {
+ func(cur, user_data);
+ }
+}
+
+
+void
+ibnd_iter_nodes_type(ibnd_fabric_t *fabric,
+ ibnd_iter_node_func_t func,
+ int node_type,
+ void *user_data)
+{
+ struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric);
+ struct ibnd_node *list = NULL;
+ struct ibnd_node *cur = NULL;
+
+ switch (node_type) {
+ case IB_NODE_SWITCH:
+ list = f->switches;
+ break;
+ case IB_NODE_CA:
+ list = f->ch_adapters;
+ break;
+ case IB_NODE_ROUTER:
+ list = f->routers;
+ break;
+ default:
+ IBND_DEBUG("Invalid node_type specified %d\n", node_type);
+ break;
+ }
+
+ for (cur = list; cur; cur = cur->type_next) {
+ func((ibnd_node_t *)cur, user_data);
+ }
+}
+
--- /dev/null
+/*\r
+ * Copyright (c) 2008-2009 Intel Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+LIBRARY LIBIBNETDISC.DLL\r
+\r
+EXPORTS\r
+ DllCanUnloadNow PRIVATE\r
+ DllGetClassObject PRIVATE\r
--- /dev/null
+#if DBG\r
+LIBRARY libibnetdiscd.dll\r
+#else\r
+LIBRARY libibnetdisc.dll\r
+#endif\r
+\r
+#ifndef _WIN64\r
+EXPORTS\r
+ ibnd_debug;\r
+ ibnd_show_progress;\r
+ ibnd_discover_fabric;\r
+ ibnd_destroy_fabric;\r
+ ibnd_find_node_guid;\r
+ ibnd_update_node;\r
+ ibnd_find_node_dr;\r
+ ibnd_is_xsigo_guid;\r
+ ibnd_is_xsigo_tca;\r
+ ibnd_is_xsigo_hca;\r
+ ibnd_get_chassis_guid;\r
+ ibnd_get_chassis_type;\r
+ ibnd_get_chassis_slot_str;\r
+ ibnd_iter_nodes;\r
+ ibnd_iter_nodes_type;\r
+#endif\r
--- /dev/null
+/*\r
+ * Copyright (c) 2008-2009 Intel Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include <windows.h>\r
+\r
+BOOLEAN WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)\r
+{\r
+ UNREFERENCED_PARAMETER(hInstance);\r
+ UNREFERENCED_PARAMETER(dwReason);\r
+ UNREFERENCED_PARAMETER(lpReserved);\r
+\r
+ return TRUE;\r
+}\r
--- /dev/null
+/*
+ * Copyright (c) 2008 Lawrence Livermore National Laboratory
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+/** =========================================================================
+ * Define the internal data structures.
+ */
+
+#ifndef _INTERNAL_H_
+#define _INTERNAL_H_
+
+#include <infiniband/ibnetdisc.h>
+
+#define MAXHOPS 63
+
+#define IBND_DEBUG(fmt, ...) \
+ if (ibdebug) { \
+ printf("%s:%u; " fmt, __FILE__, __LINE__, ## __VA_ARGS__); \
+ }
+#define IBND_ERROR(fmt, ...) \
+ fprintf(stderr, "%s:%u; " fmt, __FILE__, __LINE__, ## __VA_ARGS__)
+
+struct ibnd_node {
+ /* This member MUST BE FIRST */
+ ibnd_node_t node;
+
+ /* internal use only */
+ unsigned char ch_found;
+ struct ibnd_node *htnext; /* hash table list */
+ struct ibnd_node *dnext; /* nodesdist next */
+ struct ibnd_node *type_next; /* next based on type */
+};
+#define CONV_NODE_INTERNAL(node) ((struct ibnd_node *)node)
+
+struct ibnd_port {
+ /* This member MUST BE FIRST */
+ ibnd_port_t port;
+
+ /* internal use only */
+ struct ibnd_port *htnext;
+};
+#define CONV_PORT_INTERNAL(port) ((struct ibnd_port *)port)
+
+/* HASH table defines */
+#define HASHGUID(guid) ((uint32_t)(((uint32_t)(guid) * 101) ^ ((uint32_t)((guid) >> 32) * 103)))
+#define HTSZ 137
+
+struct ibnd_fabric {
+ /* This member MUST BE FIRST */
+ ibnd_fabric_t fabric;
+
+ /* internal use only */
+ void *ibmad_port;
+ struct ibnd_node *nodestbl[HTSZ];
+ struct ibnd_port *portstbl[HTSZ];
+ struct ibnd_node *nodesdist[MAXHOPS+1];
+ ibnd_chassis_t *first_chassis;
+ ibnd_chassis_t *current_chassis;
+ ibnd_chassis_t *last_chassis;
+ struct ibnd_node *switches;
+ struct ibnd_node *ch_adapters;
+ struct ibnd_node *routers;
+};
+#define CONV_FABRIC_INTERNAL(fabric) ((struct ibnd_fabric *)fabric)
+
+#endif /* _INTERNAL_H_ */
--- /dev/null
+#\r
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source\r
+# file to this component. This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\inc\openib.def\r
#define VER_FILESUBTYPE VFT2_UNKNOWN\r
\r
#ifdef _DEBUG_\r
-#define VER_FILEDESCRIPTION_STR "OpenFabrics Network Direct Provider (Debug)"\r
+#define VER_FILEDESCRIPTION_STR "OpenFabrics Network Direct Infiniband Provider (Debug)"\r
#define VER_INTERNALNAME_STR "ibndprov.dll"\r
#define VER_ORIGINALFILENAME_STR "ibndprov.dll"\r
#else\r
-#define VER_FILEDESCRIPTION_STR "OpenFabrics Network Direct Provider"\r
+#define VER_FILEDESCRIPTION_STR "OpenFabrics Network Direct Infiniband Provider"\r
#define VER_INTERNALNAME_STR "ibndprov.dll"\r
#define VER_ORIGINALFILENAME_STR "ibndprov.dll"\r
#endif\r
\r
+[04-23-09] stan\r
+ Network Direct InfiniBand provider sources building here, no more fake builds.\r
+ ND builds require\r
+ MS WIndows SDK v6.1\r
+ Server 2008 HPC SDK (defines ND_INC env var in WDK build env)\r
+ If HPC SDK not present then all arch builds for ND are skipped, comment in .log file.\r
+ If HPC SKD present then build for ND provider for x86 & x64, no IA64 until ND over winverbs.\r
+\r
[10-22-08]\r
Latest MS ND source (delivered to Qlogic 10-22-08 from Microsoft). \r
ND binaries (ndinstall.exe + ndibprov.dll) generated from WinOF svn.1684 tree by Alex Estrin.\r
-TARGETNAME=fake\r
+TARGETNAME=ibndprov\r
+\r
TARGETPATH=..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
-TARGETTYPE=PROGRAM\r
-UMTYPE=console\r
-USE_MSVCRT=1\r
+TARGETTYPE=DYNLINK\r
+DLLDEF = $(OBJ_PATH)\$O\NdProv.def\r
+DLLENTRY=DllMain\r
+\r
+!if $(FREEBUILD)\r
+ENABLE_EVENT_TRACING=1\r
+#!else\r
+#ENABLE_EVENT_TRACING=1\r
+!endif\r
+\r
+USE_NTDLL=1\r
+\r
+# MS HPC 2008 SDK required to build!\r
+# ND IB Provider is skipped if not installed - see makefile\r
+ND_INC_S=C:\PROGRA~1\MICROS~4\NetworkDirect\Include\r
+PLATFORM_SDK_PATH_S=C:\PROGRA~1\MICROS~3\Windows\v6.1\r
+\r
+SOURCES= \\r
+ NdProv.rc \\r
+ NdAdapter.cpp \\r
+ NdCq.cpp \\r
+ NdEndpoint.cpp \\r
+ NdListen.cpp \\r
+ NdMr.cpp \\r
+ NdMw.cpp \\r
+ NdProv.cpp \\r
+ NdConnector.cpp\r
+\r
+\r
+INCLUDES=$(SDK_INC_PATH);..;..\..\..\inc;..\..\..\inc\user;..\..\..\core\al;..\..\..\core\al\user;$(ND_INC_S);$(PLATFORM_SDK_PATH_S)\Include\r
+\r
+USER_C_FLAGS=$(USER_C_FLAGS) -DEXPORT_AL_SYMBOLS -DCL_NO_TRACK_MEM -DWPP_OLDCC\r
+\r
+#/GL\r
+#LINKER_FLAGS=$(LINKER_FLAGS) /LTCG\r
+\r
+TARGETLIBS= \\r
+ $(SDK_LIB_PATH)\Kernel32.lib\\r
+ $(SDK_LIB_PATH)\Advapi32.lib\\r
+ $(SDK_LIB_PATH)\ws2_32.lib \\r
+ $(SDK_LIB_PATH)\iphlpapi.lib \\r
+ $(TARGETPATH)\*\ibat.lib \\r
+!if $(FREEBUILD)\r
+ $(TARGETPATH)\*\complib.lib \\r
+!else\r
+ $(TARGETPATH)\*\complibd.lib\\r
+!endif\r
+ $(SDK_LIB_PATH)\uuid.lib\r
+\r
\r
-SOURCES=fake.c\r
+!IFDEF ENABLE_EVENT_TRACING\r
\r
-INCLUDES=..;..\..\..\inc;..\..\..\inc\kernel;\r
+C_DEFINES = $(C_DEFINES) -DEVENT_TRACING\r
\r
-ND_TARGET1=..\..\..\bin\user\$(O)\ibndprov.dll\r
-ND_TARGET2=..\..\..\bin\user\$(O)\ndinstall.exe\r
+RUN_WPP = $(SOURCES) -ext: .c .h .C .H \\r
+ -scan:nddebug.h \\r
+ -func:ND_PRINT(LEVEL,FLAGS,(MSG,...)) \\r
+ -func:ND_PRINT_EXIT(LEVEL,FLAGS,(MSG,...)) \\r
+ -dll\r
+!ENDIF\r
\r
-NTTARGETFILES=$(ND_TARGET1) $(ND_TARGET2)\r
+BUILD_PRODUCES=NetworkDirect\r
\r
+MSC_WARNING_LEVEL= /W4\r
#\r
-# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source\r
+# Edit .\sources. if you want to add a new source\r
# file to this component. This file merely indirects to the real make file\r
# that is shared by all the driver components of the OpenIB Windows project.\r
#\r
\r
+# If ND SDK installed then Defeat IA64 builds\r
+# Otherwise, default all builds.\r
+\r
+!IFDEF ND_INC\r
+DDK_BLOCK_ON_IA64=1\r
+!ELSE\r
+!MESSAGE Skipping ibndprov.dll build: HPC SDK not installed.\r
DDK_BLOCK_ON_IA64=1\r
+DDK_BLOCK_ON_X86=1\r
+DDK_BLOCK_ON_AMD64=1\r
+!ENDIF\r
+\r
!INCLUDE ..\..\..\inc\openib.def\r
projects / ~shefty / rdma-win.git / commitdiff