+++ /dev/null
-#!/bin/sh
-#
-# Copyright (c) 2016 Intel Corporation. All rights reserved.
-#
-# This Software is licensed under one of the following licenses:
-#
-# 1) under the terms of the "Common Public License 1.0" a copy of which is
-# in the file LICENSE.txt in the root directory. The license is also
-# available from the Open Source Initiative, see
-# http://www.opensource.org/licenses/cpl.php.
-#
-# 2) under the terms of the "The BSD License" a copy of which is in the file
-# LICENSE2.txt in the root directory. The license is also available from
-# the Open Source Initiative, see
-# http://www.opensource.org/licenses/bsd-license.php.
-#
-# 3) under the terms of the "GNU General Public License (GPL) Version 2" a
-# copy of which is in the file LICENSE3.txt in the root directory. The
-# license is also available from the Open Source Initiative, see
-# http://www.opensource.org/licenses/gpl-license.php.
-#
-# Licensee has the right to choose one of the above licenses.
-#
-# Redistributions of source code must retain the above copyright
-# notice and one of the license notices.
-#
-# Redistributions in binary form must reproduce both the above copyright
-# notice, one of the license notices in the documentation
-# and/or other materials provided with the distribution.
-#
-# Test Suite to test uDAPL Providers and CCL Proxy on MICs and Hosts
-#
-# Sample Usage, all providers, one loop, fast:
-#
-# ./dtest_suite.sh -P ALL -l 1 -f
-#
-
-### --- user input section --- ###
-server_list="cst-kc1 cst-kc1-mic0 cst-kc1-mic1"
-client_list="cst-kc2 cst-kc2-mic0 cst-kc2-mic1 cst-kc1 cst-kc1-mic0 cst-kc1-mic1"
-### --- dtest test cases fine tune zone --- ###
-# Note: value zero indicacte dtest will use the test default value
-b_options="0 1 4096"
-u_options="0 1"
-w_options="0 1"
-S_options="0 9"
-B_options="0 1"
-D_options="0 1"
-W_options="0"
-# test defaults
-def_provider="ofa-v2-mlx4_0-1u"
-dat_conf="/etc/dat.conf"
-### --- End of user input section --- ###
-
-script_version="1.05"
-
-# History log
-# 1.05 - Disable data validation mode when using scif provider
-# From: Amir Hanania <amir.hanania@intel.com>
-# 1.04 - Add data validation for dtest ping pong
-# Add option not to use CPU mask in performance test
-# From: Amir Hanania <amir.hanania@intel.com>
-# 1.03 - Add dapl tests
-# From: Amir Hanania <amir.hanania@intel.com>
-# 1.02 - Change performane test to use dtest -W case for latency.
-# Note: You must have a dtesr version that support -W to run performane test.
-# From: Amir Hanania <amir.hanania@intel.com>
-# 1.01 - Add multi provider test
-# From: Amir Hanania <amir.hanania@intel.com>
-# 1.00 - Initial Version
-# From: Amir Hanania <amir.hanania@intel.com>
-# Test script to test dapl.
-# Run dtest test in multiple options.
-# Notes:
-# 1. For performance test. Same dtest configuration is used twice.
-# Once with -W for latency and once without for BW.
-#
-
-user_provider=$def_provider
-server_client_list=$server_list" "$client_list
-host_list=`for i in $server_client_list; do echo $i | awk -F "-mic" '{ print $1 }'; done | sort | uniq`
-provider_search_debug=0
-dapl_test_user_input="y"
-ran_one_dapltest=0
-dapl_test_rep_max=100
-dapl_test_rep=$dapl_test_rep_max
-mfo_test=0
-fast_test=0
-fast_test_str=""
-perf_test=0
-no_inline_data=0
-debug_info=0
-v_for_test=""
-user_srting=""
-ctrl_c=0
-runs=0
-max_run_time=0
-dapl_mtu=0
-loops=0
-log_file_dir="dtest_perf_logs"
-log_file="$log_file_dir/dtest_performance_"
-unidirection_test=0
-cpu_mask="no_cpu_mask"
-user_b_options="none"
-dog_file=/tmp/dog.log
-dog_ser=/tmp/dog.ser
-dog_cli=/tmp/dog.cli
-i=1
-while [ $i -lt 5000000 ]; do
- b_options_for_perf_test+=" $i"
- i=$(( $i*2 ))
-done
-mkdir -p $log_file_dir
-
-control_c()
-# run if user hits control-c
-{
- echo -en "\n*** ^c ***\n"
- if [ $ctrl_c -ne 0 ]; then
- echo -ne "\n*** Forced EXIT! ***\n\n"
- for s in $server_list; do
- ssh root@$s "killall dtest" > /dev/null 2>&1
- ssh root@$s "killall dapltest" > /dev/null 2>&1
- done
- for c in $client_list; do
- ssh root@$c "killall dtest" > /dev/null 2>&1
- ssh root@$c "killall dapltest" > /dev/null 2>&1
- done
- exit 1
- fi
- let "ctrl_c+=1"
- echo -en "\n*** Will break after this test case ***\n\n"
-}
-
-# trap keyboard interrupt (control-c)
-trap control_c SIGINT
-
-exit_control()
-{
- # if dog killed us. Clean up the dtest still working.
- for s in $server_list; do
- ssh root@$c "killall dtest" > /dev/null 2>&1
- done
- for c in $client_list; do
- ssh root@$c "killall dtest" > /dev/null 2>&1
- done
-
- echo "2" > $dog_file
- sleep 2
- #kill dog
- # jobs -p | xargs kill
-}
-# trap exit to kill dog when script exit
-#trap 'jobs -p | xargs kill' EXIT
-trap exit_control EXIT
-
-function dog(){
- while true; do
- val=`cat $dog_file`
- if [ $val -eq 2 ]; then
- exit
- fi
- if [ $val -eq 1 ]; then
- server=`cat $dog_ser`
- client=`cat $dog_cli`
- server_err=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c ERR"`
- client_err=`ssh root@$client "cat /tmp/dtest_cli_run.log | grep -c ERR"`
- server_fail=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c FAIL"`
- client_fail=`ssh root@$client "cat /tmp/dtest_cli_run.log | grep -c FAIL"`
- if [ $server_err -gt 0 ] || [ $client_err -gt 0 ] || [ $server_fail -gt 0 ] || [ $client_fail -gt 0 ]; then
- sleep 2
- echo -e "\n\n\twatchdog bark - validation test failed\n\n"
- killall ${0##*/}
- fi
- echo -n "."
- fi
- sleep 1
- done
-}
-
-function wait_for_server_to_be_ready(){
- i=99
- echo -ne "Waiting to servers to come up... $i \r"
- until [ $i -eq 0 ]; do
- up=0
- file_found="NOT found"
- ssh root@$server [ -f /tmp/dtest_ser_run.log ] && file_found="file found"
- if [ "$file_found" == "file found" ]; then
- up=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c waiting"`
- fi
- if [ $up -eq 1 ]; then
- break;
- fi
- let "i = i - 1"
- echo -ne "Waiting to servers to come up... $i \r"
- sleep 0.1
- done
-}
-
-
-u=0
-w=0
-B=0
-b=0
-S=0
-D=0
-
-function testcase(){
- # Setting the dtest options
- if [ $u -ne 0 ]; then
- u_for_test="-u"
- else
- u_for_test=""
- fi
- if [ $w -ne 0 ]; then
- w_for_test="-w"
- else
- w_for_test=""
- fi
- if [ $B -ne 0 ]; then
- B_for_test="-B $B"
- else
- B_for_test=""
- fi
- if [ $b -ne 0 ]; then
- b_for_test="-b $b"
- else
- b_for_test=""
- fi
- if [ $S -ne 0 ]; then
- S_for_test="-S $S"
- else
- S_for_test=""
- fi
- if [ $W -ne 0 ]; then
- W_for_test="-W"
- else
- W_for_test=""
- fi
- if [ $D -ne 0 ]; then
- if [ $do_not_validate_data_with_scif -eq 1 ]; then
- return 0
- fi
- D_for_test="-D -a -B 10"
- else
- D_for_test=""
- fi
-
- if [ $ctrl_c -ne 0 ]; then
- echo -ne "\n*** Stop test due to ctrl c ***\n\n"
- exit 1
- fi
-
- # in case the prev test failed. The files will be still there for debug. Delete them for the new run.
- ssh root@$server "rm /tmp/dtest_ser_run.log" > /dev/null 2>&1
- ssh root@$client "rm /tmp/dtest_cli_run.log" > /dev/null 2>&1
-
- if [ $D -eq 1 ]; then
- support_data_validation
- if [ $dtest_support_data_val -ne 1 ]; then
- return
- fi
- fi
-
- #Start the server
- echo "----------------------------------------------------------"
- echo "Test case: $W_for_test $D_for_test $u_for_test $w_for_test $B_for_test $b_for_test $S_for_test $v_for_test $user_srting"
- echo -ne "Start $taskset_4_server dtest -P $provider server $server\r"
- ssh root@$server "$export_str $taskset_4_server dtest -P $provider $W_for_test $u_for_test $w_for_test $B_for_test $b_for_test $S_for_test $v_for_test $user_srting $D_for_test >& /tmp/dtest_ser_run.log" &
- ser_pid=$!
-
- # Wait for server to be ready
- wait_for_server_to_be_ready
-
- if [ $i -eq 0 ]; then
- echo $server dtest failed - did not start
- ssh root@$server "killall dtest"
- ssh root@$client "killall dtest"
- exit 1
- fi
-
- # Start client
- echo -ne "Start $taskset_4_client dtest -P $provider client \r"
- ssh root@$client "$export_str $taskset_4_client dtest -P $provider -h $server $W_for_test $u_for_test $w_for_test $B_for_test $b_for_test $S_for_test $v_for_test $user_srting $D_for_test >& /tmp/dtest_cli_run.log" &
- cli_pid=$!
-
- if [ $D -eq 1 ]; then
- echo $server > $dog_ser
- echo $client > $dog_cli
- echo "1" > $dog_file
- fi
-
- # Wait for Server and Client to be done
- wait $ser_pid $cli_pid
-
- if [ $D -eq 1 ]; then
- echo "0" > $dog_file
- fi
-
- # Check results from log files
- server_pass=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c PASSED"`
- client_pass=`ssh root@$client "cat /tmp/dtest_cli_run.log | grep -c PASSED"`
- server_err=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c ERR"`
- client_err=`ssh root@$client "cat /tmp/dtest_cli_run.log | grep -c ERR"`
- do_exit=0
- if [ $ctrl_c -ne 0 ]; then
- ssh root@$server "killall -9 dtest" > /dev/null 2>&1
- ssh root@$client "killall -9 dtest" > /dev/null 2>&1
- do_exit=1
- fi
-
- if [ $server_pass -ne 1 ] || [ $server_err -ne 0 ]; then
- echo "****** ERROR - $server server failed (with $client client) *******"
- echo " log file: /tmp/dtest_ser_run.log on $server"
- do_exit=1
- fi
-
- if [ $client_pass -ne 1 ] || [ $client_err -ne 0 ]; then
- echo "****** ERROR - $client client failed (with $server server) *******"
- echo " log file: /tmp/dtest_cli_run.log on $client"
- do_exit=1
- fi
-
- if [ $do_exit -eq 1 ]; then
- echo
- exit 1
- fi
-
- # Print to screen or file the results if needed
- if [ $perf_test -eq 1 ]; then
- echo -ne " \r"
- if [ $fast_test -eq 1 ]; then
- if [ $W -ne 0 ]; then
- # second run is latency test called with -W
- lat=`ssh root@$client cat /tmp/dtest_cli_run.log | grep PingPong | awk -F "latency " '{print $2}' | awk -F " us" '{ print $1 }'`
- res="$lat, Tx size=$res"
- echo "latency: $res"
- echo $res >> $log_file
- else
- # First test for BW
- res=`ssh root@$client cat /tmp/dtest_cli_run.log | grep direction | awk -F "00 x " '{ print $2 }'`
- fi
- else
- if [ $W -ne 0 ]; then
- # second run is latency test called with -W
- lat=`ssh root@$client cat /tmp/dtest_cli_run.log | grep PingPong | awk -F "latency " '{print $2}' | awk -F " us" '{ print $1 }'`
- echo -e "Byte size: $b\t\tlatency: $lat\t\tBW: $res"
- res=`echo $res | awk -F " MB" '{ print $1 }'`
- res=$(printf "%15s" $res)
- lat=$(printf "%10s" $lat)
- echo -e "$b\t\t$lat\t\t$res" >> $log_file
- else
- # First test for BW
- res=`ssh root@$client cat /tmp/dtest_cli_run.log | grep direction | awk -F "00 x $b, " '{ print $2 }'`
- fi
- fi
- fi
- ssh root@$server "rm /tmp/dtest_ser_run.log"
- ssh root@$client "rm /tmp/dtest_cli_run.log"
-
- echo "Test case passed "
-
- read -t 0.01 -n 1 -s u_input
- ret=$?
- if [ $ret -eq 0 ] && [ "$u_input" == "i" ]; then
- print_round_info
- fi
-
- return 0
-}
-
-
-function wait_for_it(){
- max_wait=900
- i=$max_wait
- sleep_for=0.1
- test_start_time=`date +%s`
- until [ $i -eq 0 ]; do
- echo -n "."
- sleep $sleep_for
- up=`ssh root@$wait_for_it_machine cat $wait_for_it_file | grep -c "$wait_for_it_string"`
- if [ $up -eq 1 ]; then
- break;
- fi
- let "i = i - 1"
- if [ $ctrl_c -ne 0 ]; then
- i=0
- fi
- if [ $i -eq $(( $max_wait - 20 )) ]; then
- sleep_for=1
- fi
- if [ $i -eq $(( $max_wait - 40 )) ]; then
- sleep_for=3
- fi
- done
-
- if [ $i -eq 0 ]; then
- if [ $ctrl_c -ne 0 ]; then
- echo -ne "\n\t*** Stop test due to ctrl c ***\n\n"
- else
- echo " failed"
- echo -e "\n\n\tDid not find $wait_for_it_string string on machine: $wait_for_it_machine at file $wait_for_it_file - EXIT\n\n"
- fi
- ssh root@$server killall dapltest > /dev/null 2>&1
- ssh root@$client killall dapltest > /dev/null 2>&1
- exit
- fi
- test_end_time=`date +%s`
- test_run_time=$(($test_end_time-$test_start_time))
- echo " done in $test_run_time sec"
-}
-
-
-function print_round_info(){
- now=`date +%s`
- run_time=$(($now-$start_time))
- ss=$(($run_time%60))
- mm=$(($run_time/60))
- mm=$(($mm%60))
- hh=$(($run_time/3600))
- echo "**************************************************************"
- echo -e "\tin round $runs - $hh h $mm m $ss s"
- echo "**************************************************************"
-}
-
-
-# Check if client and server dtest support data validation
-function support_data_validation() {
- dtest_support_data_val=0
-
- ssh root@$server "dtest -U >& /tmp/dtest_ser_run.log"
- ssh root@$client "dtest -U >& /tmp/dtest_cli_run.log"
- sleep .1
- ser_is_valid=`ssh root@$server cat /tmp/dtest_ser_run.log | grep -c "validate data"`
- if [ $ser_is_valid -ne 1 ]; then
- return 0
- fi
- cli_is_valid=`ssh root@$client cat /tmp/dtest_cli_run.log | grep -c "validate data"`
- if [ $cli_is_valid -ne 1 ]; then
- return 0
- fi
- dtest_support_data_val=1
-}
-
-
-# Run dtest in all data size ping pong test with data validation mode between client and server
-function server_client_data_validation_test(){
-
- echo -e "\n\n\n\t**** dtest data validation test\t\tprovider: $provider\t\tserver: $server $taskset_4_server\t\tclient: $client $taskset_4_client ****\n"
- support_data_validation
- if [ $dtest_support_data_val -ne 1 ]; then
- echo -e "\t**** $client or $server dtest does not support data validation - skipping ****"
- return
- fi
-
- echo -e " Start $taskset_4_server dtest -P $provider -D -a on server $server"
- ssh root@$server "$export_str $taskset_4_server dtest -P $provider -D -a -B 100 >& /tmp/dtest_ser_run.log" &
- ser_pid=$!
- wait_for_server_to_be_ready
-
- echo -e " Start $taskset_4_client dtest -P $provider -D -a on client $client"
- ssh root@$client "$export_str $taskset_4_client dtest -P $provider -h $server -D -a -B 100 >& /tmp/dtest_cli_run.log" &
- cli_pid=$!
- # just wait a bit for files on server and clien be ready before waking up the dog
- sleep 1
-
- echo $server > $dog_ser
- echo $client > $dog_cli
- echo "1" > $dog_file
-
- # Wait for Server and Client to be done
- wait $ser_pid $cli_pid
-
- echo "0" > $dog_file
- echo
- # Check results from log files
- server_pass=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c PASSED"`
- client_pass=`ssh root@$client "cat /tmp/dtest_cli_run.log | grep -c PASSED"`
- server_err=`ssh root@$server "cat /tmp/dtest_ser_run.log | grep -c ERR"`
- client_err=`ssh root@$client "cat /tmp/dtest_cli_run.log | grep -c ERR"`
- do_exit=0
- if [ $ctrl_c -ne 0 ]; then
- ssh root@$server "killall -9 dtest" > /dev/null 2>&1
- ssh root@$client "killall -9 dtest" > /dev/null 2>&1
- do_exit=1
- fi
-
- if [ $server_pass -ne 1 ] || [ $server_err -ne 0 ]; then
- echo "****** ERROR - $server server failed (with $client client) *******"
- echo " log file: /tmp/dtest_ser_run.log on $server"
- do_exit=1
- fi
-
- if [ $client_pass -ne 1 ] || [ $client_err -ne 0 ]; then
- echo "****** ERROR - $client client failed (with $server server) *******"
- echo " log file: /tmp/dtest_cli_run.log on $client"
- do_exit=1
- fi
-
- if [ $do_exit -eq 1 ]; then
- echo
- exit 1
- fi
-
- echo -e "\n\tdtest data validation test\t\tserver: $server\t\tclient: $client\t\tprovider: $provider\t\tTEST PASSED\n\n"
-
-}
-
-
-# Run dapltest between client and server
-function server_client_dapl_test(){
- ofa_post=""
- dapl_test_rep=$dapl_test_rep_max
- if [ $ctrl_c -ne 0 ]; then
- echo -ne "\n*** Stop test due to ctrl c ***\n\n"
- exit 1
- fi
-
- echo "----------------------------------------------------------"
- echo -ne "\t**** dapltest\t\tprovider: $provider\t\tserver: $server\t\tclient: $client "
-
- # in case the prev test failed. The files will be still there for debug. Delete them for the new run.
- ssh root@$server "rm /tmp/dapltest_ser_run.log" > /dev/null 2>&1
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
-
- # 1. skip if roc
- # 2. check that provider is ofa or scm
- is_roe=`echo $provider | grep -c roe`
- if [ $is_roe -eq 1 ]; then
- good_provider_for_dapltest=0
- echo -e " - provider $provider not supported - skipping ****"
- echo "----------------------------------------------------------"
- return 0
- fi
- is_ofa=`ssh root@$server cat $dat_conf | grep $provider | grep -c libdaplofa`
- is_scm=`ssh root@$server cat $dat_conf | grep $provider | grep -c libdaploscm`
- if [ $is_ofa -eq 0 ] && [ $is_scm -eq 0 ]; then
- good_provider_for_dapltest=0
- echo -e " - provider $provider not supported - skipping ****"
- echo "----------------------------------------------------------"
- return 0
- fi
- if [ $is_ofa -eq 1 ]; then
- dat_line=`ssh root@$server cat $dat_conf | grep $provider`
- ofa_post=`echo $dat_line | grep lofa | awk '{ print $1 }' | awk -F "ofa-v2" '{ print $2 }'`
- fi
- ran_one_dapltest=1
-
- # start server
- wait_for_it_machine=$server
- wait_for_it_file="/tmp/dapltest_ser_run.log"
- wait_for_it_string="Dapltest: Service Point Ready"
- echo -e " ****\n----------------------------------------------------------"
- echo -e "dapltest\tprovider: $provider\tserver: $server\tclient: $client"
- echo -ne "start dapltest server..."
- ssh root@$server "dapltest -T S -D $provider >& /tmp/dapltest_ser_run.log" &
- wait_for_it
-
- # tests
- wait_for_it_machine=$client
- wait_for_it_file="/tmp/dapltest_cli_run.log"
- wait_for_it_string="Total WQE"
- # test 1
- echo -ne "start dapltest client test 1 ..."
- ssh root@$client "dapltest -T T -s $server$ofa_post -D $provider -i $dapl_test_rep -t 1 -w 1 client SR 256 server SR 256 >& /tmp/dapltest_cli_run.log" &
- wait_for_it
-
- if [ $fast_test -eq 0 ]; then
- # test 2
- if [ $dapl_test_rep -ne 1 ] && [ $test_run_time -ge 4 ]; then
- dapl_test_rep=$(($dapl_test_rep/$test_run_time/8))
- if [ $dapl_test_rep -eq 0 ]; then
- dapl_test_rep=1
- fi
- echo Reduce rep to $dapl_test_rep
- fi
- echo -ne "start dapltest client test 2 ..."
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
- ssh root@$client "dapltest -T T -s $server$ofa_post -D $provider -i $dapl_test_rep -t 1 -w 1 client SR 256 server RW 4096 server SR 256 >& /tmp/dapltest_cli_run.log" &
- wait_for_it
-
- # test 3
- echo -ne "start dapltest client test 3 ..."
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
- ssh root@$client "dapltest -T T -s $server$ofa_post -D $provider -i $dapl_test_rep -t 1 -w 1 client SR 256 server RR 4096 server SR 256 >& /tmp/dapltest_cli_run.log" &
- wait_for_it
-
- # test 4
- echo -ne "start dapltest client test 4 ..."
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
- ssh root@$client "dapltest -T T -s $server$ofa_post -D $provider -i $dapl_test_rep -t 1 -w 1 client SR 256 server RW 4096 server SR 256 client SR 256 server RW 4096 server SR 256 client SR 4096 server SR 256 >& /tmp/dapltest_cli_run.log" &
- wait_for_it
-
- # test 5
- if [ $dapl_test_rep -ne 1 ] && [ $test_run_time -ge 2 ]; then
- dapl_test_rep=$(($dapl_test_rep/8))
- if [ $dapl_test_rep -eq 0 ]; then
- dapl_test_rep=1
- fi
- echo Reduce rep to $dapl_test_rep
- fi
- echo -ne "start dapltest client test 5 ..."
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
- ssh root@$client "dapltest -T T -s $server$ofa_post -D $provider -i $dapl_test_rep -t 1 -w 8 client SR 256 server RW 4096 server SR 256 client SR 256 server RW 4096 server SR 256 client SR 4096 server SR 256 >& /tmp/dapltest_cli_run.log" &
- wait_for_it
-
- if [ $dapl_test_rep -ne 1 ] && [ $test_run_time -ge 2 ]; then
- dapl_test_rep=$(($dapl_test_rep/4))
- if [ $dapl_test_rep -eq 0 ]; then
- dapl_test_rep=1
- fi
- echo Reduce rep to $dapl_test_rep
- fi
- # test 6
- echo -ne "start dapltest client test 6 ..."
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
- ssh root@$client "dapltest -T T -s $server$ofa_post -D $provider -i $dapl_test_rep -t 4 -w 8 client SR 256 server RW 4096 server SR 256 client SR 256 server RW 4096 server SR 256 client SR 4096 server SR 256 >& /tmp/dapltest_cli_run.log" &
- wait_for_it
- fi
-
- # stop server
- echo -n "stop dapltest server..."
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
- ssh root@$client "dapltest -T Q -s $server$ofa_post -D $provider >& /tmp/dapltest_cli_run.log" &
- cli_pid=$!
-
- wait_for_it_machine=$server
- wait_for_it_file="/tmp/dapltest_ser_run.log"
- wait_for_it_string="Exiting"
- echo -n "wait for dapltest server to stop..."
- wait_for_it
-
- # Wait for Server and Client to be done
- wait $cli_pid
-
- # clean up
- ssh root@$server "rm /tmp/dapltest_ser_run.log" > /dev/null 2>&1
- ssh root@$client "rm /tmp/dapltest_cli_run.log" > /dev/null 2>&1
-
- echo -e "\tdapltest\t\tserver: $server\t\tclient: $client\t\tprovider: $provider\t\tTESTS PASSED"
- echo -e "----------------------------------------------------------\n"
-}
-
-
-# Run all the test cases between two machines.
-function server_host_test(){
- taskset_4_server=""
- taskset_4_client=""
- if [ $perf_test -eq 1 ]; then
- is_mic=`echo $server | grep -c mic`
- if [ $is_mic -eq 0 ] && [ "$cpu_mask" != "no_cpu_mask" ]; then
- taskset_4_server="taskset $cpu_mask "
- fi
- is_mic=`echo $client | grep -c mic`
- if [ $is_mic -eq 0 ] && [ "$cpu_mask" != "no_cpu_mask" ]; then
- taskset_4_client="taskset $cpu_mask "
- fi
-
- echo -e "\n**** dtest: provider: $provider \tserver: $server \tclient: $client ****\n" >> $log_file
- if [ $fast_test -eq 0 ]; then
- echo -e "\nBytes\t\t Latency\t\t\t MB/s" >> $log_file
- fi
- fi
-
- if [ "$dapl_test_user_input" != "o" ]; then
- echo -e "\n\n\n\t**** dtest\t\tprovider: $provider\t\tserver: $server $taskset_4_server\t\tclient: $client $taskset_4_client ****"
-
- #set var value to zero in order to use dtest default value for that option.
- for u in $u_options; do
- for w in $w_options; do
- for b in $b_options; do
- for S in $S_options; do
- for B in $B_options; do
- for D in $D_options; do
- for W in $W_options; do # Always keep last. See Note 1.
- # Run one test case between Client and Server.
- testcase
- ret=$?
- if [ $ret -ne 0 ]; then
- echo TEST FAILED
- exit 1
- fi
- sleep 1
- done
- done
- done
- done
- done
- done
- done
-
- echo -e "\n\tdtest\t\tserver: $server\t\tclient: $client\t\tprovider: $provider\t\tTEST PASSED\n\n"
-
- if [ $perf_test -ne 1 ] && [ $do_not_validate_data_with_scif -eq 0 ] && [ $fast_test -ne 1 ]; then
- server_client_data_validation_test
- fi
- fi
-
- if [ "$dapl_test_user_input" != "n" ] && [ $good_provider_for_dapltest -eq 1 ] && [ $fast_test -ne 1 ]; then
- server_client_dapl_test
- fi
-
-}
-
-function help(){
- echo -e "\n\tRun dtest and dapltest accross cluster - from each client to each server\n"
- echo -e "\t\tServer list: $server_list"
- echo -e "\t\tClient list: $client_list\n"
- echo -e "\t-P <PROVIDER NAME> : Provider name or 'ALL' for all prividers (default $def_provider)"
- echo -e "\t-f: Fast test"
- echo -e "\t-l <NUM> : How many test loops to run. Def forever"
- echo -e "\t-t <NUM> : How many minutes to run. Def forever"
- echo -e "\t-p <CPUs mask> or \"no_cpu_mask\": Performance test"
- echo -e "\t\tMask in 0xHEX format. should match host's /sys/class/mic/mic0/device/local_cpus"
- echo -e "\t\tFor no CPU mask enter \"no_cpu_mask\""
- echo -e "\t\tConsider also: taskset mpxyd, set mcm_affinity to 2 in /etc/mpxyd.conf, performance mode at the host scaling_governor"
- echo -e "\t\tConsider also to change DAPL MTU (-M optoin)"
- echo -e "\t-w: Write only test"
- echo -e "\t-u: uni-direction only test"
- echo -e "\t-d <n|y|o> : dapl test options. \"n\": No dapl tests. \"y\": Run dapl tests. \"o\": Run Only dapl tests (no dtest). Def: Run dapl_test"
- echo -e "\t-M <NUM> : DAPL MTU"
- echo -e "\t-b <NUM> : data size. Can be: one size, many sizes as a string or type \`all\` for all sizes power of 2"
- echo -e "\t-U: \"user string\". user dtest option string ( -w -b -u and -S dtest options )"
- echo -e "\t-z: use zero for -w -b -u and -S dtest options (zero mean test default value)"
- echo -e "\t-i: No inline data test"
- echo -e "\t-m: Force MFO test"
- echo -e "\t-D: DAPL debug print in log files"
- echo -e "\t-v: dtest verbose mode"
- echo -e "\t-q: qib test over mlx4 (same as -m and -i options)"
- echo -e "\t-V: Print the script version"
- echo -e "\t-h: help"
- echo -e "\n\tWhile test is running:"
- echo -e "\t^c: Exit gracefully"
- echo -e "\t^c^c: Forced exit"
- echo -e "\ti: Print round number and time duration"
- echo -e "\n\n"
- exit 1
-}
-
-
-function log(){
- if [ $provider_search_debug -eq 1 ]; then
- echo -e "$@"
- else
- echo -n "."
- fi
-}
-
-
-function providers_search(){
- echo -e "\nSearching for devices"
- first_host=1
- for host in $host_list; do
- # make sure host dat file exist
- dat_conf_found="NOT found"
- ssh root@$host "[ -f $dat_conf ]" && dat_conf_found="dat_conf_found"
- if [ "$dat_conf_found" == "dat_conf_found" ]; then
- log "$dat_conf found on $host"
- else
- echo -e "\n\t$dat_conf was not found on $host.\n\n"
- exit 1
- fi
-
- #ib devices list
- dev_list=`ssh root@$host ibv_devices | tail -n +3 | awk '{ print $1 }'`
- for dev in $dev_list; do
- # for each device
- log Found $dev device
- port_cnt=`ssh root@$host ibv_devinfo -d $dev | grep phys_port_cnt | awk '{print$2 }'`
- log " $dev phys_port_cnt: $port_cnt"
- for port in $(seq 1 $port_cnt); do
- # for each post
- log " checking $dev port $port status"
- up=`ssh root@$host ibv_devinfo -d $dev -i $port | grep state | grep -c PORT_ACTIVE`
- if [ $up -ne 1 ]; then
- log " $dev port $i is not active"
- continue
- fi
- log " $dev port $port is active"
- log " add it to the list"
- # get a list of providers that this device can use
- providers+=`ssh root@$host cat $dat_conf | grep "$dev $port" | awk '{ print $1 }'`
- providers+=" "
- done
- done
-
- #add network ib devices
- net_dev_list=` ssh root@$host netstat -i | grep -v "no statistics available" | tail -n +3 | awk '{ print $1 }'`
- for net_dev in $net_dev_list; do
- # for each net device
- log Found $net_dev net device
- is_ib=`ssh root@$host ip addr show $net_dev | grep -c infiniband`
- if [ $is_ib -ne 1 ]; then
- log " $net_dev net device is not ib device"
- continue
- fi
- log " $net_dev is infiniband device"
- has_ip_addr=`ssh root@$host ip addr show $net_dev | grep inet | grep -vc inet6`
- if [ $has_ip_addr -ne 1 ]; then
- log " $net_dev net device has no ip addr"
- continue
- fi
- log " $net_dev net device has IP address"
- log " add it to the list"
- # get a list of providers that this device can use
- providers+=`ssh root@$host cat $dat_conf | grep "$net_dev 0" | awk '{ print $1 }'`
- providers+=" "
- done
-
- log; log -n "$host povider list: "; for i in $providers; do log -n "$i "; done; log
- if [ $first_host -eq 1 ]; then
- # just save providers from first host
- hosts_providers_list=$providers
- first_host=0
- else
- # Merge providers from prev hosts with the one from the new host
- # Keep only the providers that are on both lists
- log hosts p from prev hosts: $hosts_providers_list
- hosts_providers_list+=$providers
- hosts_providers_list=`for p in $hosts_providers_list; do echo $p; done | sort | uniq -d`
- log hosts p after merge: $hosts_providers_list
- fi
- providers=""
- done
- cnt=0
- echo -e "\nPovider list:"
- for i in $hosts_providers_list; do
- echo $i
- let cnt+=1
- done
- if [ $cnt -eq 0 ]; then
- echo -e "no devices where found\n\n"
- exit
- fi
- echo -e "Total $cnt providers\n\n"
-}
-
-
-# check if the "server-client-provider" combination is OK
-# Set server_client_provider_is_not_valid_combo to one if not a valid combo
-function check_provider_server_client_combo(){
- server_client_provider_is_not_valid_combo=0
- #check the following:
- # 1. scif providers can only run on the same machine.
- is_scif=`echo $provider | grep -c scif`
- if [ $is_scif -eq 1 ]; then
- server_host=`echo $server | awk -F "-mic" '{ print $1 }'`
- client_host=`echo $client | awk -F "-mic" '{ print $1 }'`
- if [ $server_host == $client_host ]; then
- return
- else
- server_client_provider_is_not_valid_combo=1
- return
- fi
- fi
- # 2. MIC qib can only run mcm provider
- is_ser_mic=`echo $server | grep -c mic`
- is_cli_mic=`echo $client | grep -c mic`
- if [ $is_ser_mic -eq 1 ] || [ $is_cli_mic -eq 1 ]; then
- # MIC Server or Client
- is_qib_provider=`echo $provider | grep -c qib`
- if [ $is_qib_provider -eq 1 ]; then
- # Server or Client is MIC AND qib provider - make sure provider is MCM
- is_mcm=`echo $provider | grep -c m`
- if [ $is_mcm -eq 1 ]; then
- return
- else
- server_client_provider_is_not_valid_combo=1
- return
- fi
- fi
- fi
- # 3. check if MICs ib interface is UP
- is_ib_provider=`echo $provider | grep -ce -ib`
- if [ $is_ib_provider -eq 1 ]; then
- interface=`echo $provider | awk -F "ofa-v2-" '{ print $2 }'`
- if [ $is_ser_mic -eq 1 ]; then
- up=`ssh root@$server ifconfig | grep -c $interface`
- if [ $up -eq 1 ]; then
- return
- else
- server_client_provider_is_not_valid_combo=1
- return
- fi
- fi
- if [ $is_cli_mic -eq 1 ]; then
- up=`ssh root@$client ifconfig | grep -c $interface`
- if [ $up -eq 1 ]; then
- return
- else
- server_client_provider_is_not_valid_combo=1
- return
- fi
- fi
- fi
-}
-
-
-
-
-
-
-while getopts uviVzDmfwhiql:t:P:U:p:d:M:b: option
-do
- case "${option}"
- in
- P) user_provider=${OPTARG};;
- m) no_inline_data=1 ; mfo_test=1;;
- f) fast_test=1; loops=1; fast_test_str=" fast test";;
- p) cpu_mask=${OPTARG}; perf_test=1; W_options="0 1";;
- U) user_srting=${OPTARG}; b_options="0"; u_options="0"; S_options="0"; w_options="0"; B_options="0";;
- z) b_options="0"; u_options="0"; S_options="0"; w_options="0"; B_options="0";;
- w) w_options="1";;
- u) unidirection_test=1; u_options="1";;
- D) debug_info=1;;
- d) dapl_test_user_input=${OPTARG};;
- v) v_for_test=" -v ";;
- i) no_inline_data=1;;
- q) no_inline_data=1 ; mfo_test=1;;
- t) max_run_time=${OPTARG};;
- M) dapl_mtu=${OPTARG};;
- l) loops=${OPTARG};;
- b) user_b_options=${OPTARG};;
- V) echo -e "\n\t${0##*/} version $script_version\n\n"; exit;;
- h) help;;
- esac
-done
-
-if [ $fast_test -eq 1 ]; then
- b_options="0"; u_options="0"; S_options="0"; w_options="0"; B_options="0"; D_options="0";
-fi
-
-if [ $perf_test -eq 1 ]; then
- b_options=$b_options_for_perf_test; u_options="0"; S_options="0"; loops=1; w_options="1"; B_options="0"; user_srting="$user_string -p";dapl_test_user_input="n"; D_options="0";
- legit_input=`echo $cpu_mask | grep -ci 0x`
- if [ $legit_input -ne 1 ] && [ "$cpu_mask" != "no_cpu_mask" ]; then
- echo -e "\n\t< 0xCPUs_mask > or \"no_cpu_mask\" in option -p is missing - input=$cpu_mask - Exit\n\n"
- exit
- fi
-fi
-
-if [ $fast_test -eq 1 ] && [ $perf_test -eq 1 ]; then
- b_options="0"
-fi
-
-if [ $unidirection_test -eq 1 ]; then
- u_options="1"
-fi
-
-if [ "$user_b_options" != "none" ]; then
- if [ "$user_b_options" == "all" ]; then
- b_options=$b_options_for_perf_test
- else
- b_options="$user_b_options"
- fi
-fi
-
-if [ "$dapl_test_user_input" != "n" ] && [ "$dapl_test_user_input" != "y" ] && [ "$dapl_test_user_input" != "o" ]; then
- echo -e "\n\tdapl test option must be n/y/o - Exit\n\n"
- exit
-fi
-
-# check mpxyd is running on host machines.
-for host in $host_list; do
- up=`ssh root@$host "ps ax | grep -c mpxyd"`
- if [ $up -ne 3 ]; then
- echo -e "\n\tERROR - mpxyd is not running on $host\n\n"
- exit
- fi
- if [ $no_inline_data -eq 1 ]; then
- up=`ssh root@$host cat /var/log/mpxyd.log | grep -c "RDMA IB inline threshold 0"`
- if [ $up -ne 1 ]; then
- echo on host $host you need to run mpxyd with mcm_ib_inline 0 in /etc/mpxyd.conf file for no inline data test
- exit 1
- fi
- fi
-done
-
-if [ $user_provider == "ALL" ] || [ $user_provider == "all" ]; then
- providers_search
-else
- hosts_providers_list=$user_provider
-fi
-
-echo -e "\nServer list: $server_list"
-echo -e "Client list: $client_list"
-echo -e "Host list:"
-for i in $host_list; do
- echo $i
-done
-echo
-
-if [ $mfo_test -eq 1 ]; then
- export_str="export DAPL_MAX_INLINE=0 ; export DAPL_MCM_MFO=1 ; "
- echo -ne "\n\t\t**** Running MFO test case \t\t$export_str ****\n\n"
-elif [ $no_inline_data -eq 1 ]; then
- export_str="export DAPL_MAX_INLINE=0 ; "
- echo -ne "\n\t\t**** Running no inline data test case \t\t$export_str ****\n\n"
-else
- export_str=""
-fi
-
-if [ $debug_info -eq 1 ]; then
- export_str="$export_str export DAPL_DBG_TYPE=0xffffffff ; "
- echo -ne "\n\t\t**** Running in debug mode\t\texport value: $export_str ****\n\n"
-fi
-if [ $dapl_mtu -ne 0 ]; then
- export_str="$export_str export DAPL_IB_MTU=$dapl_mtu ; "
- echo -ne "\n\t\t**** Setting DAPL_IB_MTU to $dapl_mtu \t\texport value: $export_str ****\n\n"
-fi
-
-if [ $loops -ne 0 ]; then
- echo -e "\n\tRunning$fast_test_str for $loops iterations"
-fi
-
-if [ $max_run_time -ne 0 ]; then
- echo -e "\n\tRunning$fast_test_str for $max_run_time minutes"
-fi
-
-if [ $loops -eq 0 ] && [ $max_run_time -eq 0 ]; then
- echo -ne "\n\tRunning$fast_test_str forever\n\n"
-fi
-
-if [ $perf_test -eq 1 ]; then
- if [ $unidirection_test -eq 1 ]; then
- log_file+="unidirection_test-"
- else
- log_file+="bidirection_test-"
- fi
- log_file+=`date +%F-%H-%M-%S`
- echo -e "\n\tRunning performance test with cpu mask: $cpu_mask\n\tOutput file: $log_file"
- echo "Server list: $server_list" > $log_file
- echo "Client list: $client_list" >> $log_file
- echo "CPU mask: $cpu_mask" >> $log_file
- if [ $dapl_mtu -ne 0 ]; then
- echo "DAPL_IB_MTU: $dapl_mtu" >> $log_file
- else
- echo "DAPL_IB_MTU: Default value" >> $log_file
- fi
- if [ $unidirection_test -eq 1 ]; then
- echo "Test type: unidirection test" >> $log_file
- else
- echo "Test type:bidirection test" >> $log_file
- fi
- for host in $host_list; do
- op_poll=`ssh root@$host cat /var/log/mpxyd.log | grep -c "OP thread polling enabled"`
- if [ $op_poll -ne 1 ]; then
- echo "OP thread polling on $host: disabled" >> $log_file
- echo -e "\tOP thread polling on $host: disabled"
- else
- echo "OP thread polling on $host: enabled" >> $log_file
- echo -e "\tOP thread polling on $host: enabled"
- fi
- done
- echo -e "\n\n"
-fi
-echo "0" > $dog_file
-dog &
-
-sleep 1
-start_time=`date +%s`
-
-while [ 1 ]; do
- now=`date +%s`
- run_time=$(($now-$start_time))
- ss=$(($run_time%60))
- mm=$(($run_time/60))
- total_run_time_in_min=$mm
- mm=$(($mm%60))
- hh=$(($run_time/3600))
- dd=$(($hh/24))
- hh=$(($hh%24))
- pp=$(printf "%d days %d hours %d min and %d sec" $dd $hh $mm $ss)
-
- echo
- echo
- echo "**************************************************************"
- echo "**************************************************************"
- echo Run time: $pp
- if [ $max_run_time -ne 0 ] && [ $total_run_time_in_min -ge $max_run_time ]; then
- echo -e "Ran for the $max_run_time minute requested by the user - Exiting\n\n"
- break;
- fi
- if [ $loops -ne 0 ] && [ $runs -eq $loops ]; then
- echo -e "Ran for the $loops iterations requested by the user - Exiting\n\n"
- break;
- fi
- runs=$(( $runs + 1 ))
- echo Starting round $runs
- date
- echo "**************************************************************"
- echo "**************************************************************"
- echo
-
- # Runinng
- for provider in $hosts_providers_list; do
- do_not_validate_data_with_scif=`echo $provider | grep -c scif`
- good_provider_for_dapltest=1
- for server in $server_list; do
- for client in $client_list; do
- check_provider_server_client_combo
- if [ $server_client_provider_is_not_valid_combo -ne 0 ]; then
- #echo -e "***** ***** skipping test case: Server:$server Client:$client provider:$provider ***** *****"
- continue
- fi
- # Run all test cases between Client and Server.
- server_host_test
- done
- done
-
- if [ "$dapl_test_user_input" == "o" ] && [ $ran_one_dapltest -eq 0 ]; then
- echo -e "\n\n\n\n\t\t***** ***** WARNING: only dapltest was set up but no dapltest was done with $provider provider $export_str ***** *****\n\n"
- else
- echo -e "\n\n\n\n\t\t***** ***** server client tests with $provider provider $export_str - TEST PASSED ***** *****\n\n"
- fi
- done
-done
projects / ~ardavis / dapl.git / commitdiff