/
STABILITY test script for DilOs 4
STABILITY test script for DilOs 4
- Vitaly Shaburov (Unlicensed)
Owned by Vitaly Shaburov (Unlicensed)
#!/bin/bash
get_duration() {
IFS=':' read -r -a tm <<< "$1"
num=${#tm[@]}
delta=$((${tm[0]} * 3600))
if [[ $num -gt 1 ]]; then
delta1=$((${tm[1]} * 60))
(( delta += $delta1 ))
if [[ $num -gt 2 ]]; then
(( delta += ${tm[2]} ))
fi
fi
echo $delta
}
get_time_diff() {
local cur_time=$(date +%s)
echo $(($cur_time - $1))
}
print_time_diff() {
local delta=$(get_time_diff $1)
local sec=$(($delta % 60))
let "amin = $delta / 60"
let "hour = $amin / 60"
local min=$(($amin%60))
if [ -n "$2" ]; then
printf " Running time: %02d:%02d:%02d\n" $hour $min $sec >> ${ULOG}
else
printf " Running time: %02d:%02d:%02d\n" $hour $min $sec
fi
}
boot_zone() {
echo Shutting down zone $1...
sudo zoneadm -z $1 halt
echo Cleaning zone $1 dataset...
sudo zoneadm -z $1 uninstall -F
sudo zfs destroy -fr rpool/zones/$1
echo Booting zone $1...
sudo zoneadm -z $1 clone $1-orig && sudo zoneadm -z $1 boot
ip=""
while [ "${ip}" == "" ] || [ "${ip}" == "0.0.0.0" ]
do
ip=$(sudo zlogin -l root $1 ifconfig | grep -A 1 "${2}" | sed "/${2}/d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//")
done
echo "Zone:$1 ------> $2:$ip"
}
get_random_test() {
local tests=(libc elf os util crypto net)
local len=${#tests[*]}
local idx=$(($RANDOM % $len))
echo "${tests[$idx]}"
}
log_extra() {
echo "---------------- ZPOOL LIST ------------------" >> $1
zpool list >> $1
echo "---------------- ZFS LIST ------------------" >> $1
zfs list >> $1
echo "---------------- PS AX ------------------" >> $1
ps ax >> $1
echo "---------------- TOP -n 10 ------------------" >> $1
top -n 10 >> $1
}
test_results() {
# $1 - test name
# $2 - pass number
# $3 - test log
# $4 - execution log
local percent="$(grep 'Percent passed:' $4 | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
local folder="$(grep 'Log directory:' $4 | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
local rt="$(grep 'Running Time:' $4 | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
if [[ "${percent}" == "" ]]; then
percent="0.0%"
fi
if [[ "${rt}" == "" ]]; then
rt="0 sec"
fi
echo " Percent passed: ${percent}" >> $3
echo " Running Time: ${rt}" >> $3
sed -n '/Results Summary/,/Log directory:/p' $4 > a.tmp
echo "---------------------------------------------------------" >> a.tmp
echo "" >> a.tmp
cat $4 >> a.tmp
mv -f a.tmp $4
if [[ "${percent}" == "100.0%" ]]; then
sudo rm -f $4
sudo rm -r -f ${folder}
return 0
else
cat $4 | grep "\[FAIL\]" >> ${TOTALFAIL}
print_time_diff ${GLOB_START_TIME}
echo " PASS $2 IS FAILED! (See $4)" >> $3
log_extra $3
echo "Fault on $2 pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a $4
echo "${1^^} test is failed on $2 pass at $(date). Fail information is in ${TOTALFAIL}" >> ${TOTALLOG}
return 1
fi
}
execute_test() {
# $1 - test name
# $2 - test path
# $3 - pass number
# $4 - test log
# $5 - IP address
local percent
local folder
local rt
local LOG="/var/tmp/$1-test.$3.txt"
echo "------------------ Pass $3 ---------------------" 2>&1 | /usr/bin/tee -a $4
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : $5" >> ${LOG}
$2 2>&1 | /usr/bin/tee -a ${LOG}
test_results $1 $3 $4 ${LOG}
}
i=1
testnm="__ALL__"
shuffle=0
mode=0
TOTALLOG="/var/tmp/stability.$(date +%F-%T).txt"
TOTALFAIL="/var/tmp/stability-fails.$(date +%F-%T).txt"
echo "------------ Stability test started at $(date +%F-%T) --------------" > ${TOTALLOG}
if [[ $# < 2 ]]; then
echo "Missing parameters in the command" | /usr/bin/tee -a ${TOTALLOG}
exit 1
fi
case "$1" in
-i) mode=1
# number of iterations is provided
maxiter=$2
echo "Stability Test : launched for $maxiter iterations at $(date)." | /usr/bin/tee -a ${TOTALLOG}
;;
-d) mode=2
# duration of tests is provided
delta=$(get_duration $2)
echo "Stability Test : launched for $2 interval ($delta seconds) at $(date)." | /usr/bin/tee -a ${TOTALLOG}
;;
-t) mode=3
# stop date/time is provided
stopdt=$2
;;
*) echo "Stability Test : $1 is a wrong option" | /usr/bin/tee -a ${TOTALLOG}
echo "Usage : stability.sh (-i <iterations> | -t <HH:MM[:SS]> | -d <HH:MM[:SS] [DAY-MONTH-YEAR]>]) [-s | <test name>]"
exit 1
;;
esac
shift
shift
if [[ $mode -eq 3 ]]; then
# stop time/date mode
if [[ $# > 0 ]]; then
date -d $1 +%s
stopat=$?
if [[ $stopat -eq 0 ]]; then
# stop date is entered after the time
# if it is ommited then date is treated as current one
stopdt="$stopdt $1"
shift
fi
fi
echo "Stability Test : launched until $stopdt at $(date)." | /usr/bin/tee -a ${TOTALLOG}
stopat=$(date -d "$stopdt" "+%s")
fi
if [[ $# > 0 ]]; then
# shuffle mode or test name is entered
if [[ "$1" == "-s" ]]; then
echo "Shuffle mode is turned on." | /usr/bin/tee -a ${TOTALLOG}
shuffle=1
else
testnm=$1
echo "Only $testnm test will be running." | /usr/bin/tee -a ${TOTALLOG}
fi
fi
LLOG="/var/tmp/libc-stability.$(date +%F-%T).txt"
echo "================== Stability libc tests $(date +%F-%T) ==================" > ${LLOG}
ELOG="/var/tmp/elf-stability.$(date +%F-%T).txt"
echo "================== Stability elf tests $(date +%F-%T) ==================" > ${ELOG}
OLOG="/var/tmp/os-stability.$(date +%F-%T).txt"
echo "================== Stability os tests $(date +%F-%T) ==================" > ${OLOG}
ULOG="/var/tmp/util-stability.$(date +%F-%T).txt"
echo "================== Stability util & mdb tests $(date +%F-%T) ==================" > ${ULOG}
NLOG="/var/tmp/net-stability.$(date +%F-%T).txt"
echo "================== Stability net tests $(date +%F-%T) ==================" > ${NLOG}
CLOG="/var/tmp/crypto-stability.$(date +%F-%T).txt"
echo "================== Stability crypto tests $(date +%F-%T) ==================" > ${CLOG}
ipa=$(sudo ifconfig | grep -A 1 vmxnet3s0 | sed '1d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//')
GLOB_START_TIME=$(date +%s)
while [[ $mode -gt 0 ]]
do
case "$mode" in
1) # number of iterations is provided
if [[ $i -gt $maxiter ]]; then
break
fi
;;
2) # duration of tests is provided
difftm=$(get_time_diff $GLOB_START_TIME)
if [[ $difftm -gt $delta ]]; then
break
fi
;;
3) # stop date/time is provided
tmpat=$(date +%s)
if [[ $stopat -le $tmpat ]]; then
break
fi
;;
*) break
;;
esac
if [[ $shuffle -eq 1 ]]; then
testnm=$(get_random_test)
fi
# ------------------- Tests execution ----------------------------------
if [[ "$testnm" == "libc" ]] || [[ "$testnm" == "__ALL__" ]]; then
execute_test "libc" "/opt/libc-tests/bin/libctest" ${i} ${LLOG} ${ipa}
fi
if [[ "$testnm" == "elf" ]] || [[ "$testnm" == "__ALL__" ]]; then
execute_test "elf" "/opt/elf-tests/bin/elftest" ${i} ${ELOG} ${ipa}
fi
if [[ "$testnm" == "os" ]] || [[ "$testnm" == "__ALL__" ]]; then
execute_test "os" "/opt/os-tests/bin/ostest" ${i} ${OLOG} ${ipa}
sudo rm -f /tmp/otst*
fi
if [[ "$testnm" == "util" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "--------------------------------------- Pass ${i} ------------------------------------" 2>&1 | /usr/bin/tee -a ${ULOG}
START_TIME=$(date +%s)
LOG="/var/tmp/util-test.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : \${ipa}" >> ${LOG}
echo "---------------------------------------- UTILS --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
/opt/util-tests/bin/utiltest 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
if [[ "${percent}" == "" ]]; then
percent="0.0%"
fi
echo " UTILS percent passed: ${percent}" >> ${ULOG}
folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" != "100.0%" ]; then
cat ${LOG} | grep "\[FAIL\]" >> ${TOTALFAIL}
print_time_diff ${START_TIME} ${ULOG}
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
log_extra ${ULOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "UTIL test is failed on ${i} pass at $(date). Fail information is in ${TOTALFAIL}" >> ${TOTALLOG}
exit 1
fi
sudo rm -r -f ${folder}
echo "---------------------------------------- MDB --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
/opt/util-tests/bin/utiltest -c /opt/util-tests/runfiles/mdb.run 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Percent passed:' | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
if [[ "${percent}" == "" ]]; then
percent="0.0%"
fi
echo " MDB percent passed: ${percent}" >> ${ULOG}
folder="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Log directory:' | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
print_time_diff ${START_TIME} ${ULOG}
sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
echo "---------------------------------------------------------" >> a.tmp
echo "" >> a.tmp
cat ${LOG} >> a.tmp
mv -f a.tmp ${LOG}
if [ "${percent}" == "100.0%" ]; then
sudo rm -f ${LOG}
sudo rm -r -f ${folder}
else
cat ${LOG} | grep "\[FAIL\]" >> ${TOTALFAIL}
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
log_extra ${ULOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "MDB test is failed on ${i} pass at $(date). Fail information is in ${TOTALFAIL}" >> ${TOTALLOG}
exit 1
fi
fi
if [[ "$testnm" == "crypto" ]] || [[ "$testnm" == "__ALL__" ]]; then
execute_test "crypto" "/opt/crypto-tests/bin/cryptotest" ${i} ${CLOG} ${ipa}
fi
if [[ "$testnm" == "net" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${NLOG}
LOG="/var/tmp/net-test.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : ${ipa}" >> ${LOG}
echo "" >> ${LOG}
echo ----------------------------- IP Address ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo ifconfig | grep -A 1 vmxnet3s0 2>&1 | /usr/bin/tee -a ${LOG}
boot_zone cz int1 ${LOG}
boot_zone sz int2 ${LOG}
boot_zone rz int3 ${LOG}
echo ------------------------------- Zones ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo zoneadm list -v 2>&1 | /usr/bin/tee -a ${LOG}
echo ---------------------------- DHCP Server ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo svcs -z dhcp isc-dhcp-server 2>&1 | /usr/bin/tee -a ${LOG}
echo ------------------------------- Tests ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo /opt/net-tests/bin/nettest /var/tmp 2>&1 | /usr/bin/tee -a /var/tmp/a.tmp
test_results "net" ${i} ${NLOG} ${LOG}
fi
(( i++ ))
done
echo "Tests are finished successfully at $(date) with ${i} passes." >> ${TOTALLOG}
print_time_diff ${GLOB_START_TIME} | /usr/bin/tee -a ${TOTALLOG}
, multiple selections available,
Related content
STABILITY test script
STABILITY test script
More like this
ZFS stability script
ZFS stability script
More like this
Run ZFS tests in stability mode
Run ZFS tests in stability mode
More like this
Запуск ZFS тестов в режиме stability
Запуск ZFS тестов в режиме stability
More like this
213
More like this
159
More like this