STABILITY test script
- Vitaly Shaburov
Owned by Vitaly Shaburov
#!/bin/bash
get_duration() {
IFS=':' read -r -a tm <<< "$1"
num=${#tm[@]}
delta=$((${tm[0]} * 3600))
if [[ $num -gt 1 ]]; then
delta1=$((${tm[1]} * 60))
(( delta += $delta1 ))
if [[ $num -gt 2 ]]; then
(( delta += ${tm[2]} ))
fi
fi
echo $delta
}
get_time_diff() {
local cur_time=$(date +%s)
echo $(($cur_time - $1))
}
print_time_diff() {
local delta=$(get_time_diff $1)
local sec=$(($delta % 60))
let "amin = $delta / 60"
let "hour = $amin / 60"
local min=$(($amin%60))
if [ -n "$2" ]; then
printf " Running time: %02d:%02d:%02d\n" $hour $min $sec >> ${ULOG}
else
printf " Running time: %02d:%02d:%02d\n" $hour $min $sec
fi
}
boot_zone() {
echo Shutting down zone $1...
sudo zoneadm -z $1 halt
echo Cleaning zone $1 dataset...
sudo zoneadm -z $1 uninstall -F
sudo zfs destroy -fr rpool/zones/$1
echo Booting zone $1...
sudo zoneadm -z $1 clone $1-orig && sudo zoneadm -z $1 boot
ip=""
while [ "${ip}" == "" ] || [ "${ip}" == "0.0.0.0" ]
do
ip=$(sudo zlogin -l root $1 ifconfig | grep -A 1 "${2}" | sed "/${2}/d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//")
done
echo "Zone:$1 ------> $2:$ip"
}
get_random_test() {
local tests=(libc elf os util net)
local len=${#tests[*]}
local idx=$(($RANDOM % $len))
echo "${tests[$idx]}"
}
log_extra() {
echo "---------------- ZPOOL LIST ------------------" >> $1
zpool list >> $1
echo "---------------- ZFS LIST ------------------" >> $1
zfs list >> $1
echo "---------------- PS AX ------------------" >> $1
ps ax >> $1
echo "---------------- TOP -n 10 ------------------" >> $1
top -n 10 >> $1
}
i=1
testnm="__ALL__"
shuffle=0
mode=0
TOTALLOG="/var/tmp/stability.$(date +%F-%T).txt"
echo "------------ Stability test started at $(date +%F-%T) --------------" > ${TOTALLOG}
if [[ $# < 2 ]]; then
echo "Missing parameters in the command" | /usr/bin/tee -a ${TOTALLOG}
exit 1
fi
case "$1" in
-i) mode=1
# number of iterations is provided
maxiter=$2
echo "Stability Test : launched for $maxiter iterations at $(date)." | /usr/bin/tee -a ${TOTALLOG}
;;
-d) mode=2
# duration of tests is provided
delta=$(get_duration $2)
echo "Stability Test : launched for $2 interval ($delta seconds) at $(date)." | /usr/bin/tee -a ${TOTALLOG}
;;
-t) mode=3
# stop date/time is provided
stopdt=$2
;;
*) echo "Stability Test : $1 is a wrong option" | /usr/bin/tee -a ${TOTALLOG}
echo "Usage : stability.sh (-i <iterations> | -t <HH:MM[:SS]> | -d <HH:MM[:SS] [DAY-MONTH-YEAR]>]) [-s | <test name>]"
exit 1
;;
esac
shift
shift
if [[ $mode -eq 3 ]]; then
# stop time/date mode
if [[ $# > 0 ]]; then
date -d $1 +%s
stopat=$?
if [[ $stopat -eq 0 ]]; then
# stop date is entered after the time
# if it is ommited then date is treated as current one
stopdt="$stopdt $1"
shift
fi
fi
echo "Stability Test : launched until $stopdt at $(date)." | /usr/bin/tee -a ${TOTALLOG}
stopat=$(date -d "$stopdt" "+%s")
fi
if [[ $# > 0 ]]; then
# shuffle mode or test name is entered
if [[ "$1" == "-s" ]]; then
echo "Shuffle mode is turned on." | /usr/bin/tee -a ${TOTALLOG}
shuffle=1
else
testnm=$1
echo "Only $testnm test will be running." | /usr/bin/tee -a ${TOTALLOG}
fi
fi
LLOG="/var/tmp/libc-stability.$(date +%F-%T).txt"
echo "================== Stability libc tests $(date +%F-%T) ==================" > ${LLOG}
ELOG="/var/tmp/elf-stability.$(date +%F-%T).txt"
echo "================== Stability elf tests $(date +%F-%T) ==================" > ${ELOG}
OLOG="/var/tmp/os-stability.$(date +%F-%T).txt"
echo "================== Stability os tests $(date +%F-%T) ==================" > ${OLOG}
ULOG="/var/tmp/util-stability.$(date +%F-%T).txt"
echo "================== Stability util & mdb tests $(date +%F-%T) ==================" > ${ULOG}
NLOG="/var/tmp/net-stability.$(date +%F-%T).txt"
echo "================== Stability net tests $(date +%F-%T) ==================" > ${NLOG}
ipa=$(sudo ifconfig | grep -A 1 vmxnet3s0 | sed '1d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//')
GLOB_START_TIME=$(date +%s)
while [[ $mode -gt 0 ]]
do
case "$mode" in
1) # number of iterations is provided
if [[ $i -gt $maxiter ]]; then
break
fi
;;
2) # duration of tests is provided
difftm=$(get_time_diff $GLOB_START_TIME)
if [[ $difftm -gt $delta ]]; then
break
fi
;;
3) # stop date/time is provided
tmpat=$(date +%s)
if [[ $stopat -le $tmpat ]]; then
break
fi
;;
*) break
;;
esac
if [[ $shuffle -eq 1 ]]; then
testnm=$(get_random_test)
fi
if [[ "$testnm" == "libc" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${LLOG}
LOG="/var/tmp/libc-test.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : ${ipa}" >> ${LOG}
/opt/libc-tests/bin/libctest 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" == "" ]; then
percent="0.0%"
fi
if [ "${rt}" == "" ]; then
rt="0 sec"
fi
echo " Percent passed: ${percent}" >> ${LLOG}
echo " Running Time: ${rt}" >> ${LLOG}
sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
echo "---------------------------------------------------------" >> a.tmp
echo "" >> a.tmp
cat ${LOG} >> a.tmp
mv -f a.tmp ${LOG}
if [ "${percent}" == "100.0%" ]; then
sudo rm -f ${LOG}
sudo rm -r -f ${folder}
else
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${LLOG}
log_extra ${LLOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "LIBC test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
exit 1
fi
fi
if [[ "$testnm" == "elf" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${ELOG}
LOG="/var/tmp/elftest.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
sudo /opt/elf-tests/bin/elftest 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" == "" ]; then
percent="0.0%"
fi
if [ "${rt}" == "" ]; then
rt="0 sec"
fi
echo " Percent passed: ${percent}" >> ${ELOG}
echo " Running Time: ${rt}" >> ${ELOG}
sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
echo "---------------------------------------------------------" >> a.tmp
echo "" >> a.tmp
cat ${LOG} >> a.tmp
mv -f a.tmp ${LOG}
if [ "${percent}" == "100.0%" ]; then
sudo rm -f ${LOG}
sudo rm -f /var/tmp/*.o /var/tmp/*.c /var/tmp/*.le
sudo rm -r -f ${folder}
else
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ELOG}
log_extra ${ELOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "ELF test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
exit 1
fi
fi
if [[ "$testnm" == "os" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${OLOG}
LOG="/var/tmp/ostest-test.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : ${ipa}" >> ${LOG}
sudo /opt/os-tests/bin/ostest /var/tmp 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" == "" ]; then
percent="0.0%"
fi
if [ "${rt}" == "" ]; then
rt="0 sec"
fi
echo " Percent passed: ${percent}" >> ${OLOG}
echo " Running Time: ${rt}" >> ${OLOG}
sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
echo "---------------------------------------------------------" >> a.tmp
echo "" >> a.tmp
cat ${LOG} >> a.tmp
mv -f a.tmp ${LOG}
if [ "${percent}" == "100.0%" ]; then
sudo rm -f ${LOG}
sudo rm -f /tmp/otst*
sudo rm -r -f ${folder}
else
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${OLOG}
log_extra ${OLOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "OS test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
exit 1
fi
fi
if [[ "$testnm" == "util" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "--------------------------------------- Pass ${i} ------------------------------------" 2>&1 | /usr/bin/tee -a ${ULOG}
START_TIME=$(date +%s)
LOG="/var/tmp/util-test.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : ${ipa}" >> ${LOG}
echo "---------------------------------------- UTILS --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
/opt/util-tests/bin/utiltest 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" == "" ]; then
percent="0.0%"
fi
echo " UTILS percent passed: ${percent}" >> ${ULOG}
folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" != "100.0%" ]; then
print_time_diff ${START_TIME} ${ULOG}
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
log_extra ${ULOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "UTIL test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
exit 1
fi
sudo rm -r -f ${folder}
echo "---------------------------------------- MDB --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
/opt/util-tests/bin/utiltest -c /opt/util-tests/runfiles/mdb.run 2>&1 | /usr/bin/tee -a ${LOG}
percent="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Percent passed:' | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" == "" ]; then
percent="0.0%"
fi
echo " MDB percent passed: ${percent}" >> ${ULOG}
folder="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Log directory:' | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
print_time_diff ${START_TIME} ${ULOG}
sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
echo "---------------------------------------------------------" >> a.tmp
echo "" >> a.tmp
cat ${LOG} >> a.tmp
mv -f a.tmp ${LOG}
if [ "${percent}" == "100.0%" ]; then
sudo rm -f ${LOG}
sudo rm -r -f ${folder}
else
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
log_extra ${ULOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "MDB test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
exit 1
fi
fi
if [[ "$testnm" == "net" ]] || [[ "$testnm" == "__ALL__" ]]; then
echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${NLOG}
LOG="/var/tmp/net-test.$i.txt"
sudo rm -f ${LOG}
uname -a > ${LOG}
echo "IP Address : ${ipa}" >> ${LOG}
echo "" >> ${LOG}
echo ----------------------------- IP Address ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo ifconfig | grep -A 1 vmxnet3s0 2>&1 | /usr/bin/tee -a ${LOG}
boot_zone cz int1 ${LOG}
boot_zone sz int2 ${LOG}
boot_zone rz int3 ${LOG}
echo ------------------------------- Zones ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo zoneadm list -v 2>&1 | /usr/bin/tee -a ${LOG}
echo ---------------------------- DHCP Server ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo svcs -z dhcp isc-dhcp-server 2>&1 | /usr/bin/tee -a ${LOG}
echo ------------------------------- Tests ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
sudo /opt/net-tests/bin/nettest /var/tmp 2>&1 | /usr/bin/tee -a /var/tmp/a.tmp
percent="$(grep 'Percent passed:' /var/tmp/a.tmp | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
folder="$(grep 'Log directory:' /var/tmp/a.tmp | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
rt="$(grep 'Running Time:' /var/tmp/a.tmp | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
if [ "${percent}" == "" ]; then
percent="0.0%"
fi
if [ "${rt}" == "" ]; then
rt="0 sec"
fi
echo " Percent passed: ${percent}" >> ${NLOG}
echo " Running Time: ${rt}" >> ${NLOG}
sed -n '/Results Summary/,/Log directory:/p' /var/tmp/a.tmp >> ${LOG}
echo "" >> ${LOG}
cat /var/tmp/a.tmp >> ${LOG}
rm -f /var/tmp/a.tmp
if [ "${percent}" == "100.0%" ]; then
sudo rm -f ${LOG}
sudo rm -r -f ${folder}
else
print_time_diff ${GLOB_START_TIME}
echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${NLOG}
log_extra ${NLOG}
echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
echo "NET test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
exit 1
fi
fi
(( i++ ))
done
echo "Tests are finished successfully at $(date) with ${i} passes." >> ${TOTALLOG}
print_time_diff ${GLOB_START_TIME} | /usr/bin/tee -a ${TOTALLOG}