STABILITY test script

#!/bin/bash

get_duration() {
  IFS=':' read -r -a tm <<< "$1"
  num=${#tm[@]}
  delta=$((${tm[0]} * 3600))
  if [[ $num -gt 1 ]]; then
    delta1=$((${tm[1]} * 60))
    (( delta += $delta1 ))
    if [[ $num -gt 2 ]]; then
      (( delta += ${tm[2]} ))
    fi
  fi
  echo $delta
}

get_time_diff() {
  local cur_time=$(date +%s)
  echo $(($cur_time - $1))
}

print_time_diff() {
  local delta=$(get_time_diff $1)
  local sec=$(($delta % 60))
  let "amin = $delta / 60"
  let "hour = $amin / 60"
  local min=$(($amin%60))
  if [ -n "$2" ]; then
    printf " Running time: %02d:%02d:%02d\n" $hour $min $sec >> ${ULOG}
  else
    printf " Running time: %02d:%02d:%02d\n" $hour $min $sec
  fi
}

boot_zone() {
  echo Shutting down zone $1...
  sudo zoneadm -z $1 halt
  echo Cleaning zone $1 dataset...
  sudo zoneadm -z $1 uninstall -F
  sudo zfs destroy -fr rpool/zones/$1
  echo Booting zone $1...
  sudo zoneadm -z $1 clone $1-orig && sudo zoneadm -z $1 boot
  ip=""
  while [ "${ip}" == "" ] || [ "${ip}" == "0.0.0.0" ]
  do
    ip=$(sudo zlogin -l root $1 ifconfig | grep -A 1 "${2}" | sed "/${2}/d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//")
  done
  echo "Zone:$1 ------> $2:$ip"
}

get_random_test() {
  local tests=(libc elf os util net)
  local len=${#tests[*]}
  local idx=$(($RANDOM % $len))
  echo "${tests[$idx]}"
}

log_extra() {
  echo "---------------- ZPOOL LIST ------------------" >> $1
  zpool list >> $1
  echo "----------------  ZFS LIST  ------------------" >> $1
  zfs list >> $1
  echo "----------------    PS AX   ------------------" >> $1
  ps ax >> $1
  echo "---------------- TOP  -n 10 ------------------" >> $1
  top -n 10 >> $1
}

i=1
testnm="__ALL__"
shuffle=0
mode=0
TOTALLOG="/var/tmp/stability.$(date +%F-%T).txt"

echo "------------ Stability test started at $(date +%F-%T) --------------" > ${TOTALLOG}
if [[ $# < 2 ]]; then
  echo "Missing parameters in the command" | /usr/bin/tee -a ${TOTALLOG}
  exit 1
fi

case "$1" in
  -i) mode=1
      # number of iterations is provided
      maxiter=$2
      echo "Stability Test : launched for $maxiter iterations at $(date)." | /usr/bin/tee -a ${TOTALLOG}
      ;;
  -d) mode=2
      # duration of tests is provided
      delta=$(get_duration $2)
      echo "Stability Test : launched for $2 interval ($delta seconds) at $(date)." | /usr/bin/tee -a ${TOTALLOG}
      ;;
  -t) mode=3
      # stop date/time is provided
      stopdt=$2
      ;;
  *)  echo "Stability Test : $1 is a wrong option" | /usr/bin/tee -a ${TOTALLOG}
      echo "Usage : stability.sh (-i <iterations> | -t <HH:MM[:SS]> | -d <HH:MM[:SS] [DAY-MONTH-YEAR]>]) [-s | <test name>]"
      exit 1
      ;;
esac
shift
shift

if [[ $mode -eq 3 ]]; then
  # stop time/date mode
  if [[ $# > 0 ]]; then
    date -d $1 +%s
    stopat=$?
    if [[ $stopat -eq 0 ]]; then
      # stop date is entered after the time
      # if it is ommited then date is treated as current one
      stopdt="$stopdt $1"
      shift
    fi
  fi
  echo "Stability Test : launched until $stopdt at $(date)." | /usr/bin/tee -a ${TOTALLOG}
  stopat=$(date -d "$stopdt" "+%s")
fi

if [[ $# > 0 ]]; then
  # shuffle mode or test name is entered
  if [[ "$1" == "-s" ]]; then
    echo "Shuffle mode is turned on." | /usr/bin/tee -a ${TOTALLOG}
    shuffle=1
  else
    testnm=$1
    echo "Only $testnm test will be running." | /usr/bin/tee -a ${TOTALLOG}
  fi
fi

LLOG="/var/tmp/libc-stability.$(date +%F-%T).txt"
echo "==================    Stability libc tests $(date +%F-%T)    ==================" > ${LLOG}
ELOG="/var/tmp/elf-stability.$(date +%F-%T).txt"
echo "==================    Stability elf tests $(date +%F-%T)     ==================" > ${ELOG}
OLOG="/var/tmp/os-stability.$(date +%F-%T).txt"
echo "==================    Stability os tests $(date +%F-%T)      ==================" > ${OLOG}
ULOG="/var/tmp/util-stability.$(date +%F-%T).txt"
echo "================== Stability util & mdb tests $(date +%F-%T) ==================" > ${ULOG}
NLOG="/var/tmp/net-stability.$(date +%F-%T).txt"
echo "==================    Stability net tests $(date +%F-%T)    ==================" > ${NLOG}

ipa=$(sudo ifconfig | grep -A 1 vmxnet3s0 | sed '1d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//')

GLOB_START_TIME=$(date +%s)

while [[ $mode -gt 0 ]]
do
  case "$mode" in
  1) # number of iterations is provided
     if [[ $i -gt $maxiter ]]; then
       break
     fi
     ;;
  2) # duration of tests is provided
     difftm=$(get_time_diff $GLOB_START_TIME)
     if [[ $difftm -gt $delta ]]; then
       break
     fi
     ;;
  3) # stop date/time is provided
     tmpat=$(date +%s)
     if [[ $stopat -le $tmpat ]]; then
       break
     fi
     ;;
  *) break
     ;;
  esac

  if [[ $shuffle -eq 1 ]]; then
    testnm=$(get_random_test)
  fi

  if [[ "$testnm" == "libc" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${LLOG}
    LOG="/var/tmp/libc-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    /opt/libc-tests/bin/libctest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo "    Percent passed: ${percent}" >> ${LLOG}
    echo "    Running Time:   ${rt}" >> ${LLOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
      sudo rm -f ${LOG}
      sudo rm -r -f ${folder}
    else
      print_time_diff ${GLOB_START_TIME}
      echo "    PASS ${i} IS FAILED! (See ${LOG})" >> ${LLOG}
      log_extra ${LLOG}
      echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
      echo "LIBC test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
      exit 1
    fi
  fi

  if [[ "$testnm" == "elf" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${ELOG}
    LOG="/var/tmp/elftest.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    sudo /opt/elf-tests/bin/elftest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo " Percent passed: ${percent}" >> ${ELOG}
    echo " Running Time: ${rt}" >> ${ELOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
      sudo rm -f ${LOG}
      sudo rm -f /var/tmp/*.o /var/tmp/*.c /var/tmp/*.le
      sudo rm -r -f ${folder}
    else
      print_time_diff ${GLOB_START_TIME}
      echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ELOG}
      log_extra ${ELOG}
      echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
      echo "ELF test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
      exit 1
    fi
  fi

  if [[ "$testnm" == "os" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${OLOG}
    LOG="/var/tmp/ostest-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    sudo /opt/os-tests/bin/ostest /var/tmp 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo " Percent passed: ${percent}" >> ${OLOG}
    echo " Running Time: ${rt}" >> ${OLOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo rm -f ${LOG}
        sudo rm -f /tmp/otst*
        sudo rm -r -f ${folder}
    else
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${OLOG}
        log_extra ${OLOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        echo "OS test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
        exit 1
    fi
  fi
  if [[ "$testnm" == "util" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "--------------------------------------- Pass ${i} ------------------------------------" 2>&1 | /usr/bin/tee -a ${ULOG}
    START_TIME=$(date +%s)
    LOG="/var/tmp/util-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    echo "---------------------------------------- UTILS --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
    /opt/util-tests/bin/utiltest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    echo " UTILS percent passed: ${percent}" >> ${ULOG}
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" != "100.0%" ]; then
        print_time_diff ${START_TIME} ${ULOG}
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
        log_extra ${ULOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        echo "UTIL test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
        exit 1
    fi
    sudo rm -r -f ${folder}
    echo "---------------------------------------- MDB --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
    /opt/util-tests/bin/utiltest -c /opt/util-tests/runfiles/mdb.run 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Percent passed:' | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    echo " MDB percent passed: ${percent}" >> ${ULOG}
    folder="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Log directory:' | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    print_time_diff ${START_TIME} ${ULOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo rm -f ${LOG}
        sudo rm -r -f ${folder}
    else
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
        log_extra ${ULOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        echo "MDB test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
        exit 1
    fi
  fi
  if [[ "$testnm" == "net" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${NLOG}
    LOG="/var/tmp/net-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    echo "" >> ${LOG}
    echo ----------------------------- IP Address ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo ifconfig | grep -A 1 vmxnet3s0 2>&1 | /usr/bin/tee -a ${LOG}
    boot_zone cz int1 ${LOG}
    boot_zone sz int2 ${LOG}
    boot_zone rz int3 ${LOG}
    echo ------------------------------- Zones ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo zoneadm list -v 2>&1 | /usr/bin/tee -a ${LOG}
    echo ---------------------------- DHCP Server ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo svcs -z dhcp isc-dhcp-server 2>&1 | /usr/bin/tee -a ${LOG}
    echo ------------------------------- Tests ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo /opt/net-tests/bin/nettest /var/tmp 2>&1 | /usr/bin/tee -a /var/tmp/a.tmp
    percent="$(grep 'Percent passed:' /var/tmp/a.tmp | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' /var/tmp/a.tmp | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' /var/tmp/a.tmp | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo "    Percent passed: ${percent}" >> ${NLOG}
    echo "    Running Time:   ${rt}" >> ${NLOG}
    sed -n '/Results Summary/,/Log directory:/p' /var/tmp/a.tmp >> ${LOG}
    echo "" >> ${LOG}
    cat /var/tmp/a.tmp >> ${LOG}
    rm -f /var/tmp/a.tmp
    if [ "${percent}" == "100.0%" ]; then
       sudo rm -f ${LOG}
       sudo rm -r -f ${folder}
    else
       print_time_diff ${GLOB_START_TIME}
       echo "    PASS ${i} IS FAILED! (See ${LOG})" >> ${NLOG}
       log_extra ${NLOG}
       echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
       echo "NET test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
       exit 1
    fi
  fi
  (( i++ ))
done
echo "Tests are finished successfully at $(date) with ${i} passes." >> ${TOTALLOG}
print_time_diff ${GLOB_START_TIME} | /usr/bin/tee -a ${TOTALLOG}