Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.
Code Block
breakoutModefull-width
languagebash
#!/bin/bash

printget_time_diffduration() {
  IFS=':' read -r -a tm  end_time=$(date +%s)
 <<< "$1"
  num=${#tm[@]}
  delta=$(($end_time - $1${tm[0]} * 3600))
  if  sec=$(($delta % 60))
 [[ $num -gt 1 ]]; then
  let "amin delta1= $delta / 60"$((${tm[1]} * 60))
    let(( "hourdelta += $amin / 60"
    min=$(($amin%60$delta1 ))
    if [[ $num -ngt "$2"2 ]]; then
      (( delta printf " Running time: %02d:%02d:%02d\n" $hour $min $sec >> ${ULOG}
    else
        printf " Running time: %02d:%02d:%02d\n" $hour $min $sec
    fi
}

boot_zone+= ${tm[2]} ))
    fi
  fi
  echo $delta
}

get_time_diff() {
  local cur_time=$(date +%s)
  echo $(($cur_time - $1))
}

print_time_diff() {
    echo Shutting down zone $1...local delta=$(get_time_diff $1)
    sudo zoneadm -z $1 halt
    echo Cleaning zone $1 dataset...
    sudo zoneadm -z $1 uninstall -F
    sudo zfs destroy -fr rpool/zones/$1
    echo Booting zone $1...
    sudo zoneadm -z $1 clone $1-orig && sudo zoneadm -z $1 boot
    ip=""
    while [ "${ip}" == "" ] || [ "${ip}" == "0.0.0.0" ]
    do
        ip=$(sudo zlogin -l root $1 ifconfig | grep -A 1 "${2}" | sed "/${2}/d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//")
    done
    echo "Zone:$1 ------> $2:$ip"
}

LLOG="/var/tmp/libc-stability.$(date +%F-%T).txt"
echo "==================    Stability libc tests $(date +%F-%T)    ==================" > ${LLOG}
ELOG="/var/tmp/elf-stability.$(date +%F-%T).txt"
echo "==================    Stability elf tests $(date +%F-%T)     ==================" > ${ELOG}
OLOG="/var/tmp/os-stability.$(date +%F-%T).txt"
echo "==================    Stability os tests $(date +%F-%T)      ==================" > ${OLOG}
ULOG="/var/tmp/util-stability.$(date +%F-%T).txt"
echo "================== Stability util & mdb tests $(date +%F-%T) ==================" > ${ULOG}
NLOG="/var/tmp/net-stability.$(date +%F-%T).txt"
echo "==================    Stability net tests \$(date +%F-%T)    ==================" > ${NLOG}

ipa=$(sudo ifconfig | grep -A 1 vmxnet3s0 | sed '1d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//')

GLOB_START_TIME=$(date +%s)
for (( i=1; i<=$1; i++ ))
do
    echo "-local sec=$(($delta % 60))
  let "amin = $delta / 60"
  let "hour = $amin / 60"
  local min=$(($amin%60))
  if [ -n "$2" ]; then
    printf " Running time: %02d:%02d:%02d\n" $hour $min $sec >> ${ULOG}
  else
    printf " Running time: %02d:%02d:%02d\n" $hour $min $sec
  fi
}

boot_zone() {
  echo Shutting down zone $1...
  sudo zoneadm -z $1 halt
  echo Cleaning zone $1 dataset...
  sudo zoneadm -z $1 uninstall -F
  sudo zfs destroy -fr rpool/zones/$1
  echo Booting zone $1...
  sudo zoneadm -z $1 clone $1-orig && sudo zoneadm -z $1 boot
  ip=""
  while [ "${ip}" == "" ] || [ "${ip}" == "0.0.0.0" ]
  do
    ip=$(sudo zlogin -l root $1 ifconfig | grep -A 1 "${2}" | sed "/${2}/d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//")
  done
  echo "Zone:$1 ------> $2:$ip"
}

get_random_test() {
  local tests=(libc elf os util net)
  local len=${#tests[*]}
  local idx=$(($RANDOM % $len))
  echo "${tests[$idx]}"
}

log_extra() {
  echo "---------------- ZPOOL LIST ------------------" >> $1
  zpool list >> $1
  echo "----------------  ZFS LIST  ------------------" >> $1
  zfs list >> $1
  echo "----------------    PS AX   ------------------" >> $1
  ps ax >> $1
  echo "---------------- TOP  -n 10 ------------------" >> $1
  top -n 10 >> $1
}

i=1
testnm="__ALL__"
shuffle=0
mode=0
TOTALLOG="/var/tmp/stability.$(date +%F-%T).txt"

echo "----------------- Pass ${i} --- Stability test started at $(date +%F-%T) ------------------" 2>&1 | /usr/bin/tee -a> ${LLOGTOTALLOG}
if [[ $# < 2  LOG="/var/tmp/libc-test.$i.txt"
    uname]]; then
  echo "Missing parameters in the command" | /usr/bin/tee -a > ${LOGTOTALLOG}
  exit 1
echofi
"IP
Address : \${ipa}" >> ${LOG}
 case "$1" in
  /opt/libc-tests/bin/libctest 2>&1 | /usr/bin/tee -a ${LOG}-i) mode=1
      #  percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"number of iterations is provided
      maxiter=$2
      echo folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    echo "    Percent passed: ${percent}" >> ${LLOG}
    echo "    Running Time:   ${rt}" >> ${LLOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo rm -r -f ${folder}
    else
        print_time_diff ${GLOB_START_TIME}
        echo "    PASS ${i} IS FAILED! (See ${LOG})" >> ${LLOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        exit 1
    fi

    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${ELOG}
    LOG="/var/tmp/elftest.$i.txt"
    uname -a > ${LOG}
    sudo /opt/elf-tests/bin/elftest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    echo " Percent passed: ${percent}" >> ${ELOG}
    echo " Running Time: ${rt}" >> ${ELOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo rm -f /var/tmp/*.o /var/tmp/*.c /var/tmp/*.le
        sudo rm -r -f ${folder}
    else
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ELOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        exit 1
    fi

    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${OLOG}
    LOG="/var/tmp/ostest-test.$i.txt"
    uname -a > ${LOG}
    echo "IP Address : \${ipa}" >> ${LOG}
    sudo /opt/os-tests/bin/ostest /var/tmp 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    echo " Percent passed: ${percent}" >> ${OLOG}
    echo " Running Time: ${rt}" >> ${OLOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo rm -f /tmp/otst*
        sudo rm -r -f ${folder}
    else
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${OLOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}."Stability Test : launched for $maxiter iterations at $(date)." | /usr/bin/tee -a ${TOTALLOG}
      ;;
  -d) mode=2
      # duration of tests is provided
      delta=$(get_duration $2)
      echo "Stability Test : launched for $2 interval ($delta seconds) at $(date)." | /usr/bin/tee -a ${TOTALLOG}
      ;;
  -t) mode=3
      # stop date/time is provided
      stopdt=$2
      ;;
  *)  echo "Stability Test : $1 is a wrong option" | /usr/bin/tee -a ${TOTALLOG}
      echo "Usage : stability.sh (-i <iterations> | -t <HH:MM[:SS]> | -d <HH:MM[:SS] [DAY-MONTH-YEAR]>]) [-s | <test name>]"
      exit 1
      ;;
esac
shift
shift

if [[ $mode -eq 3 ]]; then
  # stop time/date mode
  if [[ $# > 0 ]]; then
    date -d $1 +%s
    stopat=$?
    if [[ $stopat -eq 0 ]]; then
      # stop date is entered after the time
      # if it is ommited then date is treated as current one
      stopdt="$stopdt $1"
      shift
    fi
  fi
  echo "Stability Test : launched until $stopdt at $(date)." | /usr/bin/tee -a ${TOTALLOG}
  stopat=$(date -d "$stopdt" "+%s")
fi

if [[ $# > 0 ]]; then
  # shuffle mode or test name is entered
  if [[ "$1" == "-s" ]]; then
    echo "Shuffle mode is turned on." | /usr/bin/tee -a ${TOTALLOG}
    shuffle=1
  else
    testnm=$1
    echo "Only $testnm test will be running." | /usr/bin/tee -a ${TOTALLOG}
  fi
fi

LLOG="/var/tmp/libc-stability.$(date +%F-%T).txt"
echo "==================    Stability libc tests $(date +%F-%T)    ==================" > ${LLOG}
ELOG="/var/tmp/elf-stability.$(date +%F-%T).txt"
echo "==================    Stability elf tests $(date +%F-%T)     ==================" > ${ELOG}
OLOG="/var/tmp/os-stability.$(date +%F-%T).txt"
echo "==================    Stability os tests $(date +%F-%T)      ==================" > ${OLOG}
ULOG="/var/tmp/util-stability.$(date +%F-%T).txt"
echo "================== Stability util & mdb tests $(date +%F-%T) ==================" > ${ULOG}
NLOG="/var/tmp/net-stability.$(date +%F-%T).txt"
echo "==================    Stability net tests $(date +%F-%T)    ==================" > ${NLOG}

ipa=$(sudo ifconfig | grep -A 1 vmxnet3s0 | sed '1d; s/^[ \t]*inet[ \t]*//; s/[ \t]*netmask.*$//')

GLOB_START_TIME=$(date +%s)

while [[ $mode -gt 0 ]]
do
  case "$mode" in
  1) # number of iterations is provided
     if [[ $i -gt $maxiter ]]; then
       break
     fi
     ;;
  2) # duration of tests is provided
     difftm=$(get_time_diff $GLOB_START_TIME)
     if [[ $difftm -gt $delta ]]; then
       break
     fi
     ;;
  3) # stop date/time is provided
     tmpat=$(date +%s)
     if [[ $stopat -le $tmpat ]]; then
       break
     fi
     ;;
  *) break
     ;;
  esac

  if [[ $shuffle -eq 1 ]]; then
    testnm=$(get_random_test)
  fi

  if [[ "$testnm" == "libc" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${LLOG}
    LOG="/var/tmp/libc-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    /opt/libc-tests/bin/libctest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} exit| 1sed 's/Percent passed:[ \t]*//;  fis/[ \t]*$//')"
     echo "--------------------------------------- Pass ${i} ------------------------------------folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo "    Percent passed: ${percent}" >> ${LLOG}
    echo "    Running Time:   ${rt}" >> ${LLOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
      sudo rm -f ${LOG}
      sudo rm -r -f ${folder}
    else
      print_time_diff ${GLOB_START_TIME}
      echo "    PASS ${i} IS FAILED! (See ${LOG})" >> ${LLOG}
      log_extra ${LLOG}
      echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
      echo "LIBC test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
      exit 1
    fi
  fi

  if [[ "$testnm" == "elf" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${ELOG}
    LOG="/var/tmp/elftest.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    sudo /opt/elf-tests/bin/elftest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo " Percent passed: ${percent}" >> ${ELOG}
    echo " Running Time: ${rt}" >> ${ELOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
      sudo rm -f ${LOG}
      sudo rm -f /var/tmp/*.o /var/tmp/*.c /var/tmp/*.le
      sudo rm -r -f ${folder}
    else
      print_time_diff ${GLOB_START_TIME}
      echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ELOG}
      log_extra ${ELOG}
      echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
      echo "ELF test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
      exit 1
    fi
  fi

  if [[ "$testnm" == "os" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee -a ${OLOG}
    LOG="/var/tmp/ostest-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    sudo /opt/os-tests/bin/ostest /var/tmp 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' ${LOG} | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo " Percent passed: ${percent}" >> ${OLOG}
    echo " Running Time: ${rt}" >> ${OLOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo "---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo rm -f ${LOG}
        sudo rm -f /tmp/otst*
        sudo rm -r -f ${folder}
    else
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${OLOG}
        log_extra ${OLOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        echo "OS test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
        exit 1
    fi
  fi
  if [[ "$testnm" == "util" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "--------------------------------------- Pass ${i} ------------------------------------" 2>&1 | /usr/bin/tee -a ${ULOG}
    START_TIME=$(date +%s)
    LOG="/var/tmp/util-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
    echo "IP Address : ${ipa}" >> ${LOG}
    echo "---------------------------------------- UTILS --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
    /opt/util-tests/bin/utiltest 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grep 'Percent passed:' ${LOG} | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    echo " UTILS percent passed: ${percent}" >> ${ULOG}
    folder="$(grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" != "100.0%" ]; then
        print_time_diff ${START_TIME} ${ULOG}
        print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
        log_extra ${ULOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${ULOGLOG}

   START_TIME=$(date +%s)    echo LOG="/var/tmp/util-test.$i.txt""UTIL test is failed on ${i} pass at $(date)." >> ${TOTALLOG}
        exit uname1
-a > ${LOG}  fi
  echo "IP Addresssudo : \${ipa}" >>rm -r -f ${LOGfolder}
    echo "---------------------------------------- UTILSMDB --------------------------------------" 2>&1 | /usr/bin/tee -a ${LOG}
    /opt/util-tests/bin/utiltest -c /opt/util-tests/runfiles/mdb.run 2>&1 | /usr/bin/tee -a ${LOG}
    percent="$(grepsed 'Percent passed:/- UTILS -/,/- MDB -/d' ${LOG}LOG} | grep 'Percent passed:' | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    echo " UTILSMDB percent passed: ${percent}" >> ${ULOG}
    folder="$(: ${percent}" >> ${ULOG}
    folder="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Log directory:' ${LOG} | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" != "100.0%" ]; then

       print_time_diff ${START_TIME} ${ULOG}
    sed -n '/Results  print_time_diff ${GLOB_START_TIME}
   Summary/,/Log directory:/p' ${LOG} > a.tmp
    echo " PASS ${i} IS FAILED! (See ${LOG})" >> ${ULOG}
        echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}---------------------------------------------------------" >> a.tmp
    echo "" >> a.tmp
    cat ${LOG} >> a.tmp
    mv -f a.tmp ${LOG}
    if [ "${percent}" == "100.0%" ]; then
        sudo exitrm 1
-f ${LOG}
   fi     sudo rm -r -f ${folder}
    else
    echo "---------------------------------------- MDB --------------------------------------" 2>&1 | /usr/bin/tee -a    print_time_diff ${GLOB_START_TIME}
        echo " PASS ${i} IS FAILED! (See ${LOG})" >>  ${ULOG}
 /opt/util-tests/bin/utiltest -c /opt/util-tests/runfiles/mdb.run 2>&1 | /usr/bin/tee -a ${LOG}      log_extra ${ULOG}
        echo percent="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Percent passed:' | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
        echo " MDB percenttest passed: ${percent}" >>is failed on ${ULOGi} pass   at folder="$(sed '/- UTILS -/,/- MDB -/d' ${LOG} | grep 'Log directory:' | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    print_time_diff ${START_TIME} ${ULOG}
    sed -n '/Results Summary/,/Log directory:/p' ${LOG} > a.tmpdate)." >> ${TOTALLOG}
        exit 1
    fi
  fi
  if [[ "$testnm" == "net" ]] || [[ "$testnm" == "__ALL__" ]]; then
    echo "------------------ Pass ${i} ---------------------" 2>&1 | /usr/bin/tee ------------------" >> a.tmp
    echo "" >> a.tmp
    cata ${NLOG}
    LOG="/var/tmp/net-test.$i.txt"
    sudo rm -f ${LOG}
    uname -a > ${LOG}
>> a.tmp   echo "IP mv -f a.tmpAddress : ${ipa}" >> ${LOG}
    echo if"" [>> "${percent}" == "100.0%" ]; then
        sudo rm -r -f ${folderLOG}
    echo ----------------------------- IP Address ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    elsesudo ifconfig | grep -A 1 vmxnet3s0 2>&1 |  print_time_diff/usr/bin/tee -a ${GLOB_START_TIMELOG}
    boot_zone cz int1 ${LOG}
echo " PASS ${i} IS FAILED! (See boot_zone sz int2 ${LOG})"
>> ${ULOG}   boot_zone rz int3 ${LOG}
  echo "Fault on ${i} pass! Just ${percent} are successfull. You can find logs in ${folder}."echo ------------------------------- Zones ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo zoneadm list -v exit 2>&1 | /usr/bin/tee -a ${LOG}
fi      echo "echo ---------------------------- Pass ${i} DHCP Server ----------------------" 2>&1 | /usr/bin/tee -a ${NLOG}
    boot_zone cz int1
    boot_zone sz int2
    boot_zone rz int3
    LOG="/var/tmp/net-test.$i.txt"
    uname -a > ${LOG}
    echosudo "IPsvcs Address-z : ${ipa}" >> ${LOG}
    echo "" >>dhcp isc-dhcp-server 2>&1 | /usr/bin/tee -a ${LOG}
    echo ------------------------------- IP Address Tests ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo ifconfig | grep -A 1 vmxnet3s0/opt/net-tests/bin/nettest /var/tmp 2>&1 | /usr/bin/tee -a ${LOG}/tee -a /var/tmp/a.tmp
     echo ------------------------------- Zones ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo zoneadm list -v 2>&1 | /usr/bin/tee -a ${LOG}
    echo ---------------------------- DHCP Server ---------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo svcs -z dhcp isc-dhcp-server 2>&1 | /usr/bin/tee -a ${LOG}
    echo ------------------------------- Tests ------------------------- 2>&1 | /usr/bin/tee -a ${LOG}
    sudo /opt/net-tests/bin/nettest /var/tmp 2>&1 | /usr/bin/tee -a /var/tmp/a.tmp
    percent="$(grep 'Percent passed:' /var/tmp/a.tmp | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' /var/tmp/a.tmp | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:percent="$(grep 'Percent passed:' /var/tmp/a.tmp | sed 's/Percent passed:[ \t]*//; s/[ \t]*$//')"
    folder="$(grep 'Log directory:' /var/tmp/a.tmp | sed 's/Log directory:[ \t]*//; s/[ \t]*$//')"
    rt="$(grep 'Running Time:' /var/tmp/a.tmp | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')"
    if [ "${percent}" == "" ]; then
      percent="0.0%"
    fi
    if [ "${rt}" == "" ]; then
      rt="0 sec"
    fi
    echo "    Percent passed: ${percent}" >> ${NLOG}
    echo "    Running Time:   ${rt}" >> ${NLOG}
    sed -n '/Results Summary/,/Log directory:/p' /var/tmp/a.tmp | sed 's/Running Time:[ \t]*//; s/[ \t]*$//')" >> ${LOG}
    echo "" >> ${LOG}
    cat /var/tmp/a.tmp >> ${LOG}
    echorm " -f /var/tmp/a.tmp
  Percent passed: ${percent}" >> ${NLOG}
    echo " if [ "${percent}" == "100.0%" ]; then
     Running Time:  sudo ${rt}" >>rm -f ${NLOGLOG}
    sed -n '/Results Summary/,/Log directory:/p' /var/tmp/a.tmp >> sudo rm -r -f ${LOGfolder}
    echo "" >> ${LOG}
else
   cat /var/tmp/a.tmp >>    print_time_diff ${LOGGLOB_START_TIME}
    rm -f /var/tmp/a.tmp
  echo "   if [PASS "${percenti}" == "100.0%" ]; then
 IS FAILED! (See ${LOG})" >> ${NLOG}
     sudo rm -flog_extra ${LOGNLOG}
       echo "Fault sudo rm -r -fon ${i} pass! Just ${folderpercent} are successfull. You can elsefind logs in ${folder}." 2>&1 |   print_time_diff/usr/bin/tee -a ${GLOB_START_TIMELOG}
       echo "NET test is failed PASSon ${i} ISpass FAILED!at (See ${LOG})$(date)." >> ${NLOGTOTALLOG}
  
    echo "Fault on ${i} pass! Just ${percent} areexit successfull.1
You can find logs in ${folder}." 2>&1 | /usr/bin/tee -a ${LOG}
       exit 1
    fi
donefi
  fi
  (( i++ ))
done
echo "Tests are finished successfully at $(date) with ${i} passes." >> ${TOTALLOG}
print_time_diff ${GLOB_START_TIME} | /usr/bin/tee -a ${TOTALLOG}