#!/bin/ash
#
# This script does various diagnostics of the SP.
# The results are displayed as three different status types:
# [ OK ]
# [ WARNING ]
# [ ERROR]
#
#set -x

check_ipmi_stack() {
     if [ -f "/tmp/ipmiMsgHdlrUp" ]
     then
         if [ -f "/tmp/ipmi_pid_log"  ]
         then
             ipmi_good=1
             tail -n 12 "/tmp/ipmi_pid_log" | awk '{split($NF,a,"="); print substr(a[2],1,length(a[2]-1))}' | (
             while  read line
             do
                 #Just checks if the PID is running by sending signal 0. No PID is killed.
                 if [ $line -ge 1000 ]
                 then
                     echo "[ WARNING ] PID $line is > 1000 and may have been restarted"
                 fi
                 if [ ! -d "/proc/"$line ]
                 then
                     echo "[ ERROR ] IPMI stack does not operate properly. A PID ($line) is not running."
                     ipmi_good=0
                 fi
             done
             if [ $ipmi_good -eq 1 ]
             then
                     echo "[ OK ] IPMI stack works normally."
             else
                     echo "[ ERROR ] IPMI stack has a problem."
             fi
             )
         else
            echo "ipmi_pid_lock file does not exist! skipping tests for IPMI stack pids"
         fi
     else
         echo "[ ERROR ] The IPMI stack has not been started !"
     fi
}

check_running_processes() {
    processes_good=1
    cat "/etc/watcher.conf" | awk '{lst=split($NF,a,"/"); if (length(a[lst]) > 0 && $1 !~ /#/) print a[lst];}' | (
       while read line
       do
           echo $line
           # Skip these processes for RazorL
           if [ $line = "isvcd" ] || [ $line = "spcs" ]
           then
               if [ $platform = "RazorL" ]
               then
                  continue
               fi
           fi
           # Find the parent process
           pid=$(pidof $line | awk '{split($0,arr);asort(arr);print arr[1]}');
           if [ $pid -ge 0 ]
           then
              if [ $pid -ge 1200 ]
              then
                 echo "[ WARNING ] $line has been restarted - pid is now: $pid."
              fi
           else
               processes_good=0
               echo $line "[ ERROR ] process is not running!"
           fi
       done
       #Just checks if the process is running by sending signal 0. No process is killed.
       if [ $processes_good -eq 1 ]
       then
          echo "[ OK ] All SP processes run normally."
       else
          echo "[ ERROR ] Not all SP processes are running."
       fi
    )
}

check_storage_os() {
    bbsp.sh
}

check_filesystems() {
    df | awk '{print $0; var=substr($5,1,length($5)-1); if(int(var) > 50) print "[ WARNING ] file system exceeds expected value!";   }'
}

check_memory() {
    free | awk '{ print $0; if(($4 < 50000) && ($1 == "Mem:")) print "[ WARNING ] memory exceeded expected value!"; }'
}

print_line() {
   echo "________________________________________________________________________________________"
}

print_double_line() {
   echo "========================================================================================"
}

#Script starts here:
echo
echo "SP DIAGNOSTICS TOOL"
platform=$(cat /var/run/platform)
print_double_line
#Check version and network status
echo "Checking network parameters"
rlmcli_status.sh
print_line
#Check SP load
echo "Checking Linux load"
cat /proc/loadavg
sp_load.sh
echo
if [ $? -ne 0 ]
then
    echo "[ WARNING ] SP's Linux load is too high. Check SP events and run Linux top command."
else
    echo "[ OK ] SP's Linux load is normal"
fi
print_line

#Check used and free memory
echo
echo "Checking memory usage"
check_memory
print_line

#Check the file systems' space
echo
echo "Checking filesystems's usage"
check_filesystems
print_line


#Check if the IPMI stack is running
echo
echo "Checking IPMI stack"
check_ipmi_stack
print_line


#Check the watchdog status
echo
echo "Checking system watchdog status"
system_watchdog status
print_line

#Check all SP processes
echo
echo "Checking if SP processes are running"
check_running_processes
print_line

#Check if a storage OS is running
echo
echo "Checking if a storage OS is running"
check_storage_os


