#!/bin/ksh

# 13 march

EXCLUDE_list=" "

WORKDIR="/h/bmetzger/s"
TMPDIR=${WORKDIR}/tmp/all
BACKUPDIR="/var/backups"

GLOBAL_SUMMARY=${TMPDIR}/global.summary.out     # summary/results of all tests
GLOBAL_PROBLEMS=${TMPDIR}/global.problems.out   # problems only 
GLOBAL_DETAILS=${TMPDIR}/global.details.out      # all details 

CS_list=`cat /etc/opt/p7install/servers.conf`
RSP_list=`/opt/platform7/lbin/list_processors -s |sort`
WST_list=`/opt/platform7/lbin/list_processors -w |grep -v m7ossca4 | sort `
DSP_list=`/opt/platform7/lbin/list_processors -d |sort`
BPP_list=`/opt/platform7/lbin/list_processors -b | sort `

CDT_time=`TZ=CDT7; date`  # change offset for daylight savings
GMT_time=`TZ=GMT0; date`
DOW=`/bin/date "+%a"`

# Current ETR and commsConfig are placed in TMPDIR by nightly backup script
commsConfig="${TMPDIR}/commsConfig" 
etr="${TMPDIR}/etr"


clear

if [ ! -d ${TMPDIR} ]; then mkdir ${TMPDIR} ; fi
if [ -f ${GLOBAL_SUMMARY} ]; then rm ${GLOBAL_SUMMARY}; fi
if [ -f ${GLOBAL_PROBLEMS} ]; then rm ${GLOBAL_PROBLEMS}; fi
if [ -f ${GLOBAL_DETAILS} ]; then rm ${GLOBAL_DETAILS}; fi

function summarize {
# send output to appropriate files

 if ((${bad_count} > 0))  # there was a problem
 then 
     echo "$header"           # to the screen
     cat ${LOCAL_PROBLEMS}

     echo "$header"        >> ${GLOBAL_SUMMARY}

     echo "$header"        >> ${GLOBAL_PROBLEMS} 
     cat ${LOCAL_PROBLEMS} >> ${GLOBAL_PROBLEMS}

     echo "$header"        >> ${GLOBAL_DETAILS}
     cat ${LOCAL_DETAILS}  >> ${GLOBAL_DETAILS}

  else
       echo "$header"        # to the screen
       echo "$header"    >> ${GLOBAL_SUMMARY}  # global results file 
       echo "$header"    >> ${GLOBAL_DETAILS}   # problems list 
       cat ${LOCAL_DETAILS}  >> ${GLOBAL_DETAILS}
  fi
} # summarize

function create_exclude_lists {

JUNKFILE=${TMPDIR}/exclude.junkfile
LOCAL_SUMMARY=${TMPDIR}/exclude.results
COMMSFILE=/var/opt/platform7/tmp/eng_logging_C32_0

if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi
if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${COMMSFILE} ]; then rm ${COMMSFILE}; fi

   p7dumpstatus c32 > /dev/null 2>&1 
   grep not ${COMMSFILE} | sed -e "s/[(|,|)]//g" | sort -n -k 5,7 > ${JUNKFILE}

   results_count=`wc -l ${JUNKFILE}| awk '{print $1}'`

   grep not ${COMMSFILE} | grep Datastore | sed -e 's/,//g' |sed -e 's/)//g' | cut -d" " -f 5,9 > $junkfile

    if [ -s $junkfile ]  #file is not zero length
    then 
         while read site dsp 
         do
            grep ds0 ${etr_file} | grep "^$site $dsp" | awk '{print $NF}'  >>${out_of_comms_list}
         done  < ${junkfile}    
    else     # file is zero length
         > ${out_of_comms_list}
    fi
}  # build_exclude_lists

function out_of_comms {

COMMSFILE=/var/opt/platform7/tmp/eng_logging_C32_0

JUNKFILE=${TMPDIR}/commscheck.junk
LOCAL_DETAILS=${TMPDIR}/commscheck.details
LOCAL_PROBLEMS=${TMPDIR}/commscheck.problems

integer bad_count=0

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS $COMMSFILE
do
   if [ -f ${f} ]; then rm ${f}; fi
done

 p7dumpstatus c32 > /dev/null 2>&1 
 sleep 5

 grep not ${COMMSFILE} | sed -e "s/[(|,|)]//g" | sort -n -k 5,7 > ${LOCAL_PROBLEMS}
 cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}

 bad_count=`wc -l ${LOCAL_PROBLEMS}| awk '{print $1}'`

 header="\n--- ${bad_count} Elements Out of Comms "

#LOCAL_PROBLEMS >> GLOBAL_PROBLEMS
#LOCAL_DETAILS  >> GLOBAL_DETAILS
#header  >> GLOBAL_SUMMARY
summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS $COMMSFILE
do
   if [ -f ${f} ]; then rm ${f}; fi
done
} #end out_of_comms


function top_processes {

JUNKFILE=${TMPDIR}/top.junk
LOCAL_DETAILS=${TMPDIR}/top.details
LOCAL_PROBLEMS=${TMPDIR}/top.problems

integer bad_count=0
integer THRESHOLD=30

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS 
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

for f in ${RSP_list} ${WST_list} ${CS_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then

        # top output
        #host  TTY  PID USERNAME PRI NI   SIZE    RES STATE    TIME %WCPU  %CPU COMMAND
        #s001   ? 11888 a7run    155 20   206M  2324K sleep  109:10  1.32  1.32 m61.run

        remsh $f  "touch /tmp/junk; rm /tmp/junk*; top -d 1 -h -n 1 -f /tmp/junk;  tail -1 /tmp/junk" |
                 awk '{printf ("%-9s %-8s   %8s    %-s\n","'$f'", $3, $(NF-1), $NF) }'  >> ${LOCAL_DETAILS}
   else
        print "$f not pingable" >> ${LOCAL_PROBLEMS}
   fi
done

   awk '($3 >= '$THRESHOLD') {printf ("%s   %s       %s      %s\n",$1, $2, $3, $4)}' ${LOCAL_DETAILS} >> ${LOCAL_PROBLEMS}
 
   bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`

   header="\n--- ${bad_count} Top processes which exceed ${THRESHOLD}% CPU Load "

   if ((${bad_count} > 0))  
   then
         # add additional header lines
         print "host      Username     %CPU    Command"  > $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "host      Username     %CPU    Command"  > $JUNKFILE
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

#LOCAL_PROBLEMS >> GLOBAL_PROBLEMS
#LOCAL_DETAILS  >> GLOBAL_DETAILS
#header  >> GLOBAL_SUMMARY
summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS 
do
   if [ -f ${f} ]; then rm ${f}; fi
done
} # top_processes


function swapinfo {

JUNKFILE=${TMPDIR}/swapinfo.junk
LOCAL_DETAILS=${TMPDIR}/swapinfo.details
LOCAL_PROBLEMS=${TMPDIR}/swapinfo.problems

integer bad_count=0
integer THRESHOLD=50

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS 
do
   if [ -f ${f} ]; then rm ${f}; fi
done

for f in ${RSP_list} ${WST_list} ${CS_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
               #                      Mb      Mb      Mb   PCT  START/      Mb
               #          TYPE      AVAIL    USED    FREE  USED   LIMIT RESERVE  PRI  NAME
               #hostname  total      5600    3321    2279   59%       -       0    -

       remsh $f -n "/usr/sbin/swapinfo -mta |grep total" | sed -e s/\%//g  |  
                   awk '{printf ("%-10s %6s  %6s %6s %4s\n", "'$f'", $2, $3, $4, $5) }' >> ${LOCAL_DETAILS}
   else   
         print "$f not pingable" >> ${LOCAL_PROBLEMS}
   fi
done

   # find the boxes which exceed threshold
   awk '($5 >= '$THRESHOLD') {print  }' ${LOCAL_DETAILS} >> ${LOCAL_PROBLEMS}

   bad_count=`wc -l ${LOCAL_PROBLEMS}| awk '{print $1}'`

   header="\n-- ${bad_count} boxes where swap exceeded ${THRESHOLD}% "

   if ((${bad_count} > 0))  
   then
         # add additional header lines
         print "              Mb     Mb     Mb     % "  > $JUNKFILE 
         print "host        Avail   Used   Free   Used" >> $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "              Mb     Mb     Mb     % "  > $JUNKFILE 
         print "host          Avail   Used   Free   Used" >> $JUNKFILE
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

#LOCAL_PROBLEMS >> GLOBAL_PROBLEMS
#LOCAL_DETAILS  >> GLOBAL_DETAILS
#header  >> GLOBAL_SUMMARY
summarize;  #calls function to write to files

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS 
do
   if [ -f ${f} ]; then rm ${f}; fi
done
}  #swapinfo


function ntp_check {

JUNKFILE=${TMPDIR}/ntp.junk
LOCAL_DETAILS=${TMPDIR}/ntp.details
LOCAL_PROBLEMS=${TMPDIR}/ntp.problems

integer bad_count=0  
integer good_count=0

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

for f in ${RSP_list} ${CS_list} 
#for f in ${RSP_list} ${WST_list} ${CS_list} 
 do
    if ping $f -n2 2>&1 >/dev/null
    then
          remsh $f -n "/usr/sbin/ntpq -p |grep \* "  > ${JUNKFILE} 2>&1
          good_count=`grep \* ${JUNKFILE} | wc -l | awk '{print $1}'`

          # looking for high stratum number (which is a problem)
          cat ${JUNKFILE} |  awk '($3 >= "3") {printf ("%-12s %-16s %-14 s %8s  \n", "'$f'", $1, $2, $3) }'  >> ${LOCAL_PROBLEMS}

          if ((${good_count} == 1))  # found a *
          then 
              cat ${JUNKFILE} | awk '{printf ("%-12s %-16s %-14 s %8s\n", "'$f'", $1, $2, $3) }'  >> ${LOCAL_DETAILS}
          else      
               print "$f    bad_timesync" >> ${LOCAL_PROBLEMS}
          fi
    else
         print "$f not pingable" >> ${LOCAL_PROBLEMS}
    fi
 done


  bad_count=`wc -l  ${LOCAL_PROBLEMS} |  awk '{print $1}'`
  header="\n--- ${bad_count} NTP Syncronization issues"

   if ((${bad_count} > 0))  
   then
         # add additional header lines
         print "hostname    NTP_Source         refid            stratum"  > ${JUNKFILE}
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "hostname    NTP_Source         refid            stratum"  > ${JUNKFILE}
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

#LOCAL_PROBLEMS >> GLOBAL_PROBLEMS
#LOCAL_DETAILS  >> GLOBAL_DETAILS
#header  >> GLOBAL_SUMMARY
summarize;  #calls function to write to files

    for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS 
    do
       if [ -f ${f} ]; then rm ${f}; fi
    done
} # ntp_check


function ntpconf_check {

JUNKFILE=${TMPDIR}/ntpconf.junk
LOCAL_DETAILS=${TMPDIR}/ntpconf.details
LOCAL_PROBLEMS=${TMPDIR}/ntpconf.problems

integer bad_count=0 prefer_count fudge_count

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done


for f in ${RSP_list} ${WST_list} ${CS_list} 
do
    if ping $f -n2 2>&1 >/dev/null
    then
         remsh $f -n "grep -v \# /etc/ntp.conf | grep -vE "^$" "  > ${JUNKFILE} 2>&1

         prefer_count=`grep prefer ${JUNKFILE} | wc -l | awk '{print $1}'`
         fudge_count=`grep -E "fudge 127.127.1.1 stratum 10"  ${JUNKFILE} | wc -l | awk '{print $1}'`

         if ((${prefer_count} < 1))
         then 
               echo "$f    no prefered server defined  " >> ${LOCAL_PROBLEMS}
         fi

         if ((${fudge_count} < 1))
         then 
               echo "$f    no fudge server defined  " >> ${LOCAL_PROBLEMS}
         fi
    else
         print "$f not pingable" >> ${LOCAL_PROBLEMS} 
    fi
done

   bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`

   header="\n--- ${bad_count}  ntp.conf issues "

   if ((${bad_count} > 0))  
   then
         # add additional header lines
        print "> Need (1)prefer entry  and (1)fudge 127.127.1.1 stratum 10 entry"   > ${JUNKFILE}
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         cp ${JUNKFILE} ${LOCAL_PROBLEMS}
         mv ${JUNKFILE} ${LOCAL_DETAILS}

   fi

#LOCAL_PROBLEMS >> GLOBAL_PROBLEMS
#LOCAL_DETAILS  >> GLOBAL_DETAILS
#header  >> GLOBAL_SUMMARY
summarize;  #calls function to write to files

    for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS 
    do
       if [ -f ${f} ]; then rm ${f}; fi
    done
} # ntpconf_check


function rpc_check {
JUNKFILE=${TMPDIR}/rpc.junk
LOCAL_DETAILS=${TMPDIR}/rpc.details
LOCAL_PROBLEMS=${TMPDIR}/rpc.problems

integer bad_count=0

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

  rpcinfo -m |grep 20440 |sort  >> ${LOCAL_DETAILS}
  cat ${LOCAL_DETAILS} |  awk '($5 > 0) {print}'  >> ${LOCAL_PROBLEMS}

  bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`

  header="\n--- ${bad_count} failed access7 RPC calls to CS "

   if ((${bad_count} > 0))  
   then
         # add additional header lines
         print " prog          vers    netid     success       failure " >  ${JUNKFILE}
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}

         print " prog          vers    netid     success       failure " >  ${JUNKFILE}
         cat ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

   fi
#LOCAL_PROBLEMS >> GLOBAL_PROBLEMS
#LOCAL_DETAILS  >> GLOBAL_DETAILS
#header  >> GLOBAL_SUMMARY
summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

}  #rpc_check

function bpp_corefiles {
JUNKFILE=${TMPDIR}/corefile.junk
LOCAL_DETAILS=${TMPDIR}/corefile.details
LOCAL_PROBLEMS=${TMPDIR}/corefile.problems

integer bad_count=0

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

for f in  ${RSP_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
         count=`remsh $f -n "find /var/opt/platform7/tmp/ -mtime -1 -name *BPP* | wc -l "`
         if ((count > 0))
         then 
             echo  "$f            ${count}"  >> ${LOCAL_PROBLEMS}
         fi
    else
         print "$f not pingable" >> ${LOCAL_PROBLEMS}
    fi
done

  bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`
 
  header="\n--- ${bad_count} BPP corefiles created in past 24 hours "

   if ((${bad_count} > 0))  
   then
         # add additional header lines
         print "hostname   Numb_of_corefiles "  > ${JUNKFILE}
         echo "--------  ------------------ " >> ${JUNKFILE}
         cat ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         cp ${JUNKFILE} ${LOCAL_PROBLEMS} 
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

}  #bpp_corefiles


function diskspace_check {
JUNKFILE=${TMPDIR}/diskspace.junk
LOCAL_DETAILS=${TMPDIR}/diskspace.details
LOCAL_PROBLEMS=${TMPDIR}/diskspace.problems

integer bad_count=0
integer THRESHOLD=85  # level we're looking for

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done


for f in  ${RSP_list} ${WST_list} ${CS_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
          remsh $f  "bdf -l" | grep -v Filesystem | sed -e s/\%//g > ${JUNKFILE}
          cat ${JUNKFILE} | awk '(($5 >= '$THRESHOLD')) {printf ("%-9s %-16s %11s  %11s  %11s  %4s     %s\n","'$f'", $1, $2, $3, $4, $5, $6) }' >> ${LOCAL_PROBLEMS}

   else
       print "$f not pingable" >> ${LOCAL_PROBLEMS} 
   fi
done

 bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`
 header="\n---  ${bad_count} diskspace partitions exceeded ${THRESHOLD}% capacity "

if ((${bad_count} > 0))
   then
         # add additional header lines
         print " hostname    Filesystem        kbytes         used        avail     %used   Mounted on " > $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}
         cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done
 

} # end of diskspace_check 

function bpp_timesync {
JUNKFILE=${TMPDIR}/timesync.junk
LOCAL_DETAILS=${TMPDIR}/timesync.details
LOCAL_PROBLEMS=${TMPDIR}/timesync.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer timesync_count=0 
integer bad_count=0

for HOSTNAME in `list_processors -b `
#for HOSTNAME in `list_processors -b | grep m003ilm1'`
do
    # does it exist in CS hosts file
    exists_count=`grep -E "${HOSTNAME} " /etc/hosts | wc -l | awk '{print $1}'`
    if ((${exists_count} > 0))
    then
          if ping $HOSTNAME -n2 2>&1 >/dev/null
          then
                remsh $HOSTNAME level 1      > /dev/null 2>&1 
                remsh $HOSTNAME forward 1    > /dev/null 2>&1 
                remsh $HOSTNAME enableTelnet > /dev/null 2>&1 

                remsh $HOSTNAME  "showNTPStats" | grep \*   > ${JUNKFILE}
      
                stratum=`awk '{print $3}' ${JUNKFILE}`
                timesync_count=`wc -l ${JUNKFILE} | awk '{print $1}'`

                if ((${timesync_count} > 0))    # is timesynced
                then
                      awk '{ printf("%-12s    %14s  %4s        %s\n","'$HOSTNAME'", $1, $3, $8) }' ${JUNKFILE}  >> ${LOCAL_DETAILS}

                      if ((${stratum} > 3))   #somthings wrong 
                      then
                          awk '{ printf("%-12s  %s  %s  %s  stratum not=3\n","'$HOSTNAME'", $1, $3, $8) }' ${JUNKFILE}  >> ${LOCAL_PROBLEMS}
                       fi
              else
                  echo "$HOSTNAME not_timesyncd"  >> ${LOCAL_PROBLEMS}
              fi

          else
               print "$HOSTNAME  not_pingable " >> ${LOCAL_PROBLEMS}
          fi
   else
        print "$HOSTNAME  not_in_CS_hosts_file " >> ${LOCAL_PROBLEMS}
   fi
done

bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`

header="\n-----  ${bad_count} BPP timesync issues "
if ((${bad_count} > 0))
   then
         # add additional header lines
         print "hostname            remote       stratum    delay"     > $JUNKFILE
         print "===========      ==============  =======   =========" >> $JUNKFILE
         grep not ${LOCAL_PROBLEMS} | grep timesync >> ${JUNKFILE}
         grep not ${LOCAL_PROBLEMS} | grep stratum  >> ${JUNKFILE}
         grep not ${LOCAL_PROBLEMS} | grep ping     >> ${JUNKFILE}
         grep not ${LOCAL_PROBLEMS} | grep hosts    >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "hostname            remote       stratum    delay"     > $JUNKFILE
         print "===========      ==============  =======   =========" >> $JUNKFILE
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

    # cleanup
    for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
    do
       if [ -f ${f} ]; then rm ${f}; fi
    done
} # end bpp_timesync


function suspend_check {
JUNKFILE=${TMPDIR}/suspend.junk
LOCAL_DETAILS=${TMPDIR}/suspend.details
LOCAL_PROBLEMS=${TMPDIR}/suspend.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

today=`date "+%b %e"`
integer bad_count=0  problems_count=0 

for f in ${RSP_list} 
do
   if ping $f -n2 2>&1 >/dev/null
   then
          remsh $f -n "grep -hi Suspend /var/adm/syslog/*syslog.* |egrep '$today' " > ${LOCAL_DETAILS}

          total_count=`wc -l ${LOCAL_DETAILS} | awk '{print $1}' `   #all errors
          p7ctf_count=`grep p7ctf ${LOCAL_DETAILS} |wc -l | awk '{print $1}' `      # just ctf
          nonctf_count=`grep -v p7ctf ${LOCAL_DETAILS} | wc -l | awk '{print $1}' ` # everything else

         if ((${total_count} > 0))
         then
              date | awk '{ printf ("%-12s \t %4s \t\t\t %6s \t\t\t %s\n", "'$f'", '$total_count', '$p7ctf_count', '$nonctf_count') }' >> ${LOCAL_PROBLEMS}
         fi

         if ((${nonctf_count} > 0))
         then
               grep -v p7ctf ${LOCAL_DETAILS}  >> ${LOCAL_PROBLEMS}
         fi
  else
        print "$f not pingable" >> ${LOCAL_PROBLEMS} 
  fi

done


 bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`      # If any process was suspended, a line would exist. 
 problems_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`  # file contains suspened non-ctf processes

 header="\n--- ${bad_count} processes suspended on ${today} "

if ((${bad_count} > 0))
   then
         # add additional header lines
         print "hostname   total_processes_suspended   p7ctf_process_suspended  non_ctf_processes" > $JUNKFILE
         print "---------  -------------------------   -----------------------  -----------------" >> $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done
} #end suspend_check 


function iopp_check {
JUNKFILE=${TMPDIR}/iopp.junk
LOCAL_DETAILS=${TMPDIR}/iopp.details
LOCAL_PROBLEMS=${TMPDIR}/iopp.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer bad_count=0
integer count

today=`date "+%b %e"`

for f in ${RSP_list} 
 do
    if ping $f -n2 2>&1 >/dev/null
    then
           remsh $f -n "grep -hi iopp /var/adm/syslog/*syslog.* |egrep '$today' " > ${JUNKFILE}
           count=`wc -l ${JUNKFILE} | awk '{print $1}' `
       
           if ((${count} > 0)); then cat ${JUNKFILE} >> ${LOCAL_PROBLEMS}; fi
       
     else
           print "$f not pingable" >> ${LOCAL_PROBLEMS}
     fi
 done

cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}

bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`  

header="\n--- ${bad_count} IOPP errors found for ${today} ----"

   if ((${bad_count} > 0))
   then
         # add additional header lines
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

} #end iopp_check 


function find_corrupt_bootptab {
JUNKFILE=${TMPDIR}/bootptab.junk
LOCAL_DETAILS=${TMPDIR}/bootptab.details
LOCAL_PROBLEMS=${TMPDIR}/bootptab.problems

integer bad_count=0
integer count

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

 # resource: http://www.joelonsoftware.com/articles/Unicode.html
 # http://www.asciitable.com/

#   print -n  "Checking test file:   "    # just to make sure this works
#   perl -ne  'print if m/[\x80-\xA5]/  ;' ${TMPDIR}/corrupt_bootptab.sample   | tail -1  
   
 for f in  ${RSP_list}
 do
     if ping $f -n2 2>&1 >/dev/null
     then
           # Look for extended ascii characters 128 - 165 octal

           remsh $f "perl -ne  'print if m/[\x80-\xA5]/  ;' /etc/bootptab"  > ${JUNKFILE}
           count=`wc -l ${JUNKFILE} | awk '{print $1}' `

           if ((${count} > 0)) 
           then
                   print -n "$f "  >> ${LOCAL_PROBLEMS}
                   cat ${JUNKFILE} >> ${LOCAL_PROBLEMS}
           fi
     else
          print "$f not pingable " >> ${LOCAL_PROBLEMS}
     fi

 done

 bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`  
 header="\n---  ${bad_count} corrupt bootptab files identified "

  if ((${bad_count} > 0))
  then
         # add additional header lines
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

} #end of corrupt_bootptab

function find_corrupt_inetd_sec {
JUNKFILE=${TMPDIR}/inetd_sec.junk
LOCAL_DETAILS=${TMPDIR}/inetd_sec.details
LOCAL_PROBLEMS=${TMPDIR}/inetd_sec.problems

integer bad_count=0
integer count

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

 # resource: http://www.joelonsoftware.com/articles/Unicode.html
 # http://www.asciitable.com/

#   print -n  "Checking test file:   "    # just to make sure this works
#   perl -ne  'print if m/[\x80-\xA5]/  ;' ${TMPDIR}/corrupt_bootptab.sample   | tail -1  
   
 for f in  ${RSP_list}
 do
     if ping $f -n2 2>&1 >/dev/null
     then
           # Look for extended ascii characters 128 - 165 octal

           remsh $f "perl -ne  'print if m/[\x80-\xA5]/  ;' /var/adm/inetd.sec"  > ${JUNKFILE}
           count=`wc -l ${JUNKFILE} | awk '{print $1}' `

           if ((${count} > 0)) 
           then
                   print -n "$f "  >> ${LOCAL_PROBLEMS}
                   cat ${JUNKFILE} >> ${LOCAL_PROBLEMS}
           fi
     else
          print "$f not pingable " >> ${LOCAL_PROBLEMS}
     fi
 done

 bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`  
 header="\n---  ${bad_count} corrupt inetd.sec files identified "

  if ((${bad_count} > 0))
  then
         # add additional header lines
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

} #end of corrupt_inetdsec


function epp_cpu_load {

# not finished
# Hmmmm epp's seem to all be at 100%
#m003ilm13p8  03 13 08 CPU PEAK 100 2007-11-20 10:53:14


JUNKFILE=${TMPDIR}/eppload.junk
LOCAL_SUMMARY=${TMPDIR}/eppload.results
DETAIL=${TMPDIR}/eppload.problems
SUSPECTFILE=${TMPDIR}/eppload.suspect

integer THRESHOLD=30
integer bad_count=0 exists_count=0

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi
if [ -f ${SUSPECTFILE} ]; then rm ${SUSPECTFILE}; fi
if [ -f ${DETAIL} ]; then rm ${DETAIL}; fi

# Need to break out site, cc, bpp to run p7syshealthstat 

# grep 172.17.250.121 /etc/hosts
#172.17.250.121  m001ilm1p1   #  BPP  cc

# grep m001ilm1p1 etr
# J6741A 1 0030D30ED66A 172.17.250.121 m001ilm1p1 acceSS7_ILM 1

# grep 172.17.250.121 commsConfig
# 14BPP1C1S1:13.1.1.1:172.17.250.121

#for HOSTNAME in `list_processors -e `
for HOSTNAME in `list_processors -e | grep m003'`
do
    # does it exist in CS hosts file
    exists_count=`grep -E "${HOSTNAME} " /etc/hosts | wc -l | awk '{print $1}'`
    if ((${exists_count} > 0)) 
    then
          if ping $HOSTNAME -n2 2>&1 >/dev/null
          then
                ip=`grep -E "${HOSTNAME} " /etc/hosts | awk '{print $1}'`
                grep $ip ${TMPDIR}/commsConfig | awk '{FS=":"}; {print $2}' | sed -e 's/\./ /g' | read junk1 site cc eppnumb

                echo "$HOSTNAME  site:$site  cc:$cc numb:$eppnumb"

                print -n "${HOSTNAME}  " >> ${JUNKFILE}

                #                    site cc epp          peak
                # junkfile:HOSTNAME   01  06  06  CPU PEAK  0  2008-03-19 18:14:32

                p7syshealthstat -site $site -cc $cc -epp $eppnumb -cpu  -peak  -raw |
                     grep "CPU,PEAK" | sed -e "s/\.00//g" |  sed -e "s/,/ /g" >> ${JUNKFILE}

          else
               print "${HOSTNAME} not_pingable" >> ${DETAIL}
          fi
   else
        print "${HOSTNAME} not_in_CS_hostsfile" >> ${DETAIL}
   fi
done

  cat ${JUNKFILE} | awk '{if ($7 > '${THRESHOLD}') printf ("%s   %s  %s  %s\n", $1, $7, $8, $NF) }' >> ${LOCAL_SUMMARY}

cat ${LOCAL_SUMMARY}

#    grep -v `date +%Y` ${JUNKFILE} >> $SUSPECTFILE
 
#     bad_count=`wc -l ${LOCAL_SUMMARY} | awk '{print $1}' `
#     suspect_count=`wc -l ${SUSPECTFILE} | awk '{print $1}' `
#
#header1="\n-----  ${bad_count} BPPs CPU Peak Load > ${THRESHOLD}%   -----"
#header2="site cc bpp %Peak     date"
#header3="*****   Suspected Problems (date)  *****" 
#
# if ((${bad_count} > 0)) 
# then 
#        echo "${header1}"      # to the screen
#        echo "$header2"      
#        cat ${LOCAL_SUMMARY} 
#
#        if ((${suspect_count} > 0)) # not 2007.  Timesync issue?
#        then 
#              echo "$header3"      # to the screen
#              cat ${SUSPECTFILE} 
#        fi
#
#        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
#        echo "$header2" >> ${GLOBAL_DETAIL}       
#        cat ${LOCAL_SUMMARY}   >> ${GLOBAL_DETAIL}
#
#        echo "$header1" >> ${GLOBAL_SUMMARY}
#
#        if ((${suspect_count} > 1)) # not 2007.  Timesync issue?
#        then 
#              echo "$header3" >> ${GLOBAL_DETAIL}       # why it's a problem 
#              cat ${SUSPECTFILE} >> ${GLOBAL_DETAIL}    # actual errors
#
#              echo "${suspect_count} BPPs had year other than `date +%Y`"   >> ${GLOBAL_SUMMARY}  
#        fi
#
#  else
#        echo "$header1"         # to the screen
#
#        echo "$header1" >> ${GLOBAL_SUMMARY}        # identify the test
#
#        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
#  fi
#
#if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi
#if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
#if [ -f ${SUSPECTFILE} ]; then rm ${SUSPECTFILE}; fi

} # epp_cpu_load


function bpp_cpu_load {

#not finished ) (create output errors)
# m003ilm5p4 ???

JUNKFILE=${TMPDIR}/bppload.junk
LOCAL_SUMMARY=${TMPDIR}/bppload.results
DETAIL=${TMPDIR}/bppload.problems
SUSPECTFILE=${TMPDIR}/bppload.suspect

integer THRESHOLD=30
integer bad_count=0 exists_count=0

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi
if [ -f ${SUSPECTFILE} ]; then rm ${SUSPECTFILE}; fi
if [ -f ${DETAIL} ]; then rm ${DETAIL}; fi

# Need to break out site, cc, bpp to run p7syshealthstat 

# list_processors -bn 
#s001c01p1

# list_processors -b 
#m001ilm1p1

# grep 172.17.250.121 /etc/hosts
#172.17.250.121  m001ilm1p1   #  BPP  cc

# grep m001ilm1p1 etr
# J6741A 1 0030D30ED66A 172.17.250.121 m001ilm1p1 acceSS7_ILM 1

# grep 172.17.250.121 commsConfig
# 14BPP1C1S1:13.1.1.1:172.17.250.121

#for HOSTNAME in `list_processors -b `
for HOSTNAME in `list_processors -b | grep m003'`
do
    # does it exist in CS hosts file
    exists_count=`grep -E "${HOSTNAME} " /etc/hosts | wc -l | awk '{print $1}'`
    if ((${exists_count} > 0)) 
    then
          if ping $HOSTNAME -n2 2>&1 >/dev/null
          then
                ip=`grep -E "${HOSTNAME} " /etc/hosts | awk '{print $1}'`
                grep $ip ${TMPDIR}/commsConfig | awk '{FS=":"}; {print $2}' | sed -e 's/\./ /g' | read junk1 site cc bppnumb

                echo "$HOSTNAME  site:$site  cc:$cc numb:$bppnumb"

                print -n "${HOSTNAME}  " >> ${JUNKFILE}

                #                    site cc bpp          peak
                # junkfile:HOSTNAME   01  06  06  CPU PEAK  0  2008-03-19 18:14:32

                p7syshealthstat -site $site -cc $cc -bpp $bppnumb -cpu  -peak  -raw |
                     grep "CPU,PEAK" | sed -e "s/\.00//g" |  sed -e "s/,/ /g" >> ${JUNKFILE}

          else
               print "${HOSTNAME} not_pingable" >> ${DETAIL}
          fi
   else
        print "${HOSTNAME} not_in_CS_hostsfile" >> ${DETAIL}
   fi
done

  cat ${JUNKFILE} | awk '{if ($7 > '${THRESHOLD}') printf ("%s   %s  %s  %s\n", $1, $7, $8, $NF) }' >> ${LOCAL_SUMMARY}

cat ${LOCAL_SUMMARY}

#    grep -v `date +%Y` ${JUNKFILE} >> $SUSPECTFILE
 
#     bad_count=`wc -l ${LOCAL_SUMMARY} | awk '{print $1}' `
#     suspect_count=`wc -l ${SUSPECTFILE} | awk '{print $1}' `
#
#header1="\n-----  ${bad_count} BPPs CPU Peak Load > ${THRESHOLD}%   -----"
#header2="site cc bpp %Peak     date"
#header3="*****   Suspected Problems (date)  *****" 
#
# if ((${bad_count} > 0)) 
# then 
#        echo "${header1}"      # to the screen
#        echo "$header2"      
#        cat ${LOCAL_SUMMARY} 
#
#        if ((${suspect_count} > 0)) # not 2007.  Timesync issue?
#        then 
#              echo "$header3"      # to the screen
#              cat ${SUSPECTFILE} 
#        fi
#
#        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
#        echo "$header2" >> ${GLOBAL_DETAIL}       
#        cat ${LOCAL_SUMMARY}   >> ${GLOBAL_DETAIL}
#
#        echo "$header1" >> ${GLOBAL_SUMMARY}
#
#        if ((${suspect_count} > 1)) # not 2007.  Timesync issue?
#        then 
#              echo "$header3" >> ${GLOBAL_DETAIL}       # why it's a problem 
#              cat ${SUSPECTFILE} >> ${GLOBAL_DETAIL}    # actual errors
#
#              echo "${suspect_count} BPPs had year other than `date +%Y`"   >> ${GLOBAL_SUMMARY}  
#        fi
#
#  else
#        echo "$header1"         # to the screen
#
#        echo "$header1" >> ${GLOBAL_SUMMARY}        # identify the test
#
#        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
#  fi
#
#if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi
#if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
#if [ -f ${SUSPECTFILE} ]; then rm ${SUSPECTFILE}; fi

} # bpp_cpu_load


function p7Site_timesync {
JUNKFILE=${TMPDIR}/timesync.junk
LOCAL_DETAILS=${TMPDIR}/timesync.details
LOCAL_PROBLEMS=${TMPDIR}/timesync.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer bad_count=0 rsp_count=0
NUMB_of_SITES=`echo ${RSP_list} | wc -w` 

for f in ${RSP_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
         date | awk '{printf ("%-12s ", "'$f'")}' > ${JUNKFILE}
         remsh $f "grep PClockType  /etc/opt/platform7/resources/timesync/scalar/p7Site | grep -v \# " >> ${JUNKFILE}
         cat ${JUNKFILE} | awk '{ if ($NF != "LocalSite") {print}}' >> ${LOCAL_PROBLEMS}
         cat ${JUNKFILE} >> ${LOCAL_DETAILS}
   else
         print "$f: not pingable" >> ${LOCAL_PROBLEMS}
   fi
done

 bad_count=`wc -l ${LOCAL_PROBLEMS}| awk '{print $1}'`

header="\n---  PClockType problems on ${bad_count} of ${NUMB_of_SITES} RSPs "
 if ((${bad_count} > 0))
 then
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
 fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done
} #  p7Site_timesync


function check_hardware_mirror {

JUNKFILE=${TMPDIR}/hwmirror.junk
LOCAL_SUMMARY=${TMPDIR}/hwmirror.results

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

# reference:  http://docs.hp.com/en/J6369-90026/ch05s02.html

   target_list=${CS_list}

 header1="\n-----  Checking hardware mirror status on ${target_list}"

 echo "${header1}"      # identify the test to the screen

  device_file=`ioscan -kfn | grep RAID | awk '{print "/dev/ciss"$2}'`
  count=`/usr/sbin/saconfig $device_file | grep Internal | grep OK |wc -l`

  if ((${count} < 2))   # should be 2 (two disks, both okay)
  then
        echo "`hostname` appears to have a bad mirror disk"  # to the screen

        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
        echo "`hostname` appears to have a bad mirror disk"  >> ${GLOBAL_DETAIL}

        echo "$header1" >> ${GLOBAL_SUMMARY}       
        echo "`hostname` appears to have a bad mirror disk"  >> ${GLOBAL_SUMMARY}
  else 
        echo "${count} of 2 mirror disks on `hostname` are okay"  # to the screen

        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
        echo "${count} of 2 mirror disks on `hostname` are okay" >> ${GLOBAL_DETAIL}

        echo "$header1" >> ${GLOBAL_SUMMARY}       # identify the test
        echo "${count} of 2 mirror disks on `hostname` are okay" >> ${GLOBAL_SUMMARY}
  fi

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

} # check_hardware_mirror



function r75_running_check {

   links_per_dsp_file=${WORKDIR}/link_stuff/number_of_links_per_dsp.out
   etr_file=${WORKDIR}/link_stuff/wm.etr
   junkfile=${WORKDIR}/junkfile
   no_links_list=${WORKDIR}/no_links_list
   out_of_comms_list=${WORKDIR}/out_of_comms_list
   combined_exclude_list=${WORKDIR}/combined_exclude_list

   echo " "
   echo "------------------------------------------ "
   echo " r75 is not running on the following DSPs  "
   echo "------------------------------------------ "

   if [ -f ${junkfile} ]; then rm ${junkfile}; fi
   if [ -f ${no_links_list} ]; then rm ${no_links_list}; fi
   if [ -f ${out_of_comms_list} ]; then rm ${out_of_comms_list}; fi
   if [ -f ${no_links_list} ]; then rm ${no_links_list}; fi
   if [ -f ${combined_exclude_list} ]; then rm ${combined_exclude_list}; fi

   # Extract Datastore info from c32 
   rm /var/opt/platform7/tmp/eng_logging_C32_0
   p7dumpstatus c32 > /dev/null 2>&1 

   grep not /var/opt/platform7/tmp/eng_logging_C32_0 | grep Datastore | sed -e 's/,//g' |sed -e 's/)//g' | cut -d" " -f 5,9 > $junkfile

    if [ -s $junkfile ]  #file is not zero length
    then 
         while read site dsp 
         do
            grep ds0 ${etr_file} | grep "^$site $dsp" | awk '{print $NF}'  >>${out_of_comms_list}
         done  < ${junkfile}    
    else     # file is zero length
         > ${out_of_comms_list}
    fi

   # Build list of DSPs which have no links configured (r75 would not be running)
   egrep -E "^  0" ${links_per_dsp_file} | awk ' NF > 7 {print $NF }'  >>${no_links_list}

   # Sort the two, remove blank links
   cat ${out_of_comms_list} ${no_links_list} | sort | grep -v "![^$]"  > ${combined_exclude_list}


   # Building list of boxes to be excluded is done.
   # Now check legitimate boxes for r75

   integer bad_count=0   # initialize known bad

   for f in `list_processors -d`
#for f in s001ds05
   do
      # check to see if box is in exclude list
       integer exclude_count=0

       exclude_count=`grep $f ${combined_exclude_list} | wc -l'`

       if ((exclude_count < 1))    # not excluded
       then
             print -n "."
             r75_count=`ssh $f "ps  -ef|grep r75.run|grep -v grep |grep -v bash | wc -l"`
             if ((r75_count < 1))
             then 
                   print  "$f"
                  ((bad_count = bad_count + 1))
#                  ssh $f  ". /opt/platform7/lbin/p7profile;  /opt/platform7/bin/p7stop -s" > /dev/null 2>&1
#                  sleep 5
#                  ssh $f  ". /opt/platform7/lbin/p7profile;  /opt/platform7/bin/p7start -s" > /dev/null 2>&1
             fi
      fi
  done

   print
   print -n  "no links configured on: " ; egrep -E "^  0" ${links_per_dsp_file} | awk ' NF > 7 {printf ("%s ", $NF) }' 
   print
   print -n  "DSPs out of comms: " ; cat ${out_of_comms_list} | awk '{printf ("%s ", $0)}'
#   echo  "\n\nr75 was not running on $bad_count DSPs.  Cycled platform to attempt restart."
   echo  "\n\nr75 was not running on $bad_count DSPs.  "
   echo " "

   if [ -f ${junkfile} ]; then rm ${junkfile}; fi
   if [ -f ${no_links_list} ]; then rm ${no_links_list}; fi
   if [ -f ${out_of_comms_list} ]; then rm ${out_of_comms_list}; fi
   if [ -f ${no_links_list} ]; then rm ${no_links_list}; fi
   if [ -f ${combined_exclude_list} ]; then rm ${combined_exclude_list}; fi

} # end r75_check 


function amc_lan_check {
JUNKFILE=${TMPDIR}/amccheck.junk
LOCAL_DETAILS=${TMPDIR}/amccheck.details
LOCAL_PROBLEMS=${TMPDIR}/amccheck.problems

integer bad_count=0

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done


# run nightly from cron.
AMC_SUMMARY="${WORKDIR}/amc/lanconfigs_all"

 grep Half ${AMC_SUMMARY} >> ${LOCAL_PROBLEMS}
 awk '($NF == "Duplex") && ($9 != "100") {printf ("%s\n",$0)}' ${AMC_SUMMARY} >> ${LOCAL_PROBLEMS}

 bad_count=`wc -l ${LOCAL_PROBLEMS}| awk '{print $1}'`

 timestamp=`ll ${AMC_SUMMARY} | awk '{printf ("%s %s %s",$6, $7, $8) }'`
 header="\n--- ${bad_count} AMC LAN configs were not correct on ${timestamp} "

 if ((${bad_count} > 0))
   then
         # add additional header lines
         print " >Should be 100 Mb/s Full Duplex"  > $JUNKFILE
         cat ${LOCAL_PROBLEMS} >> $JUNKFILE
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}
    
         cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done
 
}  #end of amc_lan_check


function ping_check {
JUNKFILE=${TMPDIR}/ping.junk
LOCAL_DETAILS=${TMPDIR}/ping.details
LOCAL_PROBLEMS=${TMPDIR}/ping.problems

integer bad_count=0
integer ping_time_threshold=80
integer packet_loss_threshold=5

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done


for f in ${RSP_list} ${WST_list} ${CS_list} 
#for f in ${RSP_list} ${WST_list} ${CS_list} ${BPP_list}
 do
    if ping $f -n2 2>&1 >/dev/null
    then
          ping $f -n 5 -m 2 | tail -2   > ${JUNKFILE}
          packet_loss="`grep loss ${JUNKFILE} | sed -e 's/\%/ /g' | awk '{print $7}'"
          avg_ping_time="`grep round ${JUNKFILE} | sed -e 's/\// /g' | awk '{print $8}'"

          if ((${packet_loss} > ${packet_loss_threshold}))  || ((${avg_ping_time} > ${ping_time_threshold}))
          then
                echo "$f            $avg_ping_time           $packet_loss"  >> ${LOCAL_PROBLEMS}
          else
                echo "$f            $avg_ping_time           $packet_loss"  >> ${LOCAL_DETAILS}
          fi
    else
          print "$f not pingable" >> ${LOCAL_PROBLEMS} 
    fi
  done
 
  bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`

 header="\n--- ${bad_count} boxes with ping times > ${ping_time_threshold}ms or packet loss > ${packet_loss_threshold}% "

  if ((${bad_count} > 0))
   then
         # add additional header lines
         print "hostname   ping_time(ms) packet_loss%"  > $JUNKFILE
         print "--------   ------------  ------------" >> $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "hostname   ping_time(ms) packet_loss%"  > $JUNKFILE
         print "--------   ------------  ------------" >> $JUNKFILE
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep not ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

} # end of ping_check

function bad_links_summary {
JUNKFILE=${TMPDIR}/badlinks.junk
LOCAL_DETAILS=${TMPDIR}/badlinks.details
LOCAL_PROBLEMS=${TMPDIR}/badlinks.problems

ERROR_LIST=${TMPDIR}/badlinks.errorlist
WARN_LIST=${TMPDIR}/badlinks.warnlist
CREATE_LIST=${TMPDIR}/badlinks.createlist

DOW=`/bin/date +%a`   # Mon Tues ...
BADLINKS_FILE=${BACKUPDIR}/badlinks.${DOW}

integer bad_count=0

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS ${ERROR_LIST} ${WARN_LIST} ${CREATE_LIST}
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

 # get unique list of errors
  egrep -e "ERR"  ${BADLINKS_FILE} | awk 'BEGIN {FS="ERR:"} {print $NF}' | sed -e s/\|//g | sort -u  > ${ERROR_LIST}
  cat ${ERROR_LIST}  | perl -ne 's/\s+\n//; print "print \-n \"ERR:$_: \"\;grep -e \"$_\" '"${BADLINKS_FILE}"' \| wc -l \n";' > ${CREATE_LIST} 

   # get unique list of warnings
   egrep -e "WARN"  ${BADLINKS_FILE} | awk 'BEGIN {FS="WARN:"} {print $NF}' | sed -e s/\|//g | sort -u   > ${WARN_LIST}
   
   echo 'print " " ' >> ${CREATE_LIST} #just to seperate warns and errors   
   cat ${WARN_LIST}  | perl -ne 's/\s+\n//; print "print \-n \"WARN:$_: \"\;grep -e \"$_\" '"${BADLINKS_FILE}"' \| wc -l \n";' >> ${CREATE_LIST} 

   # change permissions.  Create the list  
   chmod +x ${CREATE_LIST}
   ${CREATE_LIST}  >> ${LOCAL_DETAILS}

 header="\n--- Badlinks Summary for ${DOW} "

 bad_count=1   # hardcoded
 cp ${LOCAL_DETAILS} ${LOCAL_PROBLEMS}

summarize;  #calls function to write to files

#cleanup
rm ${TMPDIR}/badlinks*
} # bad_links_summary


function check_alarm_log {

   FILE=${WORKDIR}/a7log/Sys*.csv

   echo " "
   echo "----------------------------------- "
   echo "Alarm Log Summary  "
   echo "(Fan warnings, and Critical alarms) "
   echo "----------------------------------- "

    if [ -f ${WORKDIR}/a7log/S* ]; then rm ${WORKDIR}/a7log/S*; fi
    if [ -f ${WORKDIR}/a7log/T* ]; then rm ${WORKDIR}/a7log/T*; fi

     ${WORKDIR}/a7log/alarm > /dev/null 2>&1  # generate files

    head -1  ${FILE}
    grep -i fan  ${FILE}
    grep CRITICAL  ${FILE}

  echo " "
} # end of check_alarm_log


function check_error_log {

   FILE=${WORKDIR}/a7log/SVR*.csv

   echo " "
   echo "----------------------  "
   echo "CS Error Log Summary    "
   echo "(anything but warnings) "
   echo "----------------------- "

    if [ -f ${WORKDIR}/a7log/S* ]; then rm ${WORKDIR}/a7log/S*; fi
    if [ -f ${WORKDIR}/a7log/T* ]; then rm ${WORKDIR}/a7log/T*; fi

     ${WORKDIR}/a7log/error > /dev/null 2>&1  # generate files

    grep -i fan  ${FILE}
    grep -v WARN  ${FILE}

  echo " "
} # end of check_error_log

function datastore_disk_health {

JUNKFILE=${WORKDIR}/datastore_disks.junk
LOCAL_SUMMARY=${WORKDIR}/datastore_disks.results

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

 header1="\n-----  Looking for failed Datastore disks   -----"

 echo "$header1"         #identify the test
 touch ${LOCAL_SUMMARY}  # just in case there is no problem

for f in  ${DSP_list}
do
    ssh $f "/sbin/lvmdiskscan | grep failed" > ${JUNKFILE}   2>&1 

    count=`wc -l ${JUNKFILE} | awk '{print $1}' `

    if [ ${count} -gt  0 ];
    then
          awk '{printf ("%s  %s\n","'$f'", $0)}' ${JUNKFILE} >> ${LOCAL_SUMMARY}
    fi
done

   bad_count=`wc -l ${LOCAL_SUMMARY}| awk '{print $1}'`

   if ((${bad_count} > 0))
   then
        echo " ${bad_count} Datastore disk failures found"  # to the screen
        cat ${LOCAL_SUMMARY}                        

        echo "$header1" >> ${GLOBAL_SUMMARY}       # identify the test
        echo " ${bad_count} Datastore disk failures found"  >> ${GLOBAL_SUMMARY} 

        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
        cat ${LOCAL_SUMMARY} >> ${GLOBAL_DETAIL}  # supply the gory details
  else
        echo "No datastore disk failures found"  # to screen

       echo "$header1" >> ${GLOBAL_SUMMARY}     # identify the test
        echo "No datastore disk failures found"  >> ${GLOBAL_SUMMARY}

        echo "$header1" >> ${GLOBAL_DETAIL}     # identify the test
        echo "No datastore disk failures found"  >> ${GLOBAL_DETAIL}
 fi

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

} # datastore_disk_health


function lanadmin_check {
JUNKFILE=${TMPDIR}/lanadmin.junk
LOCAL_DETAILS=${TMPDIR}/lanadmin.details
LOCAL_PROBLEMS=${TMPDIR}/lanadmin.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

REMOTE_SCRIPT=${TMPDIR}/lanadmin.remote.sh
integer bad_count=0 

# build the script which will be rcp'd to the remote box
#
    print "ppid=\`netstat -i |grep lan | cut -b 4\` 
    /usr/sbin/lanadmin <<INPUT
    lan
    ppa \$ppid
    disp


    q
    INPUT
    " > ${REMOTE_SCRIPT}

chmod 555 ${REMOTE_SCRIPT}

#for f in m001
for f in ${RSP_list} ${WST_list} ${CS_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
          remsh $f "if [ ! -d /tmp/wm/ ]; then mkdir /tmp/wm/ ; fi"
          rcp -p ${REMOTE_SCRIPT}  $f:/tmp/wm
          remsh $f "/tmp/wm/lanadmin.remote.sh"  > ${JUNKFILE} 2> /dev/null   # who cares about the error
          fcs_errrors=`grep FCS ${JUNKFILE} | awk '{print $NF}'`

          remsh $f 'ppid=`netstat -i |grep lan | cut -c 4`; /usr/sbin/lanadmin -x $ppid' | sed -e 's/\.//g'  >  ${JUNKFILE}
          grep Speed ${JUNKFILE} | read junk1 junk speed duplex
          grep negoti ${JUNKFILE} | read junk1 junk auto_setting
          date | awk '{printf("%-10s %3s          %3s   %s   %s\n", "'$f'", "'$fcs_errrors'", "'$speed'", "'$duplex'",  "'$auto_setting'") }' >> ${LOCAL_DETAILS}

          remsh $f "rm /tmp/wm/lanadmin.remote.sh" 
          

   else
          print "$f not pingable" >> ${LOCAL_PROBLEMS}
   fi
done

      awk '{ if ($2 != "0" )    {print $0}}' ${LOCAL_DETAILS} >> ${LOCAL_PROBLEMS}
      awk '{ if ($3 != "100" )  {print $0}}' ${LOCAL_DETAILS} >> ${LOCAL_PROBLEMS}
      awk '{ if ($4 !~ /Full/ ) {print $0}}' ${LOCAL_DETAILS} >> ${LOCAL_PROBLEMS}
      awk '{ if ($NF !~ /ff/ )  {print $0}}' ${LOCAL_DETAILS} >> ${LOCAL_PROBLEMS}
#      awk '{ if ($2 != "1" || $3 != "99") {print $0}}' ${LOCAL_DETAILS}
   bad_count=`wc -l ${LOCAL_PROBLEMS}| awk '{print $1}'`

   header="\n--- ${bad_count} processor lan config issues "
   if ((${bad_count} > 0))
   then
         # add additional header lines
         print "  >Should be 100 Full, Auto Off                   "  > $JUNKFILE
         print "host      FCS_errors   Speed     Duplex    Autoneg"  >> $JUNKFILE
         echo "--------  ----------   -----  ------------  -------"  >> $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "  >Should be 100 Full, Auto Off                   "  > $JUNKFILE
         print "host      FCS_errors   Speed     Duplex    Autoneg"  > $JUNKFILE
         echo  "--------  ----------   -----  ------  -------"  >> $JUNKFILE
         cat ${LOCAL_DETAILS}        >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   else 
         print "  >Should be 100 Full, Auto Off                   "  > $JUNKFILE
         print "host      FCS_errors   Speed     Duplex    Autoneg"  > $JUNKFILE
         echo "--------  ----------   -----  ------------  -------"  >> $JUNKFILE
         cat ${LOCAL_DETAILS}        >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
  fi
summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS $REMOTE_SCRIPT
do
   if [ -f ${f} ]; then rm ${f}; fi
done

} #lanadmin_check

function check_software_mirrors {
JUNKFILE=${TMPDIR}/stale.junk
LOCAL_DETAILS=${TMPDIR}/stale.details
LOCAL_PROBLEMS=${TMPDIR}/stale.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

REMOTE_SCRIPT=${TMPDIR}/remote.sh
integer bad_count installed diskcount

# build the script which will be rcp'd to the remote box
#
    print "#\!/bin/ksh
    cd /dev 
    for logical_vol in \`ls vg*/lvol*\`
    do
    /usr/sbin/lvdisplay  /dev/\${logical_vol}
    done " > ${REMOTE_SCRIPT}

chmod 555 ${REMOTE_SCRIPT}

#for f in m001
for f in ${RSP_list} ${WST_list} ${CS_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
          # test mirror software is installed
          installed=`remsh $f "/usr/sbin/swlist -l bundle |grep -i mirror" | wc -l`
       
          if ((${installed} > 0)) # it's installed
          then
                 remsh $f "if [ ! -d /tmp/wm/ ]; then mkdir /tmp/wm/ ; fi" 
                  rcp -p ${REMOTE_SCRIPT}  $f:/tmp/wm
                 remsh $f "/tmp/wm/remote.sh"  > ${JUNKFILE} 2> /dev/null   # who cares about the error

              awk '{ if ( $2 ~ /Logical/ ) {new_lvol = $2}  #start of new logical volume section
                     if ( /LV Name/ ) {lv_name = $NF}
                     if ( /LV Status/ ) {FS = "\/"; lv_status = $NF; FS = " "}
                     if ( /Mirror/ ) {mirror_copies = $NF}
                     if ( /^$/ && lv_status ~ /stale/ ) {printf ("%8s  %-18s  %s  %s\n", "'$f'", lv_name, lv_status, mirror_copies)} 
                     if ( /^$/ && mirror_copies ~ /0/ ) {printf ("%8s  %-18s  %s  %s\n", "'$f'", lv_name, lv_status, mirror_copies)} 
                   }' ${JUNKFILE} >> ${LOCAL_PROBLEMS}

                  remsh $f "if [ -f /tmp/wm/remote.sh ]; then rm /tmp/wm/remote.sh ; fi" 
         else
              diskcount=`remsh $f "/usr/sbin/ioscan -fnC disk | grep HP" | wc -l`
              print "$f  has ${diskcount} disks, but MirrorDisk not installed"  >> ${LOCAL_PROBLEMS}
         fi
   else
         print "$f not pingable" >> ${LOCAL_PROBLEMS}
   fi
done


  bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`
  header="\n--- ${bad_count} mirroring issues found "

   if ((${bad_count} > 0))
   then
         # add additional header lines
         print "                            partition        mirror "  > $JUNKFILE
         print "hostname   partition         status          copies " >> $JUNKFILE
         echo  "---------  ---------       -----------     -------- " >> $JUNKFILE
         grep   syncd     ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep   installed ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep   ping      ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE}  ${LOCAL_PROBLEMS}
   fi

         cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS $REMOTE_SCRIPT
do
   if [ -f ${f} ]; then rm ${f}; fi
done
} #end of check_software_mirrors


function check_for_stopped_dsp_sessions   {
   echo " "
   echo "------------------------------------------ "
   echo " Checking for Inactive Datastore sessions  "
   echo "------------------------------------------ "

   ignore_list=      # list of BPPs with no links installd (there will be no r75)

   stopped_junk=${WORKDIR}/stopped.junk
   stopped_remote=${WORKDIR}/stopped.remote.sh
   stopped_results=${WORKDIR}/stopped.results

   integer total_stopped=0   # 

# Build the file to be copied to remote boxes  
cat > ${stopped_remote} <<EOF
#! /usr/bin/ksh 
. /opt/platform7/lbin/p7profile; 
integer r75_running ;
r75_running=\`ps  -ef|grep r75.run|grep -v grep |grep -v bash | wc -l\`;
if  ((r75_running > 0)) ; 
 then   
     if [[ -f /var/opt/platform7/tmp/eng_logging_R75_0 ]];
      then rm /var/opt/platform7/tmp/eng_logging_R75_0;
     fi  
     /opt/platform7/bin/p7dumpstatus r75;
     cat /var/opt/platform7/tmp/eng_logging_R75_0;
 fi
EOF

for f in  ${DSP_list}
#for f in s028ds02 
   do
      if [ -f ${stopped_junk} ]; then rm ${stopped_junk}; fi
     
      print -n "."     
      scp ${stopped_remote} $f:/tmp/wm  2> /dev/null   # hide scp progress  window
      ssh $f  "chmod +x /tmp/wm/stopped*; /tmp/wm/stopped*" >  ${stopped_junk}
      ssh $f  "if [ -f /tmp/wm/stop*} ]; then rm /tmp/wm/stop* ; fi"     # cleanup

      stopped_count=`grep STOPPED ${stopped_junk} | wc -l`  # how many stopped sessions di r80 find?

      if ((${stopped_count} > 1))   # at least one card stopped
         then 
              cat ${stopped_junk} |
              awk '{ if ( $1 ~ /DCC/ ) {dcc_id = $NF}  
                   if ( /State/ ) {card_state = $NF}
                   if ( /^$/ && card_state ~ /STOPPED/ ) 
                         {printf ("%8s  %-18s  %s\n", "'$f'", dcc_id, card_state)
                          dcc_id = ""
                          card_state = ""
                         } 
              }'  >> ${stopped_results}
        fi
   done

   if [ -f ${stopped_results} ]; then sort ${stopped_results}; rm ${stopped_results}; fi
   if [ -f ${stopped_remote} ]; then rm ${stopped_remote}; fi
   if [ -f ${stopped_junk} ]; then rm ${stopped_junk}; fi

   echo " "
   echo " "
} #check_for_stopped_dsp_sessions

function check_for_corrupt_filesets {
JUNKFILE=${TMPDIR}/corrupt.junk
LOCAL_DETAILS=${TMPDIR}/corrupt.details
LOCAL_PROBLEMS=${TMPDIR}/corrupt.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer bad_count=0 count=0
today=`date "+%b %e"`

for f in  ${RSP_list} ${WST_list} ${CS_list}
do
   if ping $f -n2 2>&1 >/dev/null
   then
          remsh $f "/usr/sbin/swlist -l fileset -a state -a revision |grep -i orrup" > ${JUNKFILE}
          count=`wc -l ${JUNKFILE}| awk '{print $1}'`

          if ((${count} > 0))
          then
               echo "$f "      >> ${LOCAL_PROBLEMS}
               cat ${JUNKFILE} >> ${LOCAL_PROBLEMS}
          fi
   else
         print "$f: not pingable" >> ${LOCAL_PROBLEMS}
   fi
done

 cp ${LOCAL_PROBLEMS} ${LOCAL_DETAILS}
 bad_count=`grep orrupt ${LOCAL_PROBLEMS} | wc -l | awk '{print $1}'`

 header="\n--- ${bad_count} Corrupt filesets found "

   if ((${bad_count} > 0))
   then
         # add additional header lines
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

} # check_for_corrupt_filesets

function check_for_ip_in_CS_hosts_file {

JUNKFILE=${TMPDIR}/hostsfile.junk
LOCAL_DETAILS=${TMPDIR}/hostsfile.details
LOCAL_PROBLEMS=${TMPDIR}/hostsfile.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer bad_count=0

# build list of hardware types
  for f in `grep : ${commsConfig} | grep -v \#`
  do
     echo $f | sed -e 's/:/ /g' > ${JUNKFILE}
  done


bad_count=`wc -l ${JUNKFILE}| awk '{print $1}'`
header="\n ${bad_count} IP's not in CS Hosts file"

   if ((${bad_count} > 0))
   then
         # add additional header lines
         print "host      Username     %CPU    Command"  > $JUNKFILE
         grep -v ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         print "host      Username     %CPU    Command"  > $JUNKFILE
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
  else
         print "host      Username     %CPU    Command"  > $JUNKFILE
         cat ${LOCAL_DETAILS} >> ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
  fi

#summarize;  #calls function to write to files

    # cleanup
#    for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
#    do
#       if [ -f ${f} ]; then rm ${f}; fi
#    done
} #check_for_ip_in_CS_hosts_file

function sendmail_server_name {

JUNKFILE=${WORKDIR}/sendmail_server.junkfile
LOCAL_SUMMARY=${WORKDIR}/sendmail_server.results

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

 header1="\n----  RSP Sendmail server name check  --- "

echo "$header1"    # to the screen
   
for f in  ${RSP_list} 
do
   print -n "$f "  > ${JUNKFILE}
   remsh $f " grep -v \# /etc/rc.config.d/mailservs  | grep SENDMAIL_SERVER_NAME " >> ${JUNKFILE}
   cat ${JUNKFILE} | awk '{FS="="}; { if ($NF < 2 ) {print}}' >> ${LOCAL_SUMMARY}   #wrong but seems to work. Should be NF
done

   count=`wc -l ${LOCAL_SUMMARY}| awk '{print $1}'`
   if ((${count} > 0))
   then
        cat ${LOCAL_SUMMARY}                        # errors to the screen

        echo "$header1" >> ${GLOBAL_SUMMARY}       # identify the test
        echo "Sendmail server name not configured on ${count} boxes" >> ${GLOBAL_SUMMARY}

        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
        cat ${LOCAL_SUMMARY} >> ${GLOBAL_DETAIL}  # supply the gory details
  else
       echo "Sendmail server name configured "  # to the screen

       echo "$header1" >> ${GLOBAL_SUMMARY}     # identify the test
       echo "Sendmail server_name not blank "  >> ${GLOBAL_SUMMARY}

       echo "$header1" >> ${GLOBAL_DETAIL}     # identify the test
       echo "Sendmail server_name not blank "  >> ${GLOBAL_DETAIL}
 fi

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

}  # check_sendmail_servername

function check_sendmailcf {
   
JUNKFILE=${WORKDIR}/sendmailcf.junkfile
LOCAL_SUMMARY=${WORKDIR}/sendmailcf.results

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

 header1="\n----  Check DM in RSP sendmail.cf  --- "

echo "$header1"    # to the screen

for f in  ${RSP_list} 
do
   print -n "$f "  > ${JUNKFILE}
   remsh $f " grep ^DM /etc/mail/sendmail.cf " >> ${JUNKFILE}
   cat ${JUNKFILE} | awk '{FS="."}; { if ($NF < 2) {print}}' >> ${LOCAL_SUMMARY}
done

   rsp_count=`list_processors -s | wc -l | awk '{print $1}'`
   count=`wc -l ${LOCAL_SUMMARY}| awk '{print $1}'`
   if ((${count} > 0))
   then
        cat ${LOCAL_SUMMARY}                        # errors to the screen

        echo "$header1" >> ${GLOBAL_SUMMARY}       # identify the test
        echo "sendmail.cf DM not configured on ${count} boxes" >> ${GLOBAL_SUMMARY}

        echo "$header1" >> ${GLOBAL_DETAIL}       # identify the test
        cat ${LOCAL_SUMMARY} >> ${GLOBAL_DETAIL}  # supply the gory details
  else
       echo "sendmail.cf DM is configured on ${rsp_count} RSPs"  # to the screen

       echo "$header1" >> ${GLOBAL_SUMMARY}     # identify the test
       echo "sendmail.cf DM is configured on ${rsp_count} RSPs"   >> ${GLOBAL_SUMMARY}

       echo "$header1" >> ${GLOBAL_DETAIL}     # identify the test
       echo "sendmail.cf DM is configured on ${rsp_count} RSPs"   >> ${GLOBAL_DETAIL}
 fi

if [ -f ${LOCAL_SUMMARY} ]; then rm ${LOCAL_SUMMARY}; fi
if [ -f ${JUNKFILE} ]; then rm ${JUNKFILE}; fi

}  # check_sendmailcf

function check_diagnostics_config {
JUNKFILE=${TMPDIR}/commscheck.junk
LOCAL_DETAILS=${TMPDIR}/commscheck.details
LOCAL_PROBLEMS=${TMPDIR}/commscheck.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer bad_count=0


for f in ${RSP_list} ${WST_list}
do
  if ping $f -n2 2>&1 >/dev/null
  then
        print -n "$f "  > ${JUNKFILE}
        remsh $f " grep -v \# /etc/rc.config.d/diagnostic | sed -e 's/=/ /g' " >> ${JUNKFILE}
        cat ${JUNKFILE} | awk '{ if ($NF != "1") {print}}' >> ${LOCAL_PROBLEMS}
        cat ${JUNKFILE} >> ${LOCAL_DETAILS}
  else
        print "$f not pingable"  >> ${LOCAL_PROBLEMS}
  fi
done

bad_count=`wc -l ${LOCAL_PROBLEMS} | awk '{print $1}'`

 header="\n---- OnlineDiags not configured on ${bad_count} boxes "

   if ((${bad_count} > 0))
   then
         # add additional header lines
         grep -v ping ${LOCAL_PROBLEMS}  > ${JUNKFILE}
         grep    ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_PROBLEMS}

         cat ${LOCAL_DETAILS}         > ${JUNKFILE}
         grep ping ${LOCAL_PROBLEMS} >> ${JUNKFILE}
         mv ${JUNKFILE} ${LOCAL_DETAILS}
   fi

summarize;  #calls function to write to files

# cleanup
for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
done

}  # check_diagnostics_config


function alarmlog_rollover_period {

JUNKFILE=${TMPDIR}/alarmlog.junk
LOCAL_DETAILS=${TMPDIR}/alarmlog.details
LOCAL_PROBLEMS=${TMPDIR}/alarmlog.problems

for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
do
   if [ -f ${f} ]; then rm ${f}; fi
   touch $f
done

integer bad_count=0
integer hh mm ss
 
# /var/opt/platform7/logs/archived
#                                                                    YYYYMMDDHHMMSS
#-rw-rw-r--   1 a7run      a7grp       256000 Feb 12 08:42 alarm_log_20070212075831
#-rw-rw-r--   1 a7run      a7grp       256000 Feb 12 09:25 alarm_log_20070212084209

  # which file is oldest 
  oldest=`ls -u1 /var/opt/platform7/logs/current/alarm* | tail -1`
  p7alarmview  $oldest  > $JUNKFILE

  # in junkfile
  #>>> 03/21/08 09:44:04 PM  MINOR        H/W alarm
  #>>> 03/21/08 09:52:34 PM  MINOR        H/W alarm

  number_of_alarms=`grep ">>>" $JUNKFILE | wc -l`
  first_time=`grep ">>>" $JUNKFILE | head -1 | awk '{print $3}'`
  last_time=`grep ">>>" $JUNKFILE | tail -1 | awk '{print $3}'`

  echo $first_time | sed -e 's/:/ /g' | read hh mm ss
  ((first_in_minutes = ($hh * 60) + $mm  ))

  echo $last_time | sed -e 's/:/ /g' | read hh mm ss
  ((last_in_minutes = ($hh * 60) + $mm  ))

  ((delta = ($last_in_minutes - $first_in_minutes)  ))

  print "Total of ${number_of_alarms} alarms from ${first_time} thru ${last_time}\n" >> ${LOCAL_DETAILS}

  print "Severity   Quantity" >> ${LOCAL_DETAILS}
  echo "---------  ---------" >> ${LOCAL_DETAILS} 
  for severity in `grep ">>>" $JUNKFILE | awk '{print $5}' | sort -u`
  do 
      count=`grep ">>>" $JUNKFILE | grep $severity | wc -l`
      date |awk '{printf ("%-8s   %4s\n", "'$severity'",  "'$count'")}' >> ${LOCAL_DETAILS}
  done

  print "\nType     Quantity" >> ${LOCAL_DETAILS} 
  echo "-------   ---------" >> ${LOCAL_DETAILS}
  for type in `grep ">>>" $JUNKFILE | awk '{print $6}' | sort -u`
  do 
      count=`grep ">>>" $JUNKFILE | grep $type | wc -l`
      date |awk '{printf ("%-8s   %4s\n", "'$type'",  "'$count'")}' >> ${LOCAL_DETAILS}
  done

 cp ${LOCAL_DETAILS} ${LOCAL_PROBLEMS}
  header="\n--- Alarm log rolls over every ${delta} minutes"

summarize;  #calls function to write to files

    for f in $JUNKFILE $LOCAL_DETAILS $LOCAL_PROBLEMS
    do
       if [ -f ${f} ]; then rm ${f}; fi
    done
} #  alarmlog_rollover_period


function links_per_bpp {

   ETR=/h/bmetzger/s/link_stuff/wm.etr
   SUMMARY_FILE=/h/bmetzger/s/link_stuff/number_of_links_per_dsp.out

   echo " "
   echo "------------------------------------------------------------------------- "
   print  "DSPs with no links configured  "
   print -n "Using ETR from  "
            ll ${ETR} | awk '{printf ("%s %s %s",$6, $7, $8) }'
   print  " the following DSPs have no configured links      "
   echo "------------------------------------------------------------------------- "
   
   egrep -E "^  0" ${SUMMARY_FILE} | awk ' NF > 7 {print } '
   echo " "
   print  " Total Number of Datastores: `list_processors -d | wc -l` "
#   echo   " links/dsp summary file is `wc -l ${SUMMARY_FILE} | awk '{print $1}'` lines long "
#   echo  " To all results:  more ${SUMMARY_FILE}"
#   echo   " "

} #links_per_bpp 

function dcc_status_check {

   junkfile=${WORKDIR}/junkfile
   all_cards_status=${WORKDIR}/all_cards_status
   bad_dcc_cards=${WORKDIR}/bad_dcc_cards

   echo " "
   echo "------------------ "
   echo " DCC problems on:  "
   echo "------------------- "

   if [ -f ${junkfile} ]; then rm ${junkfile}; fi
   if [ -f ${all_cards_status} ]; then rm ${all_cards_status}; fi
   if [ -f ${bad_dcc_cards} ]; then rm ${bad_dcc_cards}; fi

   for f in `list_processors -d`
#for f in s027ds02
   do
             print $f
#             print -n "."
             ssh $f "if [ -f /var/opt/platform7/tmp/eng_log*75* ] ; then rm /var/opt/platform7/tmp/*75*; fi ; /opt/platform7/bin/p7dumpstatus r75 >/dev/null 2>&1;"
             ssh $f 'grep -E "DCC Id|State" /var/opt/platform7/tmp/eng_logging_R75_0 | grep -v "^$" ' > ${junkfile}
             ssh $f 'rm /var/opt/platform7/tmp/eng_logging_R75_0'
             cat ${junkfile} | awk '{if ( $NF ~ /^s/ ) {printf ("\n%s ", $NF) } else {printf ("%s ", $NF) }}' >> ${all_cards_status}
   done

   print
   grep STOP ${all_cards_status}       

} # dcc_status_check 

#################################


#out_of_comms
#ntp_check
#ping_check
#amc_lan_check
#bad_links_summary
#bpp_corefiles
#diskspace_check
#iopp_check 

#top_processes
#swapinfo
#ntpconf_check
#rpc_check
#suspend_check
#find_corrupt_bootptab
#find_corrupt_inetd_sec
#p7Site_timesync
#check_for_corrupt_filesets
#check_diagnostics_config
#bpp_timesync
#check_software_mirrors
#lanadmin_check
#alarmlog_rollover_period  
check_for_ip_in_CS_hosts_file

   # everything in commsconfig is in CS hosts file
   #  connectivity to mgmt ports


#check_hardware_mirror
#bpp_cpu_load        # nothing seen over 10%
#epp_cpu_load   # all seem to be at 100%
#sendmail_server_name
#check_sendmailcf
#check_alarm_log
#check_error_log
   #  patch status
   #  users last logins

#datastore_disk_health
#links_per_bpp
#check_for_stopped_dsp_sessions  # check.  needs work
#r75_running_check
#dcc_status_check

#mailx -s "Daily AcceSS7 Report"  sprinta7,bmetzger <  ${GLOBAL_PROBLEMS}
#mailx -s "Sprint healthcheck Problems $CDT_time" bmetzger <  ${GLOBAL_PROBLEMS}
#mailx -s "Sprint healthcheck Details" bmetzger  <  ${GLOBAL_DETAILS}
