#!/usr/bin/perl  

#ModulePurpose: Check mirror, quorum, and physical disk status on all WSTs, STE boxes
#ModuleName:    check_software_mirrors
#TimeRequired:  2 minutes

# Does NOT check either CS (hardware mirror)

#Problems Report:
#   -- 5 mirror or PV issues - 08.14.2009 
#   m005        lvol1    LV Status = stale    
#   m005        lvol3    LV Status = stale    
#   m005        lvol4    LV Status = stale    
#   m005        disk /dev/dsk/c2t1d0s2 is unavailable
#   m005       /dev/rdsk/c2t0d0s2 -- Low quorum not set
 

use lib '/h/bmetzger/s/p/hc/';
use HC qw(:ALL);  #functions available to all modules 

use English;
use Carp;
use Cwd;
use DBI;
use Net::Ping;
use File::Basename;
use File::Path;
use File::Glob;
use File::Copy;

###############3###
# define location of, and open files
##################

  my ($TmpDir,$sitelist,$global_problems,$global_details,$global_summary) = InitVars();

  $SSH_OPTIONS=" -o BatchMode=yes -o ConnectTimeout=2";
  $ENV{'PATH'} = 'misc_tools:/usr/bin:/usr/local/bin:/usr/sbin/:/usr/ucb:/etc/:/opt/platform7/lbin/:.';
  &initvars();

  #create list of boxes to check
#configurable
  qx[list_processors -sw > $sitelist];
#  qx[cat /etc/opt/OSSInstall/servers.conf >> $sitelist];
#  qx[list_processors -s | grep m005 > $sitelist];

### checks
 ($header, $local_problems, $local_details)= &check(); 
  summarize($header, $local_problems, $local_details);

###################33
sub initvars {

   ($sys_day,$sys_mon,$sys_year) = (localtime)[3,4,5];
   $sys_mon += 1;  #jan is 0
   $sys_year += 1900;
   $today = sprintf "%02d.%02d.%d",$sys_mon,$sys_day,$sys_year;  # zero fill date

   $SSH_OPTIONS=" -o BatchMode=yes -o ConnectTimeout=2";
   $ENV{'PATH'} = '/usr/bin:/usr/local/bin:/usr/sbin/:/usr/ucb:/etc/:.';
} #initvars

####################
sub line_count {
####################
# perform a  wc -l

  my $file = shift;
    open my $fh, "<", $file or die "could not open $file:$!";
    my $lines = 0;
    $lines++ while <$fh>;
    return $lines;
    close ($fh);
} #line_count

############################
sub check{    #(main program)
############################

     print "software mirror status \n";

     ($p, $box) =();
     $local_problems = "$TmpDir/mirror_problems";
     $local_details  = "$TmpDir/mirror_details";
     $local_junk1    = "$TmpDir/junk1";    # temp file for problems
     $local_junk2    = "$TmpDir/junk2";    # temp file for details

    # open local output files
    open PROBLEMS, ">>$local_problems" or croak "Can't open local_problems for write: $!\n";
    open DETAILS, ">>$local_details" or croak "Can't open local_details for write: $!\n";

    open SITE, "<$sitelist" or croak "Can't open sitelist file for read $!\n";

    # check each box 
    while (defined ($box = <SITE>)) {
          chomp($box);

          open PROBLEMS, ">>$local_problems" or croak "Can't open local_problems for write: $!\n";

          $p = Net::Ping->new("icmp");   #need icmp or fails to tra1 and tra2
          if ( $p->ping($box) ) {  #pingable

                 &check_mirror($box);
                 &check_pv_status($box);   #check disk health even if mirrorux is not installed
        
         } else {
               printf PROBLEMS "    $10s NOT pingable\n",$box ;
               printf  "$10s NOT pingable\n",$box ;
        }
        $p->close();
        close(PROBLEMS); 
        unlink("$local_junk1") || print $!;
        unlink("$local_junk2") || print $!;
  } #for each box in sitelist

    #gotta close, so lines can be counted
    close(DETAILS);  
    close(PROBLEMS); 

    my $badcount = line_count($local_problems);

    # open local output files
    open JUNK1, ">>$local_junk1" or croak "Can't open file $local_junk1 for write: $!\n";
    open JUNK2, ">>$local_junk2" or croak "Can't open file $local_junk2 for write: $!\n";

    my $header = "\n--- $badcount software mirror or PV issues -$today \n";

    if ( $badcount > 0 ){  # at least one problem
         #format and header
         print JUNK1 $header;
             qx[grep installed $local_problems >> $local_junk1];
             qx[grep stale $local_problems >> $local_junk1];
             qx[grep available $local_problems >> $local_junk1];
             qx[grep quorum $local_problems >> $local_junk1];
             qx[grep    ping $local_problems >> $local_junk1];
         close(JUNK1);

         print JUNK2 $header;
         close(JUNK2);
    }else {
         print JUNK2 $header;
         close(JUNK2);
    }

  copy($local_junk1,$local_problems) or die "junk1 cannot be copied";
  copy($local_junk2,$local_details) or die "junk2 cannot be copied";

   unlink("$local_junk1") || print $!;
   unlink("$local_junk2") || print $!;
   unlink("$sitelist") || print $!;

  return($header, $local_problems, $local_details);
} #check


######
sub check_mirror{ 
######
  
   my ($box) = @_; 

   $software_count = qx[remsh $box "/usr/sbin/swlist -l bundle |grep Mirror | wc -l" ];
   if ($software_count == 1) { #mirror-ux is installed 

          # check each lvol for Mirror copies and status
          qx[remsh $box "cd /dev/vg00; ls lvol* " > $local_junk1 ];
                
          open JUNK1, "<$local_junk1" or croak "Can't open junk1 for read $!\n";
          while (my $lvol = <JUNK1>) {
                 chomp $lvol;
                 next if ($lvol =~ /^$/);   # eliminate blank

# future:  might want to add more VGs instead of just vg00
                 qx[remsh $box "/usr/sbin/lvdisplay /dev/vg00/$lvol" > $local_junk2 ];

                 open JUNK2, "<$local_junk2" or croak "Can't open junk2 for read $!\n";
                 while (my $line = <JUNK2>) {
                        my @arr = split(/\s+/,$line);

                        if ($line =~ /stale/){  # LV Status  available/stale
                             $lv_status = $arr[-1];
                                $lv_status =~ s/available\///g;
                              printf PROBLEMS "    %-10s  %-8s LV Status = %-8s \n", $box, $lvol, $lv_status;
                         }

                         if ($line =~ /Mirror/){  # Mirror copies  0
                             my $mirror_copies = $arr[-1];
                             if ( $mirror_copies == 0){
                                  printf PROBLEMS "    %-10s %-8s     --Not Mirrored  \n", $box, $lvol;
                             }
                         }
                } #done with lvdisplay stats for each lvols
          } #all vg00 lvols
    } else {
            printf PROBLEMS "    $10s -- mirrorUX not installed\n",$box ;
    }
   close(JUNK1);
   close(JUNK2);
} #check_mirror

##########################
sub check_pv_status {
##########################
# check diskhealth even if mirrorux is not installed

   my ($box) = @_; 


    # check PV status, and maybe quorum
    qx[remsh $box "/usr/sbin/vgdisplay -v /dev/vg00" > $local_junk1 ];
                
    open JUNK1, "<$local_junk1" or croak "Can't open junk1 for read $!\n";
    while (<JUNK1>) {
            chomp;
            $line =~ s/\s+/ /g;
            @arr = split(/\s+/,$_);

            if (/PV Name/ .. /PV Status/) {  #range

                 if (/PV Name/){  #  PV Name /dev/dsk/c2t1d0s2
                        $pv_name = $arr[-1];  
                 }

                 if (/PV Status/){  #  PV Status unavailable 
                        $pv_status = $arr[-1];  

                        if ( $pv_status eq "unavailable") {
                               printf PROBLEMS  "    %-10s  disk %s is %s\n", $box, $pv_name, $pv_status;

                        }else { # pv_status eq "available"
                                #see if device is set to boot w/low quorum
                                # DO NOT do this if device is unavailable

                               $pv_name =~ s/dsk/rdsk/;
                               $lifcp_count = qx[remsh $box "/usr/bin/lifcp $pv_name:AUTO - | wc -w "];
                               chomp $lifcp_count;

                               if ($lifcp_count == 1){ #hpux
                                       printf PROBLEMS  "    %-10s %s -- Low quorum not set\n", $box, $pv_name;
                               }
                       }
                }                  
           } #end of range
   } #end of analyzing vgdisplay
   close(JUNK1);
   close(JUNK2);
} # check_pv_status 

