#!/usr/bin/perl # The LearningOnline Network # Housekeeping program, started by cron # # (TCP networking package # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer) # # 7/14,7/15,7/19,7/21,7/22,11/18, # 2/8 Gerd Kortemeyer use IO::File; use IO::Socket; # -------------------------------------------------- Non-critical communication sub reply { my ($cmd,$server)=@_; my $peerfile="$perlvar{'lonSockDir'}/$server"; my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", Type => SOCK_STREAM, Timeout => 10) or return "con_lost"; print $client "$cmd\n"; my $answer=<$client>; chomp($answer); if (!$answer) { $answer="con_lost"; } return $answer; } # --------------------------------------------------------- Output error status sub errout { my $fh=shift; print $fh (<
Notices$notices
Warnings$warnings
Errors$errors

Top

ENDERROUT } # ================================================================ Main Program # ------------------------------------------------------------ Read access.conf { my $config=IO::File->new("/etc/httpd/conf/access.conf"); while (my $configline=<$config>) { if ($configline =~ /PerlSetVar/) { my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); $perlvar{$varname}=$varvalue; } } } # ------------------------------------------------------------- Read hosts file { my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); while (my $configline=<$config>) { my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); $hostname{$id}=$name; $hostdom{$id}=$domain; $hostrole{$id}=$role; $hostip{$id}=$ip; if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { $libserv{$id}=$name; } } } # ------------------------------------------------------ Read spare server file { my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); while (my $configline=<$config>) { chomp($configline); if (($configline) && ($configline ne $perlvar{'lonHostID'})) { $spareid{$configline}=1; } } } # ---------------------------------------------------------------- Start report $statusdir="/home/httpd/html/lon-status"; $errors=0; $warnings=0; $notices=0; $now=time; $date=localtime($now); { my $fh=IO::File->new(">$statusdir/newstatus.html"); print $fh (< LON Status Report $perlvar{'lonHostID'}

LON Status Report $perlvar{'lonHostID'}

$date ($now)

  1. Configuration
  2. Machine Information
  3. httpd
  4. lond
  5. lonc
  6. lonnet
  7. Connections
  8. Delayed Messages
  9. Error Count

Configuration

PerlVars

ENDHEADERS foreach $varname (keys %perlvar) { print $fh "\n"; } print $fh "
$varname$perlvar{$varname}

Hosts

"; foreach $id (keys %hostname) { print $fh ""; print $fh "\n"; } print $fh "
$id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

Spare Hosts

    "; foreach $id (keys %spareid) { print $fh "
  1. $id\n"; } print $fh "
\n"; # --------------------------------------------------------------------- Machine print $fh '

Machine Information

'; print $fh "

loadavg

"; open (LOADAVGH,"/proc/loadavg"); $loadavg=; close (LOADAVGH); print $fh "$loadavg"; @parts=split(/\s+/,$loadavg); if ($parts[1]>4.0) { $errors++; } elsif ($parts[1]>2.0) { $warnings++; } elsif ($parts[1]>1.0) { $notices++; } print $fh "

df

"; print $fh "
";

open (DFH,"df|");
while ($line=) { 
   print $fh "$line"; 
   @parts=split(/\s+/,$line);
   $usage=$parts[4];
   $usage=~s/\W//g;
   if ($usage>90) { 
      $warnings++; 
   } elsif ($usage>80) {
      $warnings++;
   } elsif ($usage>60) {
      $notices++;
   }
   if ($usage>95) { $warnings++; $warnings++ }
}
close (DFH);
print $fh "
"; &errout($fh); # ----------------------------------------------------------------------- httpd print $fh '

httpd

Access Log

';

open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
while ($line=) { print $fh "$line" };
close (DFH);

print $fh "

Error Log

";

open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/\[error\]/) { $notices++; } 
};
close (DFH);
print $fh "
"; &errout($fh); # ------------------------------------------------------------------------ lonsql if ($perlvar{'lonRole'} eq "library") { print $fh '

lonsql

Log

';
    
    if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
	while ($line=) { 
	    print $fh "$line";
	    if ($line=~/INFO/) { $notices++; }
	    if ($line=~/WARNING/) { $notices++; }
	    if ($line=~/CRITICAL/) { $warnings++; }
	};
	close (DFH);
    }
    print $fh "
"; my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid"; if (-e $lonsqlfile) { my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); if (kill 0 => $lonsqlpid) { print $fh "

lonsql at pid $lonsqlpid responding

"; } else { $errors++; $errors++; print $fh "

lonsql at pid $lonsqlpid not responding

"; } } else { $errors++; print $fh "

lonsql not running, trying to start

"; system("$perlvar{'lonDaemons'}/lonsql"); sleep 120; if (-e $lonsqlfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); sleep 30; if (kill 0 => $lonsqlpid) { print $fh "

lonsql at pid $lonsqlpid responding

"; } else { $errors++; $errors++; print $fh "

lonsql at pid $lonsqlpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lonsql"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){ print $fh "

";
	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
	    while ($line=) { 
		print $fh "$line";
		if ($line=~/WARNING/) { $notices++; }
		if ($line=~/CRITICAL/) { $notices++; }
	    };
	    close (DFH);
	    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lonsql.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); } # ------------------------------------------------------------------------ lond print $fh '

lond

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/INFO/) { $notices++; }
   if ($line=~/WARNING/) { $notices++; }
   if ($line=~/CRITICAL/) { $warnings++; }
};
close (DFH);
}
print $fh "
"; my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; if (-e $londfile) { my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; } else { $errors++; $errors++; print $fh "

lond at pid $londpid not responding

"; } } else { $errors++; print $fh "

lond not running, trying to start

"; system("$perlvar{'lonDaemons'}/lond"); sleep 120; if (-e $londfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); sleep 30; if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; } else { $errors++; $errors++; print $fh "

lond at pid $londpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lond"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){ print $fh "

";
    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
    while ($line=) { 
      print $fh "$line";
      if ($line=~/WARNING/) { $notices++; }
      if ($line=~/CRITICAL/) { $notices++; }
    };
    close (DFH);
    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lond.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ------------------------------------------------------------------------ lonc print $fh '

lonc

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/INFO/) { $notices++; }
   if ($line=~/WARNING/) { $notices++; }
   if ($line=~/CRITICAL/) { $warnings++; }
};
close (DFH);
}
print $fh "
"; my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; if (-e $loncfile) { my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding, sending USR1

"; kill USR1 => $loncpid; } else { $errors++; $errors++; print $fh "

lonc at pid $loncpid not responding

"; } } else { $errors++; print $fh "

lonc not running, trying to start

"; system("$perlvar{'lonDaemons'}/lonc"); sleep 120; if (-e $loncfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); sleep 30; if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding

"; } else { $errors++; $errors++; print $fh "

lonc at pid $loncpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lonc"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") { print $fh "

";
    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
    while ($line=) { 
      print $fh "$line";
      if ($line=~/WARNING/) { $notices++; }
      if ($line=~/CRITICAL/) { $notices++; }
    };
    close (DFH);
    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lonc.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ---------------------------------------------------------------------- lonnet print $fh '

lonnet

Temp Log

';
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
while ($line=) { 
    print $fh "$line";
};
close (DFH);
}
print $fh "

Perm Log

"; if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") { open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|"); while ($line=) { print $fh "$line"; }; close (DFH); } else { print $fh "No perm log\n" } $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } print $fh ""; &errout($fh); # ----------------------------------------------------------------- Connections print $fh '

Connections

'; print $fh ""; foreach $tryserver (keys %hostname) { $answer=reply("pong",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; } else { $result=$answer; $warnings++; if ($answer eq 'con_lost') { $warnings++; } } print $fh "\n"; } print $fh "
$tryserver$result
"; &errout($fh); # ------------------------------------------------------------ Delayed messages print $fh '

Delayed Messages

'; print $fh '

Scanning Permanent Log

'; $unsend=0; { my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while ($line=<$dfh>) { ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); if ($sdf eq 'F') { $local=localtime($time); print "Failed: $time, $dserv, $dcmd
"; $warnings++; } if ($sdf eq 'S') { $unsend--; } if ($sdf eq 'D') { $unsend++; } } } print $fh "Total unsend messages: $unsend

\n"; $warnings=$warnings+5*$unsend; print $fh "

Outgoing Buffer

"; open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); while ($line=) { print $fh "$line
"; }; close (DFH); # ------------------------------------------------------------------------- End print $fh "\n"; $totalcount=$notices+4*$warnings+100*$errors; &errout($fh); print $fh "

Total Error Count: $totalcount

"; $now=time; $date=localtime($now); print $fh "
$date ($now)\n"; } rename ("$statusdir/newstatus.html","$statusdir/index.html"); if ($totalcount>200) { $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; system( "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); } 1;