#!/usr/bin/perl # The LearningOnline Network # Housekeeping program, started by cron # # (TCP networking package # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer) # # 7/14,7/15,7/19,7/21,7/22,11/18, # 2/8 Gerd Kortemeyer # Dec 00 Scott Harrison # 12/23 Gerd Kortemeyer use IO::File; use IO::Socket; # -------------------------------------------------- Non-critical communication sub reply { my ($cmd,$server)=@_; my $peerfile="$perlvar{'lonSockDir'}/$server"; my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", Type => SOCK_STREAM, Timeout => 10) or return "con_lost"; print $client "$cmd\n"; my $answer=<$client>; chomp($answer); if (!$answer) { $answer="con_lost"; } return $answer; } # --------------------------------------------------------- Output error status sub errout { my $fh=shift; print $fh (<
Notices$notices
Warnings$warnings
Errors$errors

Top

ENDERROUT } # ================================================================ Main Program # ------------------------------------------------------------ Read access.conf { my $config=IO::File->new("/etc/httpd/conf/access.conf"); while (my $configline=<$config>) { if ($configline =~ /PerlSetVar/) { my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); $perlvar{$varname}=$varvalue; } } } # ----------------------------- Make sure this process is running from user=www my $wwwid=getpwnam('www'); if ($wwwid!=$<) { print("User ID mismatch. This program must be run as user 'www'\n") unless $noprint; $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; system("echo 'User ID mismatch. loncron must be run as user www.' |\ mailto $emailto -s '$subj' > /dev/null"); exit 1; } # ------------------------------------------------------------- Read hosts file { my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); while (my $configline=<$config>) { my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); $hostname{$id}=$name; $hostdom{$id}=$domain; $hostrole{$id}=$role; $hostip{$id}=$ip; if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { $libserv{$id}=$name; } } } # ------------------------------------------------------ Read spare server file { my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); while (my $configline=<$config>) { chomp($configline); if (($configline) && ($configline ne $perlvar{'lonHostID'})) { $spareid{$configline}=1; } } } # ---------------------------------------------------------------- Start report $statusdir="/home/httpd/html/lon-status"; $errors=0; $warnings=0; $notices=0; $now=time; $date=localtime($now); { my $fh=IO::File->new(">$statusdir/newstatus.html"); print $fh (< LON Status Report $perlvar{'lonHostID'}

LON Status Report $perlvar{'lonHostID'}

$date ($now)

  1. Configuration
  2. Machine Information
  3. Temporary Files
  4. Session Tokens
  5. httpd
  6. lonsql
  7. lond
  8. lonc
  9. lonnet
  10. Connections
  11. Delayed Messages
  12. Error Count

Configuration

PerlVars

ENDHEADERS foreach $varname (keys %perlvar) { print $fh "\n"; } print $fh "
$varname$perlvar{$varname}

Hosts

"; foreach $id (keys %hostname) { print $fh ""; print $fh "\n"; } print $fh "
$id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

Spare Hosts

    "; foreach $id (keys %spareid) { print $fh "
  1. $id\n"; } print $fh "
\n"; # --------------------------------------------------------------------- Machine print $fh '

Machine Information

'; print $fh "

loadavg

"; open (LOADAVGH,"/proc/loadavg"); $loadavg=; close (LOADAVGH); print $fh "$loadavg"; @parts=split(/\s+/,$loadavg); if ($parts[1]>4.0) { $errors++; } elsif ($parts[1]>2.0) { $warnings++; } elsif ($parts[1]>1.0) { $notices++; } print $fh "

df

"; print $fh "
";

open (DFH,"df|");
while ($line=) { 
   print $fh "$line"; 
   @parts=split(/\s+/,$line);
   $usage=$parts[4];
   $usage=~s/\W//g;
   if ($usage>90) { 
      $warnings++; 
   } elsif ($usage>80) {
      $warnings++;
   } elsif ($usage>60) {
      $notices++;
   }
   if ($usage>95) { $warnings++; $warnings++ }
}
close (DFH);
print $fh "
"; &errout($fh); # --------------------------------------------------------------- clean out tmp print $fh '

Temporary Files

'; $cleaned=0; while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) { my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); $now=time; $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; unlink("$fname"); } } print $fh "Cleaned up ".$cleaned." files."; # ------------------------------------------------------------ clean out lonIDs print $fh '

Session Tokens

'; $cleaned=0; $active=0; while ($fname=<$perlvar{'lonIDsDir'}/*>) { my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); $now=time; $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; print $fh "Unlinking $fname
"; unlink("$fname"); } else { $active++; } } print $fh "

Cleaned up ".$cleaned." stale session token(s)."; print $fh "

$active open session(s)

"; # ----------------------------------------------------------------------- httpd print $fh '

httpd

Access Log

';

open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
while ($line=) { print $fh "$line" };
close (DFH);

print $fh "

Error Log

";

open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/\[error\]/) { $notices++; } 
};
close (DFH);
print $fh "
"; &errout($fh); # ---------------------------------------------------------------------- lonsql # # Do not run for now # if ($perlvar{'lonRole'} eq "library" && 1==0) { print $fh '

lonsql

Log

';
    
    if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
	while ($line=) { 
	    print $fh "$line";
	    if ($line=~/INFO/) { $notices++; }
	    if ($line=~/WARNING/) { $notices++; }
	    if ($line=~/CRITICAL/) { $warnings++; }
	};
	close (DFH);
    }
    print $fh "
"; my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid"; if (-e $lonsqlfile) { my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); if (kill 0 => $lonsqlpid) { print $fh "

lonsql at pid $lonsqlpid responding

"; } else { $errors++; $errors++; print $fh "

lonsql at pid $lonsqlpid not responding

"; } } else { $errors++; print $fh "

lonsql not running, trying to start

"; system("$perlvar{'lonDaemons'}/lonsql"); sleep 120; if (-e $lonsqlfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); sleep 30; if (kill 0 => $lonsqlpid) { print $fh "

lonsql at pid $lonsqlpid responding

"; } else { $errors++; $errors++; print $fh "

lonsql at pid $lonsqlpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lonsql"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){ print $fh "

";
	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
	    while ($line=) { 
		print $fh "$line";
		if ($line=~/WARNING/) { $notices++; }
		if ($line=~/CRITICAL/) { $notices++; }
	    };
	    close (DFH);
	    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lonsql.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); } # ------------------------------------------------------------------------ lond print $fh '

lond

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/INFO/) { $notices++; }
   if ($line=~/WARNING/) { $notices++; }
   if ($line=~/CRITICAL/) { $warnings++; }
};
close (DFH);
}
print $fh "
"; my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; my $restartflag=1; if (-e $londfile) { my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; $restartflag=0; } else { $errors++; print $fh "

lond at pid $londpid not responding

"; # Intelligently handle this. # Possibility #1: there is no process # Solution: remove .pid file and restart if (getpgrp($londpid)==-1) { unlink($londfile); $restartflag=1; } else { # Possibility #2: there is a live process that is not responding # for an unknown reason # Solution: kill parent and children processes, remove .pid and restart `killall -9 lond`; unlink($londfile); $restartflag=1; } print $fh "

Deciding to clean up stale .pid file and restart lond

"; } } if ($restartflag==1) { $errors++; print $fh "

lond not running, trying to start

"; system("$perlvar{'lonDaemons'}/lond"); sleep 120; if (-e $londfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); sleep 30; if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; } else { $errors++; $errors++; print $fh "

lond at pid $londpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lond"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){ print $fh "

";
    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
    while ($line=) { 
      print $fh "$line";
      if ($line=~/WARNING/) { $notices++; }
      if ($line=~/CRITICAL/) { $notices++; }
    };
    close (DFH);
    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lond.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ------------------------------------------------------------------------ lonc print $fh '

lonc

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/INFO/) { $notices++; }
   if ($line=~/WARNING/) { $notices++; }
   if ($line=~/CRITICAL/) { $warnings++; }
};
close (DFH);
}
print $fh "
"; my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; $restartflag=1; if (-e $loncfile) { my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding, sending USR1

"; kill USR1 => $loncpid; $restartflag=0; } else { $errors++; print $fh "

lonc at pid $loncpid not responding

"; # Intelligently handle this. # Possibility #1: there is no process # Solution: remove .pid file and restart if (getpgrp($loncpid)==-1) { unlink($loncfile); $restartflag=1; } else { # Possibility #2: there is a live process that is not responding # for an unknown reason # Solution: kill parent and children processes, remove .pid and restart `killall -9 lonc`; unlink($loncfile); $restartflag=1; } print $fh "

Deciding to clean up stale .pid file and restart lonc

"; } } if ($restartflag==1) { $errors++; print $fh "

lonc not running, trying to start

"; system("$perlvar{'lonDaemons'}/lonc"); sleep 120; if (-e $loncfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); sleep 30; if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding

"; } else { $errors++; $errors++; print $fh "

lonc at pid $loncpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lonc"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") { print $fh "

";
    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
    while ($line=) { 
      print $fh "$line";
      if ($line=~/WARNING/) { $notices++; }
      if ($line=~/CRITICAL/) { $notices++; }
    };
    close (DFH);
    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lonc.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ---------------------------------------------------------------------- lonnet print $fh '

lonnet

Temp Log

';
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
while ($line=) { 
    print $fh "$line";
};
close (DFH);
}
print $fh "

Perm Log

";

if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
    open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
while ($line=) { 
   print $fh "$line";
};
close (DFH);
} else { print $fh "No perm log\n" }

$fname="$perlvar{'lonDaemons'}/logs/lonnet.log";

                          my ($dev,$ino,$mode,$nlink,
                              $uid,$gid,$rdev,$size,
                              $atime,$mtime,$ctime,
                              $blksize,$blocks)=stat($fname);

if ($size>40000) {
    print $fh "Rotating logs ...
";
    rename("$fname.2","$fname.3");
    rename("$fname.1","$fname.2");
    rename("$fname","$fname.1");
}

print $fh "
"; &errout($fh); # ----------------------------------------------------------------- Connections print $fh '

Connections

'; print $fh ""; foreach $tryserver (keys %hostname) { $answer=reply("pong",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; } else { $result=$answer; $warnings++; if ($answer eq 'con_lost') { $warnings++; } } print $fh "\n"; } print $fh "
$tryserver$result
"; &errout($fh); # ------------------------------------------------------------ Delayed messages print $fh '

Delayed Messages

'; print $fh '

Scanning Permanent Log

'; $unsend=0; { my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while ($line=<$dfh>) { ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); if ($sdf eq 'F') { $local=localtime($time); print "Failed: $time, $dserv, $dcmd
"; $warnings++; } if ($sdf eq 'S') { $unsend--; } if ($sdf eq 'D') { $unsend++; } } } print $fh "Total unsend messages: $unsend

\n"; $warnings=$warnings+5*$unsend; print $fh "

Outgoing Buffer

"; open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); while ($line=) { print $fh "$line
"; }; close (DFH); # ------------------------------------------------------------------------- End print $fh "\n"; $totalcount=$notices+4*$warnings+100*$errors; &errout($fh); print $fh "

Total Error Count: $totalcount

"; $now=time; $date=localtime($now); print $fh "
$date ($now)\n"; } rename ("$statusdir/newstatus.html","$statusdir/index.html"); if ($totalcount>200) { $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; system( "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); } 1;