#!/usr/bin/perl # The LearningOnline Network # Housekeeping program, started by cron # # (TCP networking package # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer) # # 7/14,7/15,7/19,7/21,7/22,11/18, # 2/8 Gerd Kortemeyer # 12/6/2000,12/8 Scott Harrison # 12/23 Gerd Kortemeyer # 1/10/2001, 2/12/, 2/26, 3/15, 04/11, 04/21 Scott Harrison use IO::File; use IO::Socket; my $qflag=0; if (@ARGV) { my $arg=shift @ARGV; $qflag=1 if $arg eq 'quick'; } # -------------------------------------------------- Non-critical communication sub reply { my ($cmd,$server)=@_; my $peerfile="$perlvar{'lonSockDir'}/$server"; my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", Type => SOCK_STREAM, Timeout => 10) or return "con_lost"; print $client "$cmd\n"; my $answer=<$client>; chomp($answer); if (!$answer) { $answer="con_lost"; } return $answer; } # --------------------------------------------------------- Output error status sub errout { my $fh=shift; print $fh (<
Notices$notices
Warnings$warnings
Errors$errors

Top

ENDERROUT } # ================================================================ Main Program # ------------------------------------------------------------ Read access.conf { my $config=IO::File->new("/etc/httpd/conf/access.conf"); while (my $configline=<$config>) { if ($configline =~ /PerlSetVar/) { my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); $perlvar{$varname}=$varvalue; } } delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed } # --------------------------------------- Make sure that LON-CAPA is configured # I only test for one thing here (lonHostID). This is just a safeguard. if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { print("Unconfigured machine.\n"); $emailto=$perlvar{'lonSysEMail'}; $hostname=`/bin/hostname`; chop $hostname; $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell $subj="LON: Unconfigured machine $hostname"; system("echo 'Unconfigured machine $hostname.' |\ mailto $emailto -s '$subj' > /dev/null"); exit 1; } # ----------------------------- Make sure this process is running from user=www my $wwwid=getpwnam('www'); if ($wwwid!=$<) { print("User ID mismatch. This program must be run as user 'www'\n"); $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; system("echo 'User ID mismatch. loncron must be run as user www.' |\ mailto $emailto -s '$subj' > /dev/null"); exit 1; } # ------------------------------------------------------------- Read hosts file { my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); while (my $configline=<$config>) { my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); $hostname{$id}=$name; $hostdom{$id}=$domain; $hostrole{$id}=$role; $hostip{$id}=$ip; if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { $libserv{$id}=$name; } } } # ------------------------------------------------------ Read spare server file { my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); while (my $configline=<$config>) { chomp($configline); if (($configline) && ($configline ne $perlvar{'lonHostID'})) { $spareid{$configline}=1; } } } # ---------------------------------------------------------------- Start report $statusdir="/home/httpd/html/lon-status"; $errors=0; $warnings=0; $notices=0; $now=time; $date=localtime($now); { my $fh=IO::File->new(">$statusdir/newstatus.html"); print $fh (< LON Status Report $perlvar{'lonHostID'}

LON Status Report $perlvar{'lonHostID'}

$date ($now)

  1. Configuration
  2. Machine Information
  3. Temporary Files
  4. Session Tokens
  5. httpd
  6. lonsql
  7. lond
  8. lonc
  9. lonnet
  10. Connections
  11. Delayed Messages
  12. Error Count

Configuration

PerlVars

ENDHEADERS foreach $varname (keys %perlvar) { print $fh "\n"; } print $fh "
$varname$perlvar{$varname}

Hosts

"; foreach $id (keys %hostname) { print $fh ""; print $fh "\n"; } print $fh "
$id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

Spare Hosts

    "; foreach $id (keys %spareid) { print $fh "
  1. $id\n"; } print $fh "
\n"; # --------------------------------------------------------------------- Machine print $fh '

Machine Information

'; print $fh "

loadavg

"; open (LOADAVGH,"/proc/loadavg"); $loadavg=; close (LOADAVGH); print $fh "$loadavg"; @parts=split(/\s+/,$loadavg); if ($parts[1]>4.0) { $errors++; } elsif ($parts[1]>2.0) { $warnings++; } elsif ($parts[1]>1.0) { $notices++; } print $fh "

df

"; print $fh "
";

open (DFH,"df|");
while ($line=) { 
   print $fh "$line"; 
   @parts=split(/\s+/,$line);
   $usage=$parts[4];
   $usage=~s/\W//g;
   if ($usage>90) { 
      $warnings++; 
   } elsif ($usage>80) {
      $warnings++;
   } elsif ($usage>60) {
      $notices++;
   }
   if ($usage>95) { $warnings++; $warnings++ }
}
close (DFH);
print $fh "
"; &errout($fh); # --------------------------------------------------------------- clean out tmp print $fh '

Temporary Files

'; $cleaned=0; while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) { my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); $now=time; $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; unlink("$fname"); } } print $fh "Cleaned up ".$cleaned." files."; # ------------------------------------------------------------ clean out lonIDs print $fh '

Session Tokens

'; $cleaned=0; $active=0; while ($fname=<$perlvar{'lonIDsDir'}/*>) { my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); $now=time; $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; print $fh "Unlinking $fname
"; unlink("$fname"); } else { $active++; } } print $fh "

Cleaned up ".$cleaned." stale session token(s)."; print $fh "

$active open session(s)

"; # ----------------------------------------------------------------------- httpd print $fh '

httpd

Access Log

';

open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
while ($line=) { print $fh "$line" };
close (DFH);

print $fh "

Error Log

";

open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/\[error\]/) { $notices++; } 
};
close (DFH);
print $fh "
"; &errout($fh); # ---------------------------------------------------------------------- lonsql # # Do not run for now # if ($perlvar{'lonRole'} eq "library") { print $fh '

lonsql

Log

';
    
    if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
	while ($line=) { 
	    print $fh "$line";
	    if ($line=~/INFO/) { $notices++; }
	    if ($line=~/WARNING/) { $notices++; }
	    if ($line=~/CRITICAL/) { $warnings++; }
	};
	close (DFH);
    }
    print $fh "
"; my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid"; if (-e $lonsqlfile) { my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); if (kill 0 => $lonsqlpid) { print $fh "

lonsql at pid $lonsqlpid responding

"; } else { $errors++; $errors++; print $fh "

lonsql at pid $lonsqlpid not responding

"; } } else { $errors++; print $fh "

lonsql not running, trying to start

"; system( "$perlvar{'lonDaemons'}/lonsql 2>>$perlvar{'lonDaemons'}/logs/lonsql_errors"); sleep 120 unless $qflag; if (-e $lonsqlfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); sleep 30 unless $qflag; if (kill 0 => $lonsqlpid) { print $fh "

lonsql at pid $lonsqlpid responding

"; } else { $errors++; $errors++; print $fh "

lonsql at pid $lonsqlpid not responding

"; print $fh "Give it one more try ...

"; system( "$perlvar{'lonDaemons'}/lonsql 2>>$perlvar{'lonDaemons'}/logs/lonsql_errors"); sleep 120 unless $qflag; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){ print $fh "

";
	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
	    while ($line=) { 
		print $fh "$line";
		if ($line=~/WARNING/) { $notices++; }
		if ($line=~/CRITICAL/) { $notices++; }
	    };
	    close (DFH);
	    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lonsql.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); } # ------------------------------------------------------------------------ lond print $fh '

lond

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/INFO/) { $notices++; }
   if ($line=~/WARNING/) { $notices++; }
   if ($line=~/CRITICAL/) { $warnings++; }
};
close (DFH);
}
print $fh "
"; my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; my $restartflag=1; if (-e $londfile) { my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; $restartflag=0; } else { $errors++; print $fh "

lond at pid $londpid not responding

"; # Intelligently handle this. # Possibility #1: there is no process # Solution: remove .pid file and restart if (getpgrp($londpid)==-1) { unlink($londfile); $restartflag=1; } else { # Possibility #2: there is a live process that is not responding # for an unknown reason # Solution: kill parent and children processes, remove .pid and restart `killall -9 lond`; unlink($londfile); $restartflag=1; } print $fh "

Deciding to clean up stale .pid file and restart lond

"; } } if ($restartflag==1) { $errors++; print $fh "

lond not running, trying to start

"; system( "$perlvar{'lonDaemons'}/lond 2>>$perlvar{'lonDaemons'}/logs/lond_errors"); sleep 120 unless $qflag; if (-e $londfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); sleep 30 unless $qflag; if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; } else { $errors++; $errors++; print $fh "

lond at pid $londpid not responding

"; print $fh "Give it one more try ...

"; system( "$perlvar{'lonDaemons'}/lond 2>>$perlvar{'lonDaemons'}/logs/lond_errors"); sleep 120 unless $qflag; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){ print $fh "

";
    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
    while ($line=) { 
      print $fh "$line";
      if ($line=~/WARNING/) { $notices++; }
      if ($line=~/CRITICAL/) { $notices++; }
    };
    close (DFH);
    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lond.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ------------------------------------------------------------------------ lonc print $fh '

lonc

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/INFO/) { $notices++; }
   if ($line=~/WARNING/) { $notices++; }
   if ($line=~/CRITICAL/) { $warnings++; }
};
close (DFH);
}
print $fh "
"; my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; $restartflag=1; if (-e $loncfile) { my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding, sending USR1

"; kill USR1 => $loncpid; $restartflag=0; } else { $errors++; print $fh "

lonc at pid $loncpid not responding

"; # Intelligently handle this. # Possibility #1: there is no process # Solution: remove .pid file and restart if (getpgrp($loncpid)==-1) { unlink($loncfile); $restartflag=1; } else { # Possibility #2: there is a live process that is not responding # for an unknown reason # Solution: kill parent and children processes, remove .pid and restart `killall -9 lonc`; unlink($loncfile); $restartflag=1; } print $fh "

Deciding to clean up stale .pid file and restart lonc

"; } } if ($restartflag==1) { $errors++; print $fh "

lonc not running, trying to start

"; system( "$perlvar{'lonDaemons'}/lonc 2>>$perlvar{'lonDaemons'}/logs/lonc_errors"); sleep 120 unless $qflag; if (-e $loncfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); sleep 30 unless $qflag; if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding

"; } else { $errors++; $errors++; print $fh "

lonc at pid $loncpid not responding

"; print $fh "Give it one more try ...

"; system( "$perlvar{'lonDaemons'}/lonc 2>>$perlvar{'lonDaemons'}/logs/lonc_errors"); sleep 120 unless $qflag; } } else { print $fh "Seems like that did not work!

"; $errors++; } if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") { print $fh "

";
    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
    while ($line=) { 
      print $fh "$line";
      if ($line=~/WARNING/) { $notices++; }
      if ($line=~/CRITICAL/) { $notices++; }
    };
    close (DFH);
    print $fh "
"; } } $fname="$perlvar{'lonDaemons'}/logs/lonc.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ---------------------------------------------------------------------- lonnet print $fh '

lonnet

Temp Log

';
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
while ($line=) { 
    print $fh "$line";
};
close (DFH);
}
print $fh "

Perm Log

";

if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
    open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
while ($line=) { 
   print $fh "$line";
};
close (DFH);
} else { print $fh "No perm log\n" }

$fname="$perlvar{'lonDaemons'}/logs/lonnet.log";

                          my ($dev,$ino,$mode,$nlink,
                              $uid,$gid,$rdev,$size,
                              $atime,$mtime,$ctime,
                              $blksize,$blocks)=stat($fname);

if ($size>40000) {
    print $fh "Rotating logs ...
";
    rename("$fname.2","$fname.3");
    rename("$fname.1","$fname.2");
    rename("$fname","$fname.1");
}

print $fh "
"; &errout($fh); # ----------------------------------------------------------------- Connections print $fh '

Connections

'; print $fh ""; foreach $tryserver (keys %hostname) { $answer=reply("pong",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; } else { $result=$answer; $warnings++; if ($answer eq 'con_lost') { $warnings++; } } print $fh "\n"; } print $fh "
$tryserver$result
"; &errout($fh); # ------------------------------------------------------------ Delayed messages print $fh '

Delayed Messages

'; print $fh '

Scanning Permanent Log

'; $unsend=0; { my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while ($line=<$dfh>) { ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); if ($sdf eq 'F') { $local=localtime($time); print "Failed: $time, $dserv, $dcmd
"; $warnings++; } if ($sdf eq 'S') { $unsend--; } if ($sdf eq 'D') { $unsend++; } } } print $fh "Total unsend messages: $unsend

\n"; $warnings=$warnings+5*$unsend; print $fh "

Outgoing Buffer

"; open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); while ($line=) { print $fh "$line
"; }; close (DFH); # ------------------------------------------------------------------------- End print $fh "\n"; $totalcount=$notices+4*$warnings+100*$errors; &errout($fh); print $fh "

Total Error Count: $totalcount

"; $now=time; $date=localtime($now); print $fh "
$date ($now)\n"; } rename ("$statusdir/newstatus.html","$statusdir/index.html"); if ($totalcount>200) { $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; system( "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html") unless $qflag; } 1;