--- loncom/loncron 2005/10/06 20:35:50 1.61 +++ loncom/loncron 2007/05/31 05:20:18 1.74 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.61 2005/10/06 20:35:50 albertel Exp $ +# $Id: loncron,v 1.74 2007/05/31 05:20:18 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -32,6 +32,7 @@ use strict; use lib '/home/httpd/lib/perl/'; use LONCAPA::Configuration; +use Apache::lonnet; use IO::File; use IO::Socket; @@ -43,21 +44,6 @@ use vars qw (%perlvar %simplestatus $err my $statusdir="/home/httpd/html/lon-status"; -# -------------------------------------------------- Non-critical communication -sub reply { - my ($cmd,$server)=@_; - my $peerfile="$perlvar{'lonSockDir'}/$server"; - my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", - Type => SOCK_STREAM, - Timeout => 10) - or return "con_lost"; - print $client "$cmd\n"; - my $answer=<$client>; - chomp($answer); - if (!$answer) { $answer="con_lost"; } - return $answer; -} - # --------------------------------------------------------- Output error status sub log { @@ -76,20 +62,27 @@ sub errout { ENDERROUT } +sub rotate_logfile { + my ($file,$fh,$description) = @_; + my $size=(stat($file))[7]; + if ($size>40000) { + &log($fh,"

Rotating $description ...

"); + rename("$file.2","$file.3"); + rename("$file.1","$file.2"); + rename("$file","$file.1"); + } +} + sub start_daemon { my ($fh,$daemon,$pidfile,$args) = @_; my $progname=$daemon; - if ($daemon eq 'lonc' && $args eq 'new') { + if ($daemon eq 'lonc') { $progname='loncnew'; - print "new "; } my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; - my $size=(stat($error_fname))[7]; - if ($size>40000) { - &log($fh,"

Rotating error logs ...

"); - rename("$error_fname.2","$error_fname.3"); - rename("$error_fname.1","$error_fname.2"); - rename("$error_fname","$error_fname.1"); + &rotate_logfile($error_fname,$fh,'error logs'); + if ($daemon eq 'lonc') { + &clean_sockets($fh); } system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); sleep 1; @@ -98,7 +91,7 @@ sub start_daemon { my $lfh=IO::File->new("$pidfile"); my $daemonpid=<$lfh>; chomp($daemonpid); - if (kill 0 => $daemonpid) { + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { return 1; } else { return 0; @@ -112,6 +105,7 @@ sub start_daemon { sub checkon_daemon { my ($fh,$daemon,$maxsize,$send,$args)=@_; + my $result; &log($fh,'

'.$daemon.'

Log

'); printf("%-15s ",$daemon); if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ @@ -134,7 +128,7 @@ sub checkon_daemon { my $lfh=IO::File->new("$pidfile"); $daemonpid=<$lfh>; chomp($daemonpid); - if (kill 0 => $daemonpid) { + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { &log($fh,"

$daemon at pid $daemonpid responding"); if ($send) { &log($fh,", sending $send"); } &log($fh,"

"); @@ -142,8 +136,10 @@ sub checkon_daemon { if ($send eq 'USR2') { kill USR2 => $daemonpid; } $restartflag=0; if ($send eq 'USR2') { + $result = 'reloaded'; print "reloaded\n"; } else { + $result = 'running'; print "running\n"; } } else { @@ -169,6 +165,7 @@ sub checkon_daemon { if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; + $result = 'started'; print "started\n"; } else { $errors++; @@ -178,8 +175,10 @@ sub checkon_daemon { if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; + $result = 'started'; print "started\n"; } else { + $result = 'failed'; print " failed\n"; $simplestatus{$daemon}='failed'; $errors++; $errors++; @@ -202,20 +201,10 @@ sub checkon_daemon { } my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - - if ($size>$maxsize) { - &log($fh,"

Rotating logs ...

"); - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); - } + &rotate_logfile($fname,$fh,'logs'); &errout($fh); + return $result; } # --------------------------------------------------------------------- Machine @@ -291,7 +280,6 @@ sub log_machine_info { } sub start_logging { - my ($hostdom,$hostrole,$hostname,$spareid)=@_; my $fh=IO::File->new(">$statusdir/newstatus.html"); my %simplestatus=(); my $now=time; @@ -334,17 +322,24 @@ ENDHEADERS &encode_entities($perlvar{$varname},'<>&"')."\n"); } &log($fh,"

Hosts

"); - foreach my $id (sort(keys(%{$hostname}))) { + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $id (sort(keys(%hostname))) { + my $role = (&Apache::lonnet::is_library($id) ? 'library' + : 'access'); &log($fh, - "\n"); - } - &log($fh,"
$id".$hostdom->{$id}. - "".$hostrole->{$id}. - "".$hostname->{$id}."

Spare Hosts

    "); - foreach my $id (sort(keys(%{$spareid}))) { - &log($fh,"
  1. $id\n
    2. "); + "$id".&Apache::lonnet::host_domain($id). + "".$role. + "".&Apache::lonnet::hostname($id)."\n"); + } + &log($fh,"

    Spare Hosts

      "); + foreach my $type (sort(keys(%Apache::lonnet::spareid))) { + &log($fh,"
    • $type\n
        "); + foreach my $id (@{ $Apache::lonnet::spareid{$type} }) { + &log($fh,"
      1. $id
        2. \n"); + } + &log($fh,"
      \n
      • \n"); } - &log($fh,"
\n"); + &log($fh,"\n"); return $fh; } @@ -406,6 +401,21 @@ sub clean_lonIDs { &log($fh,"

$active open session(s)

"); } +# ----------------------------------------------------------- clean out sockets +sub clean_sockets { + my ($fh)=@_; + my $cleaned=0; + opendir(SOCKETS,$perlvar{'lonSockDir'}); + while (my $fname=readdir(SOCKETS)) { + next if (-d $fname + || $fname=~/(mysqlsock|maximasock|\Q$perlvar{'lonSockDir'}\E)/); + $cleaned++; + &log($fh,"Unlinking $fname
"); + unlink("/home/httpd/sockets/$fname"); + } + &log($fh,"

Cleaned up ".$cleaned." stale sockets.

"); +} + # ----------------------------------------------------------------------- httpd sub check_httpd_logs { @@ -452,34 +462,34 @@ sub rotate_lonnet_logs { } else { &log($fh,"No perm log\n") } my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - - if ($size>40000) { - &log($fh,"

Rotating logs ...

"); - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); - } + &rotate_logfile($fname,$fh,'lonnet log'); &log($fh,""); &errout($fh); } +sub rotate_other_logs { + my ($fh) = @_; + my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log"; + &rotate_logfile($fname,$fh,'Auto Enroll log'); + $fname="$perlvar{'lonDaemons'}/logs/autocreate.log"; + &rotate_logfile($fname,$fh,'Create Course log'); + $fname="$perlvar{'lonDaemons'}/logs/searchcat.log"; + &rotate_logfile($fname,$fh,'Search Cataloguing log'); +} + # ----------------------------------------------------------------- Connections sub test_connections { - my ($fh,$hostname)=@_; + my ($fh)=@_; &log($fh,'

Connections

'); print "testing connections\n"; &log($fh,""); my ($good,$bad)=(0,0); - foreach my $tryserver (sort(keys(%{$hostname}))) { + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $tryserver (sort(keys(%hostname))) { print("."); my $result; - my $answer=reply("ping",$tryserver); + my $answer=&Apache::lonnet::reply("ping",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; $good++; @@ -529,13 +539,22 @@ sub check_delayed_msg { if ($unsend) { $simplestatus{'unsend'}=$unsend; } &log($fh,"

Outgoing Buffer

\n
");
-
+# list directory with delayed messages and remember offline servers
+    my %servers=();
     open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
-    while (my $line=) { 
+    while (my $line=) {
+        my ($server)=($line=~/\.(\w+)$/);
+        if ($server) { $servers{$server}=1; }
 	&log($fh,&encode_entities($line,'<>&"'));
     }
     &log($fh,"
\n"); close (DFH); +# pong to all servers that have delayed messages +# this will trigger a reverse connection, which should flush the buffers + foreach my $tryserver (keys %servers) { + my $answer=&Apache::lonnet::reply("pong",$tryserver); + &log($fh,"Pong to $tryserver: $answer
"); + } } sub finish_logging { @@ -557,7 +576,7 @@ sub finish_logging { } sub log_simplestatus { - rename ("$statusdir/newstatus.html","$statusdir/index.html"); + rename("$statusdir/newstatus.html","$statusdir/index.html"); my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); foreach (keys %simplestatus) { @@ -586,8 +605,7 @@ sub usage { loncron - housekeeping program that checks up on various parts of Lon-CAPA Options: - --help Display help - --oldlonc When starting the lonc daemon use 'lonc' not 'loncnew' + --help Display --noemail Do not send the status email --justcheckconnections Only check the current status of the lonc/d connections, do not send emails do not @@ -607,10 +625,9 @@ USAGE # ================================================================ Main Program sub main () { - my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections, + my ($help,$justcheckdaemons,$noemail,$justcheckconnections, $justreload); &GetOptions("help" => \$help, - "oldlonc" => \$oldlonc, "justcheckdaemons" => \$justcheckdaemons, "noemail" => \$noemail, "justcheckconnections" => \$justcheckconnections, @@ -649,32 +666,10 @@ sub main () { exit 1; } -# ------------------------------------------------------------- Read hosts file - my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); - - my (%hostname,%hostdom,%hostrole,%spareid); - while (my $configline=<$config>) { - next if ($configline =~ /^(\#|\s*\$)/); - my ($id,$domain,$role,$name)=split(/:/,$configline); - if ($id && $domain && $role && $name) { - $name=~s/\s//g; - $hostname{$id}=$name; - $hostdom{$id}=$domain; - $hostrole{$id}=$role; - } - } - undef $config; - -# ------------------------------------------------------ Read spare server file - $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); - - while (my $configline=<$config>) { - chomp($configline); - if (($configline) && ($configline ne $perlvar{'lonHostID'})) { - $spareid{$configline}=1; - } - } - undef $config; +# -------------------------------------------- Force reload of host information + &Apache::lonnet::load_hosts_tab(1); + &Apache::lonnet::load_domain_tab(1); + &Apache::lonnet::get_iphost(1); # ---------------------------------------------------------------- Start report @@ -685,31 +680,31 @@ sub main () { my $fh; if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { - $fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); + $fh=&start_logging(); &log_machine_info($fh); &clean_tmp($fh); &clean_lonIDs($fh); &check_httpd_logs($fh); &rotate_lonnet_logs($fh); + &rotate_other_logs($fh); } if (!$justcheckconnections && !$justreload) { &checkon_daemon($fh,'lonsql',200000); - &checkon_daemon($fh,'lond',40000,'USR1'); - my $args='new'; - if ($oldlonc) { $args = ''; } - &checkon_daemon($fh,'lonc',40000,'USR1',$args); + if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { + &checkon_daemon($fh,'lond',40000,'USR2'); + } + &checkon_daemon($fh,'lonc',40000,'USR1'); &checkon_daemon($fh,'lonhttpd',40000); &checkon_daemon($fh,'lonmemcached',40000); + &checkon_daemon($fh,'lonmaxima',40000); } if ($justreload) { &checkon_daemon($fh,'lond',40000,'USR2'); - my $args='new'; - if ($oldlonc) { $args = ''; } - &checkon_daemon($fh,'lonc',40000,'USR2',$args); + &checkon_daemon($fh,'lonc',40000,'USR2'); } - if (!$justcheckdaemons && !$justreload) { - &test_connections($fh,\%hostname); + if ($justcheckconnections) { + &test_connections($fh); } if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { &check_delayed_msg($fh);