--- loncom/loncron 2005/05/26 20:25:00 1.59 +++ loncom/loncron 2007/04/03 00:47:26 1.71 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.59 2005/05/26 20:25:00 albertel Exp $ +# $Id: loncron,v 1.71 2007/04/03 00:47:26 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -45,13 +45,13 @@ my $statusdir="/home/httpd/html/lon-stat # -------------------------------------------------- Non-critical communication sub reply { - my ($cmd,$server)=@_; - my $peerfile="$perlvar{'lonSockDir'}/$server"; + my ($cmd,$server,$hostname)=@_; + my $peerfile="$perlvar{'lonSockDir'}/".$hostname->{$server}; my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", Type => SOCK_STREAM, Timeout => 10) or return "con_lost"; - print $client "$cmd\n"; + print $client "sethost:$server:$cmd\n"; my $answer=<$client>; chomp($answer); if (!$answer) { $answer="con_lost"; } @@ -79,9 +79,8 @@ ENDERROUT sub start_daemon { my ($fh,$daemon,$pidfile,$args) = @_; my $progname=$daemon; - if ($daemon eq 'lonc' && $args eq 'new') { + if ($daemon eq 'lonc') { $progname='loncnew'; - print "new "; } my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; my $size=(stat($error_fname))[7]; @@ -92,14 +91,13 @@ sub start_daemon { rename("$error_fname","$error_fname.1"); } system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); - sleep 2; + sleep 1; if (-e $pidfile) { &log($fh,"

Seems like it started ...

"); my $lfh=IO::File->new("$pidfile"); my $daemonpid=<$lfh>; chomp($daemonpid); - sleep 2; - if (kill 0 => $daemonpid) { + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { return 1; } else { return 0; @@ -113,6 +111,7 @@ sub start_daemon { sub checkon_daemon { my ($fh,$daemon,$maxsize,$send,$args)=@_; + my $result; &log($fh,'

'.$daemon.'

Log

'); printf("%-15s ",$daemon); if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ @@ -135,7 +134,7 @@ sub checkon_daemon { my $lfh=IO::File->new("$pidfile"); $daemonpid=<$lfh>; chomp($daemonpid); - if (kill 0 => $daemonpid) { + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { &log($fh,"

$daemon at pid $daemonpid responding"); if ($send) { &log($fh,", sending $send"); } &log($fh,"

"); @@ -143,8 +142,10 @@ sub checkon_daemon { if ($send eq 'USR2') { kill USR2 => $daemonpid; } $restartflag=0; if ($send eq 'USR2') { + $result = 'reloaded'; print "reloaded\n"; } else { + $result = 'running'; print "running\n"; } } else { @@ -161,7 +162,7 @@ sub checkon_daemon { if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } &log($fh,'
Killall '.$daemon.': '. `killall $kadaemon 2>&1`.' - '); - sleep 2; + sleep 1; &log($fh,unlink($pidfile).' - '. `killall -9 $kadaemon 2>&1`. '
'); @@ -170,6 +171,7 @@ sub checkon_daemon { if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; + $result = 'started'; print "started\n"; } else { $errors++; @@ -179,8 +181,10 @@ sub checkon_daemon { if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; + $result = 'started'; print "started\n"; } else { + $result = 'failed'; print " failed\n"; $simplestatus{$daemon}='failed'; $errors++; $errors++; @@ -217,6 +221,7 @@ sub checkon_daemon { } &errout($fh); + return $result; } # --------------------------------------------------------------------- Machine @@ -278,6 +283,16 @@ sub log_machine_info { if ($psproc>200) { $notices++; } if ($psproc>250) { $notices++; } + &log($fh,"

distprobe

"); + &log($fh,"
");
+    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
+    while (my $line=) { 
+	&log($fh,&encode_entities($line,'<>&"')); 
+	$psproc++;
+    }
+    close(DSH);
+    &log($fh,"
"); + &errout($fh); } @@ -470,7 +485,7 @@ sub test_connections { foreach my $tryserver (sort(keys(%{$hostname}))) { print("."); my $result; - my $answer=reply("ping",$tryserver); + my $answer=&reply("ping",$tryserver,$hostname); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; $good++; @@ -495,7 +510,7 @@ sub test_connections { # ------------------------------------------------------------ Delayed messages sub check_delayed_msg { - my ($fh)=@_; + my ($fh,$hostname)=@_; &log($fh,'

Delayed Messages

'); print "checking buffers\n"; @@ -520,13 +535,22 @@ sub check_delayed_msg { if ($unsend) { $simplestatus{'unsend'}=$unsend; } &log($fh,"

Outgoing Buffer

\n
");
-
+# list directory with delayed messages and remember offline servers
+    my %servers=();
     open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
-    while (my $line=) { 
+    while (my $line=) {
+        my ($server)=($line=~/\.(\w+)$/);
+        if ($server) { $servers{$server}=1; }
 	&log($fh,&encode_entities($line,'<>&"'));
     }
     &log($fh,"
\n"); close (DFH); +# pong to all servers that have delayed messages +# this will trigger a reverse connection, which should flush the buffers + foreach my $tryserver (keys %servers) { + my $answer=&reply("pong",$tryserver,$hostname); + &log($fh,"Pong to $tryserver: $answer
"); + } } sub finish_logging { @@ -577,8 +601,7 @@ sub usage { loncron - housekeeping program that checks up on various parts of Lon-CAPA Options: - --help Display help - --oldlonc When starting the lonc daemon use 'lonc' not 'loncnew' + --help Display --noemail Do not send the status email --justcheckconnections Only check the current status of the lonc/d connections, do not send emails do not @@ -598,10 +621,9 @@ USAGE # ================================================================ Main Program sub main () { - my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections, + my ($help,$justcheckdaemons,$noemail,$justcheckconnections, $justreload); &GetOptions("help" => \$help, - "oldlonc" => \$oldlonc, "justcheckdaemons" => \$justcheckdaemons, "noemail" => \$noemail, "justcheckconnections" => \$justcheckconnections, @@ -686,24 +708,23 @@ sub main () { } if (!$justcheckconnections && !$justreload) { &checkon_daemon($fh,'lonsql',200000); - &checkon_daemon($fh,'lond',40000,'USR1'); - my $args='new'; - if ($oldlonc) { $args = ''; } - &checkon_daemon($fh,'lonc',40000,'USR1',$args); + if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { + &checkon_daemon($fh,'lond',40000,'USR2'); + } + &checkon_daemon($fh,'lonc',40000,'USR1'); &checkon_daemon($fh,'lonhttpd',40000); &checkon_daemon($fh,'lonmemcached',40000); + &checkon_daemon($fh,'lonmaxima',40000); } if ($justreload) { &checkon_daemon($fh,'lond',40000,'USR2'); - my $args='new'; - if ($oldlonc) { $args = ''; } - &checkon_daemon($fh,'lonc',40000,'USR2',$args); + &checkon_daemon($fh,'lonc',40000,'USR2'); } - if (!$justcheckdaemons && !$justreload) { + if ($justcheckconnections) { &test_connections($fh,\%hostname); } if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { - &check_delayed_msg($fh); + &check_delayed_msg($fh,\%hostname); &finish_logging($fh); &log_simplestatus();