--- loncom/loncron 2010/03/24 18:57:49 1.83 +++ loncom/loncron 2015/09/20 18:31:21 1.103 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.83 2010/03/24 18:57:49 raeburn Exp $ +# $Id: loncron,v 1.103 2015/09/20 18:31:21 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -32,6 +32,8 @@ use strict; use lib '/home/httpd/lib/perl/'; use LONCAPA::Configuration; +use LONCAPA::Checksumming; +use LONCAPA; use Apache::lonnet; use Apache::loncommon; @@ -155,12 +157,12 @@ sub checkon_daemon { $errors++; my $kadaemon=$daemon; if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } - &log($fh,'
Killall '.$daemon.': '. + &log($fh,'
Killall '.$daemon.': '. `killall $kadaemon 2>&1`.' - '); sleep 1; &log($fh,unlink($pidfile).' - '. `killall -9 $kadaemon 2>&1`. - '
'); + '
'); &log($fh,"

$daemon not running, trying to start

"); if (&start_daemon($fh,$daemon,$pidfile,$args)) { @@ -269,12 +271,7 @@ sub log_machine_info { &log($fh,"

distprobe

"); &log($fh,"
");
-    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
-    while (my $line=) { 
-	&log($fh,&encode_entities($line,'<>&"')); 
-	$psproc++;
-    }
-    close(DSH);
+    &log($fh,&encode_entities(&LONCAPA::distro(),'<>&"'));
     &log($fh,"
"); &errout($fh); @@ -288,9 +285,11 @@ sub start_logging { &log($fh,(< + + LON Status Report $perlvar{'lonHostID'} + @@ -301,6 +300,7 @@ sub start_logging {
  • Machine Information
  • Temporary Files
  • Session Tokens
    • +
  • WebDAV Session Tokens
  • httpd
  • lonsql
  • lond
    • @@ -331,15 +331,20 @@ ENDHEADERS "".$role. "".&Apache::lonnet::hostname($id)."\n"); } - &log($fh,"

    Spare Hosts

    \n"); return $fh; } @@ -485,7 +490,7 @@ sub clean_lonIDs { my $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; - &log($fh,"Unlinking $fname
    "); + &log($fh,"Unlinking $fname
    "); unlink("$fname"); } else { $active++; @@ -495,6 +500,32 @@ sub clean_lonIDs { &log($fh,"

    $active open session(s)

    "); } +# ------------------------------------------------ clean out webDAV Session IDs +sub clean_webDAV_sessionIDs { + my ($fh)=@_; + if ($perlvar{'lonRole'} eq 'library') { + &log($fh,'

    WebDAV Session Tokens

    '); + my $cleaned=0; + my $active=0; + my $now = time; + if (-d $perlvar{'lonDAVsessDir'}) { + while (my $fname=<$perlvar{'lonDAVsessDir'}/*>) { + my @stats = stat($fname); + my $since=$now-$stats[9]; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale webDAV session token(s).

    "); + &log($fh,"

    $active open webDAV session(s)

    "); + } + } +} + # ----------------------------------------------------------- clean out sockets sub clean_sockets { my ($fh)=@_; @@ -514,21 +545,13 @@ sub clean_sockets { # ----------------------------------------------------------------------- httpd sub check_httpd_logs { my ($fh)=@_; - &log($fh,'

    httpd

    Access Log

    ');
-    
-    open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
-    while (my $line=) { &log($fh,&encode_entities($line,'<>&"')) };
-    close (DFH);
-	
-    &log($fh,"

    Error Log

    ");
-	
-    open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
-    while (my $line=) { 
-	&log($fh,"$line");
-	if ($line=~/\[error\]/) { $notices++; } 
+    if (open(PIPE,"./lchttpdlogs|")) {
+        while (my $line=) {
+            &log($fh,$line);
+            if ($line=~/\[error\]/) { $notices++; }
+        }
+        close(PIPE);
     }
-    close (DFH);
-    &log($fh,"
    "); &errout($fh); } @@ -537,7 +560,7 @@ sub check_httpd_logs { sub rotate_lonnet_logs { my ($fh)=@_; &log($fh,'

    lonnet

    Temp Log

    ');
-    print "checking logs\n";
+    print "Checking logs.\n";
     if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
 	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
 	while (my $line=) { 
@@ -581,7 +604,7 @@ sub rotate_other_logs {
 sub test_connections {
     my ($fh)=@_;
     &log($fh,'
    Connections
    ');
-    print "testing connections\n";
+    print "Testing connections.\n";
     &log($fh,"");
     my ($good,$bad)=(0,0);
     my %hostname = &Apache::lonnet::all_hostnames();
@@ -615,7 +638,7 @@ sub test_connections {
 sub check_delayed_msg {
     my ($fh)=@_;
     &log($fh,'
    Delayed Messages
    ');
-    print "checking buffers\n";
+    print "Checking buffers.\n";
     
     &log($fh,'
    Scanning Permanent Log
    ');
 
@@ -626,7 +649,7 @@ sub check_delayed_msg {
 	my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
 	if ($sdf eq 'F') { 
 	    my $local=localtime($time);
-	    &log($fh,"Failed: $time, $dserv, $dcmd
    ");
+	    &log($fh,"Failed: $time, $dserv, $dcmd
    ");
 	    $warnings++;
 	}
 	if ($sdf eq 'S') { $unsend--; }
@@ -634,7 +657,9 @@ sub check_delayed_msg {
     }
 
     &log($fh,"
    Total unsend messages: $unsend
    \n");
-    $warnings=$warnings+5*$unsend;
+    if ($unsend > 0) {
+        $warnings=$warnings+5*$unsend;
+    }
 
     if ($unsend) { $simplestatus{'unsend'}=$unsend; }
     &log($fh,"
    Outgoing Buffer
    \n
    ");
@@ -648,11 +673,28 @@ sub check_delayed_msg {
     }
     &log($fh,"
    \n");
     close (DFH);
+    my %hostname = &Apache::lonnet::all_hostnames();
+    my $numhosts = scalar(keys(%hostname));
 # pong to all servers that have delayed messages
 # this will trigger a reverse connection, which should flush the buffers
-    foreach my $tryserver (keys %servers) {
-	my $answer=&Apache::lonnet::reply("pong",$tryserver);
-	&log($fh,"Pong to $tryserver: $answer
    ");
+    foreach my $tryserver (sort(keys(%servers))) {
+        if ($hostname{$tryserver} || !$numhosts) {
+            my $answer;
+            eval {
+                local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                alarm(20);
+                $answer = &Apache::lonnet::reply("pong",$tryserver);
+                alarm(0);
+            };
+            if ($@ && $@ =~ m/TIMEOUT/) {
+                &log($fh,"Attempted pong to $tryserver timed out
    ");
+                print "Time out while contacting: $tryserver for pong.\n";
+            } else {
+                &log($fh,"Pong to $tryserver: $answer
    ");
+            }
+        } else {
+            &log($fh,"$tryserver has delayed messages, but is not part of the cluster -- skipping 'Pong'.
    ");
+        }
     }
 }
 
@@ -665,7 +707,7 @@ sub finish_logging {
     my $now=time;
     my $date=localtime($now);
     &log($fh,"
    $date ($now)\n");
-    print "lon-status webpage updated\n";
+    print "lon-status webpage updated.\n";
     $fh->close();
 
     if ($errors) { $simplestatus{'errors'}=$errors; }
@@ -685,8 +727,95 @@ sub log_simplestatus {
     $sfh->close();
 }
 
+sub write_loncaparevs {
+    print "Retrieving LON-CAPA version information.\n";
+    my %hostname = &Apache::lonnet::all_hostnames();
+    my $output;
+    foreach my $id (sort(keys(%hostname))) {
+        if ($id ne '') {
+            my $loncaparev;
+            eval {
+                local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                alarm(10);
+                $loncaparev =
+                    &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron');
+                alarm(0);
+            };
+            if ($@ && $@ =~ m/TIMEOUT/) {
+                print "Time out while contacting lonHost: $id for version.\n";   
+            }
+            if ($loncaparev =~ /^[\w.\-]+$/) {
+                $output .= $id.':'.$loncaparev."\n";
+            }
+        }
+    }
+    if ($output) {
+        if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+            print $fh $output;
+            close($fh);
+            &Apache::lonnet::load_loncaparevs();
+        }
+    }
+    return;
+}
+
+sub write_serverhomeIDs {
+    print "Retrieving LON-CAPA lonHostID information.\n";
+    my %name_to_host = &Apache::lonnet::all_names();
+    my $output;
+    foreach my $name (sort(keys(%name_to_host))) {
+        if ($name ne '') {
+            if (ref($name_to_host{$name}) eq 'ARRAY') {
+                my $serverhomeID;
+                eval {
+                    local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                    alarm(10);
+                    $serverhomeID = 
+                        &Apache::lonnet::get_server_homeID($name,1,'loncron');
+                    alarm(0);
+                };
+                if ($@ && $@ =~ m/TIMEOUT/) {
+                    print "Time out while contacting server: $name\n"; 
+                }
+                if ($serverhomeID ne '') {
+                    $output .= $name.':'.$serverhomeID."\n";
+                } else {
+                    $output .= $name.':'.$name_to_host{$name}->[0]."\n";
+                }
+            }
+        }
+    }
+    if ($output) {
+        if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+            print $fh $output;
+            close($fh);
+            &Apache::lonnet::load_serverhomeIDs();
+        }
+    }
+    return;
+}
+
+sub write_checksums {
+    my $distro = &LONCAPA::distro();
+    if ($distro) {
+        print "Retrieving file version and checksumming.\n";
+        my $numchksums = 0;
+        my ($chksumsref,$versionsref) =
+            &LONCAPA::Checksumming::get_checksums($distro,$perlvar{'lonDaemons'},
+                                                  $perlvar{'lonLib'},
+                                                  $perlvar{'lonIncludes'},
+                                                  $perlvar{'lonTabDir'});
+        if (ref($chksumsref) eq 'HASH') {
+            $numchksums = scalar(keys(%{$chksumsref}));
+        }
+        print "File version retrieved and checksumming completed for $numchksums files.\n";
+    } else {
+        print "File version retrieval and checksumming skipped - could not determine Linux distro.\n"; 
+    }
+    return;
+}
+
 sub send_mail {
-    print "sending mail\n";
     my $defdom = $perlvar{'lonDefDomain'};
     my $origmail = $perlvar{'lonAdmEMail'};
     my $emailto = &Apache::loncommon::build_recipient_list(undef,
@@ -694,17 +823,40 @@ sub send_mail {
     if ($totalcount>2500) {
 	$emailto.=",$perlvar{'lonSysEMail'}";
     }
-    my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 
-
-    my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null");
-    if ($result != 0) {
-	$result=system("mail -s '$subj' $emailto < $statusdir/index.html");
+    my $from;
+    my $hostname=`/bin/hostname`;
+    chop($hostname);
+    $hostname=~s/[^\w\.]//g;
+    if ($hostname) {
+        $from = 'www@'.$hostname;
+    }
+    my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
+    my $loncronmail = "To: $emailto\n".
+                      "From: $from\n".
+                      "Subject: ".$subj."\n".
+                      "Content-type: text/html\; charset=UTF-8\n".
+                      "MIME-Version: 1.0\n\n";
+    if (open(my $fh,"<$statusdir/index.html")) {
+        while (<$fh>) {
+            $loncronmail .= $_;
+        }
+        close($fh);
+    } else {
+        $loncronmail .= "Failed to read from http://$hostname/lon-status/index.html\n";
+    }
+    $loncronmail .= "\n\n";
+    if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) {
+        print $mailh $loncronmail;
+        close($mailh);
+        print "Sending mail.\n";
+    } else {
+        print "Sending mail failed.\n";
     }
 }
 
 sub usage {
     print(< /dev/null");
 	exit 1;
     }
 
 # -------------------------------------------- Force reload of host information
-    &Apache::lonnet::load_hosts_tab(1);
-    &Apache::lonnet::load_domain_tab(1);
-    &Apache::lonnet::get_iphost(1);
+    my $nomemcache;
+    if ($justcheckdaemons) {
+        $nomemcache=1;
+        my $memcachepidfile="$perlvar{'lonDaemons'}/logs/memcached.pid";
+        my $memcachepid;
+        if (-e $memcachepidfile) {
+            my $memfh=IO::File->new($memcachepidfile);
+            $memcachepid=<$memfh>;
+            chomp($memcachepid);
+            if ($memcachepid =~ /^\d+$/ && kill 0 => $memcachepid) {
+                undef($nomemcache);
+            }
+        }
+    }
+    &Apache::lonnet::load_hosts_tab(1,$nomemcache);
+    &Apache::lonnet::load_domain_tab(1,$nomemcache);
+    &Apache::lonnet::get_iphost(1,$nomemcache);
 
 # ----------------------------------------- Force firewall update for lond port  
 
@@ -785,9 +951,12 @@ sub main () {
                 print $fh "$key\n";
             }
             close($fh);
-            my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
-            system("$execpath $tmpfile");
-            unlink($fh);
+            if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) {
+                my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
+                system("$execpath $tmpfile");
+                unlink('/tmp/lock_lciptables');  # Remove the lock file. 
+            }
+            unlink($tmpfile);
         }
     }
 
@@ -805,6 +974,7 @@ sub main () {
 	&log_machine_info($fh);
 	&clean_tmp($fh);
 	&clean_lonIDs($fh);
+        &clean_webDAV_sessionIDs($fh);
 	&check_httpd_logs($fh);
 	&rotate_lonnet_logs($fh);
 	&rotate_other_logs($fh);
@@ -830,7 +1000,9 @@ sub main () {
 	&check_delayed_msg($fh);
 	&finish_logging($fh);
 	&log_simplestatus();
-	
+        &write_loncaparevs();
+        &write_serverhomeIDs();
+	&write_checksums();
 	if ($totalcount>200 && !$noemail) { &send_mail(); }
     }
 }
@@ -838,10 +1010,3 @@ sub main () {
 &main();
 1;
 
-
-
-
-
-
-
-