--- loncom/loncron 2013/02/02 14:42:01 1.97 +++ loncom/loncron 2020/01/11 22:39:55 1.103.2.6 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.97 2013/02/02 14:42:01 raeburn Exp $ +# $Id: loncron,v 1.103.2.6 2020/01/11 22:39:55 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -41,6 +41,7 @@ use IO::File; use IO::Socket; use HTML::Entities; use Getopt::Long; +use File::Copy; #globals use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); @@ -111,18 +112,21 @@ sub checkon_daemon { my $result; &log($fh,'

'.$daemon.'

Log

'); printf("%-15s ",$daemon); - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); - while (my $line=) { - &log($fh,"$line"); - if ($line=~/INFO/) { $notices++; } - if ($line=~/WARNING/) { $notices++; } - if ($line=~/CRITICAL/) { $warnings++; } - }; - close (DFH); + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + if (open(DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|")) { + while (my $line=) { + &log($fh,"$line"); + if ($line=~/INFO/) { $notices++; } + if ($line=~/WARNING/) { $notices++; } + if ($line=~/CRITICAL/) { $warnings++; } + } + close (DFH); + } + } + &log($fh,"

"); } - &log($fh,"

"); - + my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; my $restartflag=1; @@ -157,14 +161,14 @@ sub checkon_daemon { $errors++; my $kadaemon=$daemon; if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } - &log($fh,'
Killall '.$daemon.': '. + &log($fh,'
Killall '.$daemon.': '. `killall $kadaemon 2>&1`.' - '); sleep 1; &log($fh,unlink($pidfile).' - '. `killall -9 $kadaemon 2>&1`. - '
'); + '

'); &log($fh,"

$daemon not running, trying to start

"); - + if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; @@ -189,18 +193,20 @@ sub checkon_daemon { &log($fh,"

Unable to start $daemon

"); } } - - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - &log($fh,"

");
-	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
-	    while (my $line=) { 
-		&log($fh,"$line");
-		if ($line=~/WARNING/) { $notices++; }
-		if ($line=~/CRITICAL/) { $notices++; }
-	    };
-	    close (DFH);
-	    &log($fh,"

"); - } + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + &log($fh,"

");
+	        if (open(DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|")) {
+	            while (my $line=) { 
+		        &log($fh,"$line");
+		        if ($line=~/WARNING/) { $notices++; }
+		        if ($line=~/CRITICAL/) { $notices++; }
+	            }
+	            close (DFH);
+                }
+	        &log($fh,"

"); + } + } } my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; @@ -271,7 +277,7 @@ sub log_machine_info { &log($fh,"

distprobe

"); &log($fh,"
");
-    &get_distro($perlvar{'lonDaemons'},$fh);
+    &log($fh,&encode_entities(&LONCAPA::distro(),'<>&"'));
     &log($fh,"
"); &errout($fh); @@ -285,9 +291,11 @@ sub start_logging { &log($fh,(< + + LON Status Report $perlvar{'lonHostID'} +
@@ -298,6 +306,7 @@ sub start_logging {
  • Machine Information
  • Temporary Files
  • Session Tokens
  • +
  • WebDAV Session Tokens
  • httpd
  • lonsql
  • lond
  • @@ -328,15 +337,20 @@ ENDHEADERS "".$role. "".&Apache::lonnet::hostname($id)."\n"); } - &log($fh,"

    Spare Hosts

    \n"); return $fh; } @@ -482,7 +496,7 @@ sub clean_lonIDs { my $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; - &log($fh,"Unlinking $fname
    "); + &log($fh,"Unlinking $fname
    "); unlink("$fname"); } else { $active++; @@ -492,6 +506,60 @@ sub clean_lonIDs { &log($fh,"

    $active open session(s)

    "); } +# -------------------------------------------------------- clean out balanceIDs + +sub clean_balanceIDs { + my ($fh)=@_; + &log($fh,'

    Session Tokens

    '); + my $cleaned=0; + my $active=0; + if (-d $perlvar{'lonBalanceDir'}) { + while (my $fname=<$perlvar{'balanceDir'}/*.id>) { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale balancer files

    "); + &log($fh,"

    $active unexpired balancer files

    "); +} + +# ------------------------------------------------ clean out webDAV Session IDs +sub clean_webDAV_sessionIDs { + my ($fh)=@_; + if ($perlvar{'lonRole'} eq 'library') { + &log($fh,'

    WebDAV Session Tokens

    '); + my $cleaned=0; + my $active=0; + my $now = time; + if (-d $perlvar{'lonDAVsessDir'}) { + while (my $fname=<$perlvar{'lonDAVsessDir'}/*>) { + my @stats = stat($fname); + my $since=$now-$stats[9]; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale webDAV session token(s).

    "); + &log($fh,"

    $active open webDAV session(s)

    "); + } + } +} + # ----------------------------------------------------------- clean out sockets sub clean_sockets { my ($fh)=@_; @@ -526,7 +594,7 @@ sub check_httpd_logs { sub rotate_lonnet_logs { my ($fh)=@_; &log($fh,'

    lonnet

    Temp Log

    ');
    -    print "checking logs\n";
    +    print "Checking logs.\n";
         if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
     	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
     	while (my $line=) { 
    @@ -570,7 +638,7 @@ sub rotate_other_logs {
     sub test_connections {
         my ($fh)=@_;
         &log($fh,'

    Connections

    '); - print "testing connections\n"; + print "Testing connections.\n"; &log($fh,""); my ($good,$bad)=(0,0); my %hostname = &Apache::lonnet::all_hostnames(); @@ -604,18 +672,24 @@ sub test_connections { sub check_delayed_msg { my ($fh)=@_; &log($fh,'

    Delayed Messages

    '); - print "checking buffers\n"; + print "Checking buffers.\n"; &log($fh,'

    Scanning Permanent Log

    '); my $unsend=0; + my %hostname = &Apache::lonnet::all_hostnames(); + my $numhosts = scalar(keys(%hostname)); + my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while (my $line=<$dfh>) { my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); + if ($numhosts) { + next unless ($hostname{$dserv}); + } if ($sdf eq 'F') { my $local=localtime($time); - &log($fh,"Failed: $time, $dserv, $dcmd
    "); + &log($fh,"Failed: $time, $dserv, $dcmd
    "); $warnings++; } if ($sdf eq 'S') { $unsend--; } @@ -639,8 +713,6 @@ sub check_delayed_msg { } &log($fh,"\n"); close (DFH); - my %hostname = &Apache::lonnet::all_hostnames(); - my $numhosts = scalar(keys(%hostname)); # pong to all servers that have delayed messages # this will trigger a reverse connection, which should flush the buffers foreach my $tryserver (sort(keys(%servers))) { @@ -654,7 +726,7 @@ sub check_delayed_msg { }; if ($@ && $@ =~ m/TIMEOUT/) { &log($fh,"Attempted pong to $tryserver timed out
    "); - print "time out while contacting: $tryserver for pong\n"; + print "Time out while contacting: $tryserver for pong.\n"; } else { &log($fh,"Pong to $tryserver: $answer
    "); } @@ -673,7 +745,7 @@ sub finish_logging { my $now=time; my $date=localtime($now); &log($fh,"
    $date ($now)\n"); - print "lon-status webpage updated\n"; + print "lon-status webpage updated.\n"; $fh->close(); if ($errors) { $simplestatus{'errors'}=$errors; } @@ -694,65 +766,75 @@ sub log_simplestatus { } sub write_loncaparevs { - print "Retrieving LON-CAPA version information\n"; - if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) { - my %hostname = &Apache::lonnet::all_hostnames(); - foreach my $id (sort(keys(%hostname))) { - if ($id ne '') { - my $loncaparev; + print "Retrieving LON-CAPA version information.\n"; + my %hostname = &Apache::lonnet::all_hostnames(); + my $output; + foreach my $id (sort(keys(%hostname))) { + if ($id ne '') { + my $loncaparev; + eval { + local $SIG{ ALRM } = sub { die "TIMEOUT" }; + alarm(10); + $loncaparev = + &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron'); + alarm(0); + }; + if ($@ && $@ =~ m/TIMEOUT/) { + print "Time out while contacting lonHost: $id for version.\n"; + } + if ($loncaparev =~ /^[\w.\-]+$/) { + $output .= $id.':'.$loncaparev."\n"; + } + } + } + if ($output) { + if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) { + print $fh $output; + close($fh); + &Apache::lonnet::load_loncaparevs(); + } + } + return; +} + +sub write_serverhomeIDs { + print "Retrieving LON-CAPA lonHostID information.\n"; + my %name_to_host = &Apache::lonnet::all_names(); + my $output; + foreach my $name (sort(keys(%name_to_host))) { + if ($name ne '') { + if (ref($name_to_host{$name}) eq 'ARRAY') { + my $serverhomeID; eval { local $SIG{ ALRM } = sub { die "TIMEOUT" }; alarm(10); - $loncaparev = - &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron'); + $serverhomeID = + &Apache::lonnet::get_server_homeID($name,1,'loncron'); alarm(0); }; if ($@ && $@ =~ m/TIMEOUT/) { - print "time out while contacting lonHost: $id for version\n"; + print "Time out while contacting server: $name\n"; } - if ($loncaparev =~ /^[\w.\-]+$/) { - print $fh $id.':'.$loncaparev."\n"; + if ($serverhomeID ne '') { + $output .= $name.':'.$serverhomeID."\n"; + } else { + $output .= $name.':'.$name_to_host{$name}->[0]."\n"; } } } - close($fh); } - return; -} - -sub write_serverhomeIDs { - print "Retrieving LON-CAPA lonHostID information\n"; - if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) { - my %name_to_host = &Apache::lonnet::all_names(); - foreach my $name (sort(keys(%name_to_host))) { - if ($name ne '') { - if (ref($name_to_host{$name}) eq 'ARRAY') { - my $serverhomeID; - eval { - local $SIG{ ALRM } = sub { die "TIMEOUT" }; - alarm(10); - $serverhomeID = - &Apache::lonnet::get_server_homeID($name,1,'loncron'); - alarm(0); - }; - if ($@ && $@ =~ m/TIMEOUT/) { - print "Time out while contacting server: $name\n"; - } - if ($serverhomeID ne '') { - print $fh $name.':'.$serverhomeID."\n"; - } else { - print $fh $name.':'.$name_to_host{$name}->[0]."\n"; - } - } - } + if ($output) { + if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) { + print $fh $output; + close($fh); + &Apache::lonnet::load_serverhomeIDs(); } - close($fh); } return; } sub write_checksums { - my $distro = &get_distro($perlvar{'lonDaemons'}); + my $distro = &LONCAPA::distro(); if ($distro) { print "Retrieving file version and checksumming.\n"; my $numchksums = 0; @@ -771,8 +853,95 @@ sub write_checksums { return; } +sub write_hostips { + my $lontabdir = $perlvar{'lonTabDir'}; + my $defdom = $perlvar{'lonDefDomain'}; + my $lonhost = $perlvar{'lonHostID'}; + my $newfile = "$lontabdir/currhostips.tab"; + my $oldfile = "$lontabdir/prevhostips.tab"; + my (%prevhosts,%currhosts,%ipchange); + if ((-e $newfile) && (-s $newfile)) { + move($newfile,$oldfile); + chmod(0644,$oldfile); + if (open(my $fh,'<',$oldfile)) { + while (my $line=<$fh>) { + chomp($line); + if ($line =~ /^([^:]+):([\d.]+)$/) { + $prevhosts{$1} = $2; + } + } + close($fh); + } + } + my ($ip_info,$cached) = + &Apache::lonnet::is_cached_new('iphost','iphost'); + if (!$cached) { + &Apache::lonnet::get_iphost(); + ($ip_info,$cached) = + &Apache::lonnet::is_cached_new('iphost','iphost'); + } + if (ref($ip_info) eq 'ARRAY') { + %currhosts = %{$ip_info->[1]}; + if (open(my $fh,'>',$newfile)) { + foreach my $key (keys(%currhosts)) { + print $fh "$key:$currhosts{$key}\n"; + } + close($fh); + chmod(0644,$newfile); + } + } + if (keys(%prevhosts) && keys(%currhosts)) { + foreach my $key (keys(%prevhosts)) { + unless ($currhosts{$key} eq $prevhosts{$key}) { + $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key}; + } + } + foreach my $key (keys(%currhosts)) { + unless ($currhosts{$key} eq $prevhosts{$key}) { + $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key}; + } + } + } + if (&Apache::lonnet::domain($defdom,'primary') eq $lonhost) { + if (keys(%ipchange)) { + if (open(my $fh,'>>',$perlvar{'lonDaemons'}.'/logs/hostip.log')) { + print $fh "********************\n".localtime(time).' Changes --'."\n". + "| Hostname | Previous IP | New IP |\n". + " --------------------------------- \n"; + foreach my $hostname (sort(keys(%ipchange))) { + print $fh "| $hostname | $ipchange{$hostname} |\n"; + } + print $fh "\n*******************\n\n"; + close($fh); + } + my $emailto = &Apache::loncommon::build_recipient_list(undef, + 'hostipmail',$defdom); + if ($emailto) { + my $subject = "LON-CAPA Hostname to IP change ($perlvar{'lonHostID'})"; + my $chgmail = "To: $emailto\n". + "Subject: $subject\n". + "Content-type: text/plain\; charset=UTF-8\n". + "MIME-Version: 1.0\n\n". + "Host/IP changes\n". + " \n". + "| Hostname | Previous IP | New IP |\n". + " --------------------------------- \n"; + foreach my $hostname (sort(keys(%ipchange))) { + $chgmail .= "| $hostname | $ipchange{$hostname} |\n"; + } + $chgmail .= "\n\n"; + if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) { + print $mailh $chgmail; + close($mailh); + print "Sending mail notification of hostname/IP changes.\n"; + } + } + } + } + return; +} + sub send_mail { - print "sending mail\n"; my $defdom = $perlvar{'lonDefDomain'}; my $origmail = $perlvar{'lonAdmEMail'}; my $emailto = &Apache::loncommon::build_recipient_list(undef, @@ -780,32 +949,40 @@ sub send_mail { if ($totalcount>2500) { $emailto.=",$perlvar{'lonSysEMail'}"; } - my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; - - my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); - if ($result != 0) { - $result=system("mail -s '$subj' $emailto < $statusdir/index.html"); - } -} - -sub get_distro { - my ($dir,$fh) = @_; - my $distro; - if (open(my $disth,"$dir/distprobe |")) { - while (my $line=<$disth>) { - if ($fh) { - &log($fh,&encode_entities($line,'<>&"')); - } - $distro .= $line; + my $from; + my $hostname=`/bin/hostname`; + chop($hostname); + $hostname=~s/[^\w\.]//g; + if ($hostname) { + $from = 'www@'.$hostname; + } + my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; + my $loncronmail = "To: $emailto\n". + "From: $from\n". + "Subject: ".$subj."\n". + "Content-type: text/html\; charset=UTF-8\n". + "MIME-Version: 1.0\n\n"; + if (open(my $fh,"<$statusdir/index.html")) { + while (<$fh>) { + $loncronmail .= $_; } - close($disth); + close($fh); + } else { + $loncronmail .= "Failed to read from http://$hostname/lon-status/index.html\n"; + } + $loncronmail .= "\n\n"; + if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) { + print $mailh $loncronmail; + close($mailh); + print "Sending mail.\n"; + } else { + print "Sending mail failed.\n"; } - return $distro; } sub usage { print(< \$help, "justcheckdaemons" => \$justcheckdaemons, "noemail" => \$noemail, "justcheckconnections" => \$justcheckconnections, - "justreload" => \$justreload + "justreload" => \$justreload, + "justiptables" => \$justiptables ); if ($help) { &usage(); return; } # --------------------------------- Read loncapa_apache.conf and loncapa.conf @@ -853,26 +1034,42 @@ sub main () { chop $hostname; $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell my $subj="LON: Unconfigured machine $hostname"; - system("echo 'Unconfigured machine $hostname.' |\ - mailto $emailto -s '$subj' > /dev/null"); + system("echo 'Unconfigured machine $hostname.' |". + " mail -s '$subj' $emailto > /dev/null"); exit 1; } # ----------------------------- Make sure this process is running from user=www my $wwwid=getpwnam('www'); if ($wwwid!=$<) { - print("User ID mismatch. This program must be run as user 'www'\n"); + print("User ID mismatch. This program must be run as user 'www'.\n"); my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; - system("echo 'User ID mismatch. loncron must be run as user www.' |\ - mailto $emailto -s '$subj' > /dev/null"); + system("echo 'User ID mismatch. loncron must be run as user www.' |". + " mail -s '$subj' $emailto > /dev/null"); exit 1; } # -------------------------------------------- Force reload of host information - &Apache::lonnet::load_hosts_tab(1); - &Apache::lonnet::load_domain_tab(1); - &Apache::lonnet::get_iphost(1); + my $nomemcache; + if ($justcheckdaemons) { + $nomemcache=1; + my $memcachepidfile="$perlvar{'lonDaemons'}/logs/memcached.pid"; + my $memcachepid; + if (-e $memcachepidfile) { + my $memfh=IO::File->new($memcachepidfile); + $memcachepid=<$memfh>; + chomp($memcachepid); + if ($memcachepid =~ /^\d+$/ && kill 0 => $memcachepid) { + undef($nomemcache); + } + } + } + if (!$justiptables) { + &Apache::lonnet::load_hosts_tab(1,$nomemcache); + &Apache::lonnet::load_domain_tab(1,$nomemcache); + &Apache::lonnet::get_iphost(1,$nomemcache); + } # ----------------------------------------- Force firewall update for lond port @@ -903,17 +1100,19 @@ sub main () { my $fh; - if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { + if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) { $fh=&start_logging(); &log_machine_info($fh); &clean_tmp($fh); &clean_lonIDs($fh); + &clean_balanceIDs($fh); + &clean_webDAV_sessionIDs($fh); &check_httpd_logs($fh); &rotate_lonnet_logs($fh); &rotate_other_logs($fh); } - if (!$justcheckconnections && !$justreload) { + if (!$justcheckconnections && !$justreload && !$justiptables) { &checkon_daemon($fh,'lonmemcached',40000); &checkon_daemon($fh,'lonsql',200000); if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { @@ -930,13 +1129,14 @@ sub main () { if ($justcheckconnections) { &test_connections($fh); } - if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { + if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) { &check_delayed_msg($fh); &finish_logging($fh); &log_simplestatus(); &write_loncaparevs(); &write_serverhomeIDs(); &write_checksums(); + &write_hostips(); if ($totalcount>200 && !$noemail) { &send_mail(); } } }