--- loncom/loncron 2015/09/20 18:31:21 1.103 +++ loncom/loncron 2019/08/01 18:21:36 1.103.2.5 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.103 2015/09/20 18:31:21 raeburn Exp $ +# $Id: loncron,v 1.103.2.5 2019/08/01 18:21:36 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -41,6 +41,7 @@ use IO::File; use IO::Socket; use HTML::Entities; use Getopt::Long; +use File::Copy; #globals use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); @@ -111,18 +112,21 @@ sub checkon_daemon { my $result; &log($fh,'

'.$daemon.'

Log

'); printf("%-15s ",$daemon); - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); - while (my $line=) { - &log($fh,"$line"); - if ($line=~/INFO/) { $notices++; } - if ($line=~/WARNING/) { $notices++; } - if ($line=~/CRITICAL/) { $warnings++; } - }; - close (DFH); + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + if (open(DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|")) { + while (my $line=) { + &log($fh,"$line"); + if ($line=~/INFO/) { $notices++; } + if ($line=~/WARNING/) { $notices++; } + if ($line=~/CRITICAL/) { $warnings++; } + } + close (DFH); + } + } + &log($fh,"

"); } - &log($fh,"

"); - + my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; my $restartflag=1; @@ -164,7 +168,7 @@ sub checkon_daemon { `killall -9 $kadaemon 2>&1`. '
'); &log($fh,"

$daemon not running, trying to start

"); - + if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; @@ -189,18 +193,20 @@ sub checkon_daemon { &log($fh,"

Unable to start $daemon

"); } } - - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - &log($fh,"

");
-	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
-	    while (my $line=) { 
-		&log($fh,"$line");
-		if ($line=~/WARNING/) { $notices++; }
-		if ($line=~/CRITICAL/) { $notices++; }
-	    };
-	    close (DFH);
-	    &log($fh,"

"); - } + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + &log($fh,"

");
+	        if (open(DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|")) {
+	            while (my $line=) { 
+		        &log($fh,"$line");
+		        if ($line=~/WARNING/) { $notices++; }
+		        if ($line=~/CRITICAL/) { $notices++; }
+	            }
+	            close (DFH);
+                }
+	        &log($fh,"

"); + } + } } my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; @@ -500,6 +506,34 @@ sub clean_lonIDs { &log($fh,"

$active open session(s)

"); } +# -------------------------------------------------------- clean out balanceIDs + +sub clean_balanceIDs { + my ($fh)=@_; + &log($fh,'

Session Tokens

'); + my $cleaned=0; + my $active=0; + if (-d $perlvar{'lonBalanceDir'}) { + while (my $fname=<$perlvar{'balanceDir'}/*.id>) { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
"); + unlink("$fname"); + } else { + $active++; + } + } + } + &log($fh,"

Cleaned up ".$cleaned." stale balancer files

"); + &log($fh,"

$active unexpired balancer files

"); +} + # ------------------------------------------------ clean out webDAV Session IDs sub clean_webDAV_sessionIDs { my ($fh)=@_; @@ -644,9 +678,15 @@ sub check_delayed_msg { my $unsend=0; + my %hostname = &Apache::lonnet::all_hostnames(); + my $numhosts = scalar(keys(%hostname)); + my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while (my $line=<$dfh>) { my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); + if ($numhosts) { + next unless ($hostname{$dserv}); + } if ($sdf eq 'F') { my $local=localtime($time); &log($fh,"Failed: $time, $dserv, $dcmd
"); @@ -673,8 +713,6 @@ sub check_delayed_msg { } &log($fh,"\n"); close (DFH); - my %hostname = &Apache::lonnet::all_hostnames(); - my $numhosts = scalar(keys(%hostname)); # pong to all servers that have delayed messages # this will trigger a reverse connection, which should flush the buffers foreach my $tryserver (sort(keys(%servers))) { @@ -815,6 +853,94 @@ sub write_checksums { return; } +sub write_hostips { + my $lontabdir = $perlvar{'lonTabDir'}; + my $defdom = $perlvar{'lonDefDomain'}; + my $lonhost = $perlvar{'lonHostID'}; + my $newfile = "$lontabdir/currhostips.tab"; + my $oldfile = "$lontabdir/prevhostips.tab"; + my (%prevhosts,%currhosts,%ipchange); + if ((-e $newfile) && (-s $newfile)) { + move($newfile,$oldfile); + chmod(0644,$oldfile); + if (open(my $fh,'<',$oldfile)) { + while (my $line=<$fh>) { + chomp($line); + if ($line =~ /^([^:]+):([\d.]+)$/) { + $prevhosts{$1} = $2; + } + } + close($fh); + } + } + my ($ip_info,$cached) = + &Apache::lonnet::is_cached_new('iphost','iphost'); + if (!$cached) { + &Apache::lonnet::get_iphost(); + ($ip_info,$cached) = + &Apache::lonnet::is_cached_new('iphost','iphost'); + } + if (ref($ip_info) eq 'ARRAY') { + %currhosts = %{$ip_info->[1]}; + if (open(my $fh,'>',$newfile)) { + foreach my $key (keys(%currhosts)) { + print $fh "$key:$currhosts{$key}\n"; + } + close($fh); + chmod(0644,$newfile); + } + } + if (keys(%prevhosts) && keys(%currhosts)) { + foreach my $key (keys(%prevhosts)) { + unless ($currhosts{$key} eq $prevhosts{$key}) { + $ipchange{$key} = $prevhosts{$key}.'|'.$currhosts{$key}; + } + } + foreach my $key (keys(%currhosts)) { + unless ($currhosts{$key} eq $prevhosts{$key}) { + $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key}; + } + } + } + if (&Apache::lonnet::domain($defdom,'primary') eq $lonhost) { + if (keys(%ipchange)) { + if (open(my $fh,'>>',$perlvar{'lonDaemons'}.'/logs/hostip.log')) { + print $fh "********************\n".localtime(time).' Changes --'."\n". + "Hostname | Previous IP | New IP\n". + "--------------------------------\n"; + foreach my $hostname (sort(keys(%ipchange))) { + print $fh "$hostname | $ipchange{$hostname}\n"; + } + print $fh "\n*******************\n\n"; + close($fh); + } + my $emailto = &Apache::loncommon::build_recipient_list(undef, + 'hostipmail',$defdom); + if ($emailto) { + my $subject = "LON-CAPA Hostname to IP change ($perlvar{'lonHostID'})"; + my $chgmail = "To: $emailto\n". + "Subject: $subject\n". + "Content-type: text/plain\; charset=UTF-8\n". + "MIME-Version: 1.0\n\n". + "Host/IP changes\n". + " \n". + "Hostname | Previous IP | New IP\n". + "--------------------------------\n"; + foreach my $hostname (sort(keys(%ipchange))) { + $chgmail .= "$hostname | $ipchange{$hostname}\n"; + } + $chgmail .= "\n\n"; + if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) { + print $mailh $chgmail; + close($mailh); + print "Sending mail notification of hostname/IP changes.\n"; + } + } + } + } + return; +} + sub send_mail { my $defdom = $perlvar{'lonDefDomain'}; my $origmail = $perlvar{'lonAdmEMail'}; @@ -873,19 +999,23 @@ Options: do not send emails do not check if the daemons are running, do not generate lon-status - + --justiptables Only update the dynamic iptables rules for the + lond port; do not send emails, do not + check if the daemons are running, do not + generate lon-status USAGE } # ================================================================ Main Program sub main () { my ($help,$justcheckdaemons,$noemail,$justcheckconnections, - $justreload); + $justreload,$justiptables); &GetOptions("help" => \$help, "justcheckdaemons" => \$justcheckdaemons, "noemail" => \$noemail, "justcheckconnections" => \$justcheckconnections, - "justreload" => \$justreload + "justreload" => \$justreload, + "justiptables" => \$justiptables ); if ($help) { &usage(); return; } # --------------------------------- Read loncapa_apache.conf and loncapa.conf @@ -904,8 +1034,8 @@ sub main () { chop $hostname; $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell my $subj="LON: Unconfigured machine $hostname"; - system("echo 'Unconfigured machine $hostname.' |\ - mailto $emailto -s '$subj' > /dev/null"); + system("echo 'Unconfigured machine $hostname.' |". + " mail -s '$subj' $emailto > /dev/null"); exit 1; } @@ -915,8 +1045,8 @@ sub main () { print("User ID mismatch. This program must be run as user 'www'.\n"); my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; - system("echo 'User ID mismatch. loncron must be run as user www.' |\ - mailto $emailto -s '$subj' > /dev/null"); + system("echo 'User ID mismatch. loncron must be run as user www.' |". + " mail -s '$subj' $emailto > /dev/null"); exit 1; } @@ -935,9 +1065,11 @@ sub main () { } } } - &Apache::lonnet::load_hosts_tab(1,$nomemcache); - &Apache::lonnet::load_domain_tab(1,$nomemcache); - &Apache::lonnet::get_iphost(1,$nomemcache); + if (!$justiptables) { + &Apache::lonnet::load_hosts_tab(1,$nomemcache); + &Apache::lonnet::load_domain_tab(1,$nomemcache); + &Apache::lonnet::get_iphost(1,$nomemcache); + } # ----------------------------------------- Force firewall update for lond port @@ -968,18 +1100,19 @@ sub main () { my $fh; - if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { + if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) { $fh=&start_logging(); &log_machine_info($fh); &clean_tmp($fh); &clean_lonIDs($fh); + &clean_balanceIDs($fh); &clean_webDAV_sessionIDs($fh); &check_httpd_logs($fh); &rotate_lonnet_logs($fh); &rotate_other_logs($fh); } - if (!$justcheckconnections && !$justreload) { + if (!$justcheckconnections && !$justreload && !$justiptables) { &checkon_daemon($fh,'lonmemcached',40000); &checkon_daemon($fh,'lonsql',200000); if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { @@ -996,13 +1129,14 @@ sub main () { if ($justcheckconnections) { &test_connections($fh); } - if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { + if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) { &check_delayed_msg($fh); &finish_logging($fh); &log_simplestatus(); &write_loncaparevs(); &write_serverhomeIDs(); &write_checksums(); + &write_hostips(); if ($totalcount>200 && !$noemail) { &send_mail(); } } }