--- loncom/loncron 2014/03/17 14:47:38 1.100 +++ loncom/loncron 2018/11/06 15:37:42 1.113 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.100 2014/03/17 14:47:38 bisitz Exp $ +# $Id: loncron,v 1.113 2018/11/06 15:37:42 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -41,6 +41,8 @@ use IO::File; use IO::Socket; use HTML::Entities; use Getopt::Long; +use GDBM_File; +use Storable qw(thaw); #globals use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); @@ -111,17 +113,20 @@ sub checkon_daemon { my $result; &log($fh,'

'.$daemon.'

Log

'); printf("%-15s ",$daemon); - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); - while (my $line=) { - &log($fh,"$line"); - if ($line=~/INFO/) { $notices++; } - if ($line=~/WARNING/) { $notices++; } - if ($line=~/CRITICAL/) { $warnings++; } - }; - close (DFH); + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + if (open(DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|")) { + while (my $line=) { + &log($fh,"$line"); + if ($line=~/INFO/) { $notices++; } + if ($line=~/WARNING/) { $notices++; } + if ($line=~/CRITICAL/) { $warnings++; } + } + close (DFH); + } + } + &log($fh,"

"); } - &log($fh,"

"); my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; @@ -157,14 +162,17 @@ sub checkon_daemon { $errors++; my $kadaemon=$daemon; if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } - &log($fh,'
Killall '.$daemon.': '. + &log($fh,'
Killall '.$daemon.': '. `killall $kadaemon 2>&1`.' - '); sleep 1; &log($fh,unlink($pidfile).' - '. `killall -9 $kadaemon 2>&1`. - '
'); + '
'); + if ($kadaemon eq 'loncnew') { + &clean_lonc_childpids(); + } &log($fh,"

$daemon not running, trying to start

"); - + if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; @@ -189,17 +197,19 @@ sub checkon_daemon { &log($fh,"

Unable to start $daemon

"); } } - - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - &log($fh,"

");
-	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
-	    while (my $line=) { 
-		&log($fh,"$line");
-		if ($line=~/WARNING/) { $notices++; }
-		if ($line=~/CRITICAL/) { $notices++; }
-	    };
-	    close (DFH);
-	    &log($fh,"

"); + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + &log($fh,"

");
+	        if (open(DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|")) {
+	            while (my $line=) { 
+		        &log($fh,"$line");
+		        if ($line=~/WARNING/) { $notices++; }
+		        if ($line=~/CRITICAL/) { $notices++; }
+	            }
+	            close (DFH);
+                }
+	        &log($fh,"

"); + } } } @@ -285,9 +295,11 @@ sub start_logging { &log($fh,(< + + LON Status Report $perlvar{'lonHostID'} + @@ -298,6 +310,7 @@ sub start_logging {
  • Machine Information
  • Temporary Files
  • Session Tokens
    • +
  • WebDAV Session Tokens
  • httpd
  • lonsql
  • lond
    • @@ -328,15 +341,20 @@ ENDHEADERS "".$role. "".&Apache::lonnet::hostname($id)."\n"); } - &log($fh,"

    Spare Hosts

    \n"); return $fh; } @@ -482,7 +500,7 @@ sub clean_lonIDs { my $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; - &log($fh,"Unlinking $fname
    "); + &log($fh,"Unlinking $fname
    "); unlink("$fname"); } else { $active++; @@ -492,6 +510,32 @@ sub clean_lonIDs { &log($fh,"

    $active open session(s)

    "); } +# ------------------------------------------------ clean out webDAV Session IDs +sub clean_webDAV_sessionIDs { + my ($fh)=@_; + if ($perlvar{'lonRole'} eq 'library') { + &log($fh,'

    WebDAV Session Tokens

    '); + my $cleaned=0; + my $active=0; + my $now = time; + if (-d $perlvar{'lonDAVsessDir'}) { + while (my $fname=<$perlvar{'lonDAVsessDir'}/*>) { + my @stats = stat($fname); + my $since=$now-$stats[9]; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale webDAV session token(s).

    "); + &log($fh,"

    $active open webDAV session(s)

    "); + } + } +} + # ----------------------------------------------------------- clean out sockets sub clean_sockets { my ($fh)=@_; @@ -610,12 +654,18 @@ sub check_delayed_msg { my $unsend=0; + my %hostname = &Apache::lonnet::all_hostnames(); + my $numhosts = scalar(keys(%hostname)); + my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while (my $line=<$dfh>) { my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); + if ($numhosts) { + next unless ($hostname{$dserv}); + } if ($sdf eq 'F') { my $local=localtime($time); - &log($fh,"Failed: $time, $dserv, $dcmd
    "); + &log($fh,"Failed: $time, $dserv, $dcmd
    "); $warnings++; } if ($sdf eq 'S') { $unsend--; } @@ -639,8 +689,6 @@ sub check_delayed_msg { } &log($fh,"\n"); close (DFH); - my %hostname = &Apache::lonnet::all_hostnames(); - my $numhosts = scalar(keys(%hostname)); # pong to all servers that have delayed messages # this will trigger a reverse connection, which should flush the buffers foreach my $tryserver (sort(keys(%servers))) { @@ -665,9 +713,9 @@ sub check_delayed_msg { } sub finish_logging { - my ($fh)=@_; + my ($fh,%weights)=@_; &log($fh,"    \n"); - $totalcount=$notices+4*$warnings+100*$errors; + $totalcount=($weights{'N'}*$notices)+($weights{'W'}*$warnings)+($weights{'E'}*$errors); &errout($fh); &log($fh,"

    Total Error Count: $totalcount

    "); my $now=time; @@ -781,20 +829,300 @@ sub write_checksums { return; } +sub clean_nosslverify { + my ($fh) = @_; + my %unlinked; + if (-d "$perlvar{'lonSockDir'}/nosslverify") { + if (opendir(my $dh,"$perlvar{'lonSockDir'}/nosslverify")) { + while (my $fname=readdir($dh)) { + next if ($fname =~ /^\.+$/); + if (unlink("/home/httpd/sockets/nosslverify/$fname")) { + &log($fh,"Unlinking $fname
    "); + $unlinked{$fname} = 1; + } + } + closedir($dh); + } + } + &log($fh,"

    Removed ".scalar(keys(%unlinked))." nosslverify clients

    "); + return %unlinked; +} +sub clean_lonc_childpids { + my $childpiddir = "$perlvar{'lonDocRoot'}/lon-status/loncchld"; + if (-d $childpiddir) { + if (opendir(my $dh,$childpiddir)) { + while (my $fname=readdir($dh)) { + next if ($fname =~ /^\.+$/); + unlink("$childpiddir/$fname"); + } + closedir($dh); + } + } +} + +sub write_connection_config { + my ($domconf,%connectssl,%changes); + $domconf = &get_domain_config(); + if (ref($domconf) eq 'HASH') { + if (ref($domconf->{'ssl'}) eq 'HASH') { + foreach my $connect ('connto','connfrom') { + if (ref($domconf->{'ssl'}->{$connect}) eq 'HASH') { + my ($sslreq,$sslnoreq,$currsetting); + my %contypes; + foreach my $type ('dom','intdom','other') { + $connectssl{$connect.'_'.$type} = $domconf->{'ssl'}->{$connect}->{$type}; + } + } + } + } + if (keys(%connectssl)) { + my %currconf; + if (open(my $fh,'<',"$perlvar{'lonTabDir'}/connectionrules.tab")) { + while (my $line = <$fh>) { + chomp($line); + my ($name,$value) = split(/=/,$line); + if ($value =~ /^(?:no|yes|req)$/) { + if ($name =~ /^conn(to|from)_(dom|intdom|other)$/) { + $currconf{$name} = $value; + } + } + } + close($fh); + } + if (open(my $fh,'>',"$perlvar{'lonTabDir'}/connectionrules.tab")) { + my $count = 0; + foreach my $key (sort(keys(%connectssl))) { + print $fh "$key=$connectssl{$key}\n"; + if (exists($currconf{$key})) { + unless ($currconf{$key} eq $connectssl{$key}) { + $changes{$key} = 1; + } + } else { + $changes{$key} = 1; + } + $count ++; + } + close($fh); + print "Completed writing SSL options for lonc/lond for $count items.\n"; + } + } else { + print "Writing of SSL options skipped - no connection rules in domain configuration.\n"; + } + } else { + print "Retrieval of SSL options for lonc/lond skipped - no configuration data available for domain.\n"; + } + return %changes; +} + +sub get_domain_config { + my ($dom,$primlibserv,$isprimary,$url,%confhash); + $dom = $perlvar{'lonDefDomain'}; + $primlibserv = &Apache::lonnet::domain($dom,'primary'); + if ($primlibserv eq $perlvar{'lonHostID'}) { + $isprimary = 1; + } elsif ($primlibserv ne '') { + my $protocol = $Apache::lonnet::protocol{$primlibserv}; + my $hostname = &Apache::lonnet::hostname($primlibserv); + unless ($protocol eq 'https') { + $protocol = 'http'; + } + $url = $protocol.'://'.$hostname.'/cgi-bin/listdomconfig.pl'; + } + if ($isprimary) { + my $lonusersdir = $perlvar{'lonUsersDir'}; + my $fname = $lonusersdir.'/'.$dom.'/configuration.db'; + if (-e $fname) { + my $dbref=&LONCAPA::locking_hash_tie($fname,&GDBM_READER()); + if (ref($dbref) eq 'HASH') { + foreach my $key (sort(keys(%{$dbref}))) { + my $value = $dbref->{$key}; + if ($value =~ s/^__FROZEN__//) { + $value = thaw(&LONCAPA::unescape($value)); + } else { + $value = &LONCAPA::unescape($value); + } + $confhash{$key} = $value; + } + &LONCAPA::locking_hash_untie($dbref); + } + } + } else { + if (open(PIPE,"wget --no-check-certificate '$url?primary=$primlibserv&format=raw' |")) { + my $config = ''; + while () { + $config .= $_; + } + close(PIPE); + if ($config) { + my @pairs=split(/\&/,$config); + foreach my $item (@pairs) { + my ($key,$value)=split(/=/,$item,2); + my $what = &LONCAPA::unescape($key); + if ($value =~ s/^__FROZEN__//) { + $value = thaw(&LONCAPA::unescape($value)); + } else { + $value = &LONCAPA::unescape($value); + } + $confhash{$what}=$value; + } + } + } + } + return \%confhash; +} + +sub write_hosttypes { + my %intdom = &Apache::lonnet::all_host_intdom(); + my %hostdom = &Apache::lonnet::all_host_domain(); + my $dom = $hostdom{$perlvar{'lonHostID'}}; + my $internetdom = $intdom{$perlvar{'lonHostID'}}; + my %changes; + if (($dom ne '') && ($internetdom ne '')) { + if (keys(%hostdom)) { + my %currhosttypes; + if (open(my $fh,'<',"$perlvar{'lonTabDir'}/hosttypes.tab")) { + while (my $line = <$fh>) { + chomp($line); + my ($name,$value) = split(/:/,$line); + if (($name ne '') && ($value =~ /^(dom|intdom|other)$/)) { + $currhosttypes{$name} = $value; + } + } + close($fh); + } + if (open(my $fh,'>',"$perlvar{'lonTabDir'}/hosttypes.tab")) { + my $count = 0; + foreach my $lonid (sort(keys(%hostdom))) { + my $type = 'other'; + if ($hostdom{$lonid} eq $dom) { + $type = 'dom'; + } elsif ($intdom{$lonid} eq $internetdom) { + $type = 'intdom'; + } + print $fh "$lonid:$type\n"; + if (exists($currhosttypes{$lonid})) { + if ($type ne $currhosttypes{$lonid}) { + $changes{$lonid} = 1; + } + } else { + $changes{$lonid} = 1; + } + $count ++; + } + close($fh); + print "Completed writing host type data for $count hosts.\n"; + } + } else { + print "Writing of host types skipped - no hosts found.\n"; + } + } else { + print "Writing of host types skipped - could not determine this host's LON-CAPA domain or 'internet' domain.\n"; + } + return %changes; +} + +sub update_revocation_list { + my ($result,$changed) = &Apache::lonnet::fetch_crl_pemfile(); + if ($result eq 'ok') { + print "Certificate Revocation List (from CA) updated.\n"; + } else { + print "Certificate Revocation List from (CA) not updated.\n"; + } + return $changed; +} + +sub reset_nosslverify_pids { + my ($fh,%sslrem) = @_; + &checkon_daemon($fh,'lond',40000,'USR2'); + my $loncpidfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; + my $loncppid; + if ((-e $loncpidfile) && (open(my $pfh,'<',$loncpidfile))) { + $loncppid=<$pfh>; + chomp($loncppid); + close($pfh); + if ($loncppid =~ /^\d+$/) { + my %pids_by_host; + my $docdir = $perlvar{'lonDocRoot'}; + if (-d "$docdir/lon-status/loncchld") { + if (opendir(my $dh,"$docdir/lon-status/loncchld")) { + while (my $file = readdir($dh)) { + next if ($file =~ /^\./); + if (open(my $fh,'<',"$docdir/lon-status/loncchld/$file")) { + my $record = <$fh>; + chomp($record); + close($fh); + my ($remotehost,$authmode) = split(/:/,$record); + $pids_by_host{$remotehost}{$authmode}{$file} = 1; + } + } + closedir($dh); + if (keys(%pids_by_host)) { + foreach my $host (keys(%pids_by_host)) { + if ($sslrem{$host}) { + if (ref($pids_by_host{$host}) eq 'HASH') { + if (ref($pids_by_host{$host}{'insecure'}) eq 'HASH') { + if (keys(%{$pids_by_host{$host}{'insecure'}})) { + foreach my $pid (keys(%{$pids_by_host{$host}{'insecure'}})) { + if (open(PIPE,"ps -o ppid= -p $pid |")) { + my $ppid = ; + chomp($ppid); + close(PIPE); + $ppid =~ s/(^\s+|\s+$)//g; + if (($ppid == $loncppid) && (kill 0 => $pid)) { + kill QUIT => $pid; + } + } + } + } + } + } + } + } + } + } + } + } + } + return; +} + sub send_mail { - print "Sending mail.\n"; + my ($sysmail,$reportstatus) = @_; my $defdom = $perlvar{'lonDefDomain'}; my $origmail = $perlvar{'lonAdmEMail'}; my $emailto = &Apache::loncommon::build_recipient_list(undef, 'lonstatusmail',$defdom,$origmail); - if ($totalcount>2500) { + if (($totalcount>$sysmail) && ($reportstatus)) { $emailto.=",$perlvar{'lonSysEMail'}"; } - my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; - - my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); - if ($result != 0) { - $result=system("mail -s '$subj' $emailto < $statusdir/index.html"); + my $from; + my $hostname=`/bin/hostname`; + chop($hostname); + $hostname=~s/[^\w\.]//g; + if ($hostname) { + $from = 'www@'.$hostname; + } + my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; + my $loncronmail = "To: $emailto\n". + "From: $from\n". + "Subject: ".$subj."\n". + "Content-type: text/html\; charset=UTF-8\n". + "MIME-Version: 1.0\n\n"; + if (open(my $fh,"<$statusdir/index.html")) { + while (<$fh>) { + $loncronmail .= $_; + } + close($fh); + } else { + $loncronmail .= "Failed to read from http://$hostname/lon-status/index.html\n"; + } + $loncronmail .= "\n\n"; + if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) { + print $mailh $loncronmail; + close($mailh); + print "Sending mail.\n"; + } else { + print "Sending mail failed.\n"; } } @@ -848,8 +1176,8 @@ sub main () { chop $hostname; $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell my $subj="LON: Unconfigured machine $hostname"; - system("echo 'Unconfigured machine $hostname.' |\ - mailto $emailto -s '$subj' > /dev/null"); + system("echo 'Unconfigured machine $hostname.' |". + " mail -s '$subj' $emailto > /dev/null"); exit 1; } @@ -859,15 +1187,29 @@ sub main () { print("User ID mismatch. This program must be run as user 'www'.\n"); my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; - system("echo 'User ID mismatch. loncron must be run as user www.' |\ - mailto $emailto -s '$subj' > /dev/null"); + system("echo 'User ID mismatch. loncron must be run as user www.' |". + " mail -s '$subj' $emailto > /dev/null"); exit 1; } # -------------------------------------------- Force reload of host information - &Apache::lonnet::load_hosts_tab(1); - &Apache::lonnet::load_domain_tab(1); - &Apache::lonnet::get_iphost(1); + my $nomemcache; + if ($justcheckdaemons) { + $nomemcache=1; + my $memcachepidfile="$perlvar{'lonDaemons'}/logs/memcached.pid"; + my $memcachepid; + if (-e $memcachepidfile) { + my $memfh=IO::File->new($memcachepidfile); + $memcachepid=<$memfh>; + chomp($memcachepid); + if ($memcachepid =~ /^\d+$/ && kill 0 => $memcachepid) { + undef($nomemcache); + } + } + } + &Apache::lonnet::load_hosts_tab(1,$nomemcache); + &Apache::lonnet::load_domain_tab(1,$nomemcache); + &Apache::lonnet::get_iphost(1,$nomemcache); # ----------------------------------------- Force firewall update for lond port @@ -904,6 +1246,7 @@ sub main () { &log_machine_info($fh); &clean_tmp($fh); &clean_lonIDs($fh); + &clean_webDAV_sessionIDs($fh); &check_httpd_logs($fh); &rotate_lonnet_logs($fh); &rotate_other_logs($fh); @@ -919,6 +1262,10 @@ sub main () { &checkon_daemon($fh,'lonr',40000); } if ($justreload) { + &clean_nosslverify($fh); + &write_connection_config(); + &write_hosttypes(); + &update_revocation_list(); &checkon_daemon($fh,'lond',40000,'USR2'); &checkon_daemon($fh,'lonc',40000,'USR2'); } @@ -927,12 +1274,50 @@ sub main () { } if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { &check_delayed_msg($fh); - &finish_logging($fh); &log_simplestatus(); &write_loncaparevs(); &write_serverhomeIDs(); &write_checksums(); - if ($totalcount>200 && !$noemail) { &send_mail(); } + my %sslrem = &clean_nosslverify($fh); + my %conchgs = &write_connection_config(); + my %hosttypechgs = &write_hosttypes(); + my $hadcrlchg = &update_revocation_list(); + if ((keys(%conchgs) > 0) || (keys(%hosttypechgs) > 0) || + $hadcrlchg || (keys(%sslrem) > 0)) { + &checkon_daemon($fh,'lond',40000,'USR2'); + &reset_nosslverify_pids($fh,%sslrem); + } + my $domconf = &get_domain_config(); + my ($defaults,$names) = &Apache::loncommon::lon_status_items(); + my (%weights,$threshold); + foreach my $type ('E','W','N') { + $weights{$type} = $defaults->{$type}; + } + my $threshold = $defaults->{'threshold'}; + my $sysmail = $defaults->{'sysmail'}; + my $reportstatus = 1; + if (ref($domconf->{'contacts'}) eq 'HASH') { + if ($domconf->{'contacts'}{'reportstatus'} == 0) { + $reportstatus = 0; + } + if (ref($domconf->{'contacts'}{'lonstatus'}) eq 'HASH') { + if (ref($domconf->{'contacts'}{'lonstatus'}{weights}) eq 'HASH') { + foreach my $type ('E','W','N') { + if (exists($domconf->{'contacts'}{'lonstatus'}{weights}{$type})) { + $weights{$type} = $domconf->{'contacts'}{'lonstatus'}{weights}{$type}; + } + } + } + } + if (exists($domconf->{'contacts'}{'lonstatus'}{'threshold'})) { + $threshold = $domconf->{'contacts'}{'lonstatus'}{'threshold'}; + } + if (exists($domconf->{'contacts'}{'lonstatus'}{'sysmail'})) { + $sysmail = $domconf->{'contacts'}{'lonstatus'}{'sysmail'}; + } + } + &finish_logging($fh,%weights); + if ($totalcount>$threshold && !$noemail) { &send_mail($sysmail,$reportstatus); } } } 500 Internal Server Error

    Internal Server Error

    The server encountered an internal error or misconfiguration and was unable to complete your request.

    Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

    More information about this error may be available in the server error log.