loncom/loncron - view

File: [LON-CAPA] / loncom / loncron
Revision 1.95: download - view: text, annotated - select for diffs
Mon Nov 14 17:27:34 2011 UTC (12 years, 5 months ago) by raeburn
Branches: MAIN
CVS tags: version_2_10_X, version_2_10_1, loncapaMITrelate_1, language_hyphenation_merge, language_hyphenation, HEAD, BZ4492-merge, BZ4492-feature_horizontal_radioresponse, BZ4492-feature_Support_horizontal_radioresponse, BZ4492-Support_horizontal_radioresponse

- Do not modfy $warnings count in check_delayed_msg() if $unsend total is
  negative.
- Don't attempt pong to server in /home/httpd/sockets/delayed if
  server is not listed in cluster tables.
- Log pong timeout and also if server in delayed is not in cluster.

1: #!/usr/bin/perl 2: 3: # Housekeeping program, started by cron, loncontrol and loncron.pl 4: # 5: # $Id: loncron,v 1.95 2011/11/14 17:27:34 raeburn Exp $ 6: # 7: # Copyright Michigan State University Board of Trustees 8: # 9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA). 10: # 11: # LON-CAPA is free software; you can redistribute it and/or modify 12: # it under the terms of the GNU General Public License as published by 13: # the Free Software Foundation; either version 2 of the License, or 14: # (at your option) any later version. 15: # 16: # LON-CAPA is distributed in the hope that it will be useful, 17: # but WITHOUT ANY WARRANTY; without even the implied warranty of 18: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19: # GNU General Public License for more details. 20: # 21: # You should have received a copy of the GNU General Public License 22: # along with LON-CAPA; if not, write to the Free Software 23: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24: # 25: # /home/httpd/html/adm/gpl.txt 26: # 27: # http://www.lon-capa.org/ 28: # 29: 30: $|=1; 31: use strict; 32: 33: use lib '/home/httpd/lib/perl/'; 34: use LONCAPA::Configuration; 35: use LONCAPA; 36: use Apache::lonnet; 37: use Apache::loncommon; 38: 39: use IO::File; 40: use IO::Socket; 41: use HTML::Entities; 42: use Getopt::Long; 43: #globals 44: use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); 45: 46: my $statusdir="/home/httpd/html/lon-status"; 47: 48: 49: # --------------------------------------------------------- Output error status 50: 51: sub log { 52: my $fh=shift; 53: if ($fh) { print $fh @_ } 54: } 55: 56: sub errout { 57: my $fh=shift; 58: &log($fh,(<<ENDERROUT)); 59: <table border="2" bgcolor="#CCCCCC"> 60: <tr><td>Notices</td><td>$notices</td></tr> 61: <tr><td>Warnings</td><td>$warnings</td></tr> 62: <tr><td>Errors</td><td>$errors</td></tr> 63: </table><p><a href="#top">Top</a></p> 64: ENDERROUT 65: } 66: 67: sub rotate_logfile { 68: my ($file,$fh,$description) = @_; 69: my $size=(stat($file))[7]; 70: if ($size>40000) { 71: &log($fh,"<p>Rotating $description ...</p>"); 72: rename("$file.2","$file.3"); 73: rename("$file.1","$file.2"); 74: rename("$file","$file.1"); 75: } 76: } 77: 78: sub start_daemon { 79: my ($fh,$daemon,$pidfile,$args) = @_; 80: my $progname=$daemon; 81: if ($daemon eq 'lonc') { 82: $progname='loncnew'; 83: } 84: my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; 85: &rotate_logfile($error_fname,$fh,'error logs'); 86: if ($daemon eq 'lonc') { 87: &clean_sockets($fh); 88: } 89: system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); 90: sleep 1; 91: if (-e $pidfile) { 92: &log($fh,"<p>Seems like it started ...</p>"); 93: my $lfh=IO::File->new("$pidfile"); 94: my $daemonpid=<$lfh>; 95: chomp($daemonpid); 96: if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { 97: return 1; 98: } else { 99: return 0; 100: } 101: } 102: &log($fh,"<p>Seems like that did not work!</p>"); 103: $errors++; 104: return 0; 105: } 106: 107: sub checkon_daemon { 108: my ($fh,$daemon,$maxsize,$send,$args)=@_; 109: 110: my $result; 111: &log($fh,'<hr /><a name="'.$daemon.'" /><h2>'.$daemon.'</h2><h3>Log</h3><p style="white-space: pre;"><tt>'); 112: printf("%-15s ",$daemon); 113: if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ 114: open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); 115: while (my $line=<DFH>) { 116: &log($fh,"$line"); 117: if ($line=~/INFO/) { $notices++; } 118: if ($line=~/WARNING/) { $notices++; } 119: if ($line=~/CRITICAL/) { $warnings++; } 120: }; 121: close (DFH); 122: } 123: &log($fh,"</tt></p>"); 124: 125: my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; 126: 127: my $restartflag=1; 128: my $daemonpid; 129: if (-e $pidfile) { 130: my $lfh=IO::File->new("$pidfile"); 131: $daemonpid=<$lfh>; 132: chomp($daemonpid); 133: if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { 134: &log($fh,"<h3>$daemon at pid $daemonpid responding"); 135: if ($send) { &log($fh,", sending $send"); } 136: &log($fh,"</h3>"); 137: if ($send eq 'USR1') { kill USR1 => $daemonpid; } 138: if ($send eq 'USR2') { kill USR2 => $daemonpid; } 139: $restartflag=0; 140: if ($send eq 'USR2') { 141: $result = 'reloaded'; 142: print "reloaded\n"; 143: } else { 144: $result = 'running'; 145: print "running\n"; 146: } 147: } else { 148: $errors++; 149: &log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>"); 150: $restartflag=1; 151: &log($fh,"<h3>Decided to clean up stale .pid file and restart $daemon</h3>"); 152: } 153: } 154: if ($restartflag==1) { 155: $simplestatus{$daemon}='off'; 156: $errors++; 157: my $kadaemon=$daemon; 158: if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } 159: &log($fh,'<br><font color="red">Killall '.$daemon.': '. 160: `killall $kadaemon 2>&1`.' - '); 161: sleep 1; 162: &log($fh,unlink($pidfile).' - '. 163: `killall -9 $kadaemon 2>&1`. 164: '</font><br>'); 165: &log($fh,"<h3>$daemon not running, trying to start</h3>"); 166: 167: if (&start_daemon($fh,$daemon,$pidfile,$args)) { 168: &log($fh,"<h3>$daemon at pid $daemonpid responding</h3>"); 169: $simplestatus{$daemon}='restarted'; 170: $result = 'started'; 171: print "started\n"; 172: } else { 173: $errors++; 174: &log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>"); 175: &log($fh,"<p>Give it one more try ...</p>"); 176: print " "; 177: if (&start_daemon($fh,$daemon,$pidfile,$args)) { 178: &log($fh,"<h3>$daemon at pid $daemonpid responding</h3>"); 179: $simplestatus{$daemon}='restarted'; 180: $result = 'started'; 181: print "started\n"; 182: } else { 183: $result = 'failed'; 184: print " failed\n"; 185: $simplestatus{$daemon}='failed'; 186: $errors++; $errors++; 187: &log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>"); 188: &log($fh,"<p>Unable to start $daemon</p>"); 189: } 190: } 191: 192: if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ 193: &log($fh,"<p><pre>"); 194: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|"); 195: while (my $line=<DFH>) { 196: &log($fh,"$line"); 197: if ($line=~/WARNING/) { $notices++; } 198: if ($line=~/CRITICAL/) { $notices++; } 199: }; 200: close (DFH); 201: &log($fh,"</pre></p>"); 202: } 203: } 204: 205: my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; 206: &rotate_logfile($fname,$fh,'logs'); 207: 208: &errout($fh); 209: return $result; 210: } 211: 212: # --------------------------------------------------------------------- Machine 213: sub log_machine_info { 214: my ($fh)=@_; 215: &log($fh,'<hr /><a name="machine" /><h2>Machine Information</h2>'); 216: &log($fh,"<h3>loadavg</h3>"); 217: 218: open (LOADAVGH,"/proc/loadavg"); 219: my $loadavg=<LOADAVGH>; 220: close (LOADAVGH); 221: 222: &log($fh,"<tt>$loadavg</tt>"); 223: 224: my @parts=split(/\s+/,$loadavg); 225: if ($parts[1]>4.0) { 226: $errors++; 227: } elsif ($parts[1]>2.0) { 228: $warnings++; 229: } elsif ($parts[1]>1.0) { 230: $notices++; 231: } 232: 233: &log($fh,"<h3>df</h3>"); 234: &log($fh,"<pre>"); 235: 236: open (DFH,"df|"); 237: while (my $line=<DFH>) { 238: &log($fh,&encode_entities($line,'<>&"')); 239: @parts=split(/\s+/,$line); 240: my $usage=$parts[4]; 241: $usage=~s/\W//g; 242: if ($usage>90) { 243: $warnings++; 244: $notices++; 245: } elsif ($usage>80) { 246: $warnings++; 247: } elsif ($usage>60) { 248: $notices++; 249: } 250: if ($usage>95) { $warnings++; $warnings++; $simplestatus{'diskfull'}++; } 251: } 252: close (DFH); 253: &log($fh,"</pre>"); 254: 255: 256: &log($fh,"<h3>ps</h3>"); 257: &log($fh,"<pre>"); 258: my $psproc=0; 259: 260: open (PSH,"ps aux --cols 140 |"); 261: while (my $line=<PSH>) { 262: &log($fh,&encode_entities($line,'<>&"')); 263: $psproc++; 264: } 265: close (PSH); 266: &log($fh,"</pre>"); 267: 268: if ($psproc>200) { $notices++; } 269: if ($psproc>250) { $notices++; } 270: 271: &log($fh,"<h3>distprobe</h3>"); 272: &log($fh,"<pre>"); 273: open(DSH,"$perlvar{'lonDaemons'}/distprobe |"); 274: while (my $line=<DSH>) { 275: &log($fh,&encode_entities($line,'<>&"')); 276: $psproc++; 277: } 278: close(DSH); 279: &log($fh,"</pre>"); 280: 281: &errout($fh); 282: } 283: 284: sub start_logging { 285: my $fh=IO::File->new(">$statusdir/newstatus.html"); 286: my %simplestatus=(); 287: my $now=time; 288: my $date=localtime($now); 289: 290: 291: &log($fh,(<<ENDHEADERS)); 292: <html> 293: <head> 294: <title>LON Status Report $perlvar{'lonHostID'}</title> 295: </head> 296: <body bgcolor="#AAAAAA"> 297: <a name="top" /> 298: <h1>LON Status Report $perlvar{'lonHostID'}</h1> 299: <h2>$date ($now)</h2> 300: <ol> 301: <li><a href="#configuration">Configuration</a></li> 302: <li><a href="#machine">Machine Information</a></li> 303: <li><a href="#tmp">Temporary Files</a></li> 304: <li><a href="#tokens">Session Tokens</a></li> 305: <li><a href="#httpd">httpd</a></li> 306: <li><a href="#lonsql">lonsql</a></li> 307: <li><a href="#lond">lond</a></li> 308: <li><a href="#lonc">lonc</a></li> 309: <li><a href="#lonnet">lonnet</a></li> 310: <li><a href="#connections">Connections</a></li> 311: <li><a href="#delayed">Delayed Messages</a></li> 312: <li><a href="#errcount">Error Count</a></li> 313: </ol> 314: <hr /> 315: <a name="configuration" /> 316: <h2>Configuration</h2> 317: <h3>PerlVars</h3> 318: <table border="2"> 319: ENDHEADERS 320: 321: foreach my $varname (sort(keys(%perlvar))) { 322: &log($fh,"<tr><td>$varname</td><td>". 323: &encode_entities($perlvar{$varname},'<>&"')."</td></tr>\n"); 324: } 325: &log($fh,"</table><h3>Hosts</h3><table border='2'>"); 326: my %hostname = &Apache::lonnet::all_hostnames(); 327: foreach my $id (sort(keys(%hostname))) { 328: my $role = (&Apache::lonnet::is_library($id) ? 'library' 329: : 'access'); 330: &log($fh, 331: "<tr><td>$id</td><td>".&Apache::lonnet::host_domain($id). 332: "</td><td>".$role. 333: "</td><td>".&Apache::lonnet::hostname($id)."</td></tr>\n"); 334: } 335: &log($fh,"</table><h3>Spare Hosts</h3><ul>"); 336: foreach my $type (sort(keys(%Apache::lonnet::spareid))) { 337: &log($fh,"<li>$type\n<ol>"); 338: foreach my $id (@{ $Apache::lonnet::spareid{$type} }) { 339: &log($fh,"<li>$id</li>\n"); 340: } 341: &log($fh,"</ol>\n</li>\n"); 342: } 343: &log($fh,"</ul>\n"); 344: return $fh; 345: } 346: 347: # --------------------------------------------------------------- clean out tmp 348: sub clean_tmp { 349: my ($fh)=@_; 350: &log($fh,'<hr /><a name="tmp" /><h2>Temporary Files</h2>'); 351: my ($cleaned,$old,$removed) = (0,0,0); 352: my %errors = ( 353: dir => [], 354: file => [], 355: failopen => [], 356: ); 357: my %error_titles = ( 358: dir => 'failed to remove empty directory:', 359: file => 'failed to unlike stale file', 360: failopen => 'failed to open file or directory' 361: ); 362: ($cleaned,$old,$removed) = &recursive_clean_tmp('',$cleaned,$old,$removed,\%errors); 363: &log($fh,"Cleaned up: ".$cleaned." files; removed: $removed empty directories; (found: $old old checkout tokens)"); 364: foreach my $key (sort(keys(%errors))) { 365: if (ref($errors{$key}) eq 'ARRAY') { 366: if (@{$errors{$key}} > 0) { 367: &log($fh,"Error during cleanup ($error_titles{$key}):<ul><li>". 368: join('</li><li><tt>',@{$errors{$key}}).'</tt></li></ul><br />'); 369: } 370: } 371: } 372: } 373: 374: sub recursive_clean_tmp { 375: my ($subdir,$cleaned,$old,$removed,$errors) = @_; 376: my $base = "$perlvar{'lonDaemons'}/tmp"; 377: my $path = $base; 378: next if ($subdir =~ m{\.\./}); 379: next unless (ref($errors) eq 'HASH'); 380: unless ($subdir eq '') { 381: $path .= '/'.$subdir; 382: } 383: if (opendir(my $dh,"$path")) { 384: while (my $file = readdir($dh)) { 385: next if ($file =~ /^\.\.?$/); 386: my $fname = "$path/$file"; 387: if (-d $fname) { 388: my $innerdir; 389: if ($subdir eq '') { 390: $innerdir = $file; 391: } else { 392: $innerdir = $subdir.'/'.$file; 393: } 394: ($cleaned,$old,$removed) = 395: &recursive_clean_tmp($innerdir,$cleaned,$old,$removed,$errors); 396: my @doms = &Apache::lonnet::current_machine_domains(); 397: 398: if (open(my $dirhandle,$fname)) { 399: unless (($innerdir eq 'helprequests') || 400: (($innerdir =~ /^addcourse/) && ($innerdir !~ m{/\d+$}))) { 401: my @contents = grep {!/^\.\.?$/} readdir($dirhandle); 402: join('&&',@contents)."\n"; 403: if (scalar(grep {!/^\.\.?$/} readdir($dirhandle)) == 0) { 404: closedir($dirhandle); 405: if ($fname =~ m{^\Q$perlvar{'lonDaemons'}\E/tmp/}) { 406: if (rmdir($fname)) { 407: $removed ++; 408: } elsif (ref($errors->{dir}) eq 'ARRAY') { 409: push(@{$errors->{dir}},$fname); 410: } 411: } 412: } 413: } else { 414: closedir($dirhandle); 415: } 416: } 417: } else { 418: my ($dev,$ino,$mode,$nlink, 419: $uid,$gid,$rdev,$size, 420: $atime,$mtime,$ctime, 421: $blksize,$blocks)=stat($fname); 422: my $now=time; 423: my $since=$now-$mtime; 424: if ($since>$perlvar{'lonExpire'}) { 425: if ($subdir eq '') { 426: my $line=''; 427: if ($fname =~ /\.db$/) { 428: if (unlink($fname)) { 429: $cleaned++; 430: } elsif (ref($errors->{file}) eq 'ARRAY') { 431: push(@{$errors->{file}},$fname); 432: } 433: } elsif (open(PROBE,$fname)) { 434: my $line=''; 435: $line=<PROBE>; 436: close(PROBE); 437: if ($line=~/^CHECKOUTTOKEN\&/) { 438: if ($since>365*$perlvar{'lonExpire'}) { 439: if (unlink($fname)) { 440: $cleaned++; 441: } elsif (ref($errors->{file}) eq 'ARRAY') { 442: push(@{$errors->{file}},$fname); 443: } 444: } else { 445: $old++; 446: } 447: } else { 448: if (unlink($fname)) { 449: $cleaned++; 450: } elsif (ref($errors->{file}) eq 'ARRAY') { 451: push(@{$errors->{file}},$fname); 452: } 453: } 454: } elsif (ref($errors->{failopen}) eq 'ARRAY') { 455: push(@{$errors->{failopen}},$fname); 456: } 457: } else { 458: if (unlink($fname)) { 459: $cleaned++; 460: } elsif (ref($errors->{file}) eq 'ARRAY') { 461: push(@{$errors->{file}},$fname); 462: } 463: } 464: } 465: } 466: } 467: closedir($dh); 468: } elsif (ref($errors->{failopen}) eq 'ARRAY') { 469: push(@{$errors->{failopen}},$path); 470: } 471: return ($cleaned,$old,$removed); 472: } 473: 474: # ------------------------------------------------------------ clean out lonIDs 475: sub clean_lonIDs { 476: my ($fh)=@_; 477: &log($fh,'<hr /><a name="tokens" /><h2>Session Tokens</h2>'); 478: my $cleaned=0; 479: my $active=0; 480: while (my $fname=<$perlvar{'lonIDsDir'}/*>) { 481: my ($dev,$ino,$mode,$nlink, 482: $uid,$gid,$rdev,$size, 483: $atime,$mtime,$ctime, 484: $blksize,$blocks)=stat($fname); 485: my $now=time; 486: my $since=$now-$mtime; 487: if ($since>$perlvar{'lonExpire'}) { 488: $cleaned++; 489: &log($fh,"Unlinking $fname<br>"); 490: unlink("$fname"); 491: } else { 492: $active++; 493: } 494: } 495: &log($fh,"<p>Cleaned up ".$cleaned." stale session token(s).</p>"); 496: &log($fh,"<h3>$active open session(s)</h3>"); 497: } 498: 499: # ----------------------------------------------------------- clean out sockets 500: sub clean_sockets { 501: my ($fh)=@_; 502: my $cleaned=0; 503: opendir(SOCKETS,$perlvar{'lonSockDir'}); 504: while (my $fname=readdir(SOCKETS)) { 505: next if (-d $fname 506: || $fname=~/(mysqlsock|maximasock|rsock|\Q$perlvar{'lonSockDir'}\E)/); 507: $cleaned++; 508: &log($fh,"Unlinking $fname<br />"); 509: unlink("/home/httpd/sockets/$fname"); 510: } 511: &log($fh,"<p>Cleaned up ".$cleaned." stale sockets.</p>"); 512: } 513: 514: 515: # ----------------------------------------------------------------------- httpd 516: sub check_httpd_logs { 517: my ($fh)=@_; 518: if (open(PIPE,"./lchttpdlogs|")) { 519: while (my $line=<PIPE>) { 520: &log($fh,$line); 521: if ($line=~/\[error\]/) { $notices++; } 522: } 523: close(PIPE); 524: } 525: &errout($fh); 526: } 527: 528: # ---------------------------------------------------------------------- lonnet 529: 530: sub rotate_lonnet_logs { 531: my ($fh)=@_; 532: &log($fh,'<hr /><a name="lonnet" /><h2>lonnet</h2><h3>Temp Log</h3><pre>'); 533: print "checking logs\n"; 534: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){ 535: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|"); 536: while (my $line=<DFH>) { 537: &log($fh,&encode_entities($line,'<>&"')); 538: } 539: close (DFH); 540: } 541: &log($fh,"</pre><h3>Perm Log</h3><pre>"); 542: 543: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") { 544: open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|"); 545: while (my $line=<DFH>) { 546: &log($fh,&encode_entities($line,'<>&"')); 547: } 548: close (DFH); 549: } else { &log($fh,"No perm log\n") } 550: 551: my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; 552: &rotate_logfile($fname,$fh,'lonnet log'); 553: 554: &log($fh,"</pre>"); 555: &errout($fh); 556: } 557: 558: sub rotate_other_logs { 559: my ($fh) = @_; 560: my %logs = ( 561: autoenroll => 'Auto Enroll log', 562: autocreate => 'Create Course log', 563: searchcat => 'Search Cataloguing log', 564: autoupdate => 'Auto Update log', 565: refreshcourseids_db => 'Refresh CourseIDs db log', 566: ); 567: foreach my $item (keys(%logs)) { 568: my $fname=$perlvar{'lonDaemons'}.'/logs/'.$item.'.log'; 569: &rotate_logfile($fname,$fh,$logs{$item}); 570: } 571: } 572: 573: # ----------------------------------------------------------------- Connections 574: sub test_connections { 575: my ($fh)=@_; 576: &log($fh,'<hr /><a name="connections" /><h2>Connections</h2>'); 577: print "testing connections\n"; 578: &log($fh,"<table border='2'>"); 579: my ($good,$bad)=(0,0); 580: my %hostname = &Apache::lonnet::all_hostnames(); 581: foreach my $tryserver (sort(keys(%hostname))) { 582: print("."); 583: my $result; 584: my $answer=&Apache::lonnet::reply("ping",$tryserver); 585: if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { 586: $result="<b>ok</b>"; 587: $good++; 588: } else { 589: $result=$answer; 590: $warnings++; 591: if ($answer eq 'con_lost') { 592: $bad++; 593: $warnings++; 594: } else { 595: $good++; #self connection 596: } 597: } 598: if ($answer =~ /con_lost/) { print(" $tryserver down\n"); } 599: &log($fh,"<tr><td>$tryserver</td><td>$result</td></tr>\n"); 600: } 601: &log($fh,"</table>"); 602: print "\n$good good, $bad bad connections\n"; 603: &errout($fh); 604: } 605: 606: 607: # ------------------------------------------------------------ Delayed messages 608: sub check_delayed_msg { 609: my ($fh)=@_; 610: &log($fh,'<hr /><a name="delayed" /><h2>Delayed Messages</h2>'); 611: print "checking buffers\n"; 612: 613: &log($fh,'<h3>Scanning Permanent Log</h3>'); 614: 615: my $unsend=0; 616: 617: my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); 618: while (my $line=<$dfh>) { 619: my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); 620: if ($sdf eq 'F') { 621: my $local=localtime($time); 622: &log($fh,"<b>Failed: $time, $dserv, $dcmd</b><br>"); 623: $warnings++; 624: } 625: if ($sdf eq 'S') { $unsend--; } 626: if ($sdf eq 'D') { $unsend++; } 627: } 628: 629: &log($fh,"<p>Total unsend messages: <b>$unsend</b></p>\n"); 630: if ($unsend > 0) { 631: $warnings=$warnings+5*$unsend; 632: } 633: 634: if ($unsend) { $simplestatus{'unsend'}=$unsend; } 635: &log($fh,"<h3>Outgoing Buffer</h3>\n<pre>"); 636: # list directory with delayed messages and remember offline servers 637: my %servers=(); 638: open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); 639: while (my $line=<DFH>) { 640: my ($server)=($line=~/\.(\w+)$/); 641: if ($server) { $servers{$server}=1; } 642: &log($fh,&encode_entities($line,'<>&"')); 643: } 644: &log($fh,"</pre>\n"); 645: close (DFH); 646: my %hostname = &Apache::lonnet::all_hostnames(); 647: my $numhosts = scalar(keys(%hostname)); 648: # pong to all servers that have delayed messages 649: # this will trigger a reverse connection, which should flush the buffers 650: foreach my $tryserver (sort(keys(%servers))) { 651: if ($hostname{$tryserver} || !$numhosts) { 652: my $answer; 653: eval { 654: local $SIG{ ALRM } = sub { die "TIMEOUT" }; 655: alarm(20); 656: $answer = &Apache::lonnet::reply("pong",$tryserver); 657: alarm(0); 658: }; 659: if ($@ && $@ =~ m/TIMEOUT/) { 660: &log($fh,"Attempted pong to $tryserver timed out<br />"); 661: print "time out while contacting: $tryserver for pong\n"; 662: } else { 663: &log($fh,"Pong to $tryserver: $answer<br />"); 664: } 665: } else { 666: &log($fh,"$tryserver has delayed messages, but is not part of the cluster -- skipping 'Pong'.<br />"); 667: } 668: } 669: } 670: 671: sub finish_logging { 672: my ($fh)=@_; 673: &log($fh,"<a name='errcount' />\n"); 674: $totalcount=$notices+4*$warnings+100*$errors; 675: &errout($fh); 676: &log($fh,"<h1>Total Error Count: $totalcount</h1>"); 677: my $now=time; 678: my $date=localtime($now); 679: &log($fh,"<hr />$date ($now)</body></html>\n"); 680: print "lon-status webpage updated\n"; 681: $fh->close(); 682: 683: if ($errors) { $simplestatus{'errors'}=$errors; } 684: if ($warnings) { $simplestatus{'warnings'}=$warnings; } 685: if ($notices) { $simplestatus{'notices'}=$notices; } 686: $simplestatus{'time'}=time; 687: } 688: 689: sub log_simplestatus { 690: rename("$statusdir/newstatus.html","$statusdir/index.html"); 691: 692: my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); 693: foreach (keys %simplestatus) { 694: print $sfh $_.'='.$simplestatus{$_}.'&'; 695: } 696: print $sfh "\n"; 697: $sfh->close(); 698: } 699: 700: sub write_loncaparevs { 701: print "Retrieving LON-CAPA version information\n"; 702: if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) { 703: my %hostname = &Apache::lonnet::all_hostnames(); 704: foreach my $id (sort(keys(%hostname))) { 705: if ($id ne '') { 706: my $loncaparev; 707: eval { 708: local $SIG{ ALRM } = sub { die "TIMEOUT" }; 709: alarm(10); 710: $loncaparev = 711: &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron'); 712: alarm(0); 713: }; 714: if ($@ && $@ =~ m/TIMEOUT/) { 715: print "time out while contacting lonHost: $id for version\n"; 716: } 717: if ($loncaparev =~ /^[\w.\-]+$/) { 718: print $fh $id.':'.$loncaparev."\n"; 719: } 720: } 721: } 722: close($fh); 723: } 724: return; 725: } 726: 727: sub write_serverhomeIDs { 728: print "Retrieving LON-CAPA lonHostID information\n"; 729: if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) { 730: my %name_to_host = &Apache::lonnet::all_names(); 731: foreach my $name (sort(keys(%name_to_host))) { 732: if ($name ne '') { 733: if (ref($name_to_host{$name}) eq 'ARRAY') { 734: my $serverhomeID; 735: eval { 736: local $SIG{ ALRM } = sub { die "TIMEOUT" }; 737: alarm(10); 738: $serverhomeID = 739: &Apache::lonnet::get_server_homeID($name,1,'loncron'); 740: alarm(0); 741: }; 742: if ($@ && $@ =~ m/TIMEOUT/) { 743: print "Time out while contacting server: $name\n"; 744: } 745: if ($serverhomeID ne '') { 746: print $fh $name.':'.$serverhomeID."\n"; 747: } else { 748: print $fh $name.':'.$name_to_host{$name}->[0]."\n"; 749: } 750: } 751: } 752: } 753: close($fh); 754: } 755: return; 756: } 757: 758: sub send_mail { 759: print "sending mail\n"; 760: my $defdom = $perlvar{'lonDefDomain'}; 761: my $origmail = $perlvar{'lonAdmEMail'}; 762: my $emailto = &Apache::loncommon::build_recipient_list(undef, 763: 'lonstatusmail',$defdom,$origmail); 764: if ($totalcount>2500) { 765: $emailto.=",$perlvar{'lonSysEMail'}"; 766: } 767: my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 768: 769: my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); 770: if ($result != 0) { 771: $result=system("mail -s '$subj' $emailto < $statusdir/index.html"); 772: } 773: } 774: 775: sub usage { 776: print(<<USAGE); 777: loncron - housekeeping program that checks up on various parts of Lon-CAPA 778: 779: Options: 780: --help Display 781: --noemail Do not send the status email 782: --justcheckconnections Only check the current status of the lonc/d 783: connections, do not send emails do not 784: check if the daemons are running, do not 785: generate lon-status 786: --justcheckdaemons Only check that all of the Lon-CAPA daemons are 787: running, do not send emails do not 788: check the lonc/d connections, do not 789: generate lon-status 790: --justreload Only tell the daemons to reload the config files, 791: do not send emails do not 792: check if the daemons are running, do not 793: generate lon-status 794: 795: USAGE 796: } 797: 798: # ================================================================ Main Program 799: sub main () { 800: my ($help,$justcheckdaemons,$noemail,$justcheckconnections, 801: $justreload); 802: &GetOptions("help" => \$help, 803: "justcheckdaemons" => \$justcheckdaemons, 804: "noemail" => \$noemail, 805: "justcheckconnections" => \$justcheckconnections, 806: "justreload" => \$justreload 807: ); 808: if ($help) { &usage(); return; } 809: # --------------------------------- Read loncapa_apache.conf and loncapa.conf 810: my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf'); 811: %perlvar=%{$perlvarref}; 812: undef $perlvarref; 813: delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed 814: delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed 815: chdir($perlvar{'lonDaemons'}); 816: # --------------------------------------- Make sure that LON-CAPA is configured 817: # I only test for one thing here (lonHostID). This is just a safeguard. 818: if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { 819: print("Unconfigured machine.\n"); 820: my $emailto=$perlvar{'lonSysEMail'}; 821: my $hostname=`/bin/hostname`; 822: chop $hostname; 823: $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell 824: my $subj="LON: Unconfigured machine $hostname"; 825: system("echo 'Unconfigured machine $hostname.' |\ 826: mailto $emailto -s '$subj' > /dev/null"); 827: exit 1; 828: } 829: 830: # ----------------------------- Make sure this process is running from user=www 831: my $wwwid=getpwnam('www'); 832: if ($wwwid!=$<) { 833: print("User ID mismatch. This program must be run as user 'www'\n"); 834: my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; 835: my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; 836: system("echo 'User ID mismatch. loncron must be run as user www.' |\ 837: mailto $emailto -s '$subj' > /dev/null"); 838: exit 1; 839: } 840: 841: # -------------------------------------------- Force reload of host information 842: &Apache::lonnet::load_hosts_tab(1); 843: &Apache::lonnet::load_domain_tab(1); 844: &Apache::lonnet::get_iphost(1); 845: 846: # ----------------------------------------- Force firewall update for lond port 847: 848: if ((!$justcheckdaemons) && (!$justreload)) { 849: my $now = time; 850: my $tmpfile = $perlvar{'lonDaemons'}.'/tmp/lciptables_iphost_'. 851: $now.$$.int(rand(10000)); 852: if (open(my $fh,">$tmpfile")) { 853: my %iphosts = &Apache::lonnet::get_iphost(); 854: foreach my $key (keys(%iphosts)) { 855: print $fh "$key\n"; 856: } 857: close($fh); 858: if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) { 859: my $execpath = $perlvar{'lonDaemons'}.'/lciptables'; 860: system("$execpath $tmpfile"); 861: unlink('/tmp/lock_lciptables'); # Remove the lock file. 862: } 863: unlink($tmpfile); 864: } 865: } 866: 867: # ---------------------------------------------------------------- Start report 868: 869: $errors=0; 870: $warnings=0; 871: $notices=0; 872: 873: 874: my $fh; 875: if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { 876: $fh=&start_logging(); 877: 878: &log_machine_info($fh); 879: &clean_tmp($fh); 880: &clean_lonIDs($fh); 881: &check_httpd_logs($fh); 882: &rotate_lonnet_logs($fh); 883: &rotate_other_logs($fh); 884: } 885: if (!$justcheckconnections && !$justreload) { 886: &checkon_daemon($fh,'lonmemcached',40000); 887: &checkon_daemon($fh,'lonsql',200000); 888: if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { 889: &checkon_daemon($fh,'lond',40000,'USR2'); 890: } 891: &checkon_daemon($fh,'lonc',40000,'USR1'); 892: &checkon_daemon($fh,'lonmaxima',40000); 893: &checkon_daemon($fh,'lonr',40000); 894: } 895: if ($justreload) { 896: &checkon_daemon($fh,'lond',40000,'USR2'); 897: &checkon_daemon($fh,'lonc',40000,'USR2'); 898: } 899: if ($justcheckconnections) { 900: &test_connections($fh); 901: } 902: if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { 903: &check_delayed_msg($fh); 904: &finish_logging($fh); 905: &log_simplestatus(); 906: &write_loncaparevs(); 907: &write_serverhomeIDs(); 908: 909: if ($totalcount>200 && !$noemail) { &send_mail(); } 910: } 911: } 912: 913: &main(); 914: 1; 915: 916: 917: 918: 919: 920: 921: 922: