1: #!/usr/bin/perl
2:
3: # The LearningOnline Network
4: # Housekeeping program, started by cron
5: #
6: # (TCP networking package
7: # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30,
8: # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer)
9: #
10: # 7/14,7/15,7/19,7/21,7/22,11/18,
11: # 2/8 Gerd Kortemeyer
12: # Dec 00 Scott Harrison
13: # 12/23 Gerd Kortemeyer
14:
15: use IO::File;
16: use IO::Socket;
17:
18: # -------------------------------------------------- Non-critical communication
19: sub reply {
20: my ($cmd,$server)=@_;
21: my $peerfile="$perlvar{'lonSockDir'}/$server";
22: my $client=IO::Socket::UNIX->new(Peer =>"$peerfile",
23: Type => SOCK_STREAM,
24: Timeout => 10)
25: or return "con_lost";
26: print $client "$cmd\n";
27: my $answer=<$client>;
28: chomp($answer);
29: if (!$answer) { $answer="con_lost"; }
30: return $answer;
31: }
32:
33: # --------------------------------------------------------- Output error status
34:
35: sub errout {
36: my $fh=shift;
37: print $fh (<<ENDERROUT);
38: <p><table border=2 bgcolor="#CCCCCC">
39: <tr><td>Notices</td><td>$notices</td></tr>
40: <tr><td>Warnings</td><td>$warnings</td></tr>
41: <tr><td>Errors</td><td>$errors</td></tr>
42: </table><p><a href="#top">Top</a><p>
43: ENDERROUT
44: }
45:
46: # ================================================================ Main Program
47:
48: # ------------------------------------------------------------ Read access.conf
49: {
50: my $config=IO::File->new("/etc/httpd/conf/access.conf");
51:
52: while (my $configline=<$config>) {
53: if ($configline =~ /PerlSetVar/) {
54: my ($dummy,$varname,$varvalue)=split(/\s+/,$configline);
55: $perlvar{$varname}=$varvalue;
56: }
57: }
58: }
59:
60: # ----------------------------- Make sure this process is running from user=www
61: my $wwwid=getpwnam('www');
62: if ($wwwid!=$<) {
63: print("User ID mismatch. This program must be run as user 'www'\n") unless $noprint;
64: $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
65: $subj="LON: $perlvar{'lonHostID'} User ID mismatch";
66: system("echo 'User ID mismatch. loncron must be run as user www.' |\
67: mailto $emailto -s '$subj' > /dev/null");
68: exit 1;
69: }
70:
71: # ------------------------------------------------------------- Read hosts file
72: {
73: my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
74:
75: while (my $configline=<$config>) {
76: my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
77: $hostname{$id}=$name;
78: $hostdom{$id}=$domain;
79: $hostrole{$id}=$role;
80: $hostip{$id}=$ip;
81: if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
82: $libserv{$id}=$name;
83: }
84: }
85: }
86:
87: # ------------------------------------------------------ Read spare server file
88: {
89: my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");
90:
91: while (my $configline=<$config>) {
92: chomp($configline);
93: if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
94: $spareid{$configline}=1;
95: }
96: }
97: }
98:
99: # ---------------------------------------------------------------- Start report
100:
101: $statusdir="/home/httpd/html/lon-status";
102:
103: $errors=0;
104: $warnings=0;
105: $notices=0;
106:
107: $now=time;
108: $date=localtime($now);
109:
110: {
111: my $fh=IO::File->new(">$statusdir/newstatus.html");
112:
113: print $fh (<<ENDHEADERS);
114: <html>
115: <head>
116: <title>LON Status Report $perlvar{'lonHostID'}</title>
117: </head>
118: <body bgcolor="#AAAAAA">
119: <a name="top">
120: <h1>LON Status Report $perlvar{'lonHostID'}</h1>
121: <h2>$date ($now)</h2>
122: <ol>
123: <li><a href="#configuration">Configuration</a>
124: <li><a href="#machine">Machine Information</a>
125: <li><a href="#tmp">Temporary Files</a>
126: <li><a href="#tokens">Session Tokens</a>
127: <li><a href="#httpd">httpd</a>
128: <li><a href="#lonsql">lonsql</a>
129: <li><a href="#lond">lond</a>
130: <li><a href="#lonc">lonc</a>
131: <li><a href="#lonnet">lonnet</a>
132: <li><a href="#connections">Connections</a>
133: <li><a href="#delayed">Delayed Messages</a>
134: <li><a href="#errcount">Error Count</a>
135: </ol>
136: <hr>
137: <a name="configuration">
138: <h2>Configuration</h2>
139: <h3>PerlVars</h3>
140: <table border=2>
141: ENDHEADERS
142:
143: foreach $varname (keys %perlvar) {
144: print $fh "<tr><td>$varname</td><td>$perlvar{$varname}</td></tr>\n";
145: }
146: print $fh "</table><h3>Hosts</h3><table border=2>";
147: foreach $id (keys %hostname) {
148: print $fh
149: "<tr><td>$id</td><td>$hostdom{$id}</td><td>$hostrole{$id}</td>";
150: print $fh "<td>$hostname{$id}</td><td>$hostip{$id}</td></tr>\n";
151: }
152: print $fh "</table><h3>Spare Hosts</h3><ol>";
153: foreach $id (keys %spareid) {
154: print $fh "<li>$id\n";
155: }
156:
157: print $fh "</ol>\n";
158:
159: # --------------------------------------------------------------------- Machine
160:
161: print $fh '<hr><a name="machine"><h2>Machine Information</h2>';
162: print $fh "<h3>loadavg</h3>";
163:
164: open (LOADAVGH,"/proc/loadavg");
165: $loadavg=<LOADAVGH>;
166: close (LOADAVGH);
167:
168: print $fh "<tt>$loadavg</tt>";
169:
170: @parts=split(/\s+/,$loadavg);
171: if ($parts[1]>4.0) {
172: $errors++;
173: } elsif ($parts[1]>2.0) {
174: $warnings++;
175: } elsif ($parts[1]>1.0) {
176: $notices++;
177: }
178:
179: print $fh "<h3>df</h3>";
180: print $fh "<pre>";
181:
182: open (DFH,"df|");
183: while ($line=<DFH>) {
184: print $fh "$line";
185: @parts=split(/\s+/,$line);
186: $usage=$parts[4];
187: $usage=~s/\W//g;
188: if ($usage>90) {
189: $warnings++;
190: } elsif ($usage>80) {
191: $warnings++;
192: } elsif ($usage>60) {
193: $notices++;
194: }
195: if ($usage>95) { $warnings++; $warnings++ }
196: }
197: close (DFH);
198: print $fh "</pre>";
199: &errout($fh);
200:
201: # --------------------------------------------------------------- clean out tmp
202: print $fh '<hr><a name="tmp"><h2>Temporary Files</h2>';
203: $cleaned=0;
204: while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) {
205: my ($dev,$ino,$mode,$nlink,
206: $uid,$gid,$rdev,$size,
207: $atime,$mtime,$ctime,
208: $blksize,$blocks)=stat($fname);
209: $now=time;
210: $since=$now-$mtime;
211: if ($since>$perlvar{'lonExpire'}) {
212: $cleaned++;
213: unlink("$fname");
214: }
215:
216: }
217: print $fh "Cleaned up ".$cleaned." files.";
218:
219: # ------------------------------------------------------------ clean out lonIDs
220: print $fh '<hr><a name="tokens"><h2>Session Tokens</h2>';
221: $cleaned=0;
222: $active=0;
223: while ($fname=<$perlvar{'lonIDsDir'}/*>) {
224: my ($dev,$ino,$mode,$nlink,
225: $uid,$gid,$rdev,$size,
226: $atime,$mtime,$ctime,
227: $blksize,$blocks)=stat($fname);
228: $now=time;
229: $since=$now-$mtime;
230: if ($since>$perlvar{'lonExpire'}) {
231: $cleaned++;
232: print $fh "Unlinking $fname<br>";
233: unlink("$fname");
234: } else {
235: $active++;
236: }
237:
238: }
239: print $fh "<p>Cleaned up ".$cleaned." stale session token(s).";
240: print $fh "<h3>$active open session(s)</h3>";
241:
242: # ----------------------------------------------------------------------- httpd
243:
244: print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';
245:
246: open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
247: while ($line=<DFH>) { print $fh "$line" };
248: close (DFH);
249:
250: print $fh "</pre><h3>Error Log</h3><pre>";
251:
252: open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
253: while ($line=<DFH>) {
254: print $fh "$line";
255: if ($line=~/\[error\]/) { $notices++; }
256: };
257: close (DFH);
258: print $fh "</pre>";
259: &errout($fh);
260:
261:
262: # ---------------------------------------------------------------------- lonsql
263: #
264: # Do not run for now
265: #
266: if ($perlvar{'lonRole'} eq "library" && 1==0) {
267:
268: print $fh '<hr><a name="lonsql"><h2>lonsql</h2><h3>Log</h3><pre>';
269:
270: if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
271: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
272: while ($line=<DFH>) {
273: print $fh "$line";
274: if ($line=~/INFO/) { $notices++; }
275: if ($line=~/WARNING/) { $notices++; }
276: if ($line=~/CRITICAL/) { $warnings++; }
277: };
278: close (DFH);
279: }
280: print $fh "</pre>";
281:
282: my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";
283:
284: if (-e $lonsqlfile) {
285: my $lfh=IO::File->new("$lonsqlfile");
286: my $lonsqlpid=<$lfh>;
287: chomp($lonsqlpid);
288: if (kill 0 => $lonsqlpid) {
289: print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
290: } else {
291: $errors++; $errors++;
292: print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
293: }
294: } else {
295: $errors++;
296: print $fh "<h3>lonsql not running, trying to start</h3>";
297: system("$perlvar{'lonDaemons'}/lonsql");
298: sleep 120;
299: if (-e $lonsqlfile) {
300: print $fh "Seems like it started ...<p>";
301: my $lfh=IO::File->new("$lonsqlfile");
302: my $lonsqlpid=<$lfh>;
303: chomp($lonsqlpid);
304: sleep 30;
305: if (kill 0 => $lonsqlpid) {
306: print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
307: } else {
308: $errors++; $errors++;
309: print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
310: print $fh "Give it one more try ...<p>";
311: system("$perlvar{'lonDaemons'}/lonsql");
312: sleep 120;
313: }
314: } else {
315: print $fh "Seems like that did not work!<p>";
316: $errors++;
317: }
318: if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
319: print $fh "<p><pre>";
320: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
321: while ($line=<DFH>) {
322: print $fh "$line";
323: if ($line=~/WARNING/) { $notices++; }
324: if ($line=~/CRITICAL/) { $notices++; }
325: };
326: close (DFH);
327: print $fh "</pre>";
328: }
329: }
330:
331: $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";
332:
333: my ($dev,$ino,$mode,$nlink,
334: $uid,$gid,$rdev,$size,
335: $atime,$mtime,$ctime,
336: $blksize,$blocks)=stat($fname);
337:
338: if ($size>40000) {
339: print $fh "Rotating logs ...<p>";
340: rename("$fname.2","$fname.3");
341: rename("$fname.1","$fname.2");
342: rename("$fname","$fname.1");
343: }
344:
345: &errout($fh);
346: }
347: # ------------------------------------------------------------------------ lond
348:
349: print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';
350:
351: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
352: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
353: while ($line=<DFH>) {
354: print $fh "$line";
355: if ($line=~/INFO/) { $notices++; }
356: if ($line=~/WARNING/) { $notices++; }
357: if ($line=~/CRITICAL/) { $warnings++; }
358: };
359: close (DFH);
360: }
361: print $fh "</pre>";
362:
363: my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";
364:
365: my $restartflag=1;
366: if (-e $londfile) {
367: my $lfh=IO::File->new("$londfile");
368: my $londpid=<$lfh>;
369: chomp($londpid);
370: if (kill 0 => $londpid) {
371: print $fh "<h3>lond at pid $londpid responding</h3>";
372: $restartflag=0;
373: } else {
374: $errors++;
375: print $fh "<h3>lond at pid $londpid not responding</h3>";
376: # Intelligently handle this.
377: # Possibility #1: there is no process
378: # Solution: remove .pid file and restart
379: if (getpgrp($londpid)==-1) {
380: unlink($londfile);
381: $restartflag=1;
382: }
383: else {
384: # Possibility #2: there is a live process that is not responding
385: # for an unknown reason
386: # Solution: kill parent and children processes, remove .pid and restart
387: `killall -9 lond`;
388: unlink($londfile);
389: $restartflag=1;
390: }
391: print $fh
392: "<h3>Deciding to clean up stale .pid file and restart lond</h3>";
393: }
394: }
395: if ($restartflag==1) {
396: $errors++;
397: print $fh "<h3>lond not running, trying to start</h3>";
398: system("$perlvar{'lonDaemons'}/lond");
399: sleep 120;
400: if (-e $londfile) {
401: print $fh "Seems like it started ...<p>";
402: my $lfh=IO::File->new("$londfile");
403: my $londpid=<$lfh>;
404: chomp($londpid);
405: sleep 30;
406: if (kill 0 => $londpid) {
407: print $fh "<h3>lond at pid $londpid responding</h3>";
408: } else {
409: $errors++; $errors++;
410: print $fh "<h3>lond at pid $londpid not responding</h3>";
411: print $fh "Give it one more try ...<p>";
412: system("$perlvar{'lonDaemons'}/lond");
413: sleep 120;
414: }
415: } else {
416: print $fh "Seems like that did not work!<p>";
417: $errors++;
418: }
419: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
420: print $fh "<p><pre>";
421: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
422: while ($line=<DFH>) {
423: print $fh "$line";
424: if ($line=~/WARNING/) { $notices++; }
425: if ($line=~/CRITICAL/) { $notices++; }
426: };
427: close (DFH);
428: print $fh "</pre>";
429: }
430: }
431:
432: $fname="$perlvar{'lonDaemons'}/logs/lond.log";
433:
434: my ($dev,$ino,$mode,$nlink,
435: $uid,$gid,$rdev,$size,
436: $atime,$mtime,$ctime,
437: $blksize,$blocks)=stat($fname);
438:
439: if ($size>40000) {
440: print $fh "Rotating logs ...<p>";
441: rename("$fname.2","$fname.3");
442: rename("$fname.1","$fname.2");
443: rename("$fname","$fname.1");
444: }
445:
446: &errout($fh);
447: # ------------------------------------------------------------------------ lonc
448:
449: print $fh '<hr><a name="lonc"><h2>lonc</h2><h3>Log</h3><pre>';
450:
451: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
452: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
453: while ($line=<DFH>) {
454: print $fh "$line";
455: if ($line=~/INFO/) { $notices++; }
456: if ($line=~/WARNING/) { $notices++; }
457: if ($line=~/CRITICAL/) { $warnings++; }
458: };
459: close (DFH);
460: }
461: print $fh "</pre>";
462:
463: my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
464:
465: $restartflag=1;
466: if (-e $loncfile) {
467: my $lfh=IO::File->new("$loncfile");
468: my $loncpid=<$lfh>;
469: chomp($loncpid);
470: if (kill 0 => $loncpid) {
471: print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";
472: kill USR1 => $loncpid;
473: $restartflag=0;
474: } else {
475: $errors++;
476: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
477: # Intelligently handle this.
478: # Possibility #1: there is no process
479: # Solution: remove .pid file and restart
480: if (getpgrp($loncpid)==-1) {
481: unlink($loncfile);
482: $restartflag=1;
483: }
484: else {
485: # Possibility #2: there is a live process that is not responding
486: # for an unknown reason
487: # Solution: kill parent and children processes, remove .pid and restart
488: `killall -9 lonc`;
489: unlink($loncfile);
490: $restartflag=1;
491: }
492: print $fh
493: "<h3>Deciding to clean up stale .pid file and restart lonc</h3>";
494: }
495: }
496: if ($restartflag==1) {
497: $errors++;
498: print $fh "<h3>lonc not running, trying to start</h3>";
499: system("$perlvar{'lonDaemons'}/lonc");
500: sleep 120;
501: if (-e $loncfile) {
502: print $fh "Seems like it started ...<p>";
503: my $lfh=IO::File->new("$loncfile");
504: my $loncpid=<$lfh>;
505: chomp($loncpid);
506: sleep 30;
507: if (kill 0 => $loncpid) {
508: print $fh "<h3>lonc at pid $loncpid responding</h3>";
509: } else {
510: $errors++; $errors++;
511: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
512: print $fh "Give it one more try ...<p>";
513: system("$perlvar{'lonDaemons'}/lonc");
514: sleep 120;
515: }
516: } else {
517: print $fh "Seems like that did not work!<p>";
518: $errors++;
519: }
520: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") {
521: print $fh "<p><pre>";
522: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
523: while ($line=<DFH>) {
524: print $fh "$line";
525: if ($line=~/WARNING/) { $notices++; }
526: if ($line=~/CRITICAL/) { $notices++; }
527: };
528: close (DFH);
529: print $fh "</pre>";
530: }
531: }
532:
533: $fname="$perlvar{'lonDaemons'}/logs/lonc.log";
534:
535: my ($dev,$ino,$mode,$nlink,
536: $uid,$gid,$rdev,$size,
537: $atime,$mtime,$ctime,
538: $blksize,$blocks)=stat($fname);
539:
540: if ($size>40000) {
541: print $fh "Rotating logs ...<p>";
542: rename("$fname.2","$fname.3");
543: rename("$fname.1","$fname.2");
544: rename("$fname","$fname.1");
545: }
546:
547:
548: &errout($fh);
549: # ---------------------------------------------------------------------- lonnet
550:
551: print $fh '<hr><a name="lonnet"><h2>lonnet</h2><h3>Temp Log</h3><pre>';
552: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
553: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
554: while ($line=<DFH>) {
555: print $fh "$line";
556: };
557: close (DFH);
558: }
559: print $fh "</pre><h3>Perm Log</h3><pre>";
560:
561: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
562: open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
563: while ($line=<DFH>) {
564: print $fh "$line";
565: };
566: close (DFH);
567: } else { print $fh "No perm log\n" }
568:
569: $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
570:
571: my ($dev,$ino,$mode,$nlink,
572: $uid,$gid,$rdev,$size,
573: $atime,$mtime,$ctime,
574: $blksize,$blocks)=stat($fname);
575:
576: if ($size>40000) {
577: print $fh "Rotating logs ...<p>";
578: rename("$fname.2","$fname.3");
579: rename("$fname.1","$fname.2");
580: rename("$fname","$fname.1");
581: }
582:
583: print $fh "</pre>";
584: &errout($fh);
585: # ----------------------------------------------------------------- Connections
586:
587: print $fh '<hr><a name="connections"><h2>Connections</h2>';
588:
589: print $fh "<table border=2>";
590: foreach $tryserver (keys %hostname) {
591:
592: $answer=reply("pong",$tryserver);
593: if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
594: $result="<b>ok</b>";
595: } else {
596: $result=$answer;
597: $warnings++;
598: if ($answer eq 'con_lost') { $warnings++; }
599: }
600: print $fh "<tr><td>$tryserver</td><td>$result</td></tr>\n";
601:
602: }
603: print $fh "</table>";
604:
605: &errout($fh);
606: # ------------------------------------------------------------ Delayed messages
607:
608: print $fh '<hr><a name="delayed"><h2>Delayed Messages</h2>';
609:
610: print $fh '<h3>Scanning Permanent Log</h3>';
611:
612: $unsend=0;
613: {
614: my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
615: while ($line=<$dfh>) {
616: ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
617: if ($sdf eq 'F') {
618: $local=localtime($time);
619: print "<b>Failed: $time, $dserv, $dcmd</b><br>";
620: $warnings++;
621: }
622: if ($sdf eq 'S') { $unsend--; }
623: if ($sdf eq 'D') { $unsend++; }
624: }
625: }
626: print $fh "Total unsend messages: <b>$unsend</b><p>\n";
627: $warnings=$warnings+5*$unsend;
628:
629: print $fh "<h3>Outgoing Buffer</h3>";
630:
631: open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
632: while ($line=<DFH>) {
633: print $fh "$line<br>";
634: };
635: close (DFH);
636:
637: # ------------------------------------------------------------------------- End
638: print $fh "<a name=errcount>\n";
639: $totalcount=$notices+4*$warnings+100*$errors;
640: &errout($fh);
641: print $fh "<h1>Total Error Count: $totalcount</h1>";
642: $now=time;
643: $date=localtime($now);
644: print $fh "<hr>$date ($now)</body></html>\n";
645:
646: }
647:
648: rename ("$statusdir/newstatus.html","$statusdir/index.html");
649:
650: if ($totalcount>200) {
651: $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
652: $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
653: system(
654: "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");
655: }
656: 1;
657:
658:
659:
660:
661:
662:
663:
664:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to loncron CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [LON-CAPA] / loncom