1: #!/usr/bin/perl
2:
3: # The LearningOnline Network
4: # Housekeeping program, started by cron
5: #
6: # (TCP networking package
7: # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30,
8: # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer)
9: #
10: # 7/14,7/15,7/19,7/21,7/22,11/18,
11: # 2/8 Gerd Kortemeyer
12: # Dec 00 Scott Harrison
13: # 12/23 Gerd Kortemeyer
14:
15: use IO::File;
16: use IO::Socket;
17:
18: # -------------------------------------------------- Non-critical communication
19: sub reply {
20: my ($cmd,$server)=@_;
21: my $peerfile="$perlvar{'lonSockDir'}/$server";
22: my $client=IO::Socket::UNIX->new(Peer =>"$peerfile",
23: Type => SOCK_STREAM,
24: Timeout => 10)
25: or return "con_lost";
26: print $client "$cmd\n";
27: my $answer=<$client>;
28: chomp($answer);
29: if (!$answer) { $answer="con_lost"; }
30: return $answer;
31: }
32:
33: # --------------------------------------------------------- Output error status
34:
35: sub errout {
36: my $fh=shift;
37: print $fh (<<ENDERROUT);
38: <p><table border=2 bgcolor="#CCCCCC">
39: <tr><td>Notices</td><td>$notices</td></tr>
40: <tr><td>Warnings</td><td>$warnings</td></tr>
41: <tr><td>Errors</td><td>$errors</td></tr>
42: </table><p><a href="#top">Top</a><p>
43: ENDERROUT
44: }
45:
46: # ================================================================ Main Program
47:
48:
49: # ------------------------------------------------------------ Read access.conf
50: {
51: my $config=IO::File->new("/etc/httpd/conf/access.conf");
52:
53: while (my $configline=<$config>) {
54: if ($configline =~ /PerlSetVar/) {
55: my ($dummy,$varname,$varvalue)=split(/\s+/,$configline);
56: $perlvar{$varname}=$varvalue;
57: }
58: }
59: }
60:
61: # ------------------------------------------------------------- Read hosts file
62: {
63: my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
64:
65: while (my $configline=<$config>) {
66: my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
67: $hostname{$id}=$name;
68: $hostdom{$id}=$domain;
69: $hostrole{$id}=$role;
70: $hostip{$id}=$ip;
71: if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
72: $libserv{$id}=$name;
73: }
74: }
75: }
76:
77: # ------------------------------------------------------ Read spare server file
78: {
79: my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");
80:
81: while (my $configline=<$config>) {
82: chomp($configline);
83: if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
84: $spareid{$configline}=1;
85: }
86: }
87: }
88:
89: # ---------------------------------------------------------------- Start report
90:
91: $statusdir="/home/httpd/html/lon-status";
92:
93: $errors=0;
94: $warnings=0;
95: $notices=0;
96:
97: $now=time;
98: $date=localtime($now);
99:
100: {
101: my $fh=IO::File->new(">$statusdir/newstatus.html");
102:
103: print $fh (<<ENDHEADERS);
104: <html>
105: <head>
106: <title>LON Status Report $perlvar{'lonHostID'}</title>
107: </head>
108: <body bgcolor="#AAAAAA">
109: <a name="top">
110: <h1>LON Status Report $perlvar{'lonHostID'}</h1>
111: <h2>$date ($now)</h2>
112: <ol>
113: <li><a href="#configuration">Configuration</a>
114: <li><a href="#machine">Machine Information</a>
115: <li><a href="#tmp">Temporary Files</a>
116: <li><a href="#tokens">Session Tokens</a>
117: <li><a href="#httpd">httpd</a>
118: <li><a href="#lonsql">lonsql</a>
119: <li><a href="#lond">lond</a>
120: <li><a href="#lonc">lonc</a>
121: <li><a href="#lonnet">lonnet</a>
122: <li><a href="#connections">Connections</a>
123: <li><a href="#delayed">Delayed Messages</a>
124: <li><a href="#errcount">Error Count</a>
125: </ol>
126: <hr>
127: <a name="configuration">
128: <h2>Configuration</h2>
129: <h3>PerlVars</h3>
130: <table border=2>
131: ENDHEADERS
132:
133: foreach $varname (keys %perlvar) {
134: print $fh "<tr><td>$varname</td><td>$perlvar{$varname}</td></tr>\n";
135: }
136: print $fh "</table><h3>Hosts</h3><table border=2>";
137: foreach $id (keys %hostname) {
138: print $fh
139: "<tr><td>$id</td><td>$hostdom{$id}</td><td>$hostrole{$id}</td>";
140: print $fh "<td>$hostname{$id}</td><td>$hostip{$id}</td></tr>\n";
141: }
142: print $fh "</table><h3>Spare Hosts</h3><ol>";
143: foreach $id (keys %spareid) {
144: print $fh "<li>$id\n";
145: }
146:
147: print $fh "</ol>\n";
148:
149: # --------------------------------------------------------------------- Machine
150:
151: print $fh '<hr><a name="machine"><h2>Machine Information</h2>';
152: print $fh "<h3>loadavg</h3>";
153:
154: open (LOADAVGH,"/proc/loadavg");
155: $loadavg=<LOADAVGH>;
156: close (LOADAVGH);
157:
158: print $fh "<tt>$loadavg</tt>";
159:
160: @parts=split(/\s+/,$loadavg);
161: if ($parts[1]>4.0) {
162: $errors++;
163: } elsif ($parts[1]>2.0) {
164: $warnings++;
165: } elsif ($parts[1]>1.0) {
166: $notices++;
167: }
168:
169: print $fh "<h3>df</h3>";
170: print $fh "<pre>";
171:
172: open (DFH,"df|");
173: while ($line=<DFH>) {
174: print $fh "$line";
175: @parts=split(/\s+/,$line);
176: $usage=$parts[4];
177: $usage=~s/\W//g;
178: if ($usage>90) {
179: $warnings++;
180: } elsif ($usage>80) {
181: $warnings++;
182: } elsif ($usage>60) {
183: $notices++;
184: }
185: if ($usage>95) { $warnings++; $warnings++ }
186: }
187: close (DFH);
188: print $fh "</pre>";
189: &errout($fh);
190:
191: # --------------------------------------------------------------- clean out tmp
192: print $fh '<hr><a name="tmp"><h2>Temporary Files</h2>';
193: $cleaned=0;
194: while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) {
195: my ($dev,$ino,$mode,$nlink,
196: $uid,$gid,$rdev,$size,
197: $atime,$mtime,$ctime,
198: $blksize,$blocks)=stat($fname);
199: $now=time;
200: $since=$now-$mtime;
201: if ($since>$perlvar{'lonExpire'}) {
202: $cleaned++;
203: unlink("$fname");
204: }
205:
206: }
207: print $fh "Cleaned up ".$cleaned." files.";
208:
209: # ------------------------------------------------------------ clean out lonIDs
210: print $fh '<hr><a name="tokens"><h2>Session Tokens</h2>';
211: $cleaned=0;
212: $active=0;
213: while ($fname=<$perlvar{'lonIDsDir'}/*>) {
214: my ($dev,$ino,$mode,$nlink,
215: $uid,$gid,$rdev,$size,
216: $atime,$mtime,$ctime,
217: $blksize,$blocks)=stat($fname);
218: $now=time;
219: $since=$now-$mtime;
220: if ($since>$perlvar{'lonExpire'}) {
221: $cleaned++;
222: print $fh "Unlinking $fname<br>";
223: unlink("$fname");
224: } else {
225: $active++;
226: }
227:
228: }
229: print $fh "<p>Cleaned up ".$cleaned." stale session token(s).";
230: print $fh "<h3>$active open session(s)</h3>";
231:
232: # ----------------------------------------------------------------------- httpd
233:
234: print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';
235:
236: open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
237: while ($line=<DFH>) { print $fh "$line" };
238: close (DFH);
239:
240: print $fh "</pre><h3>Error Log</h3><pre>";
241:
242: open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
243: while ($line=<DFH>) {
244: print $fh "$line";
245: if ($line=~/\[error\]/) { $notices++; }
246: };
247: close (DFH);
248: print $fh "</pre>";
249: &errout($fh);
250:
251:
252: # ---------------------------------------------------------------------- lonsql
253: #
254: # Do not run for now
255: #
256: if ($perlvar{'lonRole'} eq "library" && 1==0) {
257:
258: print $fh '<hr><a name="lonsql"><h2>lonsql</h2><h3>Log</h3><pre>';
259:
260: if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
261: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
262: while ($line=<DFH>) {
263: print $fh "$line";
264: if ($line=~/INFO/) { $notices++; }
265: if ($line=~/WARNING/) { $notices++; }
266: if ($line=~/CRITICAL/) { $warnings++; }
267: };
268: close (DFH);
269: }
270: print $fh "</pre>";
271:
272: my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";
273:
274: if (-e $lonsqlfile) {
275: my $lfh=IO::File->new("$lonsqlfile");
276: my $lonsqlpid=<$lfh>;
277: chomp($lonsqlpid);
278: if (kill 0 => $lonsqlpid) {
279: print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
280: } else {
281: $errors++; $errors++;
282: print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
283: }
284: } else {
285: $errors++;
286: print $fh "<h3>lonsql not running, trying to start</h3>";
287: system("$perlvar{'lonDaemons'}/lonsql");
288: sleep 120;
289: if (-e $lonsqlfile) {
290: print $fh "Seems like it started ...<p>";
291: my $lfh=IO::File->new("$lonsqlfile");
292: my $lonsqlpid=<$lfh>;
293: chomp($lonsqlpid);
294: sleep 30;
295: if (kill 0 => $lonsqlpid) {
296: print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
297: } else {
298: $errors++; $errors++;
299: print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
300: print $fh "Give it one more try ...<p>";
301: system("$perlvar{'lonDaemons'}/lonsql");
302: sleep 120;
303: }
304: } else {
305: print $fh "Seems like that did not work!<p>";
306: $errors++;
307: }
308: if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
309: print $fh "<p><pre>";
310: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
311: while ($line=<DFH>) {
312: print $fh "$line";
313: if ($line=~/WARNING/) { $notices++; }
314: if ($line=~/CRITICAL/) { $notices++; }
315: };
316: close (DFH);
317: print $fh "</pre>";
318: }
319: }
320:
321: $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";
322:
323: my ($dev,$ino,$mode,$nlink,
324: $uid,$gid,$rdev,$size,
325: $atime,$mtime,$ctime,
326: $blksize,$blocks)=stat($fname);
327:
328: if ($size>40000) {
329: print $fh "Rotating logs ...<p>";
330: rename("$fname.2","$fname.3");
331: rename("$fname.1","$fname.2");
332: rename("$fname","$fname.1");
333: }
334:
335: &errout($fh);
336: }
337: # ------------------------------------------------------------------------ lond
338:
339: print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';
340:
341: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
342: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
343: while ($line=<DFH>) {
344: print $fh "$line";
345: if ($line=~/INFO/) { $notices++; }
346: if ($line=~/WARNING/) { $notices++; }
347: if ($line=~/CRITICAL/) { $warnings++; }
348: };
349: close (DFH);
350: }
351: print $fh "</pre>";
352:
353: my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";
354:
355: my $restartflag=1;
356: if (-e $londfile) {
357: my $lfh=IO::File->new("$londfile");
358: my $londpid=<$lfh>;
359: chomp($londpid);
360: if (kill 0 => $londpid) {
361: print $fh "<h3>lond at pid $londpid responding</h3>";
362: $restartflag=0;
363: } else {
364: $errors++;
365: print $fh "<h3>lond at pid $londpid not responding</h3>";
366: # Intelligently handle this.
367: # Possibility #1: there is no process
368: # Solution: remove .pid file and restart
369: if (getpgrp($londpid)==-1) {
370: unlink($londfile);
371: $restartflag=1;
372: }
373: else {
374: # Possibility #2: there is a live process that is not responding
375: # for an unknown reason
376: # Solution: kill parent and children processes, remove .pid and restart
377: `killall -9 lond`;
378: unlink($londfile);
379: $restartflag=1;
380: }
381: print $fh
382: "<h3>Deciding to clean up stale .pid file and restart lond</h3>";
383: }
384: }
385: if ($restartflag==1) {
386: $errors++;
387: print $fh "<h3>lond not running, trying to start</h3>";
388: system("$perlvar{'lonDaemons'}/lond");
389: sleep 120;
390: if (-e $londfile) {
391: print $fh "Seems like it started ...<p>";
392: my $lfh=IO::File->new("$londfile");
393: my $londpid=<$lfh>;
394: chomp($londpid);
395: sleep 30;
396: if (kill 0 => $londpid) {
397: print $fh "<h3>lond at pid $londpid responding</h3>";
398: } else {
399: $errors++; $errors++;
400: print $fh "<h3>lond at pid $londpid not responding</h3>";
401: print $fh "Give it one more try ...<p>";
402: system("$perlvar{'lonDaemons'}/lond");
403: sleep 120;
404: }
405: } else {
406: print $fh "Seems like that did not work!<p>";
407: $errors++;
408: }
409: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
410: print $fh "<p><pre>";
411: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
412: while ($line=<DFH>) {
413: print $fh "$line";
414: if ($line=~/WARNING/) { $notices++; }
415: if ($line=~/CRITICAL/) { $notices++; }
416: };
417: close (DFH);
418: print $fh "</pre>";
419: }
420: }
421:
422: $fname="$perlvar{'lonDaemons'}/logs/lond.log";
423:
424: my ($dev,$ino,$mode,$nlink,
425: $uid,$gid,$rdev,$size,
426: $atime,$mtime,$ctime,
427: $blksize,$blocks)=stat($fname);
428:
429: if ($size>40000) {
430: print $fh "Rotating logs ...<p>";
431: rename("$fname.2","$fname.3");
432: rename("$fname.1","$fname.2");
433: rename("$fname","$fname.1");
434: }
435:
436: &errout($fh);
437: # ------------------------------------------------------------------------ lonc
438:
439: print $fh '<hr><a name="lonc"><h2>lonc</h2><h3>Log</h3><pre>';
440:
441: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
442: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
443: while ($line=<DFH>) {
444: print $fh "$line";
445: if ($line=~/INFO/) { $notices++; }
446: if ($line=~/WARNING/) { $notices++; }
447: if ($line=~/CRITICAL/) { $warnings++; }
448: };
449: close (DFH);
450: }
451: print $fh "</pre>";
452:
453: my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
454:
455: $restartflag=1;
456: if (-e $loncfile) {
457: my $lfh=IO::File->new("$loncfile");
458: my $loncpid=<$lfh>;
459: chomp($loncpid);
460: if (kill 0 => $loncpid) {
461: print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";
462: kill USR1 => $loncpid;
463: $restartflag=0;
464: } else {
465: $errors++;
466: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
467: # Intelligently handle this.
468: # Possibility #1: there is no process
469: # Solution: remove .pid file and restart
470: if (getpgrp($loncpid)==-1) {
471: unlink($loncfile);
472: $restartflag=1;
473: }
474: else {
475: # Possibility #2: there is a live process that is not responding
476: # for an unknown reason
477: # Solution: kill parent and children processes, remove .pid and restart
478: `killall -9 lonc`;
479: unlink($loncfile);
480: $restartflag=1;
481: }
482: print $fh
483: "<h3>Deciding to clean up stale .pid file and restart lonc</h3>";
484: }
485: }
486: if ($restartflag==1) {
487: $errors++;
488: print $fh "<h3>lonc not running, trying to start</h3>";
489: system("$perlvar{'lonDaemons'}/lonc");
490: sleep 120;
491: if (-e $loncfile) {
492: print $fh "Seems like it started ...<p>";
493: my $lfh=IO::File->new("$loncfile");
494: my $loncpid=<$lfh>;
495: chomp($loncpid);
496: sleep 30;
497: if (kill 0 => $loncpid) {
498: print $fh "<h3>lonc at pid $loncpid responding</h3>";
499: } else {
500: $errors++; $errors++;
501: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
502: print $fh "Give it one more try ...<p>";
503: system("$perlvar{'lonDaemons'}/lonc");
504: sleep 120;
505: }
506: } else {
507: print $fh "Seems like that did not work!<p>";
508: $errors++;
509: }
510: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") {
511: print $fh "<p><pre>";
512: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
513: while ($line=<DFH>) {
514: print $fh "$line";
515: if ($line=~/WARNING/) { $notices++; }
516: if ($line=~/CRITICAL/) { $notices++; }
517: };
518: close (DFH);
519: print $fh "</pre>";
520: }
521: }
522:
523: $fname="$perlvar{'lonDaemons'}/logs/lonc.log";
524:
525: my ($dev,$ino,$mode,$nlink,
526: $uid,$gid,$rdev,$size,
527: $atime,$mtime,$ctime,
528: $blksize,$blocks)=stat($fname);
529:
530: if ($size>40000) {
531: print $fh "Rotating logs ...<p>";
532: rename("$fname.2","$fname.3");
533: rename("$fname.1","$fname.2");
534: rename("$fname","$fname.1");
535: }
536:
537:
538: &errout($fh);
539: # ---------------------------------------------------------------------- lonnet
540:
541: print $fh '<hr><a name="lonnet"><h2>lonnet</h2><h3>Temp Log</h3><pre>';
542: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
543: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
544: while ($line=<DFH>) {
545: print $fh "$line";
546: };
547: close (DFH);
548: }
549: print $fh "</pre><h3>Perm Log</h3><pre>";
550:
551: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
552: open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
553: while ($line=<DFH>) {
554: print $fh "$line";
555: };
556: close (DFH);
557: } else { print $fh "No perm log\n" }
558:
559: $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
560:
561: my ($dev,$ino,$mode,$nlink,
562: $uid,$gid,$rdev,$size,
563: $atime,$mtime,$ctime,
564: $blksize,$blocks)=stat($fname);
565:
566: if ($size>40000) {
567: print $fh "Rotating logs ...<p>";
568: rename("$fname.2","$fname.3");
569: rename("$fname.1","$fname.2");
570: rename("$fname","$fname.1");
571: }
572:
573: print $fh "</pre>";
574: &errout($fh);
575: # ----------------------------------------------------------------- Connections
576:
577: print $fh '<hr><a name="connections"><h2>Connections</h2>';
578:
579: print $fh "<table border=2>";
580: foreach $tryserver (keys %hostname) {
581:
582: $answer=reply("pong",$tryserver);
583: if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
584: $result="<b>ok</b>";
585: } else {
586: $result=$answer;
587: $warnings++;
588: if ($answer eq 'con_lost') { $warnings++; }
589: }
590: print $fh "<tr><td>$tryserver</td><td>$result</td></tr>\n";
591:
592: }
593: print $fh "</table>";
594:
595: &errout($fh);
596: # ------------------------------------------------------------ Delayed messages
597:
598: print $fh '<hr><a name="delayed"><h2>Delayed Messages</h2>';
599:
600: print $fh '<h3>Scanning Permanent Log</h3>';
601:
602: $unsend=0;
603: {
604: my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
605: while ($line=<$dfh>) {
606: ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
607: if ($sdf eq 'F') {
608: $local=localtime($time);
609: print "<b>Failed: $time, $dserv, $dcmd</b><br>";
610: $warnings++;
611: }
612: if ($sdf eq 'S') { $unsend--; }
613: if ($sdf eq 'D') { $unsend++; }
614: }
615: }
616: print $fh "Total unsend messages: <b>$unsend</b><p>\n";
617: $warnings=$warnings+5*$unsend;
618:
619: print $fh "<h3>Outgoing Buffer</h3>";
620:
621: open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
622: while ($line=<DFH>) {
623: print $fh "$line<br>";
624: };
625: close (DFH);
626:
627: # ------------------------------------------------------------------------- End
628: print $fh "<a name=errcount>\n";
629: $totalcount=$notices+4*$warnings+100*$errors;
630: &errout($fh);
631: print $fh "<h1>Total Error Count: $totalcount</h1>";
632: $now=time;
633: $date=localtime($now);
634: print $fh "<hr>$date ($now)</body></html>\n";
635:
636: }
637:
638: rename ("$statusdir/newstatus.html","$statusdir/index.html");
639:
640: if ($totalcount>200) {
641: $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
642: $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
643: system(
644: "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");
645: }
646: 1;
647:
648:
649:
650:
651:
652:
653:
654:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to loncron CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [LON-CAPA] / loncom