1: #!/usr/bin/perl
2:
3: # The LearningOnline Network
4: # Housekeeping program, started by cron
5: #
6: # (TCP networking package
7: # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30,
8: # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer)
9: #
10: # 7/14,7/15,7/19,7/21,7/22,11/18,
11: # 2/8 Gerd Kortemeyer
12: # Dec 00 Scott Harrison
13: # 12/23 Gerd Kortemeyer
14:
15: use IO::File;
16: use IO::Socket;
17:
18: # -------------------------------------------------- Non-critical communication
19: sub reply {
20: my ($cmd,$server)=@_;
21: my $peerfile="$perlvar{'lonSockDir'}/$server";
22: my $client=IO::Socket::UNIX->new(Peer =>"$peerfile",
23: Type => SOCK_STREAM,
24: Timeout => 10)
25: or return "con_lost";
26: print $client "$cmd\n";
27: my $answer=<$client>;
28: chomp($answer);
29: if (!$answer) { $answer="con_lost"; }
30: return $answer;
31: }
32:
33: # --------------------------------------------------------- Output error status
34:
35: sub errout {
36: my $fh=shift;
37: print $fh (<<ENDERROUT);
38: <p><table border=2 bgcolor="#CCCCCC">
39: <tr><td>Notices</td><td>$notices</td></tr>
40: <tr><td>Warnings</td><td>$warnings</td></tr>
41: <tr><td>Errors</td><td>$errors</td></tr>
42: </table><p><a href="#top">Top</a><p>
43: ENDERROUT
44: }
45:
46: # ================================================================ Main Program
47:
48: # ----------------------------- Make sure this process is running from user=www
49: my $wwwid=getpwnam('www');
50: if ($wwwid!=$<) {
51: print("User ID mismatch. This program must be run as user 'www'\n") unless $noprint;
52: exit 1;
53: }
54:
55: # ------------------------------------------------------------ Read access.conf
56: {
57: my $config=IO::File->new("/etc/httpd/conf/access.conf");
58:
59: while (my $configline=<$config>) {
60: if ($configline =~ /PerlSetVar/) {
61: my ($dummy,$varname,$varvalue)=split(/\s+/,$configline);
62: $perlvar{$varname}=$varvalue;
63: }
64: }
65: }
66:
67: # ------------------------------------------------------------- Read hosts file
68: {
69: my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
70:
71: while (my $configline=<$config>) {
72: my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
73: $hostname{$id}=$name;
74: $hostdom{$id}=$domain;
75: $hostrole{$id}=$role;
76: $hostip{$id}=$ip;
77: if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
78: $libserv{$id}=$name;
79: }
80: }
81: }
82:
83: # ------------------------------------------------------ Read spare server file
84: {
85: my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");
86:
87: while (my $configline=<$config>) {
88: chomp($configline);
89: if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
90: $spareid{$configline}=1;
91: }
92: }
93: }
94:
95: # ---------------------------------------------------------------- Start report
96:
97: $statusdir="/home/httpd/html/lon-status";
98:
99: $errors=0;
100: $warnings=0;
101: $notices=0;
102:
103: $now=time;
104: $date=localtime($now);
105:
106: {
107: my $fh=IO::File->new(">$statusdir/newstatus.html");
108:
109: print $fh (<<ENDHEADERS);
110: <html>
111: <head>
112: <title>LON Status Report $perlvar{'lonHostID'}</title>
113: </head>
114: <body bgcolor="#AAAAAA">
115: <a name="top">
116: <h1>LON Status Report $perlvar{'lonHostID'}</h1>
117: <h2>$date ($now)</h2>
118: <ol>
119: <li><a href="#configuration">Configuration</a>
120: <li><a href="#machine">Machine Information</a>
121: <li><a href="#tmp">Temporary Files</a>
122: <li><a href="#tokens">Session Tokens</a>
123: <li><a href="#httpd">httpd</a>
124: <li><a href="#lonsql">lonsql</a>
125: <li><a href="#lond">lond</a>
126: <li><a href="#lonc">lonc</a>
127: <li><a href="#lonnet">lonnet</a>
128: <li><a href="#connections">Connections</a>
129: <li><a href="#delayed">Delayed Messages</a>
130: <li><a href="#errcount">Error Count</a>
131: </ol>
132: <hr>
133: <a name="configuration">
134: <h2>Configuration</h2>
135: <h3>PerlVars</h3>
136: <table border=2>
137: ENDHEADERS
138:
139: foreach $varname (keys %perlvar) {
140: print $fh "<tr><td>$varname</td><td>$perlvar{$varname}</td></tr>\n";
141: }
142: print $fh "</table><h3>Hosts</h3><table border=2>";
143: foreach $id (keys %hostname) {
144: print $fh
145: "<tr><td>$id</td><td>$hostdom{$id}</td><td>$hostrole{$id}</td>";
146: print $fh "<td>$hostname{$id}</td><td>$hostip{$id}</td></tr>\n";
147: }
148: print $fh "</table><h3>Spare Hosts</h3><ol>";
149: foreach $id (keys %spareid) {
150: print $fh "<li>$id\n";
151: }
152:
153: print $fh "</ol>\n";
154:
155: # --------------------------------------------------------------------- Machine
156:
157: print $fh '<hr><a name="machine"><h2>Machine Information</h2>';
158: print $fh "<h3>loadavg</h3>";
159:
160: open (LOADAVGH,"/proc/loadavg");
161: $loadavg=<LOADAVGH>;
162: close (LOADAVGH);
163:
164: print $fh "<tt>$loadavg</tt>";
165:
166: @parts=split(/\s+/,$loadavg);
167: if ($parts[1]>4.0) {
168: $errors++;
169: } elsif ($parts[1]>2.0) {
170: $warnings++;
171: } elsif ($parts[1]>1.0) {
172: $notices++;
173: }
174:
175: print $fh "<h3>df</h3>";
176: print $fh "<pre>";
177:
178: open (DFH,"df|");
179: while ($line=<DFH>) {
180: print $fh "$line";
181: @parts=split(/\s+/,$line);
182: $usage=$parts[4];
183: $usage=~s/\W//g;
184: if ($usage>90) {
185: $warnings++;
186: } elsif ($usage>80) {
187: $warnings++;
188: } elsif ($usage>60) {
189: $notices++;
190: }
191: if ($usage>95) { $warnings++; $warnings++ }
192: }
193: close (DFH);
194: print $fh "</pre>";
195: &errout($fh);
196:
197: # --------------------------------------------------------------- clean out tmp
198: print $fh '<hr><a name="tmp"><h2>Temporary Files</h2>';
199: $cleaned=0;
200: while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) {
201: my ($dev,$ino,$mode,$nlink,
202: $uid,$gid,$rdev,$size,
203: $atime,$mtime,$ctime,
204: $blksize,$blocks)=stat($fname);
205: $now=time;
206: $since=$now-$mtime;
207: if ($since>$perlvar{'lonExpire'}) {
208: $cleaned++;
209: unlink("$fname");
210: }
211:
212: }
213: print $fh "Cleaned up ".$cleaned." files.";
214:
215: # ------------------------------------------------------------ clean out lonIDs
216: print $fh '<hr><a name="tokens"><h2>Session Tokens</h2>';
217: $cleaned=0;
218: $active=0;
219: while ($fname=<$perlvar{'lonIDsDir'}/*>) {
220: my ($dev,$ino,$mode,$nlink,
221: $uid,$gid,$rdev,$size,
222: $atime,$mtime,$ctime,
223: $blksize,$blocks)=stat($fname);
224: $now=time;
225: $since=$now-$mtime;
226: if ($since>$perlvar{'lonExpire'}) {
227: $cleaned++;
228: print $fh "Unlinking $fname<br>";
229: unlink("$fname");
230: } else {
231: $active++;
232: }
233:
234: }
235: print $fh "<p>Cleaned up ".$cleaned." stale session token(s).";
236: print $fh "<h3>$active open session(s)</h3>";
237:
238: # ----------------------------------------------------------------------- httpd
239:
240: print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';
241:
242: open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
243: while ($line=<DFH>) { print $fh "$line" };
244: close (DFH);
245:
246: print $fh "</pre><h3>Error Log</h3><pre>";
247:
248: open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
249: while ($line=<DFH>) {
250: print $fh "$line";
251: if ($line=~/\[error\]/) { $notices++; }
252: };
253: close (DFH);
254: print $fh "</pre>";
255: &errout($fh);
256:
257:
258: # ---------------------------------------------------------------------- lonsql
259: #
260: # Do not run for now
261: #
262: if ($perlvar{'lonRole'} eq "library" && 1==0) {
263:
264: print $fh '<hr><a name="lonsql"><h2>lonsql</h2><h3>Log</h3><pre>';
265:
266: if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
267: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
268: while ($line=<DFH>) {
269: print $fh "$line";
270: if ($line=~/INFO/) { $notices++; }
271: if ($line=~/WARNING/) { $notices++; }
272: if ($line=~/CRITICAL/) { $warnings++; }
273: };
274: close (DFH);
275: }
276: print $fh "</pre>";
277:
278: my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";
279:
280: if (-e $lonsqlfile) {
281: my $lfh=IO::File->new("$lonsqlfile");
282: my $lonsqlpid=<$lfh>;
283: chomp($lonsqlpid);
284: if (kill 0 => $lonsqlpid) {
285: print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
286: } else {
287: $errors++; $errors++;
288: print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
289: }
290: } else {
291: $errors++;
292: print $fh "<h3>lonsql not running, trying to start</h3>";
293: system("$perlvar{'lonDaemons'}/lonsql");
294: sleep 120;
295: if (-e $lonsqlfile) {
296: print $fh "Seems like it started ...<p>";
297: my $lfh=IO::File->new("$lonsqlfile");
298: my $lonsqlpid=<$lfh>;
299: chomp($lonsqlpid);
300: sleep 30;
301: if (kill 0 => $lonsqlpid) {
302: print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
303: } else {
304: $errors++; $errors++;
305: print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
306: print $fh "Give it one more try ...<p>";
307: system("$perlvar{'lonDaemons'}/lonsql");
308: sleep 120;
309: }
310: } else {
311: print $fh "Seems like that did not work!<p>";
312: $errors++;
313: }
314: if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
315: print $fh "<p><pre>";
316: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
317: while ($line=<DFH>) {
318: print $fh "$line";
319: if ($line=~/WARNING/) { $notices++; }
320: if ($line=~/CRITICAL/) { $notices++; }
321: };
322: close (DFH);
323: print $fh "</pre>";
324: }
325: }
326:
327: $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";
328:
329: my ($dev,$ino,$mode,$nlink,
330: $uid,$gid,$rdev,$size,
331: $atime,$mtime,$ctime,
332: $blksize,$blocks)=stat($fname);
333:
334: if ($size>40000) {
335: print $fh "Rotating logs ...<p>";
336: rename("$fname.2","$fname.3");
337: rename("$fname.1","$fname.2");
338: rename("$fname","$fname.1");
339: }
340:
341: &errout($fh);
342: }
343: # ------------------------------------------------------------------------ lond
344:
345: print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';
346:
347: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
348: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
349: while ($line=<DFH>) {
350: print $fh "$line";
351: if ($line=~/INFO/) { $notices++; }
352: if ($line=~/WARNING/) { $notices++; }
353: if ($line=~/CRITICAL/) { $warnings++; }
354: };
355: close (DFH);
356: }
357: print $fh "</pre>";
358:
359: my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";
360:
361: my $restartflag=1;
362: if (-e $londfile) {
363: my $lfh=IO::File->new("$londfile");
364: my $londpid=<$lfh>;
365: chomp($londpid);
366: if (kill 0 => $londpid) {
367: print $fh "<h3>lond at pid $londpid responding</h3>";
368: $restartflag=0;
369: } else {
370: $errors++;
371: print $fh "<h3>lond at pid $londpid not responding</h3>";
372: # Intelligently handle this.
373: # Possibility #1: there is no process
374: # Solution: remove .pid file and restart
375: if (getpgrp($londpid)==-1) {
376: unlink($londfile);
377: $restartflag=1;
378: }
379: else {
380: # Possibility #2: there is a live process that is not responding
381: # for an unknown reason
382: # Solution: kill parent and children processes, remove .pid and restart
383: `killall -9 lond`;
384: unlink($londfile);
385: $restartflag=1;
386: }
387: print $fh
388: "<h3>Deciding to clean up stale .pid file and restart lond</h3>";
389: }
390: }
391: if ($restartflag==1) {
392: $errors++;
393: print $fh "<h3>lond not running, trying to start</h3>";
394: system("$perlvar{'lonDaemons'}/lond");
395: sleep 120;
396: if (-e $londfile) {
397: print $fh "Seems like it started ...<p>";
398: my $lfh=IO::File->new("$londfile");
399: my $londpid=<$lfh>;
400: chomp($londpid);
401: sleep 30;
402: if (kill 0 => $londpid) {
403: print $fh "<h3>lond at pid $londpid responding</h3>";
404: } else {
405: $errors++; $errors++;
406: print $fh "<h3>lond at pid $londpid not responding</h3>";
407: print $fh "Give it one more try ...<p>";
408: system("$perlvar{'lonDaemons'}/lond");
409: sleep 120;
410: }
411: } else {
412: print $fh "Seems like that did not work!<p>";
413: $errors++;
414: }
415: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
416: print $fh "<p><pre>";
417: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
418: while ($line=<DFH>) {
419: print $fh "$line";
420: if ($line=~/WARNING/) { $notices++; }
421: if ($line=~/CRITICAL/) { $notices++; }
422: };
423: close (DFH);
424: print $fh "</pre>";
425: }
426: }
427:
428: $fname="$perlvar{'lonDaemons'}/logs/lond.log";
429:
430: my ($dev,$ino,$mode,$nlink,
431: $uid,$gid,$rdev,$size,
432: $atime,$mtime,$ctime,
433: $blksize,$blocks)=stat($fname);
434:
435: if ($size>40000) {
436: print $fh "Rotating logs ...<p>";
437: rename("$fname.2","$fname.3");
438: rename("$fname.1","$fname.2");
439: rename("$fname","$fname.1");
440: }
441:
442: &errout($fh);
443: # ------------------------------------------------------------------------ lonc
444:
445: print $fh '<hr><a name="lonc"><h2>lonc</h2><h3>Log</h3><pre>';
446:
447: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
448: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
449: while ($line=<DFH>) {
450: print $fh "$line";
451: if ($line=~/INFO/) { $notices++; }
452: if ($line=~/WARNING/) { $notices++; }
453: if ($line=~/CRITICAL/) { $warnings++; }
454: };
455: close (DFH);
456: }
457: print $fh "</pre>";
458:
459: my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
460:
461: $restartflag=1;
462: if (-e $loncfile) {
463: my $lfh=IO::File->new("$loncfile");
464: my $loncpid=<$lfh>;
465: chomp($loncpid);
466: if (kill 0 => $loncpid) {
467: print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";
468: kill USR1 => $loncpid;
469: $restartflag=0;
470: } else {
471: $errors++;
472: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
473: # Intelligently handle this.
474: # Possibility #1: there is no process
475: # Solution: remove .pid file and restart
476: if (getpgrp($loncpid)==-1) {
477: unlink($loncfile);
478: $restartflag=1;
479: }
480: else {
481: # Possibility #2: there is a live process that is not responding
482: # for an unknown reason
483: # Solution: kill parent and children processes, remove .pid and restart
484: `killall -9 lonc`;
485: unlink($loncfile);
486: $restartflag=1;
487: }
488: print $fh
489: "<h3>Deciding to clean up stale .pid file and restart lonc</h3>";
490: }
491: }
492: if ($restartflag==1) {
493: $errors++;
494: print $fh "<h3>lonc not running, trying to start</h3>";
495: system("$perlvar{'lonDaemons'}/lonc");
496: sleep 120;
497: if (-e $loncfile) {
498: print $fh "Seems like it started ...<p>";
499: my $lfh=IO::File->new("$loncfile");
500: my $loncpid=<$lfh>;
501: chomp($loncpid);
502: sleep 30;
503: if (kill 0 => $loncpid) {
504: print $fh "<h3>lonc at pid $loncpid responding</h3>";
505: } else {
506: $errors++; $errors++;
507: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
508: print $fh "Give it one more try ...<p>";
509: system("$perlvar{'lonDaemons'}/lonc");
510: sleep 120;
511: }
512: } else {
513: print $fh "Seems like that did not work!<p>";
514: $errors++;
515: }
516: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") {
517: print $fh "<p><pre>";
518: open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
519: while ($line=<DFH>) {
520: print $fh "$line";
521: if ($line=~/WARNING/) { $notices++; }
522: if ($line=~/CRITICAL/) { $notices++; }
523: };
524: close (DFH);
525: print $fh "</pre>";
526: }
527: }
528:
529: $fname="$perlvar{'lonDaemons'}/logs/lonc.log";
530:
531: my ($dev,$ino,$mode,$nlink,
532: $uid,$gid,$rdev,$size,
533: $atime,$mtime,$ctime,
534: $blksize,$blocks)=stat($fname);
535:
536: if ($size>40000) {
537: print $fh "Rotating logs ...<p>";
538: rename("$fname.2","$fname.3");
539: rename("$fname.1","$fname.2");
540: rename("$fname","$fname.1");
541: }
542:
543:
544: &errout($fh);
545: # ---------------------------------------------------------------------- lonnet
546:
547: print $fh '<hr><a name="lonnet"><h2>lonnet</h2><h3>Temp Log</h3><pre>';
548: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
549: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
550: while ($line=<DFH>) {
551: print $fh "$line";
552: };
553: close (DFH);
554: }
555: print $fh "</pre><h3>Perm Log</h3><pre>";
556:
557: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
558: open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
559: while ($line=<DFH>) {
560: print $fh "$line";
561: };
562: close (DFH);
563: } else { print $fh "No perm log\n" }
564:
565: $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
566:
567: my ($dev,$ino,$mode,$nlink,
568: $uid,$gid,$rdev,$size,
569: $atime,$mtime,$ctime,
570: $blksize,$blocks)=stat($fname);
571:
572: if ($size>40000) {
573: print $fh "Rotating logs ...<p>";
574: rename("$fname.2","$fname.3");
575: rename("$fname.1","$fname.2");
576: rename("$fname","$fname.1");
577: }
578:
579: print $fh "</pre>";
580: &errout($fh);
581: # ----------------------------------------------------------------- Connections
582:
583: print $fh '<hr><a name="connections"><h2>Connections</h2>';
584:
585: print $fh "<table border=2>";
586: foreach $tryserver (keys %hostname) {
587:
588: $answer=reply("pong",$tryserver);
589: if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
590: $result="<b>ok</b>";
591: } else {
592: $result=$answer;
593: $warnings++;
594: if ($answer eq 'con_lost') { $warnings++; }
595: }
596: print $fh "<tr><td>$tryserver</td><td>$result</td></tr>\n";
597:
598: }
599: print $fh "</table>";
600:
601: &errout($fh);
602: # ------------------------------------------------------------ Delayed messages
603:
604: print $fh '<hr><a name="delayed"><h2>Delayed Messages</h2>';
605:
606: print $fh '<h3>Scanning Permanent Log</h3>';
607:
608: $unsend=0;
609: {
610: my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
611: while ($line=<$dfh>) {
612: ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
613: if ($sdf eq 'F') {
614: $local=localtime($time);
615: print "<b>Failed: $time, $dserv, $dcmd</b><br>";
616: $warnings++;
617: }
618: if ($sdf eq 'S') { $unsend--; }
619: if ($sdf eq 'D') { $unsend++; }
620: }
621: }
622: print $fh "Total unsend messages: <b>$unsend</b><p>\n";
623: $warnings=$warnings+5*$unsend;
624:
625: print $fh "<h3>Outgoing Buffer</h3>";
626:
627: open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
628: while ($line=<DFH>) {
629: print $fh "$line<br>";
630: };
631: close (DFH);
632:
633: # ------------------------------------------------------------------------- End
634: print $fh "<a name=errcount>\n";
635: $totalcount=$notices+4*$warnings+100*$errors;
636: &errout($fh);
637: print $fh "<h1>Total Error Count: $totalcount</h1>";
638: $now=time;
639: $date=localtime($now);
640: print $fh "<hr>$date ($now)</body></html>\n";
641:
642: }
643:
644: rename ("$statusdir/newstatus.html","$statusdir/index.html");
645:
646: if ($totalcount>200) {
647: $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
648: $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
649: system(
650: "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");
651: }
652: 1;
653:
654:
655:
656:
657:
658:
659:
660:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to loncron CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [LON-CAPA] / loncom