1: #!/usr/bin/perl
2:
3: # The LearningOnline Network
4: # Housekeeping program, started by cron
5: #
6: # (TCP networking package
7: # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30,
8: # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer)
9: #
10: # 7/14,7/15,7/19,7/21,7/22 Gerd Kortemeyer
11:
12: use IO::File;
13: use IO::Socket;
14:
15: # -------------------------------------------------- Non-critical communication
16: sub reply {
17: my ($cmd,$server)=@_;
18: my $peerfile="$perlvar{'lonSockDir'}/$server";
19: my $client=IO::Socket::UNIX->new(Peer =>"$peerfile",
20: Type => SOCK_STREAM,
21: Timeout => 10)
22: or return "con_lost";
23: print $client "$cmd\n";
24: my $answer=<$client>;
25: chomp($answer);
26: if (!$answer) { $answer="con_lost"; }
27: return $answer;
28: }
29:
30: # --------------------------------------------------------- Output error status
31:
32: sub errout {
33: my $fh=shift;
34: print $fh (<<ENDERROUT);
35: <p><table border=2 bgcolor="#CCCCCC">
36: <tr><td>Notices</td><td>$notices</td></tr>
37: <tr><td>Warnings</td><td>$warnings</td></tr>
38: <tr><td>Errors</td><td>$errors</td></tr>
39: </table><p><a href="#top">Top</a><p>
40: ENDERROUT
41: }
42:
43: # -------------------------------------------------------------- Permanent logs
44: sub logperm {
45: my $message=shift;
46: my $execdir=$perlvar{'lonDaemons'};
47: my $now=time;
48: my $local=localtime($now);
49: my $fh=Apache::File->new(">>$execdir/logs/lonnet.perm.log");
50: print $fh "$now:$message:$local\n";
51: return 1;
52: }
53:
54: # ------------------------------------------------ Try to send delayed messages
55: sub senddelayed {
56: my $fh=shift;
57: my $dfname;
58: my $path="$perlvar{'lonSockDir'}/delayed";
59: print $fh "<h3>Attempting to send delayed messages</h3>";
60: while ($dfname=<$path/*>) {
61: my $wcmd;
62: {
63: my $dfh=IO::File->new($dfname);
64: $wcmd=<$dfh>;
65: }
66: my ($server,$cmd)=split(/:/,$wcmd);
67: chomp($cmd);
68: my $answer=reply($cmd,$server);
69: if ($answer ne 'con_lost') {
70: unlink("$dfname");
71: print $fh "Send $cmd to $server: $answer<br>\n";
72: &logperm("S:$server:$cmd");
73: } else {
74: print $fh "Failed to deliver $cmd to $server<br>\n";
75: $warnings++;
76: }
77: }
78: }
79:
80: # ================================================================ Main Program
81:
82:
83: # ------------------------------------------------------------ Read access.conf
84: {
85: my $config=IO::File->new("/etc/httpd/conf/access.conf");
86:
87: while (my $configline=<$config>) {
88: if ($configline =~ /PerlSetVar/) {
89: my ($dummy,$varname,$varvalue)=split(/\s+/,$configline);
90: $perlvar{$varname}=$varvalue;
91: }
92: }
93: }
94:
95: # ------------------------------------------------------------- Read hosts file
96: {
97: my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
98:
99: while (my $configline=<$config>) {
100: my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
101: $hostname{$id}=$name;
102: $hostdom{$id}=$domain;
103: $hostrole{$id}=$role;
104: $hostip{$id}=$ip;
105: if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
106: $libserv{$id}=$name;
107: }
108: }
109: }
110:
111: # ------------------------------------------------------ Read spare server file
112: {
113: my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");
114:
115: while (my $configline=<$config>) {
116: chomp($configline);
117: if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
118: $spareid{$configline}=1;
119: }
120: }
121: }
122:
123: # ---------------------------------------------------------------- Start report
124:
125: $statusdir="/home/httpd/html/lon-status";
126:
127: $errors=0;
128: $warnings=0;
129: $notices=0;
130:
131: $now=time;
132: $date=localtime($now);
133:
134: {
135: my $fh=IO::File->new(">$statusdir/newstatus.html");
136:
137: print $fh (<<ENDHEADERS);
138: <html>
139: <head>
140: <title>LON Status Report $perlvar{'lonHostID'}</title>
141: </head>
142: <body bgcolor="#FFFFFF">
143: <a name="top">
144: <h1>LON Status Report $perlvar{'lonHostID'}</h1>
145: <h2>$date ($now)</h2>
146: <ol>
147: <li><a href="#configuration">Configuration</a>
148: <li><a href="#machine">Machine Information</a>
149: <li><a href="#httpd">httpd</a>
150: <li><a href="#lond">lond</a>
151: <li><a href="#lonc">lonc</a>
152: <li><a href="#lonnet">lonnet</a>
153: <li><a href="#connections">Connections</a>
154: <li><a href="#delayed">Delayed Messages</a>
155: <li><a href="#errcount">Error Count</a>
156: </ol>
157: <hr>
158: <a name="configuration">
159: <h2>Configuration</h2>
160: <h3>PerlVars</h3>
161: <table border=2>
162: ENDHEADERS
163:
164: foreach $varname (keys %perlvar) {
165: print $fh "<tr><td>$varname</td><td>$perlvar{$varname}</td></tr>\n";
166: }
167: print $fh "</table><h3>Hosts</h3><table border=2>";
168: foreach $id (keys %hostname) {
169: print $fh
170: "<tr><td>$id</td><td>$hostdom{$id}</td><td>$hostrole{$id}</td>";
171: print $fh "<td>$hostname{$id}</td><td>$hostip{$id}</td></tr>\n";
172: }
173: print $fh "</table><h3>Spare Hosts</h3><ol>";
174: foreach $id (keys %spareid) {
175: print $fh "<li>$id\n";
176: }
177:
178: print $fh "</ol>\n";
179:
180: # --------------------------------------------------------------------- Machine
181:
182: print $fh '<hr><a name="machine"><h2>Machine Information</h2>';
183: print $fh "<h3>loadavg</h3>";
184:
185: open (LOADAVGH,"/proc/loadavg");
186: $loadavg=<LOADAVGH>;
187: close (LOADAVGH);
188:
189: print $fh "<tt>$loadavg</tt>";
190:
191: @parts=split(/\s+/,$loadavg);
192: if ($parts[1]>3.0) {
193: $errors++;
194: } elsif ($parts[1]>2.0) {
195: $warnings++;
196: } elsif ($parts[1]>1.0) {
197: $notices++;
198: }
199:
200: print $fh "<h3>df</h3>";
201: print $fh "<pre>";
202:
203: open (DFH,"df|");
204: while ($line=<DFH>) {
205: print $fh "$line";
206: @parts=split(/\s+/,$line);
207: $usage=$parts[4];
208: $usage=~s/\W//g;
209: if ($usage>90) {
210: $errors++;
211: } elsif ($usage>80) {
212: $warnings++;
213: } elsif ($usage>60) {
214: $notices++;
215: }
216: if ($usage>95) { $errors++; }
217: }
218: close (DFH);
219: print $fh "</pre>";
220: &errout($fh);
221: # ----------------------------------------------------------------------- httpd
222:
223: print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';
224:
225: open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
226: while ($line=<DFH>) { print $fh "$line" };
227: close (DFH);
228:
229: print $fh "</pre><h3>Error Log</h3><pre>";
230:
231: open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
232: while ($line=<DFH>) {
233: print $fh "$line";
234: if ($line=~/\[error\]/) { $notices++; }
235: };
236: close (DFH);
237: print $fh "</pre>";
238: &errout($fh);
239: # ------------------------------------------------------------------------ lond
240:
241: print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';
242:
243: if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
244: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lond.log|");
245: while ($line=<DFH>) {
246: print $fh "$line";
247: if ($line=~/giving up/) { $notices++; }
248: };
249: close (DFH);
250: }
251: print $fh "</pre>";
252:
253: my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";
254:
255: if (-e $londfile) {
256: my $lfh=IO::File->new("$londfile");
257: my $londpid=<$lfh>;
258: chomp($londpid);
259: if (kill 0 => $londpid) {
260: print $fh "<h3>lond at pid $londpid responding</h3>";
261: } else {
262: $errors++; $errors++;
263: print $fh "<h3>lond at pid $londpid not responding</h3>";
264: }
265: } else {
266: $errors++;
267: print $fh "<h3>lond not running, trying to start</h3>";
268: system("$perlvar{'lonDaemons'}/lond");
269: sleep 120;
270: if (-e $londfile) {
271: print $fh "Seems like it started ...<p>";
272: my $lfh=IO::File->new("$londfile");
273: my $londpid=<$lfh>;
274: chomp($londpid);
275: sleep 30;
276: if (kill 0 => $londpid) {
277: print $fh "<h3>lond at pid $londpid responding</h3>";
278: } else {
279: $errors++; $errors++;
280: print $fh "<h3>lond at pid $londpid not responding</h3>";
281: print $fh "Give it one more try ...<p>";
282: system("$perlvar{'lonDaemons'}/lond");
283: sleep 120;
284: }
285: } else {
286: print $fh "Seems like that did not work!<p>";
287: $errors++;
288: }
289: }
290:
291: $fname="$perlvar{'lonDaemons'}/logs/lond.log";
292:
293: my ($dev,$ino,$mode,$nlink,
294: $uid,$gid,$rdev,$size,
295: $atime,$mtime,$ctime,
296: $blksize,$blocks)=stat($fname);
297:
298: if ($size>40000) {
299: print $fh "Rotating logs ...<p>";
300: rename("$fname.2","$fname.3");
301: rename("$fname.1","$fname.2");
302: rename("$fname","$fname.1");
303: }
304:
305: &errout($fh);
306: # ------------------------------------------------------------------------ lonc
307:
308: print $fh '<hr><a name="lonc"><h2>lonc</h2><h3>Log</h3><pre>';
309:
310: if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
311: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonc.log|");
312: while ($line=<DFH>) {
313: print $fh "$line";
314: if ($line=~/died/) { $notices++; }
315: };
316: close (DFH);
317: }
318: print $fh "</pre>";
319:
320: my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
321:
322: if (-e $loncfile) {
323: my $lfh=IO::File->new("$loncfile");
324: my $loncpid=<$lfh>;
325: chomp($loncpid);
326: if (kill 0 => $loncpid) {
327: print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";
328: kill USR1 => $loncpid;
329: } else {
330: $errors++; $errors++;
331: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
332: }
333: } else {
334: $errors++;
335: print $fh "<h3>lonc not running, trying to start</h3>";
336: system("$perlvar{'lonDaemons'}/lonc");
337: sleep 120;
338: if (-e $loncfile) {
339: print $fh "Seems like it started ...<p>";
340: my $lfh=IO::File->new("$loncfile");
341: my $loncpid=<$lfh>;
342: chomp($loncpid);
343: sleep 30;
344: if (kill 0 => $loncpid) {
345: print $fh "<h3>lonc at pid $loncpid responding</h3>";
346: } else {
347: $errors++; $errors++;
348: print $fh "<h3>lonc at pid $loncpid not responding</h3>";
349: print $fh "Give it one more try ...<p>";
350: system("$perlvar{'lonDaemons'}/lonc");
351: sleep 120;
352: }
353: } else {
354: print $fh "Seems like that did not work!<p>";
355: $errors++;
356: }
357: }
358:
359: $fname="$perlvar{'lonDaemons'}/logs/lonc.log";
360:
361: my ($dev,$ino,$mode,$nlink,
362: $uid,$gid,$rdev,$size,
363: $atime,$mtime,$ctime,
364: $blksize,$blocks)=stat($fname);
365:
366: if ($size>40000) {
367: print $fh "Rotating logs ...<p>";
368: rename("$fname.2","$fname.3");
369: rename("$fname.1","$fname.2");
370: rename("$fname","$fname.1");
371: }
372:
373:
374: &errout($fh);
375: # ---------------------------------------------------------------------- lonnet
376:
377: print $fh '<hr><a name="lonnet"><h2>lonnet</h2><h3>Temp Log</h3><pre>';
378: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
379: open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
380: while ($line=<DFH>) {
381: print $fh "$line";
382: if ($line=~/Delayed/) { $warnings++; }
383: if ($line=~/giving up/) { $warnings++; }
384: if ($line=~/FAILED/) { $errors++; }
385: };
386: close (DFH);
387: }
388: print $fh "</pre><h3>Perm Log</h3>";
389:
390: if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
391: open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
392: while ($line=<DFH>) {
393: print $fh "$line";
394: };
395: close (DFH);
396: } else { print $fh "No perm log\n" }
397:
398: $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
399:
400: my ($dev,$ino,$mode,$nlink,
401: $uid,$gid,$rdev,$size,
402: $atime,$mtime,$ctime,
403: $blksize,$blocks)=stat($fname);
404:
405: if ($size>40000) {
406: print $fh "Rotating logs ...<p>";
407: rename("$fname.2","$fname.3");
408: rename("$fname.1","$fname.2");
409: rename("$fname","$fname.1");
410: }
411:
412: print $fh "</pre>";
413: &errout($fh);
414: # ----------------------------------------------------------------- Connections
415:
416: print $fh '<hr><a name="connections"><h2>Connections</h2>';
417:
418: print $fh "<table border=2>";
419: foreach $tryserver (keys %hostname) {
420:
421: $answer=reply("pong",$tryserver);
422: if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
423: $result="<b>ok</b>";
424: } else {
425: $result=$answer;
426: $warnings++;
427: if ($answer eq 'con_lost') { $warnings++; }
428: }
429: print $fh "<tr><td>$tryserver</td><td>$result</td></tr>\n";
430:
431: }
432: print $fh "</table>";
433:
434: &errout($fh);
435: # ------------------------------------------------------------ Delayed messages
436:
437: print $fh '<hr><a name="delayed"><h2>Delayed Messages</h2>';
438:
439: &senddelayed($fh);
440:
441: print $fh '<h3>Scanning Permanent Log</h3>';
442:
443: $unsend=0;
444: {
445: my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
446: while ($line=<$dfh>) {
447: ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
448: if ($sdf eq 'F') {
449: $local=localtime($time);
450: print "<b>Failed: $time, $dserv, $dcmd</b><br>";
451: $warnings++;
452: }
453: if ($sdf eq 'S') { $unsend--; }
454: if ($sdf eq 'D') { $unsend++; }
455: }
456: }
457: print $fh "Total unsend messages: <b>$unsend</b><p>\n";
458: $warnings=$warnings+5*$unsend;
459:
460: print $fh "<h3>Outgoing Buffer</h3>";
461:
462: open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
463: while ($line=<DFH>) {
464: print $fh "$line<br>";
465: };
466: close (DFH);
467:
468: # ------------------------------------------------------------------------- End
469: print $fh "<a name=errcount>\n";
470: $totalcount=$notices+4*$warnings+100*$errors;
471: &errout($fh);
472: print $fh "<h1>Total Error Count: $totalcount</h1>";
473: $now=time;
474: $date=localtime($now);
475: print $fh "<hr>$date ($now)</body></html>\n";
476:
477: }
478:
479: rename ("$statusdir/newstatus.html","$statusdir/index.html");
480:
481: if ($totalcount>200) {
482: $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
483: $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
484: system(
485: "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");
486: }
487: 1;
488:
489:
490:
491:
492:
493:
494:
495:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to loncron CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [LON-CAPA] / loncom