File:  [LON-CAPA] / loncom / loncnew
Revision 1.18: download - view: text, annotated - select for diffs
Wed Aug 6 09:52:29 2003 UTC (20 years, 8 months ago) by foxr
Branches: MAIN
CVS tags: version_1_0_3, version_1_0_2, version_1_0_1, version_1_0_0, version_0_99_5, version_0_99_4, HEAD
Also needed to remember to fail in-flight transactions if their sends fail.

    1: #!/usr/bin/perl
    2: # The LearningOnline Network with CAPA
    3: # lonc maintains the connections to remote computers
    4: #
    5: # $Id: loncnew,v 1.18 2003/08/06 09:52:29 foxr Exp $
    6: #
    7: # Copyright Michigan State University Board of Trustees
    8: #
    9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
   10: ## LON-CAPA is free software; you can redistribute it and/or modify
   11: # it under the terms of the GNU General Public License as published by
   12: # the Free Software Foundation; either version 2 of the License, or
   13: # (at your option) any later version.
   14: #
   15: # LON-CAPA is distributed in the hope that it will be useful,
   16: # but WITHOUT ANY WARRANTY; without even the implied warranty of
   17: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   18: # GNU General Public License for more details.
   19: #
   20: # You should have received a copy of the GNU General Public License
   21: # along with LON-CAPA; if not, write to the Free Software
   22: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   23: #
   24: # /home/httpd/html/adm/gpl.txt
   25: #
   26: # http://www.lon-capa.org/
   27: #
   28: #
   29: # new lonc handles n request out bver m connections to londs.
   30: # This module is based on the Event class.
   31: #   Development iterations:
   32: #    - Setup basic event loop.   (done)
   33: #    - Add timer dispatch.       (done)
   34: #    - Add ability to accept lonc UNIX domain sockets.  (done)
   35: #    - Add ability to create/negotiate lond connections (done).
   36: #    - Add general logic for dispatching requests and timeouts. (done).
   37: #    - Add support for the lonc/lond requests.          (done).
   38: #    - Add logging/status monitoring.
   39: #    - Add Signal handling - HUP restarts. USR1 status report.
   40: #    - Add Configuration file I/O                       (done).
   41: #    - Add management/status request interface.
   42: #    - Add deferred request capability.                  (done)
   43: #    - Detect transmission timeouts.
   44: #
   45: 
   46: # Change log:
   47: #    $Log: loncnew,v $
   48: #    Revision 1.18  2003/08/06 09:52:29  foxr
   49: #    Also needed to remember to fail in-flight transactions if their sends fail.
   50: #
   51: #    Revision 1.17  2003/08/03 00:44:31  foxr
   52: #    1. Correct handling of connection failure: Assume it means the host is
   53: #       unreachable and fail all of the queued transactions.  Note that the
   54: #       inflight transactions should fail on their own time due either to timeout
   55: #       or send/receive failures.
   56: #    2. Correct handling of logs for forced death signals.  Pull the signal
   57: #       from the event watcher.
   58: #
   59: #    Revision 1.16  2003/07/29 02:33:05  foxr
   60: #    Add SIGINT processing to child processes to toggle annoying trace mode
   61: #    on/off.. will try to use this to isolate the compute boud process issue.
   62: #
   63: #    Revision 1.15  2003/07/15 02:07:05  foxr
   64: #    Added code for lonc/lond transaction timeouts.  Who knows if it works right.
   65: #    The intent is for a timeout to fail any transaction in progress and kill
   66: #    off the sockt that timed out.
   67: #
   68: #    Revision 1.14  2003/07/03 02:10:18  foxr
   69: #    Get all of the signals to work correctly.
   70: #
   71: #    Revision 1.13  2003/07/02 01:31:55  foxr
   72: #    Added kill -HUP logic (restart).
   73: #
   74: #    Revision 1.11  2003/06/25 01:54:44  foxr
   75: #    Fix more problems with transaction failure.
   76: #
   77: #    Revision 1.10  2003/06/24 02:46:04  foxr
   78: #    Put a limit on  the number of times we'll retry a connection.
   79: #    Start getting the signal stuff put in as well...note that need to get signals
   80: #    going or else 6the client will permanently give up on dead servers.
   81: #
   82: #    Revision 1.9  2003/06/13 02:38:43  foxr
   83: #    Add logging in 'expected format'
   84: #
   85: #    Revision 1.8  2003/06/11 02:04:35  foxr
   86: #    Support delayed transactions... this is done uniformly by encapsulating
   87: #    transactions in an object ... a LondTransaction that is implemented by
   88: #    LondTransaction.pm
   89: #
   90: #    Revision 1.7  2003/06/03 01:59:39  foxr
   91: #    complete coding to support deferred transactions.
   92: #
   93: #
   94: 
   95: use lib "/home/httpd/lib/perl/";
   96: use lib "/home/foxr/newloncapa/types";
   97: use Event qw(:DEFAULT );
   98: use POSIX qw(:signal_h);
   99: use POSIX;
  100: use IO::Socket;
  101: use IO::Socket::INET;
  102: use IO::Socket::UNIX;
  103: use IO::File;
  104: use IO::Handle;
  105: use Socket;
  106: use Crypt::IDEA;
  107: use LONCAPA::Queue;
  108: use LONCAPA::Stack;
  109: use LONCAPA::LondConnection;
  110: use LONCAPA::LondTransaction;
  111: use LONCAPA::Configuration;
  112: use LONCAPA::HashIterator;
  113: 
  114: 
  115: #
  116: #   Disable all signals we might receive from outside for now.
  117: #
  118: #$SIG{QUIT}  = IGNORE;
  119: #$SIG{HUP}   = IGNORE;
  120: #$SIG{USR1}  = IGNORE;
  121: #$SIG{INT}   = IGNORE;
  122: #$SIG{CHLD}  = IGNORE;
  123: #$SIG{__DIE__}  = IGNORE;
  124: 
  125: 
  126: # Read the httpd configuration file to get perl variables
  127: # normally set in apache modules:
  128: 
  129: my $perlvarref = LONCAPA::Configuration::read_conf('loncapa.conf');
  130: my %perlvar    = %{$perlvarref};
  131: 
  132: #
  133: #  parent and shared variables.
  134: 
  135: my %ChildHash;			# by pid -> host.
  136: 
  137: 
  138: my $MaxConnectionCount = 10;	# Will get from config later.
  139: my $ClientConnection = 0;	# Uniquifier for client events.
  140: 
  141: my $DebugLevel = 0;
  142: my $NextDebugLevel= 10;		# So Sigint can toggle this.
  143: my $IdleTimeout= 3600;		# Wait an hour before pruning connections.
  144: 
  145: #
  146: #  The variables below are only used by the child processes.
  147: #
  148: my $RemoteHost;			# Name of host child is talking to.
  149: my $UnixSocketDir= "/home/httpd/sockets"; 
  150: my $IdleConnections = Stack->new(); # Set of idle connections
  151: my %ActiveConnections;		# Connections to the remote lond.
  152: my %ActiveTransactions;		# LondTransactions in flight.
  153: my %ActiveClients;		# Serial numbers of active clients by socket.
  154: my $WorkQueue       = Queue->new(); # Queue of pending transactions.
  155: my $ConnectionCount = 0;
  156: my $IdleSeconds     = 0;	# Number of seconds idle.
  157: my $Status          = "";	# Current status string.
  158: my $RecentLogEntry  = "";
  159: my $ConnectionRetries=5;	# Number of connection retries allowed.
  160: my $ConnectionRetriesLeft=5;	# Number of connection retries remaining.
  161: 
  162: #
  163: #   The hash below gives the HTML format for log messages
  164: #   given a severity.
  165: #    
  166: my %LogFormats;
  167: 
  168: $LogFormats{"CRITICAL"} = "<font color=red>CRITICAL: %s</font>";
  169: $LogFormats{"SUCCESS"}  = "<font color=green>SUCCESS: %s</font>";
  170: $LogFormats{"INFO"}     = "<font color=yellow>INFO: %s</font>";
  171: $LogFormats{"WARNING"}  = "<font color=blue>WARNING: %s</font>";
  172: $LogFormats{"DEFAULT"}  = " %s ";
  173: 
  174: 
  175: 
  176: =pod
  177: 
  178: =head2 LogPerm
  179: 
  180: Makes an entry into the permanent log file.
  181: 
  182: =cut
  183: sub LogPerm {
  184:     my $message=shift;
  185:     my $execdir=$perlvar{'lonDaemons'};
  186:     my $now=time;
  187:     my $local=localtime($now);
  188:     my $fh=IO::File->new(">>$execdir/logs/lonnet.perm.log");
  189:     print $fh "$now:$message:$local\n";
  190: }
  191: 
  192: =pod
  193: 
  194: =head2 Log
  195: 
  196: Logs a message to the log file.
  197: Parameters:
  198: 
  199: =item severity
  200: 
  201: One of CRITICAL, WARNING, INFO, SUCCESS used to select the
  202: format string used to format the message.  if the severity is
  203: not a defined severity the Default format string is used.
  204: 
  205: =item message
  206: 
  207: The base message.  In addtion to the format string, the message
  208: will be appended to a string containing the name of our remote
  209: host and the time will be formatted into the message.
  210: 
  211: =cut
  212: 
  213: sub Log {
  214:     my $severity = shift;
  215:     my $message  = shift;
  216:    
  217:     if(!$LogFormats{$severity}) {
  218: 	$severity = "DEFAULT";
  219:     }
  220: 
  221:     my $format = $LogFormats{$severity};
  222:     
  223:     #  Put the window dressing in in front of the message format:
  224: 
  225:     my $now   = time;
  226:     my $local = localtime($now);
  227:     my $finalformat = "$local ($$) [$RemoteHost] [$Status] ";
  228:     my $finalformat = $finalformat.$format."\n";
  229: 
  230:     # open the file and put the result.
  231: 
  232:     my $execdir = $perlvar{'lonDaemons'};
  233:     my $fh      = IO::File->new(">>$execdir/logs/lonc.log");
  234:     my $msg = sprintf($finalformat, $message);
  235:     $RecentLogEntry = $msg;
  236:     print $fh $msg;
  237:     
  238:     
  239: }
  240: 
  241: 
  242: =pod
  243: 
  244: =head2 GetPeerName
  245: 
  246: Returns the name of the host that a socket object is connected to.
  247: 
  248: =cut
  249: 
  250: sub GetPeername {
  251:     my $connection = shift;
  252:     my $AdrFamily  = shift;
  253:     my $peer       = $connection->peername();
  254:     my $peerport;
  255:     my $peerip;
  256:     if($AdrFamily == AF_INET) {
  257: 	($peerport, $peerip) = sockaddr_in($peer);
  258: 	my $peername    = gethostbyaddr($iaddr, $AdrFamily);
  259: 	return $peername;
  260:     } elsif ($AdrFamily == AF_UNIX) {
  261: 	my $peerfile;
  262: 	($peerfile) = sockaddr_un($peer);
  263: 	return $peerfile;
  264:     }
  265: }
  266: #----------------------------- Timer management ------------------------
  267: =pod
  268: 
  269: =head2 Debug
  270: 
  271: Invoked to issue a debug message.
  272: 
  273: =cut
  274: 
  275: sub Debug {
  276:     my $level   = shift;
  277:     my $message = shift;
  278:     if ($level <= $DebugLevel) {
  279: 	Log("INFO", "-Debug- $message host = $RemotHost");
  280:     }
  281: }
  282: 
  283: sub SocketDump {
  284:     my $level = shift;
  285:     my $socket= shift;
  286:     if($level <= $DebugLevel) {
  287: 	$socket->Dump();
  288:     }
  289: }
  290: 
  291: =pod
  292: 
  293: =head2 ShowStatus
  294: 
  295:  Place some text as our pid status.
  296:  and as what we return in a SIGUSR1
  297: 
  298: =cut
  299: sub ShowStatus {
  300:     my $state = shift;
  301:     my $now = time;
  302:     my $local = localtime($now);
  303:     $Status   = $local.": ".$state;
  304:     $0='lonc: '.$state.' '.$local;
  305: }
  306: 
  307: =pod
  308: 
  309: =head 2 SocketTimeout
  310: 
  311:     Called when an action on the socket times out.  The socket is 
  312:    destroyed and any active transaction is failed.
  313: 
  314: 
  315: =cut
  316: sub SocketTimeout {
  317:     my $Socket = shift;
  318:     
  319:     KillSocket($Socket);
  320: }
  321: 
  322: =pod
  323: 
  324: =head2 Tick
  325: 
  326: Invoked  each timer tick.
  327: 
  328: =cut
  329: 
  330: 
  331: sub Tick {
  332:     my $client;
  333:     ShowStatus(GetServerHost()." Connection count: ".$ConnectionCount);
  334: 
  335:     # Is it time to prune connection count:
  336: 
  337: 
  338:     if($IdleConnections->Count()  && 
  339:        ($WorkQueue->Count() == 0)) { # Idle connections and nothing to do?
  340: 	$IdleSeconds++;
  341: 	if($IdleSeconds > $IdleTimeout) { # Prune a connection...
  342: 	    $Socket = $IdleConnections->pop();
  343: 	    KillSocket($Socket);
  344: 	}
  345:     } else {
  346: 	$IdleSeconds = 0;	# Reset idle count if not idle.
  347:     }
  348:     #
  349:     #  For each inflight transaction, tick down its timeout counter.
  350:     #
  351:     foreach $item (keys %ActiveTransactions) {
  352: 	my $Socket = $ActiveTransactions{$item}->getServer();
  353: 	$Socket->Tick();
  354:     }
  355:     # Do we have work in the queue, but no connections to service them?
  356:     # If so, try to make some new connections to get things going again.
  357:     #
  358:     
  359:     my $Requests = $WorkQueue->Count();
  360:     if (($ConnectionCount == 0)  && ($Requests > 0)) { 
  361: 	if ($ConnectionRetriesLeft > 0) {
  362: 	    my $Connections = ($Requests <= $MaxConnectionCount) ?
  363: 		$Requests : $MaxConnectionCount;
  364: 	    Debug(1,"Work but no connections, start ".$Connections." of them");
  365: 	    for ($i =0; $i < $Connections; $i++) {
  366: 		MakeLondConnection();
  367: 	    }
  368: 	} else {
  369: 	    Debug(1,"Work in queue, but gave up on connections..flushing\n");
  370: 	    EmptyQueue();	# Connections can't be established.
  371: 	}
  372:        
  373:     }
  374: }
  375: 
  376: =pod
  377: 
  378: =head2 SetupTimer
  379: 
  380: Sets up a 1 per sec recurring timer event.  The event handler is used to:
  381: 
  382: =item
  383: 
  384: Trigger timeouts on communications along active sockets.
  385: 
  386: =item
  387: 
  388: Trigger disconnections of idle sockets.
  389: 
  390: =cut
  391: 
  392: sub SetupTimer {
  393:     Debug(6, "SetupTimer");
  394:     Event->timer(interval => 1, debug => 1, cb => \&Tick );
  395: }
  396: 
  397: =pod
  398: 
  399: =head2 ServerToIdle
  400: 
  401: This function is called when a connection to the server is
  402: ready for more work.
  403: 
  404: If there is work in the Work queue the top element is dequeued
  405: and the connection will start to work on it.  If the work queue is
  406: empty, the connection is pushed on the idle connection stack where
  407: it will either get another work unit, or alternatively, if it sits there
  408: long enough, it will be shut down and released.
  409: 
  410: =cut
  411: 
  412: sub ServerToIdle {
  413:     my $Socket   = shift;	# Get the socket.
  414:     delete($ActiveTransactions{$Socket}); # Server has no transaction
  415: 
  416:     &Debug(6, "Server to idle");
  417: 
  418:     #  If there's work to do, start the transaction:
  419: 
  420:     $reqdata = $WorkQueue->dequeue(); # This is a LondTransaction
  421:     unless($reqdata eq undef)  {
  422: 	Debug(9, "Queue gave request data: ".$reqdata->getRequest());
  423: 	&StartRequest($Socket,  $reqdata);
  424: 
  425:     } else {
  426: 	
  427:     #  There's no work waiting, so push the server to idle list.
  428: 	&Debug(8, "No new work requests, server connection going idle");
  429: 	$IdleConnections->push($Socket);
  430:     }
  431: }
  432: 
  433: =pod
  434: 
  435: =head2 ClientWritable
  436: 
  437: Event callback for when a client socket is writable.
  438: 
  439: This callback is established when a transaction reponse is
  440: avaiable from lond.  The response is forwarded to the unix socket
  441: as it becomes writable in this sub.
  442: 
  443: Parameters:
  444: 
  445: =item Event
  446: 
  447: The event that has been triggered. Event->w->data is
  448: the data and Event->w->fd is the socket to write.
  449: 
  450: =cut
  451: 
  452: sub ClientWritable {
  453:     my $Event    = shift;
  454:     my $Watcher  = $Event->w;
  455:     my $Data     = $Watcher->data;
  456:     my $Socket   = $Watcher->fd;
  457: 
  458:     # Try to send the data:
  459: 
  460:     &Debug(6, "ClientWritable writing".$Data);
  461:     &Debug(9, "Socket is: ".$Socket);
  462: 
  463:     if($Socket->connected) {
  464: 	my $result = $Socket->send($Data, 0);
  465: 	
  466: 	# $result undefined: the write failed.
  467: 	# otherwise $result is the number of bytes written.
  468: 	# Remove that preceding string from the data.
  469: 	# If the resulting data is empty, destroy the watcher
  470: 	# and set up a read event handler to accept the next
  471: 	# request.
  472: 	
  473: 	&Debug(9,"Send result is ".$result." Defined: ".defined($result));
  474: 	if(defined($result)) {
  475: 	    &Debug(9, "send result was defined");
  476: 	    if($result == length($Data)) { # Entire string sent.
  477: 		&Debug(9, "ClientWritable data all written");
  478: 		$Watcher->cancel();
  479: 		#
  480: 		#  Set up to read next request from socket:
  481: 		
  482: 		my $descr     = sprintf("Connection to lonc client %d",
  483: 					$ActiveClients{$Socket});
  484: 		Event->io(cb    => \&ClientRequest,
  485: 			  poll  => 'r',
  486: 			  desc  => $descr,
  487: 			  data  => "",
  488: 			  fd    => $Socket);
  489: 		
  490: 	    } else {		# Partial string sent.
  491: 		$Watcher->data(substr($Data, $result));
  492: 		if($result == 0) {    # client hung up on us!!
  493: 		    Log("INFO", "lonc pipe client hung up on us!");
  494: 		    $Watcher->cancel;
  495: 		    $Socket->shutdown(2);
  496: 		    $Socket->close();
  497: 		}
  498: 	    }
  499: 	    
  500: 	} else {			# Error of some sort...
  501: 	    
  502: 	    # Some errnos are possible:
  503: 	    my $errno = $!;
  504: 	    if($errno == POSIX::EWOULDBLOCK   ||
  505: 	       $errno == POSIX::EAGAIN        ||
  506: 	       $errno == POSIX::EINTR) {
  507: 		# No action taken?
  508: 	    } else {		# Unanticipated errno.
  509: 		&Debug(5,"ClientWritable error or peer shutdown".$RemoteHost);
  510: 		$Watcher->cancel;	# Stop the watcher.
  511: 		$Socket->shutdown(2); # Kill connection
  512: 		$Socket->close();	# Close the socket.
  513: 	    }
  514: 	    
  515: 	}
  516:     } else {
  517: 	$Watcher->cancel();	# A delayed request...just cancel.
  518:     }
  519: }
  520: 
  521: =pod
  522: 
  523: =head2 CompleteTransaction
  524: 
  525: Called when the reply data has been received for a lond 
  526: transaction.   The reply data must now be sent to the
  527: ultimate client on the other end of the Unix socket.  This is
  528: done by setting up a writable event for the socket with the
  529: data the reply data.
  530: 
  531: Parameters:
  532: 
  533: =item Socket
  534: 
  535: Socket on which the lond transaction occured.  This is a
  536: LondConnection. The data received is in the TransactionReply member.
  537: 
  538: =item Transaction
  539: 
  540: The transaction that is being completed.
  541: 
  542: =cut
  543: 
  544: sub CompleteTransaction {
  545:     &Debug(6,"Complete transaction");
  546:     my $Socket = shift;
  547:     my $Transaction = shift;
  548: 
  549:     if (!$Transaction->isDeferred()) { # Normal transaction
  550: 	my $data   = $Socket->GetReply(); # Data to send.
  551: 	StartClientReply($Transaction, $data);
  552:     } else {			# Delete deferred transaction file.
  553: 	Log("SUCCESS", "A delayed transaction was completed");
  554: 	LogPerm("S:$Client:".$Transaction->getRequest());
  555: 	unlink $Transaction->getFile();
  556:     }
  557: }
  558: =pod
  559: =head1 StartClientReply
  560: 
  561:    Initiates a reply to a client where the reply data is a parameter.
  562: 
  563: =head2  parameters:
  564: 
  565: =item Transaction
  566: 
  567:     The transaction for which we are responding to the client.
  568: 
  569: =item data
  570: 
  571:     The data to send to apached client.
  572: 
  573: =cut
  574: sub StartClientReply {
  575:     my $Transaction   = shift;
  576:     my $data     = shift;
  577: 
  578: 
  579:     my $Client   = $Transaction->getClient();
  580: 
  581:     &Debug(8," Reply was: ".$data);
  582:     my $Serial         = $ActiveClients{$Client};
  583:     my $desc           = sprintf("Connection to lonc client %d",
  584: 
  585: 				 $Serial);
  586:     Event->io(fd       => $Client,
  587: 	      poll     => "w",
  588: 	      desc     => $desc,
  589: 	      cb       => \&ClientWritable,
  590: 	      data     => $data);
  591: }
  592: =pod
  593: =head2 FailTransaction
  594: 
  595:   Finishes a transaction with failure because the associated lond socket
  596:   disconnected.  There are two possibilities:
  597:   - The transaction is deferred: in which case we just quietly
  598:     delete the transaction since there is no client connection.
  599:   - The transaction is 'live' in which case we initiate the sending
  600:     of "con_lost" to the client.
  601: 
  602: Deleting the transaction means killing it from the 
  603: %ActiveTransactions hash.
  604: 
  605: Parameters:
  606: 
  607: =item client  
  608:  
  609:    The LondTransaction we are failing.
  610:  
  611: =cut
  612: 
  613: sub FailTransaction {
  614:     my $transaction = shift;
  615:     Log("WARNING", "Failing transaction ".$transaction->getRequest());
  616:     Debug(1, "Failing transaction: ".$transaction->getRequest());
  617:     if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it.
  618: 	my $client  = $transaction->getClient();
  619: 	Debug(1," Replying con_lost to ".$transaction->getRequest());
  620: 	StartClientReply($transaction, "con_lost\n");
  621:     }
  622: 
  623: }
  624: 
  625: =pod
  626: =head1  EmptyQueue
  627: 
  628:   Fails all items in the work queue with con_lost.
  629:   Note that each item in the work queue is a transaction.
  630: 
  631: =cut
  632: sub EmptyQueue {
  633:     while($WorkQueue->Count()) {
  634: 	my $request = $WorkQueue->dequeue(); # This is a transaction
  635: 	FailTransaction($request);
  636:     }
  637: }
  638: 
  639: =pod
  640: 
  641: =head2 CloseAllLondConnections
  642: 
  643: Close all connections open on lond prior to exit e.g.
  644: 
  645: =cut
  646: sub CloseAllLondConnections {
  647:     foreach $Socket (keys %ActiveConnections) {
  648: 	KillSocket($Socket);
  649:     }
  650: }
  651: =cut
  652: 
  653: =pod
  654: 
  655: =head2 KillSocket
  656:  
  657: Destroys a socket.  This function can be called either when a socket
  658: has died of 'natural' causes or because a socket needs to be pruned due to
  659: idleness.  If the socket has died naturally, if there are no longer any 
  660: live connections a new connection is created (in case there are transactions
  661: in the queue).  If the socket has been pruned, it is never re-created.
  662: 
  663: Parameters:
  664: 
  665: =item Socket
  666:  
  667:   The socket to kill off.
  668: 
  669: =item Restart
  670: 
  671: nonzero if we are allowed to create a new connection.
  672: 
  673: 
  674: =cut
  675: sub KillSocket {
  676:     my $Socket = shift;
  677: 
  678:     Log("WARNING", "Shutting down a socket");
  679:     $Socket->Shutdown();
  680: 
  681:     #  If the socket came from the active connection set,
  682:     #  delete its transaction... note that FailTransaction should
  683:     #  already have been called!!!
  684:     #  otherwise it came from the idle set.
  685:     #  
  686:     
  687:     if(exists($ActiveTransactions{$Socket})) {
  688: 	delete ($ActiveTransactions{$Socket});
  689:     }
  690:     if(exists($ActiveConnections{$Socket})) {
  691: 	delete($ActiveConnections{$Socket});
  692:     }
  693:     $ConnectionCount--;
  694: 
  695:     #  If the connection count has gone to zero and there is work in the
  696:     #  work queue, the work all gets failed with con_lost.
  697:     #
  698:     if($ConnectionCount == 0) {
  699: 	EmptyQueue;
  700:     }
  701: }
  702: 
  703: =pod
  704: 
  705: =head2 LondReadable
  706: 
  707: This function is called whenever a lond connection
  708: is readable.  The action is state dependent:
  709: 
  710: =head3 State=Initialized
  711: 
  712: We''re waiting for the challenge, this is a no-op until the
  713: state changes.
  714: 
  715: =head3 State=Challenged 
  716: 
  717: The challenge has arrived we need to transition to Writable.
  718: The connection must echo the challenge back.
  719: 
  720: =head3 State=ChallengeReplied
  721: 
  722: The challenge has been replied to.  The we are receiveing the 
  723: 'ok' from the partner.
  724: 
  725: =head3 State=RequestingKey
  726: 
  727: The ok has been received and we need to send the request for
  728: an encryption key.  Transition to writable for that.
  729: 
  730: =head3 State=ReceivingKey
  731: 
  732: The the key has been requested, now we are reading the new key.
  733: 
  734: =head3 State=Idle 
  735: 
  736: The encryption key has been negotiated or we have finished 
  737: reading data from the a transaction.   If the callback data has
  738: a client as well as the socket iformation, then we are 
  739: doing a transaction and the data received is relayed to the client
  740: before the socket is put on the idle list.
  741: 
  742: =head3 State=SendingRequest
  743: 
  744: I do not think this state can be received here, but if it is,
  745: the appropriate thing to do is to transition to writable, and send
  746: the request.
  747: 
  748: =head3 State=ReceivingReply
  749: 
  750: We finished sending the request to the server and now transition
  751: to readable to receive the reply. 
  752: 
  753: The parameter to this function are:
  754: 
  755: The event. Implicit in this is the watcher and its data.  The data 
  756: contains at least the lond connection object and, if a 
  757: transaction is in progress, the socket attached to the local client.
  758: 
  759: =cut
  760: 
  761: sub LondReadable {
  762: 
  763:     my $Event      = shift;
  764:     my $Watcher    = $Event->w;
  765:     my $Socket     = $Watcher->data;
  766:     my $client     = undef;
  767: 
  768:     &Debug(6,"LondReadable called state = ".$State);
  769: 
  770: 
  771:     my $State = $Socket->GetState(); # All action depends on the state.
  772: 
  773:     SocketDump(6, $Socket);
  774:     my $status = $Socket->Readable();
  775: 
  776:     &Debug(2, "Socket->Readable returned: $status");
  777: 
  778:     if($status != 0) {
  779: 	 # bad return from socket read. Currently this means that
  780: 	# The socket has become disconnected. We fail the transaction.
  781: 
  782: 	Log("WARNING",
  783: 	    "Lond connection lost.");
  784: 	if(exists($ActiveTransactions{$Socket})) {
  785: 	    FailTransaction($ActiveTransactions{$Socket});
  786: 	}
  787: 	$Watcher->cancel();
  788: 	KillSocket($Socket);
  789: 	return;
  790:     }
  791:     SocketDump(6,$Socket);
  792: 
  793:     $State = $Socket->GetState(); # Update in case of transition.
  794:     &Debug(6, "After read, state is ".$State);
  795: 
  796:    if($State eq "Initialized") {
  797: 
  798: 
  799:     } elsif ($State eq "ChallengeReceived") {
  800: 	#  The challenge must be echoed back;  The state machine
  801: 	# in the connection takes care of setting that up.  Just
  802: 	# need to transition to writable:
  803: 
  804: 	$Watcher->cb(\&LondWritable);
  805: 	$Watcher->poll("w");
  806: 
  807:     } elsif ($State eq "ChallengeReplied") {
  808: 
  809: 
  810:     } elsif ($State eq "RequestingKey") {
  811: 	#  The ok was received.  Now we need to request the key
  812: 	#  That requires us to be writable:
  813: 
  814: 	$Watcher->cb(\&LondWritable);
  815: 	$Watcher->poll("w");
  816: 
  817:     } elsif ($State eq "ReceivingKey") {
  818: 
  819:     } elsif ($State eq "Idle") {
  820: 	# If necessary, complete a transaction and then go into the
  821: 	# idle queue.
  822: 	$Watcher->cancel();
  823: 	if(exists($ActiveTransactions{$Socket})) {
  824: 	    Debug(8,"Completing transaction!!");
  825: 	    CompleteTransaction($Socket, 
  826: 				$ActiveTransactions{$Socket});
  827: 	} else {
  828: 	    Log("SUCCESS", "Connection ".$ConnectionCount." to "
  829: 		.$RemoteHost." now ready for action");
  830: 	}
  831: 	ServerToIdle($Socket);	# Next work unit or idle.
  832: 	
  833:     } elsif ($State eq "SendingRequest") {
  834: 	#  We need to be writable for this and probably don't belong
  835: 	#  here inthe first place.
  836: 
  837: 	Deubg(6, "SendingRequest state encountered in readable");
  838: 	$Watcher->poll("w");
  839: 	$Watcher->cb(\&LondWritable);
  840: 
  841:     } elsif ($State eq "ReceivingReply") {
  842: 
  843: 
  844:     } else {
  845: 	 # Invalid state.
  846: 	Debug(4, "Invalid state in LondReadable");
  847:     }
  848: }
  849: 
  850: =pod
  851: 
  852: =head2 LondWritable
  853: 
  854: This function is called whenever a lond connection
  855: becomes writable while there is a writeable monitoring
  856: event.  The action taken is very state dependent:
  857: 
  858: =head3 State = Connected 
  859: 
  860: The connection is in the process of sending the 'init' hailing to the
  861: lond on the remote end.  The connection object''s Writable member is
  862: called.  On error, ConnectionError is called to destroy the connection
  863: and remove it from the ActiveConnections hash
  864: 
  865: =head3 Initialized
  866: 
  867: 'init' has been sent, writability monitoring is removed and
  868: readability monitoring is started with LondReadable as the callback.
  869: 
  870: =head3 ChallengeReceived
  871: 
  872: The connection has received the who are you challenge from the remote
  873: system, and is in the process of sending the challenge
  874: response. Writable is called.
  875: 
  876: =head3 ChallengeReplied
  877: 
  878: The connection has replied to the initial challenge The we switch to
  879: monitoring readability looking for the server to reply with 'ok'.
  880: 
  881: =head3 RequestingKey
  882: 
  883: The connection is in the process of requesting its encryption key.
  884: Writable is called.
  885: 
  886: =head3 ReceivingKey
  887: 
  888: The connection has sent the request for a key.  Switch to readability
  889: monitoring to accept the key
  890: 
  891: =head3 SendingRequest
  892: 
  893: The connection is in the process of sending a request to the server.
  894: This request is part of a client transaction.  All the states until
  895: now represent the client setup protocol. Writable is called.
  896: 
  897: =head3 ReceivingReply
  898: 
  899: The connection has sent a request.  Now it must receive a reply.
  900: Readability monitoring is requested.
  901: 
  902: This function is an event handler and therefore receives as
  903: a parameter the event that has fired.  The data for the watcher
  904: of this event is a reference to a list of one or two elements,
  905: depending on state. The first (and possibly only) element is the
  906: socket.  The second (present only if a request is in progress)
  907: is the socket on which to return a reply to the caller.
  908: 
  909: =cut
  910: 
  911: sub LondWritable {
  912:     my $Event   = shift;
  913:     my $Watcher = $Event->w;
  914:     my $Socket  = $Watcher->data;
  915:     my $State   = $Socket->GetState();
  916: 
  917:     Debug(6,"LondWritable State = ".$State."\n");
  918: 
  919:  
  920:     #  Figure out what to do depending on the state of the socket:
  921:     
  922: 
  923: 
  924: 
  925:     SocketDump(6,$Socket);
  926: 
  927:     if      ($State eq "Connected")         {
  928: 
  929: 	if ($Socket->Writable() != 0) {
  930: 	    #  The write resulted in an error.
  931: 	    # We'll treat this as if the socket got disconnected:
  932: 	    Log("WARNING", "Connection to ".$RemoteHost.
  933: 		" has been disconnected");
  934: 	    FailTransaction($ActiveTransactions{$Socket});
  935: 	    $Watcher->cancel();
  936: 	    KillSocket($Socket);
  937: 	    return;
  938: 	}
  939: 	#  "init" is being sent...
  940: 
  941: 	
  942:     } elsif ($State eq "Initialized")       {
  943: 
  944: 	# Now that init was sent, we switch 
  945: 	# to watching for readability:
  946: 
  947: 	$Watcher->cb(\&LondReadable);
  948: 	$Watcher->poll("r");
  949: 
  950:     } elsif ($State eq "ChallengeReceived") {
  951: 	# We received the challenge, now we 
  952: 	# are echoing it back. This is a no-op,
  953: 	# we're waiting for the state to change
  954: 	
  955: 	if($Socket->Writable() != 0) {
  956: 
  957: 	    $Watcher->cancel();
  958: 	    KillSocket($Socket);
  959: 	    return;
  960: 	}
  961: 	
  962:     } elsif ($State eq "ChallengeReplied")  {
  963: 	# The echo was sent back, so we switch
  964: 	# to watching readability.
  965: 
  966: 	$Watcher->cb(\&LondReadable);
  967: 	$Watcher->poll("r");
  968: 
  969:     } elsif ($State eq "RequestingKey")     {
  970: 	# At this time we're requesting the key.
  971: 	# again, this is essentially a no-op.
  972: 	# we'll write the next chunk until the
  973: 	# state changes.
  974: 
  975: 	if($Socket->Writable() != 0) {
  976: 	    # Write resulted in an error.
  977: 
  978: 	    $Watcher->cancel();
  979: 	    KillSocket($Socket);
  980: 	    return;
  981: 
  982: 	}
  983:     } elsif ($State eq "ReceivingKey")      {
  984: 	# Now we need to wait for the key
  985: 	# to come back from the peer:
  986: 
  987: 	$Watcher->cb(\&LondReadable);
  988: 	$Watcher->poll("r");
  989: 
  990:     } elsif ($State eq "SendingRequest")    {
  991: 	# At this time we are sending a request to the
  992: 	# peer... write the next chunk:
  993: 
  994: 	if($Socket->Writable() != 0) {
  995: 
  996: 	    if(exists($ActiveTransactions{$Socket})) {
  997: 		Debug(3, "Lond connection lost, failing transactions");
  998: 		FailTransaction($ActiveTransactions{$Socket});
  999: 	    }
 1000: 	    $Watcher->cancel();
 1001: 	    KillSocket($Socket);
 1002: 	    return;
 1003: 	    
 1004: 	}
 1005: 
 1006:     } elsif ($State eq "ReceivingReply")    {
 1007: 	# The send has completed.  Wait for the
 1008: 	# data to come in for a reply.
 1009: 	Debug(8,"Writable sent request/receiving reply");
 1010: 	$Watcher->cb(\&LondReadable);
 1011: 	$Watcher->poll("r");
 1012: 
 1013:     } else {
 1014: 	#  Control only passes here on an error: 
 1015: 	#  the socket state does not match any
 1016: 	#  of the known states... so an error
 1017: 	#  must be logged.
 1018: 
 1019: 	&Debug(4, "Invalid socket state ".$State."\n");
 1020:     }
 1021:     
 1022: }
 1023: =pod
 1024:     
 1025: =cut
 1026: sub QueueDelayed {
 1027:     Debug(3,"QueueDelayed called");
 1028: 
 1029:     my $path = "$perlvar{'lonSockDir'}/delayed";
 1030: 
 1031:     Debug(4, "Delayed path: ".$path);
 1032:     opendir(DIRHANDLE, $path);
 1033:     
 1034:     @alldelayed = grep /\.$RemoteHost$/, readdir DIRHANDLE;
 1035:     Debug(4, "Got ".$alldelayed." delayed files");
 1036:     closedir(DIRHANDLE);
 1037:     my $dfname;
 1038:     my $reqfile;
 1039:     foreach $dfname (sort  @alldelayed) {
 1040: 	$reqfile = "$path/$dfname";
 1041: 	Debug(4, "queueing ".$reqfile);
 1042: 	my $Handle = IO::File->new($reqfile);
 1043: 	my $cmd    = <$Handle>;
 1044: 	chomp $cmd;		# There may or may not be a newline...
 1045: 	$cmd = $cmd."\n";	# now for sure there's exactly one newline.
 1046: 	my $Transaction = LondTransaction->new($cmd);
 1047: 	$Transaction->SetDeferred($reqfile);
 1048: 	QueueTransaction($Transaction);
 1049:     }
 1050:     
 1051: }
 1052: 
 1053: =pod
 1054: 
 1055: =head2 MakeLondConnection
 1056: 
 1057: Create a new lond connection object, and start it towards its initial
 1058: idleness.  Once idle, it becomes elligible to receive transactions
 1059: from the work queue.  If the work queue is not empty when the
 1060: connection is completed and becomes idle, it will dequeue an entry and
 1061: start off on it.
 1062: 
 1063: =cut
 1064: 
 1065: sub MakeLondConnection {     
 1066:     Debug(4,"MakeLondConnection to ".GetServerHost()." on port "
 1067: 	  .GetServerPort());
 1068: 
 1069:     my $Connection = LondConnection->new(&GetServerHost(),
 1070: 					 &GetServerPort());
 1071: 
 1072:     if($Connection == undef) {	# Needs to be more robust later.
 1073: 	Log("CRITICAL","Failed to make a connection with lond.");
 1074: 	$ConnectionRetriesLeft--;
 1075: 	return 0;		# Failure.
 1076:     }  else {
 1077: 	$ConnectionRetriesLeft = $ConnectionRetries; # success resets the count
 1078: 	# The connection needs to have writability 
 1079: 	# monitored in order to send the init sequence
 1080: 	# that starts the whole authentication/key
 1081: 	# exchange underway.
 1082: 	#
 1083: 	my $Socket = $Connection->GetSocket();
 1084: 	if($Socket == undef) {
 1085: 	    die "did not get a socket from the connection";
 1086: 	} else {
 1087: 	    &Debug(9,"MakeLondConnection got socket: ".$Socket);
 1088: 	}
 1089: 	
 1090: 	
 1091: 	$event = Event->io(fd       => $Socket,
 1092: 			   poll     => 'w',
 1093: 			   cb       => \&LondWritable,
 1094: 			   data     => $Connection,
 1095: 			   desc => 'Connection to lond server');
 1096: 	$ActiveConnections{$Connection} = $event;
 1097: 	
 1098: 	$ConnectionCount++;
 1099: 	Debug(4, "Connection count = ".$ConnectionCount);
 1100: 	if($ConnectionCount == 1) { # First Connection:
 1101: 	    QueueDelayed;
 1102: 	}
 1103: 	Log("SUCESS", "Created connection ".$ConnectionCount
 1104: 	    ." to host ".GetServerHost());
 1105: 	return 1;		# Return success.
 1106:     }
 1107:     
 1108: }
 1109: 
 1110: =pod
 1111: 
 1112: =head2 StartRequest
 1113: 
 1114: Starts a lond request going on a specified lond connection.
 1115: parameters are:
 1116: 
 1117: =item $Lond
 1118: 
 1119: Connection to the lond that will send the transaction and receive the
 1120: reply.
 1121: 
 1122: =item $Client
 1123: 
 1124: Connection to the client that is making this request We got the
 1125: request from this socket, and when the request has been relayed to
 1126: lond and we get a reply back from lond it will get sent to this
 1127: socket.
 1128: 
 1129: =item $Request
 1130: 
 1131: The text of the request to send.
 1132: 
 1133: =cut
 1134: 
 1135: sub StartRequest {
 1136:     my $Lond     = shift;
 1137:     my $Request  = shift;	# This is a LondTransaction.
 1138:     
 1139:     Debug(6, "StartRequest: ".$Request->getRequest());
 1140: 
 1141:     my $Socket = $Lond->GetSocket();
 1142:     
 1143:     $Request->Activate($Lond);
 1144:     $ActiveTransactions{$Lond} = $Request;
 1145: 
 1146:     $Lond->InitiateTransaction($Request->getRequest());
 1147:     $event = Event->io(fd      => $Socket,
 1148: 		       poll    => "w",
 1149: 		       cb      => \&LondWritable,
 1150: 		       data    => $Lond,
 1151: 		       desc    => "lond transaction connection");
 1152:     $ActiveConnections{$Lond} = $event;
 1153:     Debug(8," Start Request made watcher data with ".$event->data."\n");
 1154: }
 1155: 
 1156: =pod
 1157: 
 1158: =head2 QueueTransaction
 1159: 
 1160: If there is an idle lond connection, it is put to work doing this
 1161: transaction.  Otherwise, the transaction is placed in the work queue.
 1162: If placed in the work queue and the maximum number of connections has
 1163: not yet been created, a new connection will be started.  Our goal is
 1164: to eventually have a sufficient number of connections that the work
 1165: queue will typically be empty.  parameters are:
 1166: 
 1167: =item Socket
 1168: 
 1169: open on the lonc client.
 1170: 
 1171: =item Request
 1172: 
 1173: data to send to the lond.
 1174: 
 1175: =cut
 1176: 
 1177: sub QueueTransaction {
 1178: 
 1179:     my $requestData   = shift;	# This is a LondTransaction.
 1180:     my $cmd           = $requestData->getRequest();
 1181: 
 1182:     Debug(6,"QueueTransaction: ".$cmd);
 1183: 
 1184:     my $LondSocket    = $IdleConnections->pop();
 1185:     if(!defined $LondSocket) {	# Need to queue request.
 1186: 	Debug(8,"Must queue...");
 1187: 	$WorkQueue->enqueue($requestData);
 1188: 	if($ConnectionCount < $MaxConnectionCount) {
 1189: 	    Debug(4,"Starting additional lond connection");
 1190: 	    if(MakeLondConnection() == 0) {
 1191: 		EmptyQueue();	# Fail transactions, can't make connection.
 1192: 	    }
 1193: 	}
 1194:     } else {			# Can start the request:
 1195: 	Debug(8,"Can start...");
 1196: 	StartRequest($LondSocket,  $requestData);
 1197:     }
 1198: }
 1199: 
 1200: #-------------------------- Lonc UNIX socket handling ---------------------
 1201: 
 1202: =pod
 1203: 
 1204: =head2 ClientRequest
 1205: Callback that is called when data can be read from the UNIX domain
 1206: socket connecting us with an apache server process.
 1207: 
 1208: =cut
 1209: 
 1210: sub ClientRequest {
 1211:     Debug(6, "ClientRequest");
 1212:     my $event   = shift;
 1213:     my $watcher = $event->w;
 1214:     my $socket  = $watcher->fd;
 1215:     my $data    = $watcher->data;
 1216:     my $thisread;
 1217: 
 1218:     Debug(9, "  Watcher named: ".$watcher->desc);
 1219: 
 1220:     my $rv = $socket->recv($thisread, POSIX::BUFSIZ, 0);
 1221:     Debug(8, "rcv:  data length = ".length($thisread)
 1222: 	  ." read =".$thisread);
 1223:     unless (defined $rv && length($thisread)) {
 1224: 	 # Likely eof on socket.
 1225: 	Debug(5,"Client Socket closed on lonc for ".$RemoteHost);
 1226: 	close($socket);
 1227: 	$watcher->cancel();
 1228: 	delete($ActiveClients{$socket});
 1229: 	return;
 1230:     }
 1231:     Debug(8,"Data: ".$data." this read: ".$thisread);
 1232:     $data = $data.$thisread;	# Append new data.
 1233:     $watcher->data($data);
 1234:     if($data =~ /(.*\n)/) {	# Request entirely read.
 1235: 	if($data eq "close_connection_exit\n") {
 1236: 	    Log("CRITICAL",
 1237: 		"Request Close Connection ... exiting");
 1238: 	    CloseAllLondConnections();
 1239: 	    exit;
 1240: 	}
 1241: 	Debug(8, "Complete transaction received: ".$data);
 1242: 	my $Transaction = LondTransaction->new($data);
 1243: 	$Transaction->SetClient($socket);
 1244: 	QueueTransaction($Transaction);
 1245: 	$watcher->cancel();	# Done looking for input data.
 1246:     }
 1247: 
 1248: }
 1249: 
 1250: 
 1251: =pod
 1252: 
 1253: =head2  NewClient
 1254: 
 1255: Callback that is called when a connection is received on the unix
 1256: socket for a new client of lonc.  The callback is parameterized by the
 1257: event.. which is a-priori assumed to be an io event, and therefore has
 1258: an fd member that is the Listener socket.  We Accept the connection
 1259: and register a new event on the readability of that socket:
 1260: 
 1261: =cut
 1262: 
 1263: sub NewClient {
 1264:     Debug(6, "NewClient");
 1265:     my $event      = shift;		# Get the event parameters.
 1266:     my $watcher    = $event->w; 
 1267:     my $socket     = $watcher->fd;	# Get the event' socket.
 1268:     my $connection = $socket->accept();	# Accept the client connection.
 1269:     Debug(8,"Connection request accepted from "
 1270: 	  .GetPeername($connection, AF_UNIX));
 1271: 
 1272: 
 1273:     my $description = sprintf("Connection to lonc client %d",
 1274: 			      $ClientConnection);
 1275:     Debug(9, "Creating event named: ".$description);
 1276:     Event->io(cb      => \&ClientRequest,
 1277: 	      poll    => 'r',
 1278: 	      desc    => $description,
 1279: 	      data    => "",
 1280: 	      fd      => $connection);
 1281:     $ActiveClients{$connection} = $ClientConnection;
 1282:     $ClientConnection++;
 1283: }
 1284: 
 1285: =pod
 1286: 
 1287: =head2 GetLoncSocketPath
 1288: 
 1289: Returns the name of the UNIX socket on which to listen for client
 1290: connections.
 1291: 
 1292: =cut
 1293: 
 1294: sub GetLoncSocketPath {
 1295:     return $UnixSocketDir."/".GetServerHost();
 1296: }
 1297: 
 1298: =pod
 1299: 
 1300: =head2 GetServerHost
 1301: 
 1302: Returns the host whose lond we talk with.
 1303: 
 1304: =cut
 1305: 
 1306: sub GetServerHost {
 1307:     return $RemoteHost;		# Setup by the fork.
 1308: }
 1309: 
 1310: =pod
 1311: 
 1312: =head2 GetServerPort
 1313: 
 1314: Returns the lond port number.
 1315: 
 1316: =cut
 1317: 
 1318: sub GetServerPort {
 1319:     return $perlvar{londPort};
 1320: }
 1321: 
 1322: =pod
 1323: 
 1324: =head2 SetupLoncListener
 1325: 
 1326: Setup a lonc listener event.  The event is called when the socket
 1327: becomes readable.. that corresponds to the receipt of a new
 1328: connection.  The event handler established will accept the connection
 1329: (creating a communcations channel), that int turn will establish
 1330: another event handler to subess requests.
 1331: 
 1332: =cut
 1333: 
 1334: sub SetupLoncListener {
 1335: 
 1336:     my $socket;
 1337:     my $SocketName = GetLoncSocketPath();
 1338:     unlink($SocketName);
 1339:     unless ($socket =IO::Socket::UNIX->new(Local  => $SocketName,
 1340: 					    Listen => 10, 
 1341: 					    Type   => SOCK_STREAM)) {
 1342: 	die "Failed to create a lonc listner socket";
 1343:     }
 1344:     Event->io(cb     => \&NewClient,
 1345: 	      poll   => 'r',
 1346: 	      desc   => 'Lonc listener Unix Socket',
 1347: 	      fd     => $socket);
 1348: }
 1349: 
 1350: =pod 
 1351: 
 1352: =head2 ChildStatus
 1353:  
 1354: Child USR1 signal handler to report the most recent status
 1355: into the status file.
 1356: 
 1357: =cut
 1358: sub ChildStatus {
 1359:     my $event = shift;
 1360:     my $watcher = $event->w;
 1361: 
 1362:     Debug(2, "Reporting child status because : ".$watcher->data);
 1363:     my $docdir = $perlvar{'lonDocRoot'};
 1364:     my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt");
 1365:     print $fh $$."\t".$RemoteHost."\t".$Status."\t".
 1366: 	$RecentLogEntry."\n";
 1367: }
 1368: 
 1369: =pod
 1370: 
 1371: =head2 SignalledToDeath
 1372: 
 1373: Called in response to a signal that causes a chid process to die.
 1374: 
 1375: =cut
 1376: 
 1377: 
 1378: sub SignalledToDeath {
 1379:     my $event  = shift;
 1380:     my $watcher= $event->w;
 1381: 
 1382:     Debug(2,"Signalled to death! via ".$watcher->data);
 1383:     my ($signal) = $watcher->data;
 1384:     chomp($signal);
 1385:     Log("CRITICAL", "Abnormal exit.  Child $$ for $RemoteHost "
 1386: 	."died through "."\"$signal\"");
 1387:     LogPerm("F:lonc: $$ on $RemoteHost signalled to death: "
 1388: 	    ."\"$signal\"");
 1389:     exit 0;
 1390: 
 1391: }
 1392: 
 1393: =head2 ToggleDebug
 1394: 
 1395: This sub toggles trace debugging on and off.
 1396: 
 1397: =cut
 1398: 
 1399: sub ToggleDebug {
 1400:     my $Current    = $DebugLevel;
 1401:        $DebugLevel = $NextDebugLevel;
 1402:        $NextDebugLevel = $Current;
 1403: 
 1404:     Log("SUCCESS", "New debugging level for $RemoteHost now $DebugLevel");
 1405: 
 1406: }
 1407: 
 1408: =head2 ChildProcess
 1409: 
 1410: This sub implements a child process for a single lonc daemon.
 1411: 
 1412: =cut
 1413: 
 1414: sub ChildProcess {
 1415: 
 1416: 
 1417:     #
 1418:     #  Signals must be handled by the Event framework...
 1419: #
 1420: 
 1421:     Event->signal(signal   => "QUIT",
 1422: 		  cb       => \&SignalledToDeath,
 1423: 		  data     => "QUIT");
 1424:     Event->signal(signal   => "HUP",
 1425: 		  cb       => \&ChildStatus,
 1426: 		  data     => "HUP");
 1427:     Event->signal(signal   => "USR1",
 1428: 		  cb       => \&ChildStatus,
 1429: 		  data     => "USR1");
 1430:     Event->signal(signal   => "INT",
 1431: 		  cb       => \&ToggleDebug,
 1432: 		  data     => "INT");
 1433: 
 1434:     SetupTimer();
 1435:     
 1436:     SetupLoncListener();
 1437:     
 1438:     $Event::Debuglevel = $DebugLevel;
 1439:     
 1440:     Debug(9, "Making initial lond connection for ".$RemoteHost);
 1441: 
 1442: # Setup the initial server connection:
 1443:     
 1444:      # &MakeLondConnection(); // let first work requirest do it.
 1445: 
 1446: 
 1447:     Debug(9,"Entering event loop");
 1448:     my $ret = Event::loop();		#  Start the main event loop.
 1449:     
 1450:     
 1451:     die "Main event loop exited!!!";
 1452: }
 1453: 
 1454: #  Create a new child for host passed in:
 1455: 
 1456: sub CreateChild {
 1457:     my $sigset = POSIX::SigSet->new(SIGINT);
 1458:     sigprocmask(SIG_BLOCK, $sigset);
 1459:     my $host = shift;
 1460:     $RemoteHost = $host;
 1461:     Log("CRITICAL", "Forking server for ".$host);
 1462:     $pid          = fork;
 1463:     if($pid) {			# Parent
 1464: 	$RemoteHost = "Parent";
 1465: 	$ChildHash{$pid} = $RemoteHost;
 1466: 	sigprocmask(SIG_UNBLOCK, $sigset);
 1467: 
 1468:     } else {			# child.
 1469: 	ShowStatus("Connected to ".$RemoteHost);
 1470: 	$SIG{INT} = DEFAULT;
 1471: 	sigprocmask(SIG_UNBLOCK, $sigset);
 1472: 	ChildProcess;		# Does not return.
 1473:     }
 1474: 
 1475: }
 1476: #
 1477: #  Parent process logic pass 1:
 1478: #   For each entry in the hosts table, we will
 1479: #  fork off an instance of ChildProcess to service the transactions
 1480: #  to that host.  Each pid will be entered in a global hash
 1481: #  with the value of the key, the host.
 1482: #  The parent will then enter a loop to wait for process exits.
 1483: #  Each exit gets logged and the child gets restarted.
 1484: #
 1485: 
 1486: #
 1487: #   Fork and start in new session so hang-up isn't going to 
 1488: #   happen without intent.
 1489: #
 1490: 
 1491: 
 1492: 
 1493: 
 1494: 
 1495: 
 1496: ShowStatus("Forming new session");
 1497: my $childpid = fork;
 1498: if ($childpid != 0) {
 1499:     sleep 4;			# Give child a chacne to break to
 1500:     exit 0;			# a new sesion.
 1501: }
 1502: #
 1503: #   Write my pid into the pid file so I can be located
 1504: #
 1505: 
 1506: ShowStatus("Parent writing pid file:");
 1507: $execdir = $perlvar{'lonDaemons'};
 1508: open (PIDSAVE, ">$execdir/logs/lonc.pid");
 1509: print PIDSAVE "$$\n";
 1510: close(PIDSAVE);
 1511: 
 1512: 
 1513: 
 1514: if (POSIX::setsid() < 0) {
 1515:     print "Could not create new session\n";
 1516:     exit -1;
 1517: }
 1518: 
 1519: ShowStatus("Forking node servers");
 1520: 
 1521: Log("CRITICAL", "--------------- Starting children ---------------");
 1522: 
 1523: my $HostIterator = LondConnection::GetHostIterator;
 1524: while (! $HostIterator->end()) {
 1525: 
 1526:     $hostentryref = $HostIterator->get();
 1527:     CreateChild($hostentryref->[0]);
 1528:     $HostIterator->next();
 1529: }
 1530: $RemoteHost = "Parent Server";
 1531: 
 1532: # Maintain the population:
 1533: 
 1534: ShowStatus("Parent keeping the flock");
 1535: 
 1536: #
 1537: #   Set up parent signals:
 1538: #
 1539: 
 1540: $SIG{INT}  = \&Terminate;
 1541: $SIG{TERM} = \&Terminate; 
 1542: $SIG{HUP}  = \&Restart;
 1543: $SIG{USR1} = \&CheckKids; 
 1544: 
 1545: while(1) {
 1546:     $deadchild = wait();
 1547:     if(exists $ChildHash{$deadchild}) {	# need to restart.
 1548: 	$deadhost = $ChildHash{$deadchild};
 1549: 	delete($ChildHash{$deadchild});
 1550: 	Log("WARNING","Lost child pid= ".$deadchild.
 1551: 	      "Connected to host ".$deadhost);
 1552: 	Log("INFO", "Restarting child procesing ".$deadhost);
 1553: 	CreateChild($deadhost);
 1554:     }
 1555: }
 1556: 
 1557: 
 1558: 
 1559: =pod
 1560: 
 1561: =head1 CheckKids
 1562: 
 1563:   Since kids do not die as easily in this implementation
 1564: as the previous one, there  is no need to restart the
 1565: dead ones (all dead kids get restarted when they die!!)
 1566: The only thing this function does is to pass USR1 to the
 1567: kids so that they report their status.
 1568: 
 1569: =cut
 1570: 
 1571: sub CheckKids {
 1572:     Debug(2, "Checking status of children");
 1573:     my $docdir = $perlvar{'lonDocRoot'};
 1574:     my $fh = IO::File->new(">$docdir/lon-status/loncstatus.txt");
 1575:     my $now=time;
 1576:     my $local=localtime($now);
 1577:     print $fh "LONC status $local - parent $$ \n\n";
 1578:     foreach $pid (keys %ChildHash) {
 1579: 	Debug(2, "Sending USR1 -> $pid");
 1580: 	kill 'USR1' => $pid;	# Tell Child to report status.
 1581: 	sleep 1;		# Wait so file doesn't intermix.
 1582:     }
 1583: }
 1584: 
 1585: =pod
 1586: 
 1587: =head1 Restart
 1588: 
 1589: Signal handler for HUP... all children are killed and
 1590: we self restart.  This is an el-cheapo way to re read
 1591: the config file.
 1592: 
 1593: =cut
 1594: 
 1595: sub Restart {
 1596:     KillThemAll;		# First kill all the children.
 1597:     Log("CRITICAL", "Restarting");
 1598:     my $execdir = $perlvar{'lonDaemons'};
 1599:     unlink("$execdir/logs/lonc.pid");
 1600:     exec("$execdir/lonc");
 1601: }
 1602: 
 1603: =pod
 1604: 
 1605: =head1 KillThemAll
 1606: 
 1607: Signal handler that kills all children by sending them a 
 1608: SIGHUP.  Responds to sigint and sigterm.
 1609: 
 1610: =cut
 1611: 
 1612: sub KillThemAll {
 1613:     Debug(2, "Kill them all!!");
 1614:     local($SIG{CHLD}) = 'IGNORE';      # Our children >will< die.
 1615:     foreach $pid (keys %ChildHash) {
 1616: 	my $serving = $ChildHash{$pid};
 1617: 	Debug(2, "Killing lonc for $serving pid = $pid");
 1618: 	ShowStatus("Killing lonc for $serving pid = $pid");
 1619: 	Log("CRITICAL", "Killing lonc for $serving pid = $pid");
 1620: 	kill 'QUIT' => $pid;
 1621: 	delete($ChildHash{$pid});
 1622:     }
 1623:     my $execdir = $perlvar{'lonDaemons'};
 1624:     unlink("$execdir/logs/lonc.pid");
 1625: 
 1626: }
 1627: 
 1628: =pod
 1629: 
 1630: =head1 Terminate
 1631:  
 1632: Terminate the system.
 1633: 
 1634: =cut
 1635: 
 1636: sub Terminate {
 1637:     KillThemAll;
 1638:     Log("CRITICAL","Master process exiting");
 1639:     exit 0;
 1640: 
 1641: }
 1642: =pod
 1643: 
 1644: =head1 Theory
 1645: 
 1646: The event class is used to build this as a single process with an
 1647: event driven model.  The following events are handled:
 1648: 
 1649: =item UNIX Socket connection Received
 1650: 
 1651: =item Request data arrives on UNIX data transfer socket.
 1652: 
 1653: =item lond connection becomes writable.
 1654: 
 1655: =item timer fires at 1 second intervals.
 1656: 
 1657: All sockets are run in non-blocking mode.  Timeouts managed by the timer
 1658: handler prevents hung connections.
 1659: 
 1660: Key data structures:
 1661: 
 1662: =item RequestQueue
 1663: 
 1664: A queue of requests received from UNIX sockets that are
 1665: waiting for a chance to be forwarded on a lond connection socket.
 1666: 
 1667: =item ActiveConnections
 1668: 
 1669: A hash of lond connections that have transactions in process that are
 1670: available to be timed out.
 1671: 
 1672: =item ActiveTransactions
 1673: 
 1674: A hash indexed by lond connections that contain the client reply
 1675: socket for each connection that has an active transaction on it.
 1676: 
 1677: =item IdleConnections
 1678: 
 1679: A hash of lond connections that have no work to do.  These connections
 1680: can be closed if they are idle for a long enough time.
 1681: 
 1682: =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>