File:  [LON-CAPA] / loncom / loncnew
Revision 1.20: download - view: text, annotated - select for diffs
Mon Aug 25 18:48:11 2003 UTC (20 years, 9 months ago) by albertel
Branches: MAIN
CVS tags: HEAD
- fixing a forgotten ;

    1: #!/usr/bin/perl
    2: # The LearningOnline Network with CAPA
    3: # lonc maintains the connections to remote computers
    4: #
    5: # $Id: loncnew,v 1.20 2003/08/25 18:48:11 albertel Exp $
    6: #
    7: # Copyright Michigan State University Board of Trustees
    8: #
    9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
   10: ## LON-CAPA is free software; you can redistribute it and/or modify
   11: # it under the terms of the GNU General Public License as published by
   12: # the Free Software Foundation; either version 2 of the License, or
   13: # (at your option) any later version.
   14: #
   15: # LON-CAPA is distributed in the hope that it will be useful,
   16: # but WITHOUT ANY WARRANTY; without even the implied warranty of
   17: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   18: # GNU General Public License for more details.
   19: #
   20: # You should have received a copy of the GNU General Public License
   21: # along with LON-CAPA; if not, write to the Free Software
   22: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   23: #
   24: # /home/httpd/html/adm/gpl.txt
   25: #
   26: # http://www.lon-capa.org/
   27: #
   28: #
   29: # new lonc handles n request out bver m connections to londs.
   30: # This module is based on the Event class.
   31: #   Development iterations:
   32: #    - Setup basic event loop.   (done)
   33: #    - Add timer dispatch.       (done)
   34: #    - Add ability to accept lonc UNIX domain sockets.  (done)
   35: #    - Add ability to create/negotiate lond connections (done).
   36: #    - Add general logic for dispatching requests and timeouts. (done).
   37: #    - Add support for the lonc/lond requests.          (done).
   38: #    - Add logging/status monitoring.
   39: #    - Add Signal handling - HUP restarts. USR1 status report.
   40: #    - Add Configuration file I/O                       (done).
   41: #    - Add management/status request interface.
   42: #    - Add deferred request capability.                  (done)
   43: #    - Detect transmission timeouts.
   44: #
   45: 
   46: # Change log:
   47: #    $Log: loncnew,v $
   48: #    Revision 1.20  2003/08/25 18:48:11  albertel
   49: #    - fixing a forgotten ;
   50: #
   51: #    Revision 1.19  2003/08/19 09:31:46  foxr
   52: #    Get socket directory from configuration rather than the old hard coded test
   53: #    way that I forgot to un-hard code.
   54: #
   55: #    Revision 1.18  2003/08/06 09:52:29  foxr
   56: #    Also needed to remember to fail in-flight transactions if their sends fail.
   57: #
   58: #    Revision 1.17  2003/08/03 00:44:31  foxr
   59: #    1. Correct handling of connection failure: Assume it means the host is
   60: #       unreachable and fail all of the queued transactions.  Note that the
   61: #       inflight transactions should fail on their own time due either to timeout
   62: #       or send/receive failures.
   63: #    2. Correct handling of logs for forced death signals.  Pull the signal
   64: #       from the event watcher.
   65: #
   66: #    Revision 1.16  2003/07/29 02:33:05  foxr
   67: #    Add SIGINT processing to child processes to toggle annoying trace mode
   68: #    on/off.. will try to use this to isolate the compute boud process issue.
   69: #
   70: #    Revision 1.15  2003/07/15 02:07:05  foxr
   71: #    Added code for lonc/lond transaction timeouts.  Who knows if it works right.
   72: #    The intent is for a timeout to fail any transaction in progress and kill
   73: #    off the sockt that timed out.
   74: #
   75: #    Revision 1.14  2003/07/03 02:10:18  foxr
   76: #    Get all of the signals to work correctly.
   77: #
   78: #    Revision 1.13  2003/07/02 01:31:55  foxr
   79: #    Added kill -HUP logic (restart).
   80: #
   81: #    Revision 1.11  2003/06/25 01:54:44  foxr
   82: #    Fix more problems with transaction failure.
   83: #
   84: #    Revision 1.10  2003/06/24 02:46:04  foxr
   85: #    Put a limit on  the number of times we'll retry a connection.
   86: #    Start getting the signal stuff put in as well...note that need to get signals
   87: #    going or else 6the client will permanently give up on dead servers.
   88: #
   89: #    Revision 1.9  2003/06/13 02:38:43  foxr
   90: #    Add logging in 'expected format'
   91: #
   92: #    Revision 1.8  2003/06/11 02:04:35  foxr
   93: #    Support delayed transactions... this is done uniformly by encapsulating
   94: #    transactions in an object ... a LondTransaction that is implemented by
   95: #    LondTransaction.pm
   96: #
   97: #    Revision 1.7  2003/06/03 01:59:39  foxr
   98: #    complete coding to support deferred transactions.
   99: #
  100: #
  101: 
  102: use lib "/home/httpd/lib/perl/";
  103: use lib "/home/foxr/newloncapa/types";
  104: use Event qw(:DEFAULT );
  105: use POSIX qw(:signal_h);
  106: use POSIX;
  107: use IO::Socket;
  108: use IO::Socket::INET;
  109: use IO::Socket::UNIX;
  110: use IO::File;
  111: use IO::Handle;
  112: use Socket;
  113: use Crypt::IDEA;
  114: use LONCAPA::Queue;
  115: use LONCAPA::Stack;
  116: use LONCAPA::LondConnection;
  117: use LONCAPA::LondTransaction;
  118: use LONCAPA::Configuration;
  119: use LONCAPA::HashIterator;
  120: 
  121: 
  122: #
  123: #   Disable all signals we might receive from outside for now.
  124: #
  125: #$SIG{QUIT}  = IGNORE;
  126: #$SIG{HUP}   = IGNORE;
  127: #$SIG{USR1}  = IGNORE;
  128: #$SIG{INT}   = IGNORE;
  129: #$SIG{CHLD}  = IGNORE;
  130: #$SIG{__DIE__}  = IGNORE;
  131: 
  132: 
  133: # Read the httpd configuration file to get perl variables
  134: # normally set in apache modules:
  135: 
  136: my $perlvarref = LONCAPA::Configuration::read_conf('loncapa.conf');
  137: my %perlvar    = %{$perlvarref};
  138: 
  139: #
  140: #  parent and shared variables.
  141: 
  142: my %ChildHash;			# by pid -> host.
  143: 
  144: 
  145: my $MaxConnectionCount = 10;	# Will get from config later.
  146: my $ClientConnection = 0;	# Uniquifier for client events.
  147: 
  148: my $DebugLevel = 0;
  149: my $NextDebugLevel= 10;		# So Sigint can toggle this.
  150: my $IdleTimeout= 3600;		# Wait an hour before pruning connections.
  151: 
  152: #
  153: #  The variables below are only used by the child processes.
  154: #
  155: my $RemoteHost;			# Name of host child is talking to.
  156: my $UnixSocketDir= $perlvar{'lonSockDir'};
  157: my $IdleConnections = Stack->new(); # Set of idle connections
  158: my %ActiveConnections;		# Connections to the remote lond.
  159: my %ActiveTransactions;		# LondTransactions in flight.
  160: my %ActiveClients;		# Serial numbers of active clients by socket.
  161: my $WorkQueue       = Queue->new(); # Queue of pending transactions.
  162: my $ConnectionCount = 0;
  163: my $IdleSeconds     = 0;	# Number of seconds idle.
  164: my $Status          = "";	# Current status string.
  165: my $RecentLogEntry  = "";
  166: my $ConnectionRetries=5;	# Number of connection retries allowed.
  167: my $ConnectionRetriesLeft=5;	# Number of connection retries remaining.
  168: 
  169: #
  170: #   The hash below gives the HTML format for log messages
  171: #   given a severity.
  172: #    
  173: my %LogFormats;
  174: 
  175: $LogFormats{"CRITICAL"} = "<font color=red>CRITICAL: %s</font>";
  176: $LogFormats{"SUCCESS"}  = "<font color=green>SUCCESS: %s</font>";
  177: $LogFormats{"INFO"}     = "<font color=yellow>INFO: %s</font>";
  178: $LogFormats{"WARNING"}  = "<font color=blue>WARNING: %s</font>";
  179: $LogFormats{"DEFAULT"}  = " %s ";
  180: 
  181: 
  182: 
  183: =pod
  184: 
  185: =head2 LogPerm
  186: 
  187: Makes an entry into the permanent log file.
  188: 
  189: =cut
  190: sub LogPerm {
  191:     my $message=shift;
  192:     my $execdir=$perlvar{'lonDaemons'};
  193:     my $now=time;
  194:     my $local=localtime($now);
  195:     my $fh=IO::File->new(">>$execdir/logs/lonnet.perm.log");
  196:     print $fh "$now:$message:$local\n";
  197: }
  198: 
  199: =pod
  200: 
  201: =head2 Log
  202: 
  203: Logs a message to the log file.
  204: Parameters:
  205: 
  206: =item severity
  207: 
  208: One of CRITICAL, WARNING, INFO, SUCCESS used to select the
  209: format string used to format the message.  if the severity is
  210: not a defined severity the Default format string is used.
  211: 
  212: =item message
  213: 
  214: The base message.  In addtion to the format string, the message
  215: will be appended to a string containing the name of our remote
  216: host and the time will be formatted into the message.
  217: 
  218: =cut
  219: 
  220: sub Log {
  221:     my $severity = shift;
  222:     my $message  = shift;
  223:    
  224:     if(!$LogFormats{$severity}) {
  225: 	$severity = "DEFAULT";
  226:     }
  227: 
  228:     my $format = $LogFormats{$severity};
  229:     
  230:     #  Put the window dressing in in front of the message format:
  231: 
  232:     my $now   = time;
  233:     my $local = localtime($now);
  234:     my $finalformat = "$local ($$) [$RemoteHost] [$Status] ";
  235:     my $finalformat = $finalformat.$format."\n";
  236: 
  237:     # open the file and put the result.
  238: 
  239:     my $execdir = $perlvar{'lonDaemons'};
  240:     my $fh      = IO::File->new(">>$execdir/logs/lonc.log");
  241:     my $msg = sprintf($finalformat, $message);
  242:     $RecentLogEntry = $msg;
  243:     print $fh $msg;
  244:     
  245:     
  246: }
  247: 
  248: 
  249: =pod
  250: 
  251: =head2 GetPeerName
  252: 
  253: Returns the name of the host that a socket object is connected to.
  254: 
  255: =cut
  256: 
  257: sub GetPeername {
  258:     my $connection = shift;
  259:     my $AdrFamily  = shift;
  260:     my $peer       = $connection->peername();
  261:     my $peerport;
  262:     my $peerip;
  263:     if($AdrFamily == AF_INET) {
  264: 	($peerport, $peerip) = sockaddr_in($peer);
  265: 	my $peername    = gethostbyaddr($iaddr, $AdrFamily);
  266: 	return $peername;
  267:     } elsif ($AdrFamily == AF_UNIX) {
  268: 	my $peerfile;
  269: 	($peerfile) = sockaddr_un($peer);
  270: 	return $peerfile;
  271:     }
  272: }
  273: #----------------------------- Timer management ------------------------
  274: =pod
  275: 
  276: =head2 Debug
  277: 
  278: Invoked to issue a debug message.
  279: 
  280: =cut
  281: 
  282: sub Debug {
  283:     my $level   = shift;
  284:     my $message = shift;
  285:     if ($level <= $DebugLevel) {
  286: 	Log("INFO", "-Debug- $message host = $RemotHost");
  287:     }
  288: }
  289: 
  290: sub SocketDump {
  291:     my $level = shift;
  292:     my $socket= shift;
  293:     if($level <= $DebugLevel) {
  294: 	$socket->Dump();
  295:     }
  296: }
  297: 
  298: =pod
  299: 
  300: =head2 ShowStatus
  301: 
  302:  Place some text as our pid status.
  303:  and as what we return in a SIGUSR1
  304: 
  305: =cut
  306: sub ShowStatus {
  307:     my $state = shift;
  308:     my $now = time;
  309:     my $local = localtime($now);
  310:     $Status   = $local.": ".$state;
  311:     $0='lonc: '.$state.' '.$local;
  312: }
  313: 
  314: =pod
  315: 
  316: =head 2 SocketTimeout
  317: 
  318:     Called when an action on the socket times out.  The socket is 
  319:    destroyed and any active transaction is failed.
  320: 
  321: 
  322: =cut
  323: sub SocketTimeout {
  324:     my $Socket = shift;
  325:     
  326:     KillSocket($Socket);
  327: }
  328: 
  329: =pod
  330: 
  331: =head2 Tick
  332: 
  333: Invoked  each timer tick.
  334: 
  335: =cut
  336: 
  337: 
  338: sub Tick {
  339:     my $client;
  340:     ShowStatus(GetServerHost()." Connection count: ".$ConnectionCount);
  341: 
  342:     # Is it time to prune connection count:
  343: 
  344: 
  345:     if($IdleConnections->Count()  && 
  346:        ($WorkQueue->Count() == 0)) { # Idle connections and nothing to do?
  347: 	$IdleSeconds++;
  348: 	if($IdleSeconds > $IdleTimeout) { # Prune a connection...
  349: 	    $Socket = $IdleConnections->pop();
  350: 	    KillSocket($Socket);
  351: 	}
  352:     } else {
  353: 	$IdleSeconds = 0;	# Reset idle count if not idle.
  354:     }
  355:     #
  356:     #  For each inflight transaction, tick down its timeout counter.
  357:     #
  358:     foreach $item (keys %ActiveTransactions) {
  359: 	my $Socket = $ActiveTransactions{$item}->getServer();
  360: 	$Socket->Tick();
  361:     }
  362:     # Do we have work in the queue, but no connections to service them?
  363:     # If so, try to make some new connections to get things going again.
  364:     #
  365:     
  366:     my $Requests = $WorkQueue->Count();
  367:     if (($ConnectionCount == 0)  && ($Requests > 0)) { 
  368: 	if ($ConnectionRetriesLeft > 0) {
  369: 	    my $Connections = ($Requests <= $MaxConnectionCount) ?
  370: 		$Requests : $MaxConnectionCount;
  371: 	    Debug(1,"Work but no connections, start ".$Connections." of them");
  372: 	    for ($i =0; $i < $Connections; $i++) {
  373: 		MakeLondConnection();
  374: 	    }
  375: 	} else {
  376: 	    Debug(1,"Work in queue, but gave up on connections..flushing\n");
  377: 	    EmptyQueue();	# Connections can't be established.
  378: 	}
  379:        
  380:     }
  381: }
  382: 
  383: =pod
  384: 
  385: =head2 SetupTimer
  386: 
  387: Sets up a 1 per sec recurring timer event.  The event handler is used to:
  388: 
  389: =item
  390: 
  391: Trigger timeouts on communications along active sockets.
  392: 
  393: =item
  394: 
  395: Trigger disconnections of idle sockets.
  396: 
  397: =cut
  398: 
  399: sub SetupTimer {
  400:     Debug(6, "SetupTimer");
  401:     Event->timer(interval => 1, debug => 1, cb => \&Tick );
  402: }
  403: 
  404: =pod
  405: 
  406: =head2 ServerToIdle
  407: 
  408: This function is called when a connection to the server is
  409: ready for more work.
  410: 
  411: If there is work in the Work queue the top element is dequeued
  412: and the connection will start to work on it.  If the work queue is
  413: empty, the connection is pushed on the idle connection stack where
  414: it will either get another work unit, or alternatively, if it sits there
  415: long enough, it will be shut down and released.
  416: 
  417: =cut
  418: 
  419: sub ServerToIdle {
  420:     my $Socket   = shift;	# Get the socket.
  421:     delete($ActiveTransactions{$Socket}); # Server has no transaction
  422: 
  423:     &Debug(6, "Server to idle");
  424: 
  425:     #  If there's work to do, start the transaction:
  426: 
  427:     $reqdata = $WorkQueue->dequeue(); # This is a LondTransaction
  428:     unless($reqdata eq undef)  {
  429: 	Debug(9, "Queue gave request data: ".$reqdata->getRequest());
  430: 	&StartRequest($Socket,  $reqdata);
  431: 
  432:     } else {
  433: 	
  434:     #  There's no work waiting, so push the server to idle list.
  435: 	&Debug(8, "No new work requests, server connection going idle");
  436: 	$IdleConnections->push($Socket);
  437:     }
  438: }
  439: 
  440: =pod
  441: 
  442: =head2 ClientWritable
  443: 
  444: Event callback for when a client socket is writable.
  445: 
  446: This callback is established when a transaction reponse is
  447: avaiable from lond.  The response is forwarded to the unix socket
  448: as it becomes writable in this sub.
  449: 
  450: Parameters:
  451: 
  452: =item Event
  453: 
  454: The event that has been triggered. Event->w->data is
  455: the data and Event->w->fd is the socket to write.
  456: 
  457: =cut
  458: 
  459: sub ClientWritable {
  460:     my $Event    = shift;
  461:     my $Watcher  = $Event->w;
  462:     my $Data     = $Watcher->data;
  463:     my $Socket   = $Watcher->fd;
  464: 
  465:     # Try to send the data:
  466: 
  467:     &Debug(6, "ClientWritable writing".$Data);
  468:     &Debug(9, "Socket is: ".$Socket);
  469: 
  470:     if($Socket->connected) {
  471: 	my $result = $Socket->send($Data, 0);
  472: 	
  473: 	# $result undefined: the write failed.
  474: 	# otherwise $result is the number of bytes written.
  475: 	# Remove that preceding string from the data.
  476: 	# If the resulting data is empty, destroy the watcher
  477: 	# and set up a read event handler to accept the next
  478: 	# request.
  479: 	
  480: 	&Debug(9,"Send result is ".$result." Defined: ".defined($result));
  481: 	if(defined($result)) {
  482: 	    &Debug(9, "send result was defined");
  483: 	    if($result == length($Data)) { # Entire string sent.
  484: 		&Debug(9, "ClientWritable data all written");
  485: 		$Watcher->cancel();
  486: 		#
  487: 		#  Set up to read next request from socket:
  488: 		
  489: 		my $descr     = sprintf("Connection to lonc client %d",
  490: 					$ActiveClients{$Socket});
  491: 		Event->io(cb    => \&ClientRequest,
  492: 			  poll  => 'r',
  493: 			  desc  => $descr,
  494: 			  data  => "",
  495: 			  fd    => $Socket);
  496: 		
  497: 	    } else {		# Partial string sent.
  498: 		$Watcher->data(substr($Data, $result));
  499: 		if($result == 0) {    # client hung up on us!!
  500: 		    Log("INFO", "lonc pipe client hung up on us!");
  501: 		    $Watcher->cancel;
  502: 		    $Socket->shutdown(2);
  503: 		    $Socket->close();
  504: 		}
  505: 	    }
  506: 	    
  507: 	} else {			# Error of some sort...
  508: 	    
  509: 	    # Some errnos are possible:
  510: 	    my $errno = $!;
  511: 	    if($errno == POSIX::EWOULDBLOCK   ||
  512: 	       $errno == POSIX::EAGAIN        ||
  513: 	       $errno == POSIX::EINTR) {
  514: 		# No action taken?
  515: 	    } else {		# Unanticipated errno.
  516: 		&Debug(5,"ClientWritable error or peer shutdown".$RemoteHost);
  517: 		$Watcher->cancel;	# Stop the watcher.
  518: 		$Socket->shutdown(2); # Kill connection
  519: 		$Socket->close();	# Close the socket.
  520: 	    }
  521: 	    
  522: 	}
  523:     } else {
  524: 	$Watcher->cancel();	# A delayed request...just cancel.
  525:     }
  526: }
  527: 
  528: =pod
  529: 
  530: =head2 CompleteTransaction
  531: 
  532: Called when the reply data has been received for a lond 
  533: transaction.   The reply data must now be sent to the
  534: ultimate client on the other end of the Unix socket.  This is
  535: done by setting up a writable event for the socket with the
  536: data the reply data.
  537: 
  538: Parameters:
  539: 
  540: =item Socket
  541: 
  542: Socket on which the lond transaction occured.  This is a
  543: LondConnection. The data received is in the TransactionReply member.
  544: 
  545: =item Transaction
  546: 
  547: The transaction that is being completed.
  548: 
  549: =cut
  550: 
  551: sub CompleteTransaction {
  552:     &Debug(6,"Complete transaction");
  553:     my $Socket = shift;
  554:     my $Transaction = shift;
  555: 
  556:     if (!$Transaction->isDeferred()) { # Normal transaction
  557: 	my $data   = $Socket->GetReply(); # Data to send.
  558: 	StartClientReply($Transaction, $data);
  559:     } else {			# Delete deferred transaction file.
  560: 	Log("SUCCESS", "A delayed transaction was completed");
  561: 	LogPerm("S:$Client:".$Transaction->getRequest());
  562: 	unlink $Transaction->getFile();
  563:     }
  564: }
  565: =pod
  566: =head1 StartClientReply
  567: 
  568:    Initiates a reply to a client where the reply data is a parameter.
  569: 
  570: =head2  parameters:
  571: 
  572: =item Transaction
  573: 
  574:     The transaction for which we are responding to the client.
  575: 
  576: =item data
  577: 
  578:     The data to send to apached client.
  579: 
  580: =cut
  581: sub StartClientReply {
  582:     my $Transaction   = shift;
  583:     my $data     = shift;
  584: 
  585: 
  586:     my $Client   = $Transaction->getClient();
  587: 
  588:     &Debug(8," Reply was: ".$data);
  589:     my $Serial         = $ActiveClients{$Client};
  590:     my $desc           = sprintf("Connection to lonc client %d",
  591: 
  592: 				 $Serial);
  593:     Event->io(fd       => $Client,
  594: 	      poll     => "w",
  595: 	      desc     => $desc,
  596: 	      cb       => \&ClientWritable,
  597: 	      data     => $data);
  598: }
  599: =pod
  600: =head2 FailTransaction
  601: 
  602:   Finishes a transaction with failure because the associated lond socket
  603:   disconnected.  There are two possibilities:
  604:   - The transaction is deferred: in which case we just quietly
  605:     delete the transaction since there is no client connection.
  606:   - The transaction is 'live' in which case we initiate the sending
  607:     of "con_lost" to the client.
  608: 
  609: Deleting the transaction means killing it from the 
  610: %ActiveTransactions hash.
  611: 
  612: Parameters:
  613: 
  614: =item client  
  615:  
  616:    The LondTransaction we are failing.
  617:  
  618: =cut
  619: 
  620: sub FailTransaction {
  621:     my $transaction = shift;
  622:     Log("WARNING", "Failing transaction ".$transaction->getRequest());
  623:     Debug(1, "Failing transaction: ".$transaction->getRequest());
  624:     if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it.
  625: 	my $client  = $transaction->getClient();
  626: 	Debug(1," Replying con_lost to ".$transaction->getRequest());
  627: 	StartClientReply($transaction, "con_lost\n");
  628:     }
  629: 
  630: }
  631: 
  632: =pod
  633: =head1  EmptyQueue
  634: 
  635:   Fails all items in the work queue with con_lost.
  636:   Note that each item in the work queue is a transaction.
  637: 
  638: =cut
  639: sub EmptyQueue {
  640:     while($WorkQueue->Count()) {
  641: 	my $request = $WorkQueue->dequeue(); # This is a transaction
  642: 	FailTransaction($request);
  643:     }
  644: }
  645: 
  646: =pod
  647: 
  648: =head2 CloseAllLondConnections
  649: 
  650: Close all connections open on lond prior to exit e.g.
  651: 
  652: =cut
  653: sub CloseAllLondConnections {
  654:     foreach $Socket (keys %ActiveConnections) {
  655: 	KillSocket($Socket);
  656:     }
  657: }
  658: =cut
  659: 
  660: =pod
  661: 
  662: =head2 KillSocket
  663:  
  664: Destroys a socket.  This function can be called either when a socket
  665: has died of 'natural' causes or because a socket needs to be pruned due to
  666: idleness.  If the socket has died naturally, if there are no longer any 
  667: live connections a new connection is created (in case there are transactions
  668: in the queue).  If the socket has been pruned, it is never re-created.
  669: 
  670: Parameters:
  671: 
  672: =item Socket
  673:  
  674:   The socket to kill off.
  675: 
  676: =item Restart
  677: 
  678: nonzero if we are allowed to create a new connection.
  679: 
  680: 
  681: =cut
  682: sub KillSocket {
  683:     my $Socket = shift;
  684: 
  685:     Log("WARNING", "Shutting down a socket");
  686:     $Socket->Shutdown();
  687: 
  688:     #  If the socket came from the active connection set,
  689:     #  delete its transaction... note that FailTransaction should
  690:     #  already have been called!!!
  691:     #  otherwise it came from the idle set.
  692:     #  
  693:     
  694:     if(exists($ActiveTransactions{$Socket})) {
  695: 	delete ($ActiveTransactions{$Socket});
  696:     }
  697:     if(exists($ActiveConnections{$Socket})) {
  698: 	delete($ActiveConnections{$Socket});
  699:     }
  700:     $ConnectionCount--;
  701: 
  702:     #  If the connection count has gone to zero and there is work in the
  703:     #  work queue, the work all gets failed with con_lost.
  704:     #
  705:     if($ConnectionCount == 0) {
  706: 	EmptyQueue;
  707:     }
  708: }
  709: 
  710: =pod
  711: 
  712: =head2 LondReadable
  713: 
  714: This function is called whenever a lond connection
  715: is readable.  The action is state dependent:
  716: 
  717: =head3 State=Initialized
  718: 
  719: We''re waiting for the challenge, this is a no-op until the
  720: state changes.
  721: 
  722: =head3 State=Challenged 
  723: 
  724: The challenge has arrived we need to transition to Writable.
  725: The connection must echo the challenge back.
  726: 
  727: =head3 State=ChallengeReplied
  728: 
  729: The challenge has been replied to.  The we are receiveing the 
  730: 'ok' from the partner.
  731: 
  732: =head3 State=RequestingKey
  733: 
  734: The ok has been received and we need to send the request for
  735: an encryption key.  Transition to writable for that.
  736: 
  737: =head3 State=ReceivingKey
  738: 
  739: The the key has been requested, now we are reading the new key.
  740: 
  741: =head3 State=Idle 
  742: 
  743: The encryption key has been negotiated or we have finished 
  744: reading data from the a transaction.   If the callback data has
  745: a client as well as the socket iformation, then we are 
  746: doing a transaction and the data received is relayed to the client
  747: before the socket is put on the idle list.
  748: 
  749: =head3 State=SendingRequest
  750: 
  751: I do not think this state can be received here, but if it is,
  752: the appropriate thing to do is to transition to writable, and send
  753: the request.
  754: 
  755: =head3 State=ReceivingReply
  756: 
  757: We finished sending the request to the server and now transition
  758: to readable to receive the reply. 
  759: 
  760: The parameter to this function are:
  761: 
  762: The event. Implicit in this is the watcher and its data.  The data 
  763: contains at least the lond connection object and, if a 
  764: transaction is in progress, the socket attached to the local client.
  765: 
  766: =cut
  767: 
  768: sub LondReadable {
  769: 
  770:     my $Event      = shift;
  771:     my $Watcher    = $Event->w;
  772:     my $Socket     = $Watcher->data;
  773:     my $client     = undef;
  774: 
  775:     &Debug(6,"LondReadable called state = ".$State);
  776: 
  777: 
  778:     my $State = $Socket->GetState(); # All action depends on the state.
  779: 
  780:     SocketDump(6, $Socket);
  781:     my $status = $Socket->Readable();
  782: 
  783:     &Debug(2, "Socket->Readable returned: $status");
  784: 
  785:     if($status != 0) {
  786: 	 # bad return from socket read. Currently this means that
  787: 	# The socket has become disconnected. We fail the transaction.
  788: 
  789: 	Log("WARNING",
  790: 	    "Lond connection lost.");
  791: 	if(exists($ActiveTransactions{$Socket})) {
  792: 	    FailTransaction($ActiveTransactions{$Socket});
  793: 	}
  794: 	$Watcher->cancel();
  795: 	KillSocket($Socket);
  796: 	return;
  797:     }
  798:     SocketDump(6,$Socket);
  799: 
  800:     $State = $Socket->GetState(); # Update in case of transition.
  801:     &Debug(6, "After read, state is ".$State);
  802: 
  803:    if($State eq "Initialized") {
  804: 
  805: 
  806:     } elsif ($State eq "ChallengeReceived") {
  807: 	#  The challenge must be echoed back;  The state machine
  808: 	# in the connection takes care of setting that up.  Just
  809: 	# need to transition to writable:
  810: 
  811: 	$Watcher->cb(\&LondWritable);
  812: 	$Watcher->poll("w");
  813: 
  814:     } elsif ($State eq "ChallengeReplied") {
  815: 
  816: 
  817:     } elsif ($State eq "RequestingKey") {
  818: 	#  The ok was received.  Now we need to request the key
  819: 	#  That requires us to be writable:
  820: 
  821: 	$Watcher->cb(\&LondWritable);
  822: 	$Watcher->poll("w");
  823: 
  824:     } elsif ($State eq "ReceivingKey") {
  825: 
  826:     } elsif ($State eq "Idle") {
  827: 	# If necessary, complete a transaction and then go into the
  828: 	# idle queue.
  829: 	$Watcher->cancel();
  830: 	if(exists($ActiveTransactions{$Socket})) {
  831: 	    Debug(8,"Completing transaction!!");
  832: 	    CompleteTransaction($Socket, 
  833: 				$ActiveTransactions{$Socket});
  834: 	} else {
  835: 	    Log("SUCCESS", "Connection ".$ConnectionCount." to "
  836: 		.$RemoteHost." now ready for action");
  837: 	}
  838: 	ServerToIdle($Socket);	# Next work unit or idle.
  839: 	
  840:     } elsif ($State eq "SendingRequest") {
  841: 	#  We need to be writable for this and probably don't belong
  842: 	#  here inthe first place.
  843: 
  844: 	Deubg(6, "SendingRequest state encountered in readable");
  845: 	$Watcher->poll("w");
  846: 	$Watcher->cb(\&LondWritable);
  847: 
  848:     } elsif ($State eq "ReceivingReply") {
  849: 
  850: 
  851:     } else {
  852: 	 # Invalid state.
  853: 	Debug(4, "Invalid state in LondReadable");
  854:     }
  855: }
  856: 
  857: =pod
  858: 
  859: =head2 LondWritable
  860: 
  861: This function is called whenever a lond connection
  862: becomes writable while there is a writeable monitoring
  863: event.  The action taken is very state dependent:
  864: 
  865: =head3 State = Connected 
  866: 
  867: The connection is in the process of sending the 'init' hailing to the
  868: lond on the remote end.  The connection object''s Writable member is
  869: called.  On error, ConnectionError is called to destroy the connection
  870: and remove it from the ActiveConnections hash
  871: 
  872: =head3 Initialized
  873: 
  874: 'init' has been sent, writability monitoring is removed and
  875: readability monitoring is started with LondReadable as the callback.
  876: 
  877: =head3 ChallengeReceived
  878: 
  879: The connection has received the who are you challenge from the remote
  880: system, and is in the process of sending the challenge
  881: response. Writable is called.
  882: 
  883: =head3 ChallengeReplied
  884: 
  885: The connection has replied to the initial challenge The we switch to
  886: monitoring readability looking for the server to reply with 'ok'.
  887: 
  888: =head3 RequestingKey
  889: 
  890: The connection is in the process of requesting its encryption key.
  891: Writable is called.
  892: 
  893: =head3 ReceivingKey
  894: 
  895: The connection has sent the request for a key.  Switch to readability
  896: monitoring to accept the key
  897: 
  898: =head3 SendingRequest
  899: 
  900: The connection is in the process of sending a request to the server.
  901: This request is part of a client transaction.  All the states until
  902: now represent the client setup protocol. Writable is called.
  903: 
  904: =head3 ReceivingReply
  905: 
  906: The connection has sent a request.  Now it must receive a reply.
  907: Readability monitoring is requested.
  908: 
  909: This function is an event handler and therefore receives as
  910: a parameter the event that has fired.  The data for the watcher
  911: of this event is a reference to a list of one or two elements,
  912: depending on state. The first (and possibly only) element is the
  913: socket.  The second (present only if a request is in progress)
  914: is the socket on which to return a reply to the caller.
  915: 
  916: =cut
  917: 
  918: sub LondWritable {
  919:     my $Event   = shift;
  920:     my $Watcher = $Event->w;
  921:     my $Socket  = $Watcher->data;
  922:     my $State   = $Socket->GetState();
  923: 
  924:     Debug(6,"LondWritable State = ".$State."\n");
  925: 
  926:  
  927:     #  Figure out what to do depending on the state of the socket:
  928:     
  929: 
  930: 
  931: 
  932:     SocketDump(6,$Socket);
  933: 
  934:     if      ($State eq "Connected")         {
  935: 
  936: 	if ($Socket->Writable() != 0) {
  937: 	    #  The write resulted in an error.
  938: 	    # We'll treat this as if the socket got disconnected:
  939: 	    Log("WARNING", "Connection to ".$RemoteHost.
  940: 		" has been disconnected");
  941: 	    FailTransaction($ActiveTransactions{$Socket});
  942: 	    $Watcher->cancel();
  943: 	    KillSocket($Socket);
  944: 	    return;
  945: 	}
  946: 	#  "init" is being sent...
  947: 
  948: 	
  949:     } elsif ($State eq "Initialized")       {
  950: 
  951: 	# Now that init was sent, we switch 
  952: 	# to watching for readability:
  953: 
  954: 	$Watcher->cb(\&LondReadable);
  955: 	$Watcher->poll("r");
  956: 
  957:     } elsif ($State eq "ChallengeReceived") {
  958: 	# We received the challenge, now we 
  959: 	# are echoing it back. This is a no-op,
  960: 	# we're waiting for the state to change
  961: 	
  962: 	if($Socket->Writable() != 0) {
  963: 
  964: 	    $Watcher->cancel();
  965: 	    KillSocket($Socket);
  966: 	    return;
  967: 	}
  968: 	
  969:     } elsif ($State eq "ChallengeReplied")  {
  970: 	# The echo was sent back, so we switch
  971: 	# to watching readability.
  972: 
  973: 	$Watcher->cb(\&LondReadable);
  974: 	$Watcher->poll("r");
  975: 
  976:     } elsif ($State eq "RequestingKey")     {
  977: 	# At this time we're requesting the key.
  978: 	# again, this is essentially a no-op.
  979: 	# we'll write the next chunk until the
  980: 	# state changes.
  981: 
  982: 	if($Socket->Writable() != 0) {
  983: 	    # Write resulted in an error.
  984: 
  985: 	    $Watcher->cancel();
  986: 	    KillSocket($Socket);
  987: 	    return;
  988: 
  989: 	}
  990:     } elsif ($State eq "ReceivingKey")      {
  991: 	# Now we need to wait for the key
  992: 	# to come back from the peer:
  993: 
  994: 	$Watcher->cb(\&LondReadable);
  995: 	$Watcher->poll("r");
  996: 
  997:     } elsif ($State eq "SendingRequest")    {
  998: 	# At this time we are sending a request to the
  999: 	# peer... write the next chunk:
 1000: 
 1001: 	if($Socket->Writable() != 0) {
 1002: 
 1003: 	    if(exists($ActiveTransactions{$Socket})) {
 1004: 		Debug(3, "Lond connection lost, failing transactions");
 1005: 		FailTransaction($ActiveTransactions{$Socket});
 1006: 	    }
 1007: 	    $Watcher->cancel();
 1008: 	    KillSocket($Socket);
 1009: 	    return;
 1010: 	    
 1011: 	}
 1012: 
 1013:     } elsif ($State eq "ReceivingReply")    {
 1014: 	# The send has completed.  Wait for the
 1015: 	# data to come in for a reply.
 1016: 	Debug(8,"Writable sent request/receiving reply");
 1017: 	$Watcher->cb(\&LondReadable);
 1018: 	$Watcher->poll("r");
 1019: 
 1020:     } else {
 1021: 	#  Control only passes here on an error: 
 1022: 	#  the socket state does not match any
 1023: 	#  of the known states... so an error
 1024: 	#  must be logged.
 1025: 
 1026: 	&Debug(4, "Invalid socket state ".$State."\n");
 1027:     }
 1028:     
 1029: }
 1030: =pod
 1031:     
 1032: =cut
 1033: sub QueueDelayed {
 1034:     Debug(3,"QueueDelayed called");
 1035: 
 1036:     my $path = "$perlvar{'lonSockDir'}/delayed";
 1037: 
 1038:     Debug(4, "Delayed path: ".$path);
 1039:     opendir(DIRHANDLE, $path);
 1040:     
 1041:     @alldelayed = grep /\.$RemoteHost$/, readdir DIRHANDLE;
 1042:     Debug(4, "Got ".$alldelayed." delayed files");
 1043:     closedir(DIRHANDLE);
 1044:     my $dfname;
 1045:     my $reqfile;
 1046:     foreach $dfname (sort  @alldelayed) {
 1047: 	$reqfile = "$path/$dfname";
 1048: 	Debug(4, "queueing ".$reqfile);
 1049: 	my $Handle = IO::File->new($reqfile);
 1050: 	my $cmd    = <$Handle>;
 1051: 	chomp $cmd;		# There may or may not be a newline...
 1052: 	$cmd = $cmd."\n";	# now for sure there's exactly one newline.
 1053: 	my $Transaction = LondTransaction->new($cmd);
 1054: 	$Transaction->SetDeferred($reqfile);
 1055: 	QueueTransaction($Transaction);
 1056:     }
 1057:     
 1058: }
 1059: 
 1060: =pod
 1061: 
 1062: =head2 MakeLondConnection
 1063: 
 1064: Create a new lond connection object, and start it towards its initial
 1065: idleness.  Once idle, it becomes elligible to receive transactions
 1066: from the work queue.  If the work queue is not empty when the
 1067: connection is completed and becomes idle, it will dequeue an entry and
 1068: start off on it.
 1069: 
 1070: =cut
 1071: 
 1072: sub MakeLondConnection {     
 1073:     Debug(4,"MakeLondConnection to ".GetServerHost()." on port "
 1074: 	  .GetServerPort());
 1075: 
 1076:     my $Connection = LondConnection->new(&GetServerHost(),
 1077: 					 &GetServerPort());
 1078: 
 1079:     if($Connection == undef) {	# Needs to be more robust later.
 1080: 	Log("CRITICAL","Failed to make a connection with lond.");
 1081: 	$ConnectionRetriesLeft--;
 1082: 	return 0;		# Failure.
 1083:     }  else {
 1084: 	$ConnectionRetriesLeft = $ConnectionRetries; # success resets the count
 1085: 	# The connection needs to have writability 
 1086: 	# monitored in order to send the init sequence
 1087: 	# that starts the whole authentication/key
 1088: 	# exchange underway.
 1089: 	#
 1090: 	my $Socket = $Connection->GetSocket();
 1091: 	if($Socket == undef) {
 1092: 	    die "did not get a socket from the connection";
 1093: 	} else {
 1094: 	    &Debug(9,"MakeLondConnection got socket: ".$Socket);
 1095: 	}
 1096: 	
 1097: 	
 1098: 	$event = Event->io(fd       => $Socket,
 1099: 			   poll     => 'w',
 1100: 			   cb       => \&LondWritable,
 1101: 			   data     => $Connection,
 1102: 			   desc => 'Connection to lond server');
 1103: 	$ActiveConnections{$Connection} = $event;
 1104: 	
 1105: 	$ConnectionCount++;
 1106: 	Debug(4, "Connection count = ".$ConnectionCount);
 1107: 	if($ConnectionCount == 1) { # First Connection:
 1108: 	    QueueDelayed;
 1109: 	}
 1110: 	Log("SUCESS", "Created connection ".$ConnectionCount
 1111: 	    ." to host ".GetServerHost());
 1112: 	return 1;		# Return success.
 1113:     }
 1114:     
 1115: }
 1116: 
 1117: =pod
 1118: 
 1119: =head2 StartRequest
 1120: 
 1121: Starts a lond request going on a specified lond connection.
 1122: parameters are:
 1123: 
 1124: =item $Lond
 1125: 
 1126: Connection to the lond that will send the transaction and receive the
 1127: reply.
 1128: 
 1129: =item $Client
 1130: 
 1131: Connection to the client that is making this request We got the
 1132: request from this socket, and when the request has been relayed to
 1133: lond and we get a reply back from lond it will get sent to this
 1134: socket.
 1135: 
 1136: =item $Request
 1137: 
 1138: The text of the request to send.
 1139: 
 1140: =cut
 1141: 
 1142: sub StartRequest {
 1143:     my $Lond     = shift;
 1144:     my $Request  = shift;	# This is a LondTransaction.
 1145:     
 1146:     Debug(6, "StartRequest: ".$Request->getRequest());
 1147: 
 1148:     my $Socket = $Lond->GetSocket();
 1149:     
 1150:     $Request->Activate($Lond);
 1151:     $ActiveTransactions{$Lond} = $Request;
 1152: 
 1153:     $Lond->InitiateTransaction($Request->getRequest());
 1154:     $event = Event->io(fd      => $Socket,
 1155: 		       poll    => "w",
 1156: 		       cb      => \&LondWritable,
 1157: 		       data    => $Lond,
 1158: 		       desc    => "lond transaction connection");
 1159:     $ActiveConnections{$Lond} = $event;
 1160:     Debug(8," Start Request made watcher data with ".$event->data."\n");
 1161: }
 1162: 
 1163: =pod
 1164: 
 1165: =head2 QueueTransaction
 1166: 
 1167: If there is an idle lond connection, it is put to work doing this
 1168: transaction.  Otherwise, the transaction is placed in the work queue.
 1169: If placed in the work queue and the maximum number of connections has
 1170: not yet been created, a new connection will be started.  Our goal is
 1171: to eventually have a sufficient number of connections that the work
 1172: queue will typically be empty.  parameters are:
 1173: 
 1174: =item Socket
 1175: 
 1176: open on the lonc client.
 1177: 
 1178: =item Request
 1179: 
 1180: data to send to the lond.
 1181: 
 1182: =cut
 1183: 
 1184: sub QueueTransaction {
 1185: 
 1186:     my $requestData   = shift;	# This is a LondTransaction.
 1187:     my $cmd           = $requestData->getRequest();
 1188: 
 1189:     Debug(6,"QueueTransaction: ".$cmd);
 1190: 
 1191:     my $LondSocket    = $IdleConnections->pop();
 1192:     if(!defined $LondSocket) {	# Need to queue request.
 1193: 	Debug(8,"Must queue...");
 1194: 	$WorkQueue->enqueue($requestData);
 1195: 	if($ConnectionCount < $MaxConnectionCount) {
 1196: 	    Debug(4,"Starting additional lond connection");
 1197: 	    if(MakeLondConnection() == 0) {
 1198: 		EmptyQueue();	# Fail transactions, can't make connection.
 1199: 	    }
 1200: 	}
 1201:     } else {			# Can start the request:
 1202: 	Debug(8,"Can start...");
 1203: 	StartRequest($LondSocket,  $requestData);
 1204:     }
 1205: }
 1206: 
 1207: #-------------------------- Lonc UNIX socket handling ---------------------
 1208: 
 1209: =pod
 1210: 
 1211: =head2 ClientRequest
 1212: Callback that is called when data can be read from the UNIX domain
 1213: socket connecting us with an apache server process.
 1214: 
 1215: =cut
 1216: 
 1217: sub ClientRequest {
 1218:     Debug(6, "ClientRequest");
 1219:     my $event   = shift;
 1220:     my $watcher = $event->w;
 1221:     my $socket  = $watcher->fd;
 1222:     my $data    = $watcher->data;
 1223:     my $thisread;
 1224: 
 1225:     Debug(9, "  Watcher named: ".$watcher->desc);
 1226: 
 1227:     my $rv = $socket->recv($thisread, POSIX::BUFSIZ, 0);
 1228:     Debug(8, "rcv:  data length = ".length($thisread)
 1229: 	  ." read =".$thisread);
 1230:     unless (defined $rv && length($thisread)) {
 1231: 	 # Likely eof on socket.
 1232: 	Debug(5,"Client Socket closed on lonc for ".$RemoteHost);
 1233: 	close($socket);
 1234: 	$watcher->cancel();
 1235: 	delete($ActiveClients{$socket});
 1236: 	return;
 1237:     }
 1238:     Debug(8,"Data: ".$data." this read: ".$thisread);
 1239:     $data = $data.$thisread;	# Append new data.
 1240:     $watcher->data($data);
 1241:     if($data =~ /(.*\n)/) {	# Request entirely read.
 1242: 	if($data eq "close_connection_exit\n") {
 1243: 	    Log("CRITICAL",
 1244: 		"Request Close Connection ... exiting");
 1245: 	    CloseAllLondConnections();
 1246: 	    exit;
 1247: 	}
 1248: 	Debug(8, "Complete transaction received: ".$data);
 1249: 	my $Transaction = LondTransaction->new($data);
 1250: 	$Transaction->SetClient($socket);
 1251: 	QueueTransaction($Transaction);
 1252: 	$watcher->cancel();	# Done looking for input data.
 1253:     }
 1254: 
 1255: }
 1256: 
 1257: 
 1258: =pod
 1259: 
 1260: =head2  NewClient
 1261: 
 1262: Callback that is called when a connection is received on the unix
 1263: socket for a new client of lonc.  The callback is parameterized by the
 1264: event.. which is a-priori assumed to be an io event, and therefore has
 1265: an fd member that is the Listener socket.  We Accept the connection
 1266: and register a new event on the readability of that socket:
 1267: 
 1268: =cut
 1269: 
 1270: sub NewClient {
 1271:     Debug(6, "NewClient");
 1272:     my $event      = shift;		# Get the event parameters.
 1273:     my $watcher    = $event->w; 
 1274:     my $socket     = $watcher->fd;	# Get the event' socket.
 1275:     my $connection = $socket->accept();	# Accept the client connection.
 1276:     Debug(8,"Connection request accepted from "
 1277: 	  .GetPeername($connection, AF_UNIX));
 1278: 
 1279: 
 1280:     my $description = sprintf("Connection to lonc client %d",
 1281: 			      $ClientConnection);
 1282:     Debug(9, "Creating event named: ".$description);
 1283:     Event->io(cb      => \&ClientRequest,
 1284: 	      poll    => 'r',
 1285: 	      desc    => $description,
 1286: 	      data    => "",
 1287: 	      fd      => $connection);
 1288:     $ActiveClients{$connection} = $ClientConnection;
 1289:     $ClientConnection++;
 1290: }
 1291: 
 1292: =pod
 1293: 
 1294: =head2 GetLoncSocketPath
 1295: 
 1296: Returns the name of the UNIX socket on which to listen for client
 1297: connections.
 1298: 
 1299: =cut
 1300: 
 1301: sub GetLoncSocketPath {
 1302:     return $UnixSocketDir."/".GetServerHost();
 1303: }
 1304: 
 1305: =pod
 1306: 
 1307: =head2 GetServerHost
 1308: 
 1309: Returns the host whose lond we talk with.
 1310: 
 1311: =cut
 1312: 
 1313: sub GetServerHost {
 1314:     return $RemoteHost;		# Setup by the fork.
 1315: }
 1316: 
 1317: =pod
 1318: 
 1319: =head2 GetServerPort
 1320: 
 1321: Returns the lond port number.
 1322: 
 1323: =cut
 1324: 
 1325: sub GetServerPort {
 1326:     return $perlvar{londPort};
 1327: }
 1328: 
 1329: =pod
 1330: 
 1331: =head2 SetupLoncListener
 1332: 
 1333: Setup a lonc listener event.  The event is called when the socket
 1334: becomes readable.. that corresponds to the receipt of a new
 1335: connection.  The event handler established will accept the connection
 1336: (creating a communcations channel), that int turn will establish
 1337: another event handler to subess requests.
 1338: 
 1339: =cut
 1340: 
 1341: sub SetupLoncListener {
 1342: 
 1343:     my $socket;
 1344:     my $SocketName = GetLoncSocketPath();
 1345:     unlink($SocketName);
 1346:     unless ($socket =IO::Socket::UNIX->new(Local  => $SocketName,
 1347: 					    Listen => 10, 
 1348: 					    Type   => SOCK_STREAM)) {
 1349: 	die "Failed to create a lonc listner socket";
 1350:     }
 1351:     Event->io(cb     => \&NewClient,
 1352: 	      poll   => 'r',
 1353: 	      desc   => 'Lonc listener Unix Socket',
 1354: 	      fd     => $socket);
 1355: }
 1356: 
 1357: =pod 
 1358: 
 1359: =head2 ChildStatus
 1360:  
 1361: Child USR1 signal handler to report the most recent status
 1362: into the status file.
 1363: 
 1364: =cut
 1365: sub ChildStatus {
 1366:     my $event = shift;
 1367:     my $watcher = $event->w;
 1368: 
 1369:     Debug(2, "Reporting child status because : ".$watcher->data);
 1370:     my $docdir = $perlvar{'lonDocRoot'};
 1371:     my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt");
 1372:     print $fh $$."\t".$RemoteHost."\t".$Status."\t".
 1373: 	$RecentLogEntry."\n";
 1374: }
 1375: 
 1376: =pod
 1377: 
 1378: =head2 SignalledToDeath
 1379: 
 1380: Called in response to a signal that causes a chid process to die.
 1381: 
 1382: =cut
 1383: 
 1384: 
 1385: sub SignalledToDeath {
 1386:     my $event  = shift;
 1387:     my $watcher= $event->w;
 1388: 
 1389:     Debug(2,"Signalled to death! via ".$watcher->data);
 1390:     my ($signal) = $watcher->data;
 1391:     chomp($signal);
 1392:     Log("CRITICAL", "Abnormal exit.  Child $$ for $RemoteHost "
 1393: 	."died through "."\"$signal\"");
 1394:     LogPerm("F:lonc: $$ on $RemoteHost signalled to death: "
 1395: 	    ."\"$signal\"");
 1396:     exit 0;
 1397: 
 1398: }
 1399: 
 1400: =head2 ToggleDebug
 1401: 
 1402: This sub toggles trace debugging on and off.
 1403: 
 1404: =cut
 1405: 
 1406: sub ToggleDebug {
 1407:     my $Current    = $DebugLevel;
 1408:        $DebugLevel = $NextDebugLevel;
 1409:        $NextDebugLevel = $Current;
 1410: 
 1411:     Log("SUCCESS", "New debugging level for $RemoteHost now $DebugLevel");
 1412: 
 1413: }
 1414: 
 1415: =head2 ChildProcess
 1416: 
 1417: This sub implements a child process for a single lonc daemon.
 1418: 
 1419: =cut
 1420: 
 1421: sub ChildProcess {
 1422: 
 1423: 
 1424:     #
 1425:     #  Signals must be handled by the Event framework...
 1426: #
 1427: 
 1428:     Event->signal(signal   => "QUIT",
 1429: 		  cb       => \&SignalledToDeath,
 1430: 		  data     => "QUIT");
 1431:     Event->signal(signal   => "HUP",
 1432: 		  cb       => \&ChildStatus,
 1433: 		  data     => "HUP");
 1434:     Event->signal(signal   => "USR1",
 1435: 		  cb       => \&ChildStatus,
 1436: 		  data     => "USR1");
 1437:     Event->signal(signal   => "INT",
 1438: 		  cb       => \&ToggleDebug,
 1439: 		  data     => "INT");
 1440: 
 1441:     SetupTimer();
 1442:     
 1443:     SetupLoncListener();
 1444:     
 1445:     $Event::Debuglevel = $DebugLevel;
 1446:     
 1447:     Debug(9, "Making initial lond connection for ".$RemoteHost);
 1448: 
 1449: # Setup the initial server connection:
 1450:     
 1451:      # &MakeLondConnection(); // let first work requirest do it.
 1452: 
 1453: 
 1454:     Debug(9,"Entering event loop");
 1455:     my $ret = Event::loop();		#  Start the main event loop.
 1456:     
 1457:     
 1458:     die "Main event loop exited!!!";
 1459: }
 1460: 
 1461: #  Create a new child for host passed in:
 1462: 
 1463: sub CreateChild {
 1464:     my $sigset = POSIX::SigSet->new(SIGINT);
 1465:     sigprocmask(SIG_BLOCK, $sigset);
 1466:     my $host = shift;
 1467:     $RemoteHost = $host;
 1468:     Log("CRITICAL", "Forking server for ".$host);
 1469:     $pid          = fork;
 1470:     if($pid) {			# Parent
 1471: 	$RemoteHost = "Parent";
 1472: 	$ChildHash{$pid} = $RemoteHost;
 1473: 	sigprocmask(SIG_UNBLOCK, $sigset);
 1474: 
 1475:     } else {			# child.
 1476: 	ShowStatus("Connected to ".$RemoteHost);
 1477: 	$SIG{INT} = DEFAULT;
 1478: 	sigprocmask(SIG_UNBLOCK, $sigset);
 1479: 	ChildProcess;		# Does not return.
 1480:     }
 1481: 
 1482: }
 1483: #
 1484: #  Parent process logic pass 1:
 1485: #   For each entry in the hosts table, we will
 1486: #  fork off an instance of ChildProcess to service the transactions
 1487: #  to that host.  Each pid will be entered in a global hash
 1488: #  with the value of the key, the host.
 1489: #  The parent will then enter a loop to wait for process exits.
 1490: #  Each exit gets logged and the child gets restarted.
 1491: #
 1492: 
 1493: #
 1494: #   Fork and start in new session so hang-up isn't going to 
 1495: #   happen without intent.
 1496: #
 1497: 
 1498: 
 1499: 
 1500: 
 1501: 
 1502: 
 1503: ShowStatus("Forming new session");
 1504: my $childpid = fork;
 1505: if ($childpid != 0) {
 1506:     sleep 4;			# Give child a chacne to break to
 1507:     exit 0;			# a new sesion.
 1508: }
 1509: #
 1510: #   Write my pid into the pid file so I can be located
 1511: #
 1512: 
 1513: ShowStatus("Parent writing pid file:");
 1514: $execdir = $perlvar{'lonDaemons'};
 1515: open (PIDSAVE, ">$execdir/logs/lonc.pid");
 1516: print PIDSAVE "$$\n";
 1517: close(PIDSAVE);
 1518: 
 1519: 
 1520: 
 1521: if (POSIX::setsid() < 0) {
 1522:     print "Could not create new session\n";
 1523:     exit -1;
 1524: }
 1525: 
 1526: ShowStatus("Forking node servers");
 1527: 
 1528: Log("CRITICAL", "--------------- Starting children ---------------");
 1529: 
 1530: my $HostIterator = LondConnection::GetHostIterator;
 1531: while (! $HostIterator->end()) {
 1532: 
 1533:     $hostentryref = $HostIterator->get();
 1534:     CreateChild($hostentryref->[0]);
 1535:     $HostIterator->next();
 1536: }
 1537: $RemoteHost = "Parent Server";
 1538: 
 1539: # Maintain the population:
 1540: 
 1541: ShowStatus("Parent keeping the flock");
 1542: 
 1543: #
 1544: #   Set up parent signals:
 1545: #
 1546: 
 1547: $SIG{INT}  = \&Terminate;
 1548: $SIG{TERM} = \&Terminate; 
 1549: $SIG{HUP}  = \&Restart;
 1550: $SIG{USR1} = \&CheckKids; 
 1551: 
 1552: while(1) {
 1553:     $deadchild = wait();
 1554:     if(exists $ChildHash{$deadchild}) {	# need to restart.
 1555: 	$deadhost = $ChildHash{$deadchild};
 1556: 	delete($ChildHash{$deadchild});
 1557: 	Log("WARNING","Lost child pid= ".$deadchild.
 1558: 	      "Connected to host ".$deadhost);
 1559: 	Log("INFO", "Restarting child procesing ".$deadhost);
 1560: 	CreateChild($deadhost);
 1561:     }
 1562: }
 1563: 
 1564: 
 1565: 
 1566: =pod
 1567: 
 1568: =head1 CheckKids
 1569: 
 1570:   Since kids do not die as easily in this implementation
 1571: as the previous one, there  is no need to restart the
 1572: dead ones (all dead kids get restarted when they die!!)
 1573: The only thing this function does is to pass USR1 to the
 1574: kids so that they report their status.
 1575: 
 1576: =cut
 1577: 
 1578: sub CheckKids {
 1579:     Debug(2, "Checking status of children");
 1580:     my $docdir = $perlvar{'lonDocRoot'};
 1581:     my $fh = IO::File->new(">$docdir/lon-status/loncstatus.txt");
 1582:     my $now=time;
 1583:     my $local=localtime($now);
 1584:     print $fh "LONC status $local - parent $$ \n\n";
 1585:     foreach $pid (keys %ChildHash) {
 1586: 	Debug(2, "Sending USR1 -> $pid");
 1587: 	kill 'USR1' => $pid;	# Tell Child to report status.
 1588: 	sleep 1;		# Wait so file doesn't intermix.
 1589:     }
 1590: }
 1591: 
 1592: =pod
 1593: 
 1594: =head1 Restart
 1595: 
 1596: Signal handler for HUP... all children are killed and
 1597: we self restart.  This is an el-cheapo way to re read
 1598: the config file.
 1599: 
 1600: =cut
 1601: 
 1602: sub Restart {
 1603:     KillThemAll;		# First kill all the children.
 1604:     Log("CRITICAL", "Restarting");
 1605:     my $execdir = $perlvar{'lonDaemons'};
 1606:     unlink("$execdir/logs/lonc.pid");
 1607:     exec("$execdir/lonc");
 1608: }
 1609: 
 1610: =pod
 1611: 
 1612: =head1 KillThemAll
 1613: 
 1614: Signal handler that kills all children by sending them a 
 1615: SIGHUP.  Responds to sigint and sigterm.
 1616: 
 1617: =cut
 1618: 
 1619: sub KillThemAll {
 1620:     Debug(2, "Kill them all!!");
 1621:     local($SIG{CHLD}) = 'IGNORE';      # Our children >will< die.
 1622:     foreach $pid (keys %ChildHash) {
 1623: 	my $serving = $ChildHash{$pid};
 1624: 	Debug(2, "Killing lonc for $serving pid = $pid");
 1625: 	ShowStatus("Killing lonc for $serving pid = $pid");
 1626: 	Log("CRITICAL", "Killing lonc for $serving pid = $pid");
 1627: 	kill 'QUIT' => $pid;
 1628: 	delete($ChildHash{$pid});
 1629:     }
 1630:     my $execdir = $perlvar{'lonDaemons'};
 1631:     unlink("$execdir/logs/lonc.pid");
 1632: 
 1633: }
 1634: 
 1635: =pod
 1636: 
 1637: =head1 Terminate
 1638:  
 1639: Terminate the system.
 1640: 
 1641: =cut
 1642: 
 1643: sub Terminate {
 1644:     KillThemAll;
 1645:     Log("CRITICAL","Master process exiting");
 1646:     exit 0;
 1647: 
 1648: }
 1649: =pod
 1650: 
 1651: =head1 Theory
 1652: 
 1653: The event class is used to build this as a single process with an
 1654: event driven model.  The following events are handled:
 1655: 
 1656: =item UNIX Socket connection Received
 1657: 
 1658: =item Request data arrives on UNIX data transfer socket.
 1659: 
 1660: =item lond connection becomes writable.
 1661: 
 1662: =item timer fires at 1 second intervals.
 1663: 
 1664: All sockets are run in non-blocking mode.  Timeouts managed by the timer
 1665: handler prevents hung connections.
 1666: 
 1667: Key data structures:
 1668: 
 1669: =item RequestQueue
 1670: 
 1671: A queue of requests received from UNIX sockets that are
 1672: waiting for a chance to be forwarded on a lond connection socket.
 1673: 
 1674: =item ActiveConnections
 1675: 
 1676: A hash of lond connections that have transactions in process that are
 1677: available to be timed out.
 1678: 
 1679: =item ActiveTransactions
 1680: 
 1681: A hash indexed by lond connections that contain the client reply
 1682: socket for each connection that has an active transaction on it.
 1683: 
 1684: =item IdleConnections
 1685: 
 1686: A hash of lond connections that have no work to do.  These connections
 1687: can be closed if they are idle for a long enough time.
 1688: 
 1689: =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>