File:  [LON-CAPA] / loncom / loncnew
Revision 1.16: download - view: text, annotated - select for diffs
Tue Jul 29 02:33:05 2003 UTC (20 years, 9 months ago) by foxr
Branches: MAIN
CVS tags: HEAD
Add SIGINT processing to child processes to toggle annoying trace mode
on/off.. will try to use this to isolate the compute boud process issue.

    1: #!/usr/bin/perl
    2: # The LearningOnline Network with CAPA
    3: # lonc maintains the connections to remote computers
    4: #
    5: # $Id: loncnew,v 1.16 2003/07/29 02:33:05 foxr Exp $
    6: #
    7: # Copyright Michigan State University Board of Trustees
    8: #
    9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
   10: #
   11: # LON-CAPA is free software; you can redistribute it and/or modify
   12: # it under the terms of the GNU General Public License as published by
   13: # the Free Software Foundation; either version 2 of the License, or
   14: # (at your option) any later version.
   15: #
   16: # LON-CAPA is distributed in the hope that it will be useful,
   17: # but WITHOUT ANY WARRANTY; without even the implied warranty of
   18: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   19: # GNU General Public License for more details.
   20: #
   21: # You should have received a copy of the GNU General Public License
   22: # along with LON-CAPA; if not, write to the Free Software
   23: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   24: #
   25: # /home/httpd/html/adm/gpl.txt
   26: #
   27: # http://www.lon-capa.org/
   28: #
   29: #
   30: # new lonc handles n request out bver m connections to londs.
   31: # This module is based on the Event class.
   32: #   Development iterations:
   33: #    - Setup basic event loop.   (done)
   34: #    - Add timer dispatch.       (done)
   35: #    - Add ability to accept lonc UNIX domain sockets.  (done)
   36: #    - Add ability to create/negotiate lond connections (done).
   37: #    - Add general logic for dispatching requests and timeouts. (done).
   38: #    - Add support for the lonc/lond requests.          (done).
   39: #    - Add logging/status monitoring.
   40: #    - Add Signal handling - HUP restarts. USR1 status report.
   41: #    - Add Configuration file I/O                       (done).
   42: #    - Add management/status request interface.
   43: #    - Add deferred request capability.                  (done)
   44: #    - Detect transmission timeouts.
   45: #
   46: 
   47: # Change log:
   48: #    $Log: loncnew,v $
   49: #    Revision 1.16  2003/07/29 02:33:05  foxr
   50: #    Add SIGINT processing to child processes to toggle annoying trace mode
   51: #    on/off.. will try to use this to isolate the compute boud process issue.
   52: #
   53: #    Revision 1.15  2003/07/15 02:07:05  foxr
   54: #    Added code for lonc/lond transaction timeouts.  Who knows if it works right.
   55: #    The intent is for a timeout to fail any transaction in progress and kill
   56: #    off the sockt that timed out.
   57: #
   58: #    Revision 1.14  2003/07/03 02:10:18  foxr
   59: #    Get all of the signals to work correctly.
   60: #
   61: #    Revision 1.13  2003/07/02 01:31:55  foxr
   62: #    Added kill -HUP logic (restart).
   63: #
   64: #    Revision 1.11  2003/06/25 01:54:44  foxr
   65: #    Fix more problems with transaction failure.
   66: #
   67: #    Revision 1.10  2003/06/24 02:46:04  foxr
   68: #    Put a limit on  the number of times we'll retry a connection.
   69: #    Start getting the signal stuff put in as well...note that need to get signals
   70: #    going or else 6the client will permanently give up on dead servers.
   71: #
   72: #    Revision 1.9  2003/06/13 02:38:43  foxr
   73: #    Add logging in 'expected format'
   74: #
   75: #    Revision 1.8  2003/06/11 02:04:35  foxr
   76: #    Support delayed transactions... this is done uniformly by encapsulating
   77: #    transactions in an object ... a LondTransaction that is implemented by
   78: #    LondTransaction.pm
   79: #
   80: #    Revision 1.7  2003/06/03 01:59:39  foxr
   81: #    complete coding to support deferred transactions.
   82: #
   83: #
   84: 
   85: use lib "/home/httpd/lib/perl/";
   86: use lib "/home/foxr/newloncapa/types";
   87: use Event qw(:DEFAULT );
   88: use POSIX qw(:signal_h);
   89: use POSIX;
   90: use IO::Socket;
   91: use IO::Socket::INET;
   92: use IO::Socket::UNIX;
   93: use IO::File;
   94: use IO::Handle;
   95: use Socket;
   96: use Crypt::IDEA;
   97: use LONCAPA::Queue;
   98: use LONCAPA::Stack;
   99: use LONCAPA::LondConnection;
  100: use LONCAPA::LondTransaction;
  101: use LONCAPA::Configuration;
  102: use LONCAPA::HashIterator;
  103: 
  104: 
  105: #
  106: #   Disable all signals we might receive from outside for now.
  107: #
  108: #$SIG{QUIT}  = IGNORE;
  109: #$SIG{HUP}   = IGNORE;
  110: #$SIG{USR1}  = IGNORE;
  111: #$SIG{INT}   = IGNORE;
  112: #$SIG{CHLD}  = IGNORE;
  113: #$SIG{__DIE__}  = IGNORE;
  114: 
  115: 
  116: # Read the httpd configuration file to get perl variables
  117: # normally set in apache modules:
  118: 
  119: my $perlvarref = LONCAPA::Configuration::read_conf('loncapa.conf');
  120: my %perlvar    = %{$perlvarref};
  121: 
  122: #
  123: #  parent and shared variables.
  124: 
  125: my %ChildHash;			# by pid -> host.
  126: 
  127: 
  128: my $MaxConnectionCount = 10;	# Will get from config later.
  129: my $ClientConnection = 0;	# Uniquifier for client events.
  130: 
  131: my $DebugLevel = 0;
  132: my $NextDebugLevel= 10;		# So Sigint can toggle this.
  133: my $IdleTimeout= 3600;		# Wait an hour before pruning connections.
  134: 
  135: #
  136: #  The variables below are only used by the child processes.
  137: #
  138: my $RemoteHost;			# Name of host child is talking to.
  139: my $UnixSocketDir= "/home/httpd/sockets"; 
  140: my $IdleConnections = Stack->new(); # Set of idle connections
  141: my %ActiveConnections;		# Connections to the remote lond.
  142: my %ActiveTransactions;		# LondTransactions in flight.
  143: my %ActiveClients;		# Serial numbers of active clients by socket.
  144: my $WorkQueue       = Queue->new(); # Queue of pending transactions.
  145: my $ConnectionCount = 0;
  146: my $IdleSeconds     = 0;	# Number of seconds idle.
  147: my $Status          = "";	# Current status string.
  148: my $RecentLogEntry  = "";
  149: my $ConnectionRetries=5;	# Number of connection retries allowed.
  150: my $ConnectionRetriesLeft=5;	# Number of connection retries remaining.
  151: 
  152: #
  153: #   The hash below gives the HTML format for log messages
  154: #   given a severity.
  155: #    
  156: my %LogFormats;
  157: 
  158: $LogFormats{"CRITICAL"} = "<font color=red>CRITICAL: %s</font>";
  159: $LogFormats{"SUCCESS"}  = "<font color=green>SUCCESS: %s</font>";
  160: $LogFormats{"INFO"}     = "<font color=yellow>INFO: %s</font>";
  161: $LogFormats{"WARNING"}  = "<font color=blue>WARNING: %s</font>";
  162: $LogFormats{"DEFAULT"}  = " %s ";
  163: 
  164: 
  165: 
  166: =pod
  167: 
  168: =head2 LogPerm
  169: 
  170: Makes an entry into the permanent log file.
  171: 
  172: =cut
  173: sub LogPerm {
  174:     my $message=shift;
  175:     my $execdir=$perlvar{'lonDaemons'};
  176:     my $now=time;
  177:     my $local=localtime($now);
  178:     my $fh=IO::File->new(">>$execdir/logs/lonnet.perm.log");
  179:     print $fh "$now:$message:$local\n";
  180: }
  181: 
  182: =pod
  183: 
  184: =head2 Log
  185: 
  186: Logs a message to the log file.
  187: Parameters:
  188: 
  189: =item severity
  190: 
  191: One of CRITICAL, WARNING, INFO, SUCCESS used to select the
  192: format string used to format the message.  if the severity is
  193: not a defined severity the Default format string is used.
  194: 
  195: =item message
  196: 
  197: The base message.  In addtion to the format string, the message
  198: will be appended to a string containing the name of our remote
  199: host and the time will be formatted into the message.
  200: 
  201: =cut
  202: 
  203: sub Log {
  204:     my $severity = shift;
  205:     my $message  = shift;
  206:    
  207:     if(!$LogFormats{$severity}) {
  208: 	$severity = "DEFAULT";
  209:     }
  210: 
  211:     my $format = $LogFormats{$severity};
  212:     
  213:     #  Put the window dressing in in front of the message format:
  214: 
  215:     my $now   = time;
  216:     my $local = localtime($now);
  217:     my $finalformat = "$local ($$) [$RemoteHost] [$Status] ";
  218:     my $finalformat = $finalformat.$format."\n";
  219: 
  220:     # open the file and put the result.
  221: 
  222:     my $execdir = $perlvar{'lonDaemons'};
  223:     my $fh      = IO::File->new(">>$execdir/logs/lonc.log");
  224:     my $msg = sprintf($finalformat, $message);
  225:     $RecentLogEntry = $msg;
  226:     print $fh $msg;
  227:     
  228:     
  229: }
  230: 
  231: 
  232: =pod
  233: 
  234: =head2 GetPeerName
  235: 
  236: Returns the name of the host that a socket object is connected to.
  237: 
  238: =cut
  239: 
  240: sub GetPeername {
  241:     my $connection = shift;
  242:     my $AdrFamily  = shift;
  243:     my $peer       = $connection->peername();
  244:     my $peerport;
  245:     my $peerip;
  246:     if($AdrFamily == AF_INET) {
  247: 	($peerport, $peerip) = sockaddr_in($peer);
  248: 	my $peername    = gethostbyaddr($iaddr, $AdrFamily);
  249: 	return $peername;
  250:     } elsif ($AdrFamily == AF_UNIX) {
  251: 	my $peerfile;
  252: 	($peerfile) = sockaddr_un($peer);
  253: 	return $peerfile;
  254:     }
  255: }
  256: #----------------------------- Timer management ------------------------
  257: =pod
  258: 
  259: =head2 Debug
  260: 
  261: Invoked to issue a debug message.
  262: 
  263: =cut
  264: 
  265: sub Debug {
  266:     my $level   = shift;
  267:     my $message = shift;
  268:     if ($level <= $DebugLevel) {
  269: 	Log("INFO", "-Debug- $message host = $RemotHost");
  270:     }
  271: }
  272: 
  273: sub SocketDump {
  274:     my $level = shift;
  275:     my $socket= shift;
  276:     if($level <= $DebugLevel) {
  277: 	$socket->Dump();
  278:     }
  279: }
  280: 
  281: =pod
  282: 
  283: =head2 ShowStatus
  284: 
  285:  Place some text as our pid status.
  286:  and as what we return in a SIGUSR1
  287: 
  288: =cut
  289: sub ShowStatus {
  290:     my $state = shift;
  291:     my $now = time;
  292:     my $local = localtime($now);
  293:     $Status   = $local.": ".$state;
  294:     $0='lonc: '.$state.' '.$local;
  295: }
  296: 
  297: =pod
  298: 
  299: =head 2 SocketTimeout
  300: 
  301:     Called when an action on the socket times out.  The socket is 
  302:    destroyed and any active transaction is failed.
  303: 
  304: 
  305: =cut
  306: sub SocketTimeout {
  307:     my $Socket = shift;
  308:     
  309:     KillSocket($Socket);
  310: }
  311: 
  312: =pod
  313: 
  314: =head2 Tick
  315: 
  316: Invoked  each timer tick.
  317: 
  318: =cut
  319: 
  320: 
  321: sub Tick {
  322:     my $client;
  323:     ShowStatus(GetServerHost()." Connection count: ".$ConnectionCount);
  324: 
  325:     # Is it time to prune connection count:
  326: 
  327: 
  328:     if($IdleConnections->Count()  && 
  329:        ($WorkQueue->Count() == 0)) { # Idle connections and nothing to do?
  330: 	$IdleSeconds++;
  331: 	if($IdleSeconds > $IdleTimeout) { # Prune a connection...
  332: 	    $Socket = $IdleConnections->pop();
  333: 	    KillSocket($Socket);
  334: 	}
  335:     } else {
  336: 	$IdleSeconds = 0;	# Reset idle count if not idle.
  337:     }
  338:     #
  339:     #  For each inflight transaction, tick down its timeout counter.
  340:     #
  341:     foreach $item (keys %ActiveTransactions) {
  342: 	my $Socket = $ActiveTransactions{$item}->getServer();
  343: 	$Socket->Tick();
  344:     }
  345:     # Do we have work in the queue, but no connections to service them?
  346:     # If so, try to make some new connections to get things going again.
  347:     #
  348:     
  349:     my $Requests = $WorkQueue->Count();
  350:     if (($ConnectionCount == 0)  && ($Requests > 0)) { 
  351: 	if ($ConnectionRetriesLeft > 0) {
  352: 	    my $Connections = ($Requests <= $MaxConnectionCount) ?
  353: 		$Requests : $MaxConnectionCount;
  354: 	    Debug(1,"Work but no connections, start ".$Connections." of them");
  355: 	    for ($i =0; $i < $Connections; $i++) {
  356: 		MakeLondConnection();
  357: 	    }
  358: 	} else {
  359: 	    Debug(1,"Work in queue, but gave up on connections..flushing\n");
  360: 	    EmptyQueue();	# Connections can't be established.
  361: 	}
  362:        
  363:     }
  364: }
  365: 
  366: =pod
  367: 
  368: =head2 SetupTimer
  369: 
  370: Sets up a 1 per sec recurring timer event.  The event handler is used to:
  371: 
  372: =item
  373: 
  374: Trigger timeouts on communications along active sockets.
  375: 
  376: =item
  377: 
  378: Trigger disconnections of idle sockets.
  379: 
  380: =cut
  381: 
  382: sub SetupTimer {
  383:     Debug(6, "SetupTimer");
  384:     Event->timer(interval => 1, debug => 1, cb => \&Tick );
  385: }
  386: 
  387: =pod
  388: 
  389: =head2 ServerToIdle
  390: 
  391: This function is called when a connection to the server is
  392: ready for more work.
  393: 
  394: If there is work in the Work queue the top element is dequeued
  395: and the connection will start to work on it.  If the work queue is
  396: empty, the connection is pushed on the idle connection stack where
  397: it will either get another work unit, or alternatively, if it sits there
  398: long enough, it will be shut down and released.
  399: 
  400: =cut
  401: 
  402: sub ServerToIdle {
  403:     my $Socket   = shift;	# Get the socket.
  404:     delete($ActiveTransactions{$Socket}); # Server has no transaction
  405: 
  406:     &Debug(6, "Server to idle");
  407: 
  408:     #  If there's work to do, start the transaction:
  409: 
  410:     $reqdata = $WorkQueue->dequeue(); # This is a LondTransaction
  411:     unless($reqdata eq undef)  {
  412: 	Debug(9, "Queue gave request data: ".$reqdata->getRequest());
  413: 	&StartRequest($Socket,  $reqdata);
  414: 
  415:     } else {
  416: 	
  417:     #  There's no work waiting, so push the server to idle list.
  418: 	&Debug(8, "No new work requests, server connection going idle");
  419: 	$IdleConnections->push($Socket);
  420:     }
  421: }
  422: 
  423: =pod
  424: 
  425: =head2 ClientWritable
  426: 
  427: Event callback for when a client socket is writable.
  428: 
  429: This callback is established when a transaction reponse is
  430: avaiable from lond.  The response is forwarded to the unix socket
  431: as it becomes writable in this sub.
  432: 
  433: Parameters:
  434: 
  435: =item Event
  436: 
  437: The event that has been triggered. Event->w->data is
  438: the data and Event->w->fd is the socket to write.
  439: 
  440: =cut
  441: 
  442: sub ClientWritable {
  443:     my $Event    = shift;
  444:     my $Watcher  = $Event->w;
  445:     my $Data     = $Watcher->data;
  446:     my $Socket   = $Watcher->fd;
  447: 
  448:     # Try to send the data:
  449: 
  450:     &Debug(6, "ClientWritable writing".$Data);
  451:     &Debug(9, "Socket is: ".$Socket);
  452: 
  453:     if($Socket->connected) {
  454: 	my $result = $Socket->send($Data, 0);
  455: 	
  456: 	# $result undefined: the write failed.
  457: 	# otherwise $result is the number of bytes written.
  458: 	# Remove that preceding string from the data.
  459: 	# If the resulting data is empty, destroy the watcher
  460: 	# and set up a read event handler to accept the next
  461: 	# request.
  462: 	
  463: 	&Debug(9,"Send result is ".$result." Defined: ".defined($result));
  464: 	if(defined($result)) {
  465: 	    &Debug(9, "send result was defined");
  466: 	    if($result == length($Data)) { # Entire string sent.
  467: 		&Debug(9, "ClientWritable data all written");
  468: 		$Watcher->cancel();
  469: 		#
  470: 		#  Set up to read next request from socket:
  471: 		
  472: 		my $descr     = sprintf("Connection to lonc client %d",
  473: 					$ActiveClients{$Socket});
  474: 		Event->io(cb    => \&ClientRequest,
  475: 			  poll  => 'r',
  476: 			  desc  => $descr,
  477: 			  data  => "",
  478: 			  fd    => $Socket);
  479: 		
  480: 	    } else {		# Partial string sent.
  481: 		$Watcher->data(substr($Data, $result));
  482: 		if($result == 0) {    # client hung up on us!!
  483: 		    Log("INFO", "lonc pipe client hung up on us!");
  484: 		    $Watcher->cancel;
  485: 		    $Socket->shutdown(2);
  486: 		    $Socket->close();
  487: 		}
  488: 	    }
  489: 	    
  490: 	} else {			# Error of some sort...
  491: 	    
  492: 	    # Some errnos are possible:
  493: 	    my $errno = $!;
  494: 	    if($errno == POSIX::EWOULDBLOCK   ||
  495: 	       $errno == POSIX::EAGAIN        ||
  496: 	       $errno == POSIX::EINTR) {
  497: 		# No action taken?
  498: 	    } else {		# Unanticipated errno.
  499: 		&Debug(5,"ClientWritable error or peer shutdown".$RemoteHost);
  500: 		$Watcher->cancel;	# Stop the watcher.
  501: 		$Socket->shutdown(2); # Kill connection
  502: 		$Socket->close();	# Close the socket.
  503: 	    }
  504: 	    
  505: 	}
  506:     } else {
  507: 	$Watcher->cancel();	# A delayed request...just cancel.
  508:     }
  509: }
  510: 
  511: =pod
  512: 
  513: =head2 CompleteTransaction
  514: 
  515: Called when the reply data has been received for a lond 
  516: transaction.   The reply data must now be sent to the
  517: ultimate client on the other end of the Unix socket.  This is
  518: done by setting up a writable event for the socket with the
  519: data the reply data.
  520: 
  521: Parameters:
  522: 
  523: =item Socket
  524: 
  525: Socket on which the lond transaction occured.  This is a
  526: LondConnection. The data received is in the TransactionReply member.
  527: 
  528: =item Transaction
  529: 
  530: The transaction that is being completed.
  531: 
  532: =cut
  533: 
  534: sub CompleteTransaction {
  535:     &Debug(6,"Complete transaction");
  536:     my $Socket = shift;
  537:     my $Transaction = shift;
  538: 
  539:     if (!$Transaction->isDeferred()) { # Normal transaction
  540: 	my $data   = $Socket->GetReply(); # Data to send.
  541: 	StartClientReply($Transaction, $data);
  542:     } else {			# Delete deferred transaction file.
  543: 	Log("SUCCESS", "A delayed transaction was completed");
  544: 	LogPerm("S:$Client:".$Transaction->getRequest());
  545: 	unlink $Transaction->getFile();
  546:     }
  547: }
  548: =pod
  549: =head1 StartClientReply
  550: 
  551:    Initiates a reply to a client where the reply data is a parameter.
  552: 
  553: =head2  parameters:
  554: 
  555: =item Transaction
  556: 
  557:     The transaction for which we are responding to the client.
  558: 
  559: =item data
  560: 
  561:     The data to send to apached client.
  562: 
  563: =cut
  564: sub StartClientReply {
  565:     my $Transaction   = shift;
  566:     my $data     = shift;
  567: 
  568: 
  569:     my $Client   = $Transaction->getClient();
  570: 
  571:     &Debug(8," Reply was: ".$data);
  572:     my $Serial         = $ActiveClients{$Client};
  573:     my $desc           = sprintf("Connection to lonc client %d",
  574: 
  575: 				 $Serial);
  576:     Event->io(fd       => $Client,
  577: 	      poll     => "w",
  578: 	      desc     => $desc,
  579: 	      cb       => \&ClientWritable,
  580: 	      data     => $data);
  581: }
  582: =pod
  583: =head2 FailTransaction
  584: 
  585:   Finishes a transaction with failure because the associated lond socket
  586:   disconnected.  There are two possibilities:
  587:   - The transaction is deferred: in which case we just quietly
  588:     delete the transaction since there is no client connection.
  589:   - The transaction is 'live' in which case we initiate the sending
  590:     of "con_lost" to the client.
  591: 
  592: Deleting the transaction means killing it from the 
  593: %ActiveTransactions hash.
  594: 
  595: Parameters:
  596: 
  597: =item client  
  598:  
  599:    The LondTransaction we are failing.
  600:  
  601: =cut
  602: 
  603: sub FailTransaction {
  604:     my $transaction = shift;
  605:     Debug(1, "Failing transaction: ".$transaction->getRequest());
  606:     if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it.
  607: 	my $client  = $transaction->getClient();
  608: 	Debug(1," Replying con_lost to ".$transaction->getRequest());
  609: 	StartClientReply($transaction, "con_lost\n");
  610:     }
  611: 
  612: }
  613: 
  614: =pod
  615: =head1  EmptyQueue
  616: 
  617:   Fails all items in the work queue with con_lost.
  618:   Note that each item in the work queue is a transaction.
  619: 
  620: =cut
  621: sub EmptyQueue {
  622:     while($WorkQueue->Count()) {
  623: 	my $request = $WorkQueue->dequeue(); # This is a transaction
  624: 	FailTransaction($request);
  625:     }
  626: }
  627: 
  628: =pod
  629: 
  630: =head2 CloseAllLondConnections
  631: 
  632: Close all connections open on lond prior to exit e.g.
  633: 
  634: =cut
  635: sub CloseAllLondConnections {
  636:     foreach $Socket (keys %ActiveConnections) {
  637: 	KillSocket($Socket);
  638:     }
  639: }
  640: =cut
  641: 
  642: =pod
  643: 
  644: =head2 KillSocket
  645:  
  646: Destroys a socket.  This function can be called either when a socket
  647: has died of 'natural' causes or because a socket needs to be pruned due to
  648: idleness.  If the socket has died naturally, if there are no longer any 
  649: live connections a new connection is created (in case there are transactions
  650: in the queue).  If the socket has been pruned, it is never re-created.
  651: 
  652: Parameters:
  653: 
  654: =item Socket
  655:  
  656:   The socket to kill off.
  657: 
  658: =item Restart
  659: 
  660: nonzero if we are allowed to create a new connection.
  661: 
  662: 
  663: =cut
  664: sub KillSocket {
  665:     my $Socket = shift;
  666: 
  667:     $Socket->Shutdown();
  668: 
  669:     #  If the socket came from the active connection set,
  670:     #  delete its transaction... note that FailTransaction should
  671:     #  already have been called!!!
  672:     #  otherwise it came from the idle set.
  673:     #  
  674:     
  675:     if(exists($ActiveTransactions{$Socket})) {
  676: 	delete ($ActiveTransactions{$Socket});
  677:     }
  678:     if(exists($ActiveConnections{$Socket})) {
  679: 	delete($ActiveConnections{$Socket});
  680:     }
  681:     $ConnectionCount--;
  682: 
  683:     #  If the connection count has gone to zero and there is work in the
  684:     #  work queue, the work all gets failed with con_lost.
  685:     #
  686:     if($ConnectionCount == 0) {
  687: 	EmptyQueue;
  688:     }
  689: }
  690: 
  691: =pod
  692: 
  693: =head2 LondReadable
  694: 
  695: This function is called whenever a lond connection
  696: is readable.  The action is state dependent:
  697: 
  698: =head3 State=Initialized
  699: 
  700: We''re waiting for the challenge, this is a no-op until the
  701: state changes.
  702: 
  703: =head3 State=Challenged 
  704: 
  705: The challenge has arrived we need to transition to Writable.
  706: The connection must echo the challenge back.
  707: 
  708: =head3 State=ChallengeReplied
  709: 
  710: The challenge has been replied to.  The we are receiveing the 
  711: 'ok' from the partner.
  712: 
  713: =head3 State=RequestingKey
  714: 
  715: The ok has been received and we need to send the request for
  716: an encryption key.  Transition to writable for that.
  717: 
  718: =head3 State=ReceivingKey
  719: 
  720: The the key has been requested, now we are reading the new key.
  721: 
  722: =head3 State=Idle 
  723: 
  724: The encryption key has been negotiated or we have finished 
  725: reading data from the a transaction.   If the callback data has
  726: a client as well as the socket iformation, then we are 
  727: doing a transaction and the data received is relayed to the client
  728: before the socket is put on the idle list.
  729: 
  730: =head3 State=SendingRequest
  731: 
  732: I do not think this state can be received here, but if it is,
  733: the appropriate thing to do is to transition to writable, and send
  734: the request.
  735: 
  736: =head3 State=ReceivingReply
  737: 
  738: We finished sending the request to the server and now transition
  739: to readable to receive the reply. 
  740: 
  741: The parameter to this function are:
  742: 
  743: The event. Implicit in this is the watcher and its data.  The data 
  744: contains at least the lond connection object and, if a 
  745: transaction is in progress, the socket attached to the local client.
  746: 
  747: =cut
  748: 
  749: sub LondReadable {
  750: 
  751:     my $Event      = shift;
  752:     my $Watcher    = $Event->w;
  753:     my $Socket     = $Watcher->data;
  754:     my $client     = undef;
  755: 
  756:     &Debug(6,"LondReadable called state = ".$State);
  757: 
  758: 
  759:     my $State = $Socket->GetState(); # All action depends on the state.
  760: 
  761:     SocketDump(6, $Socket);
  762:     my $status = $Socket->Readable();
  763:     &Debug(2, "Socket->Readable returned: $status");
  764: 
  765:     if($status != 0) {
  766: 	 # bad return from socket read. Currently this means that
  767: 	# The socket has become disconnected. We fail the transaction.
  768: 
  769: 	if(exists($ActiveTransactions{$Socket})) {
  770: 	    Debug(3,"Lond connection lost failing transaction");
  771: 	    FailTransaction($ActiveTransactions{$Socket});
  772: 	}
  773: 	$Watcher->cancel();
  774: 	KillSocket($Socket);
  775: 	return;
  776:     }
  777:     SocketDump(6,$Socket);
  778: 
  779:     $State = $Socket->GetState(); # Update in case of transition.
  780:     &Debug(6, "After read, state is ".$State);
  781: 
  782:    if($State eq "Initialized") {
  783: 
  784: 
  785:     } elsif ($State eq "ChallengeReceived") {
  786: 	#  The challenge must be echoed back;  The state machine
  787: 	# in the connection takes care of setting that up.  Just
  788: 	# need to transition to writable:
  789: 
  790: 	$Watcher->cb(\&LondWritable);
  791: 	$Watcher->poll("w");
  792: 
  793:     } elsif ($State eq "ChallengeReplied") {
  794: 
  795: 
  796:     } elsif ($State eq "RequestingKey") {
  797: 	#  The ok was received.  Now we need to request the key
  798: 	#  That requires us to be writable:
  799: 
  800: 	$Watcher->cb(\&LondWritable);
  801: 	$Watcher->poll("w");
  802: 
  803:     } elsif ($State eq "ReceivingKey") {
  804: 
  805:     } elsif ($State eq "Idle") {
  806: 	# If necessary, complete a transaction and then go into the
  807: 	# idle queue.
  808: 	$Watcher->cancel();
  809: 	if(exists($ActiveTransactions{$Socket})) {
  810: 	    Debug(8,"Completing transaction!!");
  811: 	    CompleteTransaction($Socket, 
  812: 				$ActiveTransactions{$Socket});
  813: 	} else {
  814: 	    Log("SUCCESS", "Connection ".$ConnectionCount." to "
  815: 		.$RemoteHost." now ready for action");
  816: 	}
  817: 	ServerToIdle($Socket);	# Next work unit or idle.
  818: 	
  819:     } elsif ($State eq "SendingRequest") {
  820: 	#  We need to be writable for this and probably don't belong
  821: 	#  here inthe first place.
  822: 
  823: 	Deubg(6, "SendingRequest state encountered in readable");
  824: 	$Watcher->poll("w");
  825: 	$Watcher->cb(\&LondWritable);
  826: 
  827:     } elsif ($State eq "ReceivingReply") {
  828: 
  829: 
  830:     } else {
  831: 	 # Invalid state.
  832: 	Debug(4, "Invalid state in LondReadable");
  833:     }
  834: }
  835: 
  836: =pod
  837: 
  838: =head2 LondWritable
  839: 
  840: This function is called whenever a lond connection
  841: becomes writable while there is a writeable monitoring
  842: event.  The action taken is very state dependent:
  843: 
  844: =head3 State = Connected 
  845: 
  846: The connection is in the process of sending the 'init' hailing to the
  847: lond on the remote end.  The connection object''s Writable member is
  848: called.  On error, ConnectionError is called to destroy the connection
  849: and remove it from the ActiveConnections hash
  850: 
  851: =head3 Initialized
  852: 
  853: 'init' has been sent, writability monitoring is removed and
  854: readability monitoring is started with LondReadable as the callback.
  855: 
  856: =head3 ChallengeReceived
  857: 
  858: The connection has received the who are you challenge from the remote
  859: system, and is in the process of sending the challenge
  860: response. Writable is called.
  861: 
  862: =head3 ChallengeReplied
  863: 
  864: The connection has replied to the initial challenge The we switch to
  865: monitoring readability looking for the server to reply with 'ok'.
  866: 
  867: =head3 RequestingKey
  868: 
  869: The connection is in the process of requesting its encryption key.
  870: Writable is called.
  871: 
  872: =head3 ReceivingKey
  873: 
  874: The connection has sent the request for a key.  Switch to readability
  875: monitoring to accept the key
  876: 
  877: =head3 SendingRequest
  878: 
  879: The connection is in the process of sending a request to the server.
  880: This request is part of a client transaction.  All the states until
  881: now represent the client setup protocol. Writable is called.
  882: 
  883: =head3 ReceivingReply
  884: 
  885: The connection has sent a request.  Now it must receive a reply.
  886: Readability monitoring is requested.
  887: 
  888: This function is an event handler and therefore receives as
  889: a parameter the event that has fired.  The data for the watcher
  890: of this event is a reference to a list of one or two elements,
  891: depending on state. The first (and possibly only) element is the
  892: socket.  The second (present only if a request is in progress)
  893: is the socket on which to return a reply to the caller.
  894: 
  895: =cut
  896: 
  897: sub LondWritable {
  898:     my $Event   = shift;
  899:     my $Watcher = $Event->w;
  900:     my $Socket  = $Watcher->data;
  901:     my $State   = $Socket->GetState();
  902: 
  903:     Debug(6,"LondWritable State = ".$State."\n");
  904: 
  905:  
  906:     #  Figure out what to do depending on the state of the socket:
  907:     
  908: 
  909: 
  910: 
  911:     SocketDump(6,$Socket);
  912: 
  913:     if      ($State eq "Connected")         {
  914: 
  915: 	if ($Socket->Writable() != 0) {
  916: 	    #  The write resulted in an error.
  917: 	    # We'll treat this as if the socket got disconnected:
  918: 	    Log("WARNING", "Connection to ".$RemoteHost.
  919: 		" has been disconnected");
  920: 	    $Watcher->cancel();
  921: 	    KillSocket($Socket);
  922: 	    return;
  923: 	}
  924: 	#  "init" is being sent...
  925: 
  926: 	
  927:     } elsif ($State eq "Initialized")       {
  928: 
  929: 	# Now that init was sent, we switch 
  930: 	# to watching for readability:
  931: 
  932: 	$Watcher->cb(\&LondReadable);
  933: 	$Watcher->poll("r");
  934: 
  935:     } elsif ($State eq "ChallengeReceived") {
  936: 	# We received the challenge, now we 
  937: 	# are echoing it back. This is a no-op,
  938: 	# we're waiting for the state to change
  939: 	
  940: 	if($Socket->Writable() != 0) {
  941: 
  942: 	    $Watcher->cancel();
  943: 	    KillSocket($Socket);
  944: 	    return;
  945: 	}
  946: 	
  947:     } elsif ($State eq "ChallengeReplied")  {
  948: 	# The echo was sent back, so we switch
  949: 	# to watching readability.
  950: 
  951: 	$Watcher->cb(\&LondReadable);
  952: 	$Watcher->poll("r");
  953: 
  954:     } elsif ($State eq "RequestingKey")     {
  955: 	# At this time we're requesting the key.
  956: 	# again, this is essentially a no-op.
  957: 	# we'll write the next chunk until the
  958: 	# state changes.
  959: 
  960: 	if($Socket->Writable() != 0) {
  961: 	    # Write resulted in an error.
  962: 
  963: 	    $Watcher->cancel();
  964: 	    KillSocket($Socket);
  965: 	    return;
  966: 
  967: 	}
  968:     } elsif ($State eq "ReceivingKey")      {
  969: 	# Now we need to wait for the key
  970: 	# to come back from the peer:
  971: 
  972: 	$Watcher->cb(\&LondReadable);
  973: 	$Watcher->poll("r");
  974: 
  975:     } elsif ($State eq "SendingRequest")    {
  976: 	# At this time we are sending a request to the
  977: 	# peer... write the next chunk:
  978: 
  979: 	if($Socket->Writable() != 0) {
  980: 
  981: 	    if(exists($ActiveTransactions{$Socket})) {
  982: 		Debug(3, "Lond connection lost, failing transactions");
  983: 		FailTransaction($ActiveTransactions{$Socket});
  984: 	    }
  985: 	    $Watcher->cancel();
  986: 	    KillSocket($Socket);
  987: 	    return;
  988: 	    
  989: 	}
  990: 
  991:     } elsif ($State eq "ReceivingReply")    {
  992: 	# The send has completed.  Wait for the
  993: 	# data to come in for a reply.
  994: 	Debug(8,"Writable sent request/receiving reply");
  995: 	$Watcher->cb(\&LondReadable);
  996: 	$Watcher->poll("r");
  997: 
  998:     } else {
  999: 	#  Control only passes here on an error: 
 1000: 	#  the socket state does not match any
 1001: 	#  of the known states... so an error
 1002: 	#  must be logged.
 1003: 
 1004: 	&Debug(4, "Invalid socket state ".$State."\n");
 1005:     }
 1006:     
 1007: }
 1008: =pod
 1009:     
 1010: =cut
 1011: sub QueueDelayed {
 1012:     Debug(3,"QueueDelayed called");
 1013: 
 1014:     my $path = "$perlvar{'lonSockDir'}/delayed";
 1015: 
 1016:     Debug(4, "Delayed path: ".$path);
 1017:     opendir(DIRHANDLE, $path);
 1018:     
 1019:     @alldelayed = grep /\.$RemoteHost$/, readdir DIRHANDLE;
 1020:     Debug(4, "Got ".$alldelayed." delayed files");
 1021:     closedir(DIRHANDLE);
 1022:     my $dfname;
 1023:     my $reqfile;
 1024:     foreach $dfname (sort  @alldelayed) {
 1025: 	$reqfile = "$path/$dfname";
 1026: 	Debug(4, "queueing ".$reqfile);
 1027: 	my $Handle = IO::File->new($reqfile);
 1028: 	my $cmd    = <$Handle>;
 1029: 	chomp $cmd;		# There may or may not be a newline...
 1030: 	$cmd = $cmd."\n";	# now for sure there's exactly one newline.
 1031: 	my $Transaction = LondTransaction->new($cmd);
 1032: 	$Transaction->SetDeferred($reqfile);
 1033: 	QueueTransaction($Transaction);
 1034:     }
 1035:     
 1036: }
 1037: 
 1038: =pod
 1039: 
 1040: =head2 MakeLondConnection
 1041: 
 1042: Create a new lond connection object, and start it towards its initial
 1043: idleness.  Once idle, it becomes elligible to receive transactions
 1044: from the work queue.  If the work queue is not empty when the
 1045: connection is completed and becomes idle, it will dequeue an entry and
 1046: start off on it.
 1047: 
 1048: =cut
 1049: 
 1050: sub MakeLondConnection {     
 1051:     Debug(4,"MakeLondConnection to ".GetServerHost()." on port "
 1052: 	  .GetServerPort());
 1053: 
 1054:     my $Connection = LondConnection->new(&GetServerHost(),
 1055: 					 &GetServerPort());
 1056: 
 1057:     if($Connection == undef) {	# Needs to be more robust later.
 1058: 	Log("CRITICAL","Failed to make a connection with lond.");
 1059: 	$ConnectionRetriesLeft--;
 1060: 	return 0;		# Failure.
 1061:     }  else {
 1062: 	$ConnectionRetriesLeft = $ConnectionRetries; # success resets the count
 1063: 	# The connection needs to have writability 
 1064: 	# monitored in order to send the init sequence
 1065: 	# that starts the whole authentication/key
 1066: 	# exchange underway.
 1067: 	#
 1068: 	my $Socket = $Connection->GetSocket();
 1069: 	if($Socket == undef) {
 1070: 	    die "did not get a socket from the connection";
 1071: 	} else {
 1072: 	    &Debug(9,"MakeLondConnection got socket: ".$Socket);
 1073: 	}
 1074: 	
 1075: 	
 1076: 	$event = Event->io(fd       => $Socket,
 1077: 			   poll     => 'w',
 1078: 			   cb       => \&LondWritable,
 1079: 			   data     => $Connection,
 1080: 			   desc => 'Connection to lond server');
 1081: 	$ActiveConnections{$Connection} = $event;
 1082: 	
 1083: 	$ConnectionCount++;
 1084: 	Debug(4, "Connection count = ".$ConnectionCount);
 1085: 	if($ConnectionCount == 1) { # First Connection:
 1086: 	    QueueDelayed;
 1087: 	}
 1088: 	Log("SUCESS", "Created connection ".$ConnectionCount
 1089: 	    ." to host ".GetServerHost());
 1090: 	return 1;		# Return success.
 1091:     }
 1092:     
 1093: }
 1094: 
 1095: =pod
 1096: 
 1097: =head2 StartRequest
 1098: 
 1099: Starts a lond request going on a specified lond connection.
 1100: parameters are:
 1101: 
 1102: =item $Lond
 1103: 
 1104: Connection to the lond that will send the transaction and receive the
 1105: reply.
 1106: 
 1107: =item $Client
 1108: 
 1109: Connection to the client that is making this request We got the
 1110: request from this socket, and when the request has been relayed to
 1111: lond and we get a reply back from lond it will get sent to this
 1112: socket.
 1113: 
 1114: =item $Request
 1115: 
 1116: The text of the request to send.
 1117: 
 1118: =cut
 1119: 
 1120: sub StartRequest {
 1121:     my $Lond     = shift;
 1122:     my $Request  = shift;	# This is a LondTransaction.
 1123:     
 1124:     Debug(6, "StartRequest: ".$Request->getRequest());
 1125: 
 1126:     my $Socket = $Lond->GetSocket();
 1127:     
 1128:     $Request->Activate($Lond);
 1129:     $ActiveTransactions{$Lond} = $Request;
 1130: 
 1131:     $Lond->InitiateTransaction($Request->getRequest());
 1132:     $event = Event->io(fd      => $Socket,
 1133: 		       poll    => "w",
 1134: 		       cb      => \&LondWritable,
 1135: 		       data    => $Lond,
 1136: 		       desc    => "lond transaction connection");
 1137:     $ActiveConnections{$Lond} = $event;
 1138:     Debug(8," Start Request made watcher data with ".$event->data."\n");
 1139: }
 1140: 
 1141: =pod
 1142: 
 1143: =head2 QueueTransaction
 1144: 
 1145: If there is an idle lond connection, it is put to work doing this
 1146: transaction.  Otherwise, the transaction is placed in the work queue.
 1147: If placed in the work queue and the maximum number of connections has
 1148: not yet been created, a new connection will be started.  Our goal is
 1149: to eventually have a sufficient number of connections that the work
 1150: queue will typically be empty.  parameters are:
 1151: 
 1152: =item Socket
 1153: 
 1154: open on the lonc client.
 1155: 
 1156: =item Request
 1157: 
 1158: data to send to the lond.
 1159: 
 1160: =cut
 1161: 
 1162: sub QueueTransaction {
 1163: 
 1164:     my $requestData   = shift;	# This is a LondTransaction.
 1165:     my $cmd           = $requestData->getRequest();
 1166: 
 1167:     Debug(6,"QueueTransaction: ".$cmd);
 1168: 
 1169:     my $LondSocket    = $IdleConnections->pop();
 1170:     if(!defined $LondSocket) {	# Need to queue request.
 1171: 	Debug(8,"Must queue...");
 1172: 	$WorkQueue->enqueue($requestData);
 1173: 	if($ConnectionCount < $MaxConnectionCount) {
 1174: 	    Debug(4,"Starting additional lond connection");
 1175: 	    MakeLondConnection();
 1176: 	}
 1177:     } else {			# Can start the request:
 1178: 	Debug(8,"Can start...");
 1179: 	StartRequest($LondSocket,  $requestData);
 1180:     }
 1181: }
 1182: 
 1183: #-------------------------- Lonc UNIX socket handling ---------------------
 1184: 
 1185: =pod
 1186: 
 1187: =head2 ClientRequest
 1188: Callback that is called when data can be read from the UNIX domain
 1189: socket connecting us with an apache server process.
 1190: 
 1191: =cut
 1192: 
 1193: sub ClientRequest {
 1194:     Debug(6, "ClientRequest");
 1195:     my $event   = shift;
 1196:     my $watcher = $event->w;
 1197:     my $socket  = $watcher->fd;
 1198:     my $data    = $watcher->data;
 1199:     my $thisread;
 1200: 
 1201:     Debug(9, "  Watcher named: ".$watcher->desc);
 1202: 
 1203:     my $rv = $socket->recv($thisread, POSIX::BUFSIZ, 0);
 1204:     Debug(8, "rcv:  data length = ".length($thisread)
 1205: 	  ." read =".$thisread);
 1206:     unless (defined $rv && length($thisread)) {
 1207: 	 # Likely eof on socket.
 1208: 	Debug(5,"Client Socket closed on lonc for ".$RemoteHost);
 1209: 	close($socket);
 1210: 	$watcher->cancel();
 1211: 	delete($ActiveClients{$socket});
 1212: 	return;
 1213:     }
 1214:     Debug(8,"Data: ".$data." this read: ".$thisread);
 1215:     $data = $data.$thisread;	# Append new data.
 1216:     $watcher->data($data);
 1217:     if($data =~ /(.*\n)/) {	# Request entirely read.
 1218: 	if($data eq "close_connection_exit\n") {
 1219: 	    Log("CRITICAL",
 1220: 		"Request Close Connection ... exiting");
 1221: 	    CloseAllLondConnections();
 1222: 	    exit;
 1223: 	}
 1224: 	Debug(8, "Complete transaction received: ".$data);
 1225: 	my $Transaction = LondTransaction->new($data);
 1226: 	$Transaction->SetClient($socket);
 1227: 	QueueTransaction($Transaction);
 1228: 	$watcher->cancel();	# Done looking for input data.
 1229:     }
 1230: 
 1231: }
 1232: 
 1233: 
 1234: =pod
 1235: 
 1236: =head2  NewClient
 1237: 
 1238: Callback that is called when a connection is received on the unix
 1239: socket for a new client of lonc.  The callback is parameterized by the
 1240: event.. which is a-priori assumed to be an io event, and therefore has
 1241: an fd member that is the Listener socket.  We Accept the connection
 1242: and register a new event on the readability of that socket:
 1243: 
 1244: =cut
 1245: 
 1246: sub NewClient {
 1247:     Debug(6, "NewClient");
 1248:     my $event      = shift;		# Get the event parameters.
 1249:     my $watcher    = $event->w; 
 1250:     my $socket     = $watcher->fd;	# Get the event' socket.
 1251:     my $connection = $socket->accept();	# Accept the client connection.
 1252:     Debug(8,"Connection request accepted from "
 1253: 	  .GetPeername($connection, AF_UNIX));
 1254: 
 1255: 
 1256:     my $description = sprintf("Connection to lonc client %d",
 1257: 			      $ClientConnection);
 1258:     Debug(9, "Creating event named: ".$description);
 1259:     Event->io(cb      => \&ClientRequest,
 1260: 	      poll    => 'r',
 1261: 	      desc    => $description,
 1262: 	      data    => "",
 1263: 	      fd      => $connection);
 1264:     $ActiveClients{$connection} = $ClientConnection;
 1265:     $ClientConnection++;
 1266: }
 1267: 
 1268: =pod
 1269: 
 1270: =head2 GetLoncSocketPath
 1271: 
 1272: Returns the name of the UNIX socket on which to listen for client
 1273: connections.
 1274: 
 1275: =cut
 1276: 
 1277: sub GetLoncSocketPath {
 1278:     return $UnixSocketDir."/".GetServerHost();
 1279: }
 1280: 
 1281: =pod
 1282: 
 1283: =head2 GetServerHost
 1284: 
 1285: Returns the host whose lond we talk with.
 1286: 
 1287: =cut
 1288: 
 1289: sub GetServerHost {
 1290:     return $RemoteHost;		# Setup by the fork.
 1291: }
 1292: 
 1293: =pod
 1294: 
 1295: =head2 GetServerPort
 1296: 
 1297: Returns the lond port number.
 1298: 
 1299: =cut
 1300: 
 1301: sub GetServerPort {
 1302:     return $perlvar{londPort};
 1303: }
 1304: 
 1305: =pod
 1306: 
 1307: =head2 SetupLoncListener
 1308: 
 1309: Setup a lonc listener event.  The event is called when the socket
 1310: becomes readable.. that corresponds to the receipt of a new
 1311: connection.  The event handler established will accept the connection
 1312: (creating a communcations channel), that int turn will establish
 1313: another event handler to subess requests.
 1314: 
 1315: =cut
 1316: 
 1317: sub SetupLoncListener {
 1318: 
 1319:     my $socket;
 1320:     my $SocketName = GetLoncSocketPath();
 1321:     unlink($SocketName);
 1322:     unless ($socket =IO::Socket::UNIX->new(Local  => $SocketName,
 1323: 					    Listen => 10, 
 1324: 					    Type   => SOCK_STREAM)) {
 1325: 	die "Failed to create a lonc listner socket";
 1326:     }
 1327:     Event->io(cb     => \&NewClient,
 1328: 	      poll   => 'r',
 1329: 	      desc   => 'Lonc listener Unix Socket',
 1330: 	      fd     => $socket);
 1331: }
 1332: 
 1333: =pod 
 1334: 
 1335: =head2 ChildStatus
 1336:  
 1337: Child USR1 signal handler to report the most recent status
 1338: into the status file.
 1339: 
 1340: =cut
 1341: sub ChildStatus {
 1342:     my $event = shift;
 1343:     my $watcher = $event->w;
 1344: 
 1345:     Debug(2, "Reporting child status because : ".$watcher->data);
 1346:     my $docdir = $perlvar{'lonDocRoot'};
 1347:     my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt");
 1348:     print $fh $$."\t".$RemoteHost."\t".$Status."\t".
 1349: 	$RecentLogEntry."\n";
 1350: }
 1351: 
 1352: =pod
 1353: 
 1354: =head2 SignalledToDeath
 1355: 
 1356: Called in response to a signal that causes a chid process to die.
 1357: 
 1358: =cut
 1359: 
 1360: 
 1361: sub SignalledToDeath {
 1362:     my $event  = shift;
 1363:     my $watcher= $event->w;
 1364: 
 1365:     Debug(2,"Signalled to death! via ".$watcher->data);
 1366:     my ($signal) = @_;
 1367:     chomp($signal);
 1368:     Log("CRITICAL", "Abnormal exit.  Child $$ for $RemoteHost "
 1369: 	."died through "."\"$signal\"");
 1370:     LogPerm("F:lonc: $$ on $RemoteHost signalled to death: "
 1371: 	    ."\"$signal\"");
 1372:     die("Signal abnormal end");
 1373:     exit 0;
 1374: 
 1375: }
 1376: 
 1377: =head2 ToggleDebug
 1378: 
 1379: This sub toggles trace debugging on and off.
 1380: 
 1381: =cut
 1382: 
 1383: sub ToggleDebug {
 1384:     my $Current    = $DebugLevel;
 1385:        $DebugLevel = $NextDebugLevel;
 1386:        $NextDebugLevel = $Current;
 1387: 
 1388:     Log("SUCCESS", "New debugging level for $RemoteHost now $DebugLevel");
 1389: 
 1390: }
 1391: 
 1392: =head2 ChildProcess
 1393: 
 1394: This sub implements a child process for a single lonc daemon.
 1395: 
 1396: =cut
 1397: 
 1398: sub ChildProcess {
 1399: 
 1400: 
 1401:     #
 1402:     #  Signals must be handled by the Event framework...
 1403: #
 1404: 
 1405:     Event->signal(signal   => "QUIT",
 1406: 		  cb       => \&SignalledToDeath,
 1407: 		  data     => "QUIT");
 1408:     Event->signal(signal   => "HUP",
 1409: 		  cb       => \&ChildStatus,
 1410: 		  data     => "HUP");
 1411:     Event->signal(signal   => "USR1",
 1412: 		  cb       => \&ChildStatus,
 1413: 		  data     => "USR1");
 1414:     Event->signal(signal   => "INT",
 1415: 		  cb       => \&ToggleDebug,
 1416: 		  data     => "INT");
 1417: 
 1418:     SetupTimer();
 1419:     
 1420:     SetupLoncListener();
 1421:     
 1422:     $Event::Debuglevel = $DebugLevel;
 1423:     
 1424:     Debug(9, "Making initial lond connection for ".$RemoteHost);
 1425: 
 1426: # Setup the initial server connection:
 1427:     
 1428:      # &MakeLondConnection(); // let first work requirest do it.
 1429: 
 1430: 
 1431:     Debug(9,"Entering event loop");
 1432:     my $ret = Event::loop();		#  Start the main event loop.
 1433:     
 1434:     
 1435:     die "Main event loop exited!!!";
 1436: }
 1437: 
 1438: #  Create a new child for host passed in:
 1439: 
 1440: sub CreateChild {
 1441:     my $sigset = POSIX::SigSet->new(SIGINT);
 1442:     sigprocmask(SIG_BLOCK, $sigset);
 1443:     my $host = shift;
 1444:     $RemoteHost = $host;
 1445:     Log("CRITICAL", "Forking server for ".$host);
 1446:     $pid          = fork;
 1447:     if($pid) {			# Parent
 1448: 	$ChildHash{$pid} = $RemoteHost;
 1449: 	sigprocmask(SIG_UNBLOCK, $sigset);
 1450: 
 1451:     } else {			# child.
 1452: 	ShowStatus("Connected to ".$RemoteHost);
 1453: 	$SIG{INT} = DEFAULT;
 1454: 	sigprocmask(SIG_UNBLOCK, $sigset);
 1455: 	ChildProcess;		# Does not return.
 1456:     }
 1457: 
 1458: }
 1459: #
 1460: #  Parent process logic pass 1:
 1461: #   For each entry in the hosts table, we will
 1462: #  fork off an instance of ChildProcess to service the transactions
 1463: #  to that host.  Each pid will be entered in a global hash
 1464: #  with the value of the key, the host.
 1465: #  The parent will then enter a loop to wait for process exits.
 1466: #  Each exit gets logged and the child gets restarted.
 1467: #
 1468: 
 1469: #
 1470: #   Fork and start in new session so hang-up isn't going to 
 1471: #   happen without intent.
 1472: #
 1473: 
 1474: 
 1475: 
 1476: 
 1477: 
 1478: 
 1479: ShowStatus("Forming new session");
 1480: my $childpid = fork;
 1481: if ($childpid != 0) {
 1482:     sleep 4;			# Give child a chacne to break to
 1483:     exit 0;			# a new sesion.
 1484: }
 1485: #
 1486: #   Write my pid into the pid file so I can be located
 1487: #
 1488: 
 1489: ShowStatus("Parent writing pid file:");
 1490: $execdir = $perlvar{'lonDaemons'};
 1491: open (PIDSAVE, ">$execdir/logs/lonc.pid");
 1492: print PIDSAVE "$$\n";
 1493: close(PIDSAVE);
 1494: 
 1495: if (POSIX::setsid() < 0) {
 1496:     print "Could not create new session\n";
 1497:     exit -1;
 1498: }
 1499: 
 1500: ShowStatus("Forking node servers");
 1501: 
 1502: Log("CRITICAL", "--------------- Starting children ---------------");
 1503: 
 1504: my $HostIterator = LondConnection::GetHostIterator;
 1505: while (! $HostIterator->end()) {
 1506: 
 1507:     $hostentryref = $HostIterator->get();
 1508:     CreateChild($hostentryref->[0]);
 1509:     $HostIterator->next();
 1510: }
 1511: $RemoteHost = "Parent Server";
 1512: 
 1513: # Maintain the population:
 1514: 
 1515: ShowStatus("Parent keeping the flock");
 1516: 
 1517: #
 1518: #   Set up parent signals:
 1519: #
 1520: 
 1521: $SIG{INT}  = \&Terminate;
 1522: $SIG{TERM} = \&Terminate; 
 1523: $SIG{HUP}  = \&Restart;
 1524: $SIG{USR1} = \&CheckKids; 
 1525: 
 1526: while(1) {
 1527:     $deadchild = wait();
 1528:     if(exists $ChildHash{$deadchild}) {	# need to restart.
 1529: 	$deadhost = $ChildHash{$deadchild};
 1530: 	delete($ChildHash{$deadchild});
 1531: 	Log("WARNING","Lost child pid= ".$deadchild.
 1532: 	      "Connected to host ".$deadhost);
 1533: 	Log("INFO", "Restarting child procesing ".$deadhost);
 1534: 	CreateChild($deadhost);
 1535:     }
 1536: }
 1537: 
 1538: 
 1539: 
 1540: =pod
 1541: 
 1542: =head1 CheckKids
 1543: 
 1544:   Since kids do not die as easily in this implementation
 1545: as the previous one, there  is no need to restart the
 1546: dead ones (all dead kids get restarted when they die!!)
 1547: The only thing this function does is to pass USR1 to the
 1548: kids so that they report their status.
 1549: 
 1550: =cut
 1551: 
 1552: sub CheckKids {
 1553:     Debug(2, "Checking status of children");
 1554:     my $docdir = $perlvar{'lonDocRoot'};
 1555:     my $fh = IO::File->new(">$docdir/lon-status/loncstatus.txt");
 1556:     my $now=time;
 1557:     my $local=localtime($now);
 1558:     print $fh "LONC status $local - parent $$ \n\n";
 1559:     foreach $pid (keys %ChildHash) {
 1560: 	Debug(2, "Sending USR1 -> $pid");
 1561: 	kill 'USR1' => $pid;	# Tell Child to report status.
 1562: 	sleep 1;		# Wait so file doesn't intermix.
 1563:     }
 1564: }
 1565: 
 1566: =pod
 1567: 
 1568: =head1 Restart
 1569: 
 1570: Signal handler for HUP... all children are killed and
 1571: we self restart.  This is an el-cheapo way to re read
 1572: the config file.
 1573: 
 1574: =cut
 1575: 
 1576: sub Restart {
 1577:     KillThemAll;		# First kill all the children.
 1578:     Log("CRITICAL", "Restarting");
 1579:     my $execdir = $perlvar{'lonDaemons'};
 1580:     unlink("$execdir/logs/lonc.pid");
 1581:     exec("$execdir/lonc");
 1582: }
 1583: 
 1584: =pod
 1585: 
 1586: =head1 KillThemAll
 1587: 
 1588: Signal handler that kills all children by sending them a 
 1589: SIGINT.  Responds to sigint and sigterm.
 1590: 
 1591: =cut
 1592: 
 1593: sub KillThemAll {
 1594:     Debug(2, "Kill them all!!");
 1595:     local($SIG{CHLD}) = 'IGNORE';      # Our children >will< die.
 1596:     foreach $pid (keys %ChildHash) {
 1597: 	my $serving = $ChildHash{$pid};
 1598: 	Debug(2, "Killing lonc for $serving pid = $pid");
 1599: 	ShowStatus("Killing lonc for $serving pid = $pid");
 1600: 	Log("CRITICAL", "Killing lonc for $serving pid = $pid");
 1601: 	kill('INT', $pid);
 1602: 	delete($ChildeHash{$pid});
 1603:     }
 1604:     my $execdir = $perlvar{'lonDaemons'};
 1605:     unlink("$execdir/logs/lonc.pid");
 1606:     ShowStatus("Killing the master process");
 1607:     Log("CRITICAL", "Killing the master process.");
 1608: }
 1609: 
 1610: =pod
 1611: 
 1612: =head1 Terminate
 1613:  
 1614: Terminate the system.
 1615: 
 1616: =cut
 1617: 
 1618: sub Terminate {
 1619:     KillThemAll;
 1620:     exit;
 1621: 
 1622: }
 1623: =pod
 1624: 
 1625: =head1 Theory
 1626: 
 1627: The event class is used to build this as a single process with an
 1628: event driven model.  The following events are handled:
 1629: 
 1630: =item UNIX Socket connection Received
 1631: 
 1632: =item Request data arrives on UNIX data transfer socket.
 1633: 
 1634: =item lond connection becomes writable.
 1635: 
 1636: =item timer fires at 1 second intervals.
 1637: 
 1638: All sockets are run in non-blocking mode.  Timeouts managed by the timer
 1639: handler prevents hung connections.
 1640: 
 1641: Key data structures:
 1642: 
 1643: =item RequestQueue
 1644: 
 1645: A queue of requests received from UNIX sockets that are
 1646: waiting for a chance to be forwarded on a lond connection socket.
 1647: 
 1648: =item ActiveConnections
 1649: 
 1650: A hash of lond connections that have transactions in process that are
 1651: available to be timed out.
 1652: 
 1653: =item ActiveTransactions
 1654: 
 1655: A hash indexed by lond connections that contain the client reply
 1656: socket for each connection that has an active transaction on it.
 1657: 
 1658: =item IdleConnections
 1659: 
 1660: A hash of lond connections that have no work to do.  These connections
 1661: can be closed if they are idle for a long enough time.
 1662: 
 1663: =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>