Annotation of loncom/loncnew, revision 1.58

1.1       foxr        1: #!/usr/bin/perl
1.2       albertel    2: # The LearningOnline Network with CAPA
                      3: # lonc maintains the connections to remote computers
                      4: #
1.57      foxr        5: # $Id: loncnew,v 1.56 2004/09/21 10:54:43 foxr Exp $
1.2       albertel    6: #
                      7: # Copyright Michigan State University Board of Trustees
                      8: #
                      9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
1.17      foxr       10: ## LON-CAPA is free software; you can redistribute it and/or modify
1.2       albertel   11: # it under the terms of the GNU General Public License as published by
                     12: # the Free Software Foundation; either version 2 of the License, or
                     13: # (at your option) any later version.
                     14: #
                     15: # LON-CAPA is distributed in the hope that it will be useful,
                     16: # but WITHOUT ANY WARRANTY; without even the implied warranty of
                     17: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
                     18: # GNU General Public License for more details.
                     19: #
                     20: # You should have received a copy of the GNU General Public License
                     21: # along with LON-CAPA; if not, write to the Free Software
                     22: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
                     23: #
                     24: # /home/httpd/html/adm/gpl.txt
                     25: #
                     26: # http://www.lon-capa.org/
                     27: #
1.1       foxr       28: #
1.15      foxr       29: # new lonc handles n request out bver m connections to londs.
1.1       foxr       30: # This module is based on the Event class.
                     31: #   Development iterations:
                     32: #    - Setup basic event loop.   (done)
                     33: #    - Add timer dispatch.       (done)
                     34: #    - Add ability to accept lonc UNIX domain sockets.  (done)
                     35: #    - Add ability to create/negotiate lond connections (done).
1.7       foxr       36: #    - Add general logic for dispatching requests and timeouts. (done).
                     37: #    - Add support for the lonc/lond requests.          (done).
1.38      foxr       38: #    - Add logging/status monitoring.                    (done)
                     39: #    - Add Signal handling - HUP restarts. USR1 status report. (done)
1.7       foxr       40: #    - Add Configuration file I/O                       (done).
1.38      foxr       41: #    - Add management/status request interface.         (done)
1.8       foxr       42: #    - Add deferred request capability.                  (done)
1.38      foxr       43: #    - Detect transmission timeouts.                     (done)
1.7       foxr       44: #
                     45: 
1.23      foxr       46: use strict;
1.1       foxr       47: use lib "/home/httpd/lib/perl/";
                     48: use Event qw(:DEFAULT );
                     49: use POSIX qw(:signal_h);
1.12      foxr       50: use POSIX;
1.1       foxr       51: use IO::Socket;
                     52: use IO::Socket::INET;
                     53: use IO::Socket::UNIX;
1.9       foxr       54: use IO::File;
1.6       foxr       55: use IO::Handle;
1.1       foxr       56: use Socket;
                     57: use Crypt::IDEA;
                     58: use LONCAPA::Queue;
                     59: use LONCAPA::Stack;
                     60: use LONCAPA::LondConnection;
1.7       foxr       61: use LONCAPA::LondTransaction;
1.1       foxr       62: use LONCAPA::Configuration;
                     63: use LONCAPA::HashIterator;
                     64: 
                     65: 
                     66: # Read the httpd configuration file to get perl variables
                     67: # normally set in apache modules:
                     68: 
                     69: my $perlvarref = LONCAPA::Configuration::read_conf('loncapa.conf');
                     70: my %perlvar    = %{$perlvarref};
                     71: 
                     72: #
                     73: #  parent and shared variables.
                     74: 
                     75: my %ChildHash;			# by pid -> host.
1.26      foxr       76: my %HostToPid;			# By host -> pid.
                     77: my %HostHash;			# by loncapaname -> IP.
1.1       foxr       78: 
                     79: 
1.9       foxr       80: my $MaxConnectionCount = 10;	# Will get from config later.
1.1       foxr       81: my $ClientConnection = 0;	# Uniquifier for client events.
                     82: 
1.9       foxr       83: my $DebugLevel = 0;
1.29      foxr       84: my $NextDebugLevel= 2;		# So Sigint can toggle this.
1.50      albertel   85: my $IdleTimeout= 600;		# Wait 10 minutes before pruning connections.
1.1       foxr       86: 
1.39      foxr       87: my $LogTransactions = 0;	# When True, all transactions/replies get logged.
                     88: 
1.1       foxr       89: #
                     90: #  The variables below are only used by the child processes.
                     91: #
                     92: my $RemoteHost;			# Name of host child is talking to.
1.20      albertel   93: my $UnixSocketDir= $perlvar{'lonSockDir'};
1.1       foxr       94: my $IdleConnections = Stack->new(); # Set of idle connections
                     95: my %ActiveConnections;		# Connections to the remote lond.
1.7       foxr       96: my %ActiveTransactions;		# LondTransactions in flight.
1.1       foxr       97: my %ActiveClients;		# Serial numbers of active clients by socket.
                     98: my $WorkQueue       = Queue->new(); # Queue of pending transactions.
                     99: my $ConnectionCount = 0;
1.4       foxr      100: my $IdleSeconds     = 0;	# Number of seconds idle.
1.9       foxr      101: my $Status          = "";	# Current status string.
1.14      foxr      102: my $RecentLogEntry  = "";
1.30      foxr      103: my $ConnectionRetries=2;	# Number of connection retries allowed.
                    104: my $ConnectionRetriesLeft=2;	# Number of connection retries remaining.
1.40      foxr      105: my $LondVersion     = "unknown"; # Version of lond we talk with.
1.49      foxr      106: my $KeyMode         = "";       # e.g. ssl, local, insecure from last connect.
1.54      foxr      107: my $LondConnecting  = 0;       # True when a connection is being built.
1.1       foxr      108: 
1.57      foxr      109: my $DieWhenIdle     = 0;	# When true children die when trimmed -> 0.
                    110: 
1.1       foxr      111: #
1.9       foxr      112: #   The hash below gives the HTML format for log messages
                    113: #   given a severity.
                    114: #    
                    115: my %LogFormats;
                    116: 
1.45      albertel  117: $LogFormats{"CRITICAL"} = "<font color='red'>CRITICAL: %s</font>";
                    118: $LogFormats{"SUCCESS"}  = "<font color='green'>SUCCESS: %s</font>";
                    119: $LogFormats{"INFO"}     = "<font color='yellow'>INFO: %s</font>";
                    120: $LogFormats{"WARNING"}  = "<font color='blue'>WARNING: %s</font>";
1.9       foxr      121: $LogFormats{"DEFAULT"}  = " %s ";
                    122: 
1.10      foxr      123: 
1.57      foxr      124: #  UpdateStatus;
                    125: #    Update the idle status display to show how many connections
                    126: #    are left, retries and other stuff.
                    127: #
                    128: sub UpdateStatus {
                    129:     if ($ConnectionRetriesLeft > 0) {
                    130: 	ShowStatus(GetServerHost()." Connection count: ".$ConnectionCount
                    131: 		   ." Retries remaining: ".$ConnectionRetriesLeft
                    132: 		   ." ($KeyMode)");
                    133:     } else {
                    134: 	ShowStatus(GetServerHost()." >> DEAD <<");
                    135:     }
                    136: }
                    137: 
1.10      foxr      138: 
                    139: =pod
                    140: 
                    141: =head2 LogPerm
                    142: 
                    143: Makes an entry into the permanent log file.
                    144: 
                    145: =cut
                    146: sub LogPerm {
                    147:     my $message=shift;
                    148:     my $execdir=$perlvar{'lonDaemons'};
                    149:     my $now=time;
                    150:     my $local=localtime($now);
                    151:     my $fh=IO::File->new(">>$execdir/logs/lonnet.perm.log");
                    152:     print $fh "$now:$message:$local\n";
                    153: }
1.9       foxr      154: 
                    155: =pod
                    156: 
                    157: =head2 Log
                    158: 
                    159: Logs a message to the log file.
                    160: Parameters:
                    161: 
                    162: =item severity
                    163: 
                    164: One of CRITICAL, WARNING, INFO, SUCCESS used to select the
                    165: format string used to format the message.  if the severity is
                    166: not a defined severity the Default format string is used.
                    167: 
                    168: =item message
                    169: 
                    170: The base message.  In addtion to the format string, the message
                    171: will be appended to a string containing the name of our remote
                    172: host and the time will be formatted into the message.
                    173: 
                    174: =cut
                    175: 
                    176: sub Log {
1.47      foxr      177: 
                    178:     my ($severity, $message) = @_;
                    179: 
1.9       foxr      180:     if(!$LogFormats{$severity}) {
                    181: 	$severity = "DEFAULT";
                    182:     }
                    183: 
                    184:     my $format = $LogFormats{$severity};
                    185:     
                    186:     #  Put the window dressing in in front of the message format:
                    187: 
                    188:     my $now   = time;
                    189:     my $local = localtime($now);
                    190:     my $finalformat = "$local ($$) [$RemoteHost] [$Status] ";
                    191:     my $finalformat = $finalformat.$format."\n";
                    192: 
                    193:     # open the file and put the result.
                    194: 
                    195:     my $execdir = $perlvar{'lonDaemons'};
                    196:     my $fh      = IO::File->new(">>$execdir/logs/lonc.log");
                    197:     my $msg = sprintf($finalformat, $message);
1.14      foxr      198:     $RecentLogEntry = $msg;
1.9       foxr      199:     print $fh $msg;
                    200:     
1.10      foxr      201:     
1.9       foxr      202: }
1.6       foxr      203: 
1.3       albertel  204: 
1.1       foxr      205: =pod
1.3       albertel  206: 
                    207: =head2 GetPeerName
                    208: 
                    209: Returns the name of the host that a socket object is connected to.
                    210: 
1.1       foxr      211: =cut
                    212: 
                    213: sub GetPeername {
1.47      foxr      214: 
                    215: 
                    216:     my ($connection, $AdrFamily) = @_;
                    217: 
1.1       foxr      218:     my $peer       = $connection->peername();
                    219:     my $peerport;
                    220:     my $peerip;
                    221:     if($AdrFamily == AF_INET) {
                    222: 	($peerport, $peerip) = sockaddr_in($peer);
1.23      foxr      223: 	my $peername    = gethostbyaddr($peerip, $AdrFamily);
1.1       foxr      224: 	return $peername;
                    225:     } elsif ($AdrFamily == AF_UNIX) {
                    226: 	my $peerfile;
                    227: 	($peerfile) = sockaddr_un($peer);
                    228: 	return $peerfile;
                    229:     }
                    230: }
                    231: =pod
1.3       albertel  232: 
1.1       foxr      233: =head2 Debug
1.3       albertel  234: 
                    235: Invoked to issue a debug message.
                    236: 
1.1       foxr      237: =cut
1.3       albertel  238: 
1.1       foxr      239: sub Debug {
1.47      foxr      240: 
                    241:     my ($level, $message) = @_;
                    242: 
1.1       foxr      243:     if ($level <= $DebugLevel) {
1.23      foxr      244: 	Log("INFO", "-Debug- $message host = $RemoteHost");
1.1       foxr      245:     }
                    246: }
                    247: 
                    248: sub SocketDump {
1.47      foxr      249: 
                    250:     my ($level, $socket) = @_;
                    251: 
1.1       foxr      252:     if($level <= $DebugLevel) {
1.48      foxr      253: 	$socket->Dump(-1);	# Ensure it will get dumped.
1.1       foxr      254:     }
                    255: }
1.3       albertel  256: 
1.1       foxr      257: =pod
1.3       albertel  258: 
1.5       foxr      259: =head2 ShowStatus
                    260: 
                    261:  Place some text as our pid status.
1.10      foxr      262:  and as what we return in a SIGUSR1
1.5       foxr      263: 
                    264: =cut
                    265: sub ShowStatus {
1.10      foxr      266:     my $state = shift;
                    267:     my $now = time;
                    268:     my $local = localtime($now);
                    269:     $Status   = $local.": ".$state;
                    270:     $0='lonc: '.$state.' '.$local;
1.5       foxr      271: }
                    272: 
                    273: =pod
                    274: 
1.15      foxr      275: =head 2 SocketTimeout
                    276: 
                    277:     Called when an action on the socket times out.  The socket is 
                    278:    destroyed and any active transaction is failed.
                    279: 
                    280: 
                    281: =cut
                    282: sub SocketTimeout {
                    283:     my $Socket = shift;
1.38      foxr      284:     Log("WARNING", "A socket timeout was detected");
1.52      foxr      285:     Debug(5, " SocketTimeout called: ");
1.48      foxr      286:     $Socket->Dump(0);
1.42      foxr      287:     if(exists($ActiveTransactions{$Socket})) {
1.43      albertel  288: 	FailTransaction($ActiveTransactions{$Socket});
1.42      foxr      289:     }
1.22      foxr      290:     KillSocket($Socket);	# A transaction timeout also counts as
                    291:                                 # a connection failure:
                    292:     $ConnectionRetriesLeft--;
1.42      foxr      293:     if($ConnectionRetriesLeft <= 0) {
1.52      foxr      294: 	Log("CRITICAL", "Host marked DEAD: ".GetServerHost());
1.56      foxr      295: 	$LondConnecting = 0;
1.42      foxr      296:     }
                    297: 
1.15      foxr      298: }
1.35      foxr      299: #----------------------------- Timer management ------------------------
1.15      foxr      300: 
                    301: =pod
                    302: 
1.1       foxr      303: =head2 Tick
1.3       albertel  304: 
                    305: Invoked  each timer tick.
                    306: 
1.1       foxr      307: =cut
                    308: 
1.5       foxr      309: 
1.1       foxr      310: sub Tick {
1.52      foxr      311:     my ($Event)       = @_;
                    312:     my $clock_watcher = $Event->w;
                    313: 
1.1       foxr      314:     my $client;
1.57      foxr      315:     UpdateStatus();
                    316: 
1.4       foxr      317:     # Is it time to prune connection count:
                    318: 
                    319: 
                    320:     if($IdleConnections->Count()  && 
                    321:        ($WorkQueue->Count() == 0)) { # Idle connections and nothing to do?
1.52      foxr      322: 	$IdleSeconds++;
1.4       foxr      323: 	if($IdleSeconds > $IdleTimeout) { # Prune a connection...
1.23      foxr      324: 	    my $Socket = $IdleConnections->pop();
1.6       foxr      325: 	    KillSocket($Socket);
1.54      foxr      326: 	    $IdleSeconds = 0;	# Otherwise all connections get trimmed to fast.
1.57      foxr      327: 	    UpdateStatus();
                    328: 	    if(($ConnectionCount == 0) && $DieWhenIdle) {
                    329: 		#
                    330: 		#  Create a lock file since there will be a time window
                    331: 		#  between our exit and the parent's picking up the listen
                    332: 		#  during which no listens will be done on the
                    333: 		#  lonnet client socket.
                    334: 		#
                    335: 		my $lock_file = GetLoncSocketPath().".lock";
                    336: 		open(LOCK,">$lock_file");
                    337: 		print LOCK "Contents not important";
                    338: 		close(LOCK);
                    339: 		
                    340: 		exit(0);
                    341: 	    }
1.4       foxr      342: 	}
                    343:     } else {
                    344: 	$IdleSeconds = 0;	# Reset idle count if not idle.
                    345:     }
1.15      foxr      346:     #
                    347:     #  For each inflight transaction, tick down its timeout counter.
                    348:     #
1.35      foxr      349: 
1.34      albertel  350:     foreach my $item (keys %ActiveConnections) {
                    351: 	my $State = $ActiveConnections{$item}->data->GetState();
1.35      foxr      352: 	if ($State ne 'Idle') {
1.34      albertel  353: 	    Debug(5,"Ticking Socket $State $item");
                    354: 	    $ActiveConnections{$item}->data->Tick();
                    355: 	}
1.15      foxr      356:     }
1.5       foxr      357:     # Do we have work in the queue, but no connections to service them?
                    358:     # If so, try to make some new connections to get things going again.
                    359:     #
1.57      foxr      360:     #   Note this code is dead now...
                    361:     #
1.5       foxr      362:     my $Requests = $WorkQueue->Count();
1.56      foxr      363:     if (($ConnectionCount == 0)  && ($Requests > 0) && (!$LondConnecting)) { 
1.10      foxr      364: 	if ($ConnectionRetriesLeft > 0) {
1.56      foxr      365: 	    Debug(5,"Work but no connections, Make a new one");
                    366: 	    my $success;
                    367: 	    $success    = &MakeLondConnection;
                    368: 	    if($success == 0) { # All connections failed:
1.29      foxr      369: 		Debug(5,"Work in queue failed to make any connectiouns\n");
1.22      foxr      370: 		EmptyQueue();	# Fail pending transactions with con_lost.
1.42      foxr      371: 		CloseAllLondConnections(); # Should all be closed but....
1.10      foxr      372: 	    }
                    373: 	} else {
1.56      foxr      374: 	    $LondConnecting = 0;
1.22      foxr      375: 	    ShowStatus(GetServerHost()." >>> DEAD!!! <<<");
1.29      foxr      376: 	    Debug(5,"Work in queue, but gave up on connections..flushing\n");
1.10      foxr      377: 	    EmptyQueue();	# Connections can't be established.
1.42      foxr      378: 	    CloseAllLondConnections(); # Should all already be closed but...
1.5       foxr      379: 	}
                    380:        
                    381:     }
1.49      foxr      382:     if ($ConnectionCount == 0) {
                    383: 	$KeyMode = ""; 
1.52      foxr      384: 	$clock_watcher->cancel();
1.49      foxr      385:     }
1.1       foxr      386: }
                    387: 
                    388: =pod
1.3       albertel  389: 
1.1       foxr      390: =head2 SetupTimer
                    391: 
1.3       albertel  392: Sets up a 1 per sec recurring timer event.  The event handler is used to:
1.1       foxr      393: 
1.3       albertel  394: =item
                    395: 
                    396: Trigger timeouts on communications along active sockets.
                    397: 
                    398: =item
                    399: 
                    400: Trigger disconnections of idle sockets.
1.1       foxr      401: 
                    402: =cut
                    403: 
                    404: sub SetupTimer {
1.52      foxr      405:     Debug(6, "SetupTimer");
                    406:     Event->timer(interval => 1, cb => \&Tick );
1.1       foxr      407: }
1.3       albertel  408: 
1.1       foxr      409: =pod
1.3       albertel  410: 
1.1       foxr      411: =head2 ServerToIdle
1.3       albertel  412: 
                    413: This function is called when a connection to the server is
                    414: ready for more work.
                    415: 
                    416: If there is work in the Work queue the top element is dequeued
1.1       foxr      417: and the connection will start to work on it.  If the work queue is
                    418: empty, the connection is pushed on the idle connection stack where
                    419: it will either get another work unit, or alternatively, if it sits there
                    420: long enough, it will be shut down and released.
                    421: 
1.3       albertel  422: =cut
1.1       foxr      423: 
                    424: sub ServerToIdle {
                    425:     my $Socket   = shift;	# Get the socket.
1.49      foxr      426:     $KeyMode = $Socket->{AuthenticationMode};
1.7       foxr      427:     delete($ActiveTransactions{$Socket}); # Server has no transaction
1.1       foxr      428: 
1.29      foxr      429:     &Debug(5, "Server to idle");
1.1       foxr      430: 
                    431:     #  If there's work to do, start the transaction:
                    432: 
1.23      foxr      433:     my $reqdata = $WorkQueue->dequeue(); # This is a LondTransaction
1.29      foxr      434:     if ($reqdata ne undef)  {
                    435: 	Debug(5, "Queue gave request data: ".$reqdata->getRequest());
1.7       foxr      436: 	&StartRequest($Socket,  $reqdata);
1.8       foxr      437: 
1.1       foxr      438:     } else {
                    439: 	
                    440:     #  There's no work waiting, so push the server to idle list.
1.29      foxr      441: 	&Debug(5, "No new work requests, server connection going idle");
1.1       foxr      442: 	$IdleConnections->push($Socket);
                    443:     }
                    444: }
1.3       albertel  445: 
1.1       foxr      446: =pod
1.3       albertel  447: 
1.1       foxr      448: =head2 ClientWritable
1.3       albertel  449: 
                    450: Event callback for when a client socket is writable.
                    451: 
                    452: This callback is established when a transaction reponse is
                    453: avaiable from lond.  The response is forwarded to the unix socket
                    454: as it becomes writable in this sub.
                    455: 
1.1       foxr      456: Parameters:
                    457: 
1.3       albertel  458: =item Event
                    459: 
                    460: The event that has been triggered. Event->w->data is
                    461: the data and Event->w->fd is the socket to write.
1.1       foxr      462: 
                    463: =cut
1.3       albertel  464: 
1.1       foxr      465: sub ClientWritable {
                    466:     my $Event    = shift;
                    467:     my $Watcher  = $Event->w;
                    468:     my $Data     = $Watcher->data;
                    469:     my $Socket   = $Watcher->fd;
                    470: 
                    471:     # Try to send the data:
                    472: 
                    473:     &Debug(6, "ClientWritable writing".$Data);
                    474:     &Debug(9, "Socket is: ".$Socket);
                    475: 
1.6       foxr      476:     if($Socket->connected) {
                    477: 	my $result = $Socket->send($Data, 0);
                    478: 	
                    479: 	# $result undefined: the write failed.
                    480: 	# otherwise $result is the number of bytes written.
                    481: 	# Remove that preceding string from the data.
                    482: 	# If the resulting data is empty, destroy the watcher
                    483: 	# and set up a read event handler to accept the next
                    484: 	# request.
                    485: 	
                    486: 	&Debug(9,"Send result is ".$result." Defined: ".defined($result));
1.29      foxr      487: 	if($result ne undef) {
1.6       foxr      488: 	    &Debug(9, "send result was defined");
                    489: 	    if($result == length($Data)) { # Entire string sent.
                    490: 		&Debug(9, "ClientWritable data all written");
                    491: 		$Watcher->cancel();
                    492: 		#
                    493: 		#  Set up to read next request from socket:
                    494: 		
                    495: 		my $descr     = sprintf("Connection to lonc client %d",
                    496: 					$ActiveClients{$Socket});
                    497: 		Event->io(cb    => \&ClientRequest,
                    498: 			  poll  => 'r',
                    499: 			  desc  => $descr,
                    500: 			  data  => "",
                    501: 			  fd    => $Socket);
                    502: 		
                    503: 	    } else {		# Partial string sent.
                    504: 		$Watcher->data(substr($Data, $result));
1.15      foxr      505: 		if($result == 0) {    # client hung up on us!!
1.52      foxr      506: 		    # Log("INFO", "lonc pipe client hung up on us!");
1.15      foxr      507: 		    $Watcher->cancel;
                    508: 		    $Socket->shutdown(2);
                    509: 		    $Socket->close();
                    510: 		}
1.6       foxr      511: 	    }
                    512: 	    
                    513: 	} else {			# Error of some sort...
                    514: 	    
                    515: 	    # Some errnos are possible:
                    516: 	    my $errno = $!;
                    517: 	    if($errno == POSIX::EWOULDBLOCK   ||
                    518: 	       $errno == POSIX::EAGAIN        ||
                    519: 	       $errno == POSIX::EINTR) {
                    520: 		# No action taken?
                    521: 	    } else {		# Unanticipated errno.
                    522: 		&Debug(5,"ClientWritable error or peer shutdown".$RemoteHost);
                    523: 		$Watcher->cancel;	# Stop the watcher.
                    524: 		$Socket->shutdown(2); # Kill connection
                    525: 		$Socket->close();	# Close the socket.
                    526: 	    }
1.1       foxr      527: 	    
                    528: 	}
1.6       foxr      529:     } else {
                    530: 	$Watcher->cancel();	# A delayed request...just cancel.
1.1       foxr      531:     }
                    532: }
                    533: 
                    534: =pod
1.3       albertel  535: 
1.1       foxr      536: =head2 CompleteTransaction
1.3       albertel  537: 
                    538: Called when the reply data has been received for a lond 
1.1       foxr      539: transaction.   The reply data must now be sent to the
                    540: ultimate client on the other end of the Unix socket.  This is
                    541: done by setting up a writable event for the socket with the
                    542: data the reply data.
1.3       albertel  543: 
1.1       foxr      544: Parameters:
1.3       albertel  545: 
                    546: =item Socket
                    547: 
                    548: Socket on which the lond transaction occured.  This is a
                    549: LondConnection. The data received is in the TransactionReply member.
                    550: 
1.7       foxr      551: =item Transaction
1.3       albertel  552: 
1.7       foxr      553: The transaction that is being completed.
1.1       foxr      554: 
                    555: =cut
1.3       albertel  556: 
1.1       foxr      557: sub CompleteTransaction {
1.29      foxr      558:     &Debug(5,"Complete transaction");
1.47      foxr      559: 
                    560:     my ($Socket, $Transaction) = @_;
1.1       foxr      561: 
1.7       foxr      562:     if (!$Transaction->isDeferred()) { # Normal transaction
                    563: 	my $data   = $Socket->GetReply(); # Data to send.
1.39      foxr      564: 	if($LogTransactions) {
                    565: 	    Log("SUCCESS", "Reply from lond: '$data'");
                    566: 	}
1.7       foxr      567: 	StartClientReply($Transaction, $data);
                    568:     } else {			# Delete deferred transaction file.
1.9       foxr      569: 	Log("SUCCESS", "A delayed transaction was completed");
1.23      foxr      570: 	LogPerm("S:$Transaction->getClient() :".$Transaction->getRequest());
1.7       foxr      571: 	unlink $Transaction->getFile();
                    572:     }
1.6       foxr      573: }
1.42      foxr      574: 
1.6       foxr      575: =pod
1.42      foxr      576: 
1.6       foxr      577: =head1 StartClientReply
                    578: 
                    579:    Initiates a reply to a client where the reply data is a parameter.
                    580: 
1.7       foxr      581: =head2  parameters:
                    582: 
                    583: =item Transaction
                    584: 
                    585:     The transaction for which we are responding to the client.
                    586: 
                    587: =item data
                    588: 
                    589:     The data to send to apached client.
                    590: 
1.6       foxr      591: =cut
1.42      foxr      592: 
1.6       foxr      593: sub StartClientReply {
1.1       foxr      594: 
1.47      foxr      595:     my ($Transaction, $data) = @_;
1.12      foxr      596: 
1.7       foxr      597:     my $Client   = $Transaction->getClient();
                    598: 
1.1       foxr      599:     &Debug(8," Reply was: ".$data);
                    600:     my $Serial         = $ActiveClients{$Client};
                    601:     my $desc           = sprintf("Connection to lonc client %d",
                    602: 				 $Serial);
                    603:     Event->io(fd       => $Client,
                    604: 	      poll     => "w",
                    605: 	      desc     => $desc,
                    606: 	      cb       => \&ClientWritable,
                    607: 	      data     => $data);
                    608: }
1.42      foxr      609: 
1.4       foxr      610: =pod
1.42      foxr      611: 
1.4       foxr      612: =head2 FailTransaction
                    613: 
                    614:   Finishes a transaction with failure because the associated lond socket
1.7       foxr      615:   disconnected.  There are two possibilities:
                    616:   - The transaction is deferred: in which case we just quietly
                    617:     delete the transaction since there is no client connection.
                    618:   - The transaction is 'live' in which case we initiate the sending
                    619:     of "con_lost" to the client.
                    620: 
1.42      foxr      621: Deleting the transaction means killing it from the %ActiveTransactions hash.
1.4       foxr      622: 
                    623: Parameters:
                    624: 
                    625: =item client  
                    626:  
1.7       foxr      627:    The LondTransaction we are failing.
                    628:  
1.42      foxr      629: 
1.4       foxr      630: =cut
                    631: 
                    632: sub FailTransaction {
1.7       foxr      633:     my $transaction = shift;
1.52      foxr      634:     
                    635:     #  If the socket is dead, that's already logged.
                    636: 
                    637:     if ($ConnectionRetriesLeft > 0) {
                    638: 	Log("WARNING", "Failing transaction "
                    639: 	    .$transaction->getRequest());
                    640:     }
1.30      foxr      641:     Debug(1, "Failing transaction: ".$transaction->getRequest());
1.10      foxr      642:     if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it.
1.11      foxr      643: 	my $client  = $transaction->getClient();
1.30      foxr      644: 	Debug(1," Replying con_lost to ".$transaction->getRequest());
1.11      foxr      645: 	StartClientReply($transaction, "con_lost\n");
1.7       foxr      646:     }
1.4       foxr      647: 
                    648: }
                    649: 
                    650: =pod
1.6       foxr      651: =head1  EmptyQueue
1.7       foxr      652: 
1.6       foxr      653:   Fails all items in the work queue with con_lost.
1.7       foxr      654:   Note that each item in the work queue is a transaction.
                    655: 
1.6       foxr      656: =cut
                    657: sub EmptyQueue {
1.22      foxr      658:     $ConnectionRetriesLeft--;	# Counts as connection failure too.
1.6       foxr      659:     while($WorkQueue->Count()) {
1.10      foxr      660: 	my $request = $WorkQueue->dequeue(); # This is a transaction
1.7       foxr      661: 	FailTransaction($request);
1.6       foxr      662:     }
                    663: }
                    664: 
                    665: =pod
1.4       foxr      666: 
1.9       foxr      667: =head2 CloseAllLondConnections
                    668: 
                    669: Close all connections open on lond prior to exit e.g.
                    670: 
                    671: =cut
                    672: sub CloseAllLondConnections {
1.23      foxr      673:     foreach my $Socket (keys %ActiveConnections) {
1.42      foxr      674:       if(exists($ActiveTransactions{$Socket})) {
                    675: 	FailTransaction($ActiveTransactions{$Socket});
                    676:       }
                    677:       KillSocket($Socket);
1.9       foxr      678:     }
                    679: }
                    680: =cut
                    681: 
                    682: =pod
                    683: 
1.4       foxr      684: =head2 KillSocket
                    685:  
                    686: Destroys a socket.  This function can be called either when a socket
                    687: has died of 'natural' causes or because a socket needs to be pruned due to
                    688: idleness.  If the socket has died naturally, if there are no longer any 
                    689: live connections a new connection is created (in case there are transactions
                    690: in the queue).  If the socket has been pruned, it is never re-created.
                    691: 
                    692: Parameters:
1.1       foxr      693: 
1.4       foxr      694: =item Socket
                    695:  
                    696:   The socket to kill off.
                    697: 
                    698: =item Restart
                    699: 
                    700: nonzero if we are allowed to create a new connection.
                    701: 
                    702: 
                    703: =cut
                    704: sub KillSocket {
                    705:     my $Socket = shift;
                    706: 
1.17      foxr      707:     Log("WARNING", "Shutting down a socket");
1.9       foxr      708:     $Socket->Shutdown();
                    709: 
1.7       foxr      710:     #  If the socket came from the active connection set,
                    711:     #  delete its transaction... note that FailTransaction should
                    712:     #  already have been called!!!
                    713:     #  otherwise it came from the idle set.
                    714:     #  
1.4       foxr      715:     
                    716:     if(exists($ActiveTransactions{$Socket})) {
                    717: 	delete ($ActiveTransactions{$Socket});
                    718:     }
                    719:     if(exists($ActiveConnections{$Socket})) {
                    720: 	delete($ActiveConnections{$Socket});
1.37      albertel  721: 	$ConnectionCount--;
                    722: 	if ($ConnectionCount < 0) { $ConnectionCount = 0; }
1.4       foxr      723:     }
1.6       foxr      724:     #  If the connection count has gone to zero and there is work in the
                    725:     #  work queue, the work all gets failed with con_lost.
                    726:     #
                    727:     if($ConnectionCount == 0) {
1.22      foxr      728: 	EmptyQueue();
1.42      foxr      729: 	CloseAllLondConnections; # Should all already be closed but...
1.4       foxr      730:     }
                    731: }
1.1       foxr      732: 
                    733: =pod
1.3       albertel  734: 
1.1       foxr      735: =head2 LondReadable
1.3       albertel  736: 
1.1       foxr      737: This function is called whenever a lond connection
                    738: is readable.  The action is state dependent:
                    739: 
1.3       albertel  740: =head3 State=Initialized
                    741: 
                    742: We''re waiting for the challenge, this is a no-op until the
1.1       foxr      743: state changes.
1.3       albertel  744: 
1.1       foxr      745: =head3 State=Challenged 
1.3       albertel  746: 
                    747: The challenge has arrived we need to transition to Writable.
1.1       foxr      748: The connection must echo the challenge back.
1.3       albertel  749: 
1.1       foxr      750: =head3 State=ChallengeReplied
1.3       albertel  751: 
                    752: The challenge has been replied to.  The we are receiveing the 
1.1       foxr      753: 'ok' from the partner.
1.3       albertel  754: 
1.40      foxr      755: =head3  State=ReadingVersionString
                    756: 
                    757: We have requested the lond version and are reading the
                    758: version back.  Upon completion, we'll store the version away
                    759: for future use(?).
                    760: 
                    761: =head3 State=HostSet
                    762: 
                    763: We have selected the domain name of our peer (multhomed hosts)
                    764: and are getting the reply (presumably ok) back.
                    765: 
1.1       foxr      766: =head3 State=RequestingKey
1.3       albertel  767: 
                    768: The ok has been received and we need to send the request for
1.1       foxr      769: an encryption key.  Transition to writable for that.
1.3       albertel  770: 
1.1       foxr      771: =head3 State=ReceivingKey
1.3       albertel  772: 
                    773: The the key has been requested, now we are reading the new key.
                    774: 
1.1       foxr      775: =head3 State=Idle 
1.3       albertel  776: 
                    777: The encryption key has been negotiated or we have finished 
1.1       foxr      778: reading data from the a transaction.   If the callback data has
                    779: a client as well as the socket iformation, then we are 
                    780: doing a transaction and the data received is relayed to the client
                    781: before the socket is put on the idle list.
1.3       albertel  782: 
1.1       foxr      783: =head3 State=SendingRequest
1.3       albertel  784: 
                    785: I do not think this state can be received here, but if it is,
1.1       foxr      786: the appropriate thing to do is to transition to writable, and send
                    787: the request.
1.3       albertel  788: 
1.1       foxr      789: =head3 State=ReceivingReply
1.3       albertel  790: 
                    791: We finished sending the request to the server and now transition
1.1       foxr      792: to readable to receive the reply. 
                    793: 
                    794: The parameter to this function are:
1.3       albertel  795: 
1.1       foxr      796: The event. Implicit in this is the watcher and its data.  The data 
                    797: contains at least the lond connection object and, if a 
                    798: transaction is in progress, the socket attached to the local client.
                    799: 
1.3       albertel  800: =cut
1.1       foxr      801: 
                    802: sub LondReadable {
1.8       foxr      803: 
1.41      albertel  804:     my $Event      = shift;
                    805:     my $Watcher    = $Event->w;
                    806:     my $Socket     = $Watcher->data;
                    807:     my $client     = undef;
1.40      foxr      808: 
1.41      albertel  809:     &Debug(6,"LondReadable called state = ".$Socket->GetState());
1.40      foxr      810: 
                    811: 
1.41      albertel  812:     my $State = $Socket->GetState(); # All action depends on the state.
1.40      foxr      813: 
1.41      albertel  814:     SocketDump(6, $Socket);
                    815:     my $status = $Socket->Readable();
1.40      foxr      816: 
1.41      albertel  817:     &Debug(2, "Socket->Readable returned: $status");
1.40      foxr      818: 
1.41      albertel  819:     if($status != 0) {
                    820: 	# bad return from socket read. Currently this means that
                    821: 	# The socket has become disconnected. We fail the transaction.
1.40      foxr      822: 
1.41      albertel  823: 	Log("WARNING",
                    824: 	    "Lond connection lost.");
                    825: 	if(exists($ActiveTransactions{$Socket})) {
                    826: 	    FailTransaction($ActiveTransactions{$Socket});
1.56      foxr      827: 	} else {
                    828: 	    #  Socket is connecting and failed... need to mark
                    829: 	    #  no longer connecting.
                    830: 	   
                    831: 	    $LondConnecting = 0;
1.41      albertel  832: 	}
                    833: 	$Watcher->cancel();
                    834: 	KillSocket($Socket);
                    835: 	$ConnectionRetriesLeft--;       # Counts as connection failure
                    836: 	return;
                    837:     }
                    838:     SocketDump(6,$Socket);
1.17      foxr      839: 
1.41      albertel  840:     $State = $Socket->GetState(); # Update in case of transition.
                    841:     &Debug(6, "After read, state is ".$State);
1.1       foxr      842: 
1.41      albertel  843:     if($State eq "Initialized") {
1.1       foxr      844: 
                    845: 
1.41      albertel  846:     } elsif ($State eq "ChallengeReceived") {
1.1       foxr      847: 	#  The challenge must be echoed back;  The state machine
                    848: 	# in the connection takes care of setting that up.  Just
                    849: 	# need to transition to writable:
1.41      albertel  850: 	
                    851: 	$Watcher->cb(\&LondWritable);
                    852: 	$Watcher->poll("w");
1.1       foxr      853: 
1.41      albertel  854:     } elsif ($State eq "ChallengeReplied") {
1.1       foxr      855: 
1.41      albertel  856:     } elsif ($State eq "RequestingVersion") {
                    857: 	# Need to ask for the version... that is writiability:
1.1       foxr      858: 
1.41      albertel  859: 	$Watcher->cb(\&LondWritable);
                    860: 	$Watcher->poll("w");
                    861: 
                    862:     } elsif ($State eq "ReadingVersionString") {
                    863: 	# Read the rest of the version string... 
                    864:     } elsif ($State eq "SetHost") {
                    865: 	# Need to request the actual domain get set...
                    866: 
                    867: 	$Watcher->cb(\&LondWritable);
                    868: 	$Watcher->poll("w");
                    869:     } elsif ($State eq "HostSet") {
                    870: 	# Reading the 'ok' from the peer.
                    871: 
                    872:     } elsif ($State eq "RequestingKey") {
1.1       foxr      873: 	#  The ok was received.  Now we need to request the key
                    874: 	#  That requires us to be writable:
                    875: 
1.41      albertel  876: 	$Watcher->cb(\&LondWritable);
                    877: 	$Watcher->poll("w");
1.1       foxr      878: 
1.41      albertel  879:     } elsif ($State eq "ReceivingKey") {
1.1       foxr      880: 
1.41      albertel  881:     } elsif ($State eq "Idle") {
1.40      foxr      882:    
1.41      albertel  883: 	# This is as good a spot as any to get the peer version
                    884: 	# string:
1.40      foxr      885:    
1.41      albertel  886: 	if($LondVersion eq "unknown") {
                    887: 	    $LondVersion = $Socket->PeerVersion();
                    888: 	    Log("INFO", "Connected to lond version: $LondVersion");
                    889: 	}
1.1       foxr      890: 	# If necessary, complete a transaction and then go into the
                    891: 	# idle queue.
1.22      foxr      892: 	#  Note that a trasition to idle indicates a live lond
                    893: 	# on the other end so reset the connection retries.
                    894: 	#
1.41      albertel  895: 	$ConnectionRetriesLeft = $ConnectionRetries; # success resets the count
                    896: 	$Watcher->cancel();
                    897: 	if(exists($ActiveTransactions{$Socket})) {
                    898: 	    Debug(5,"Completing transaction!!");
                    899: 	    CompleteTransaction($Socket, 
                    900: 				$ActiveTransactions{$Socket});
                    901: 	} else {
                    902: 	    Log("SUCCESS", "Connection ".$ConnectionCount." to "
                    903: 		.$RemoteHost." now ready for action");
                    904: 	}
                    905: 	ServerToIdle($Socket);	# Next work unit or idle.
1.54      foxr      906: 
                    907: 	#
                    908: 	$LondConnecting = 0;	# Best spot I can think of for this.
                    909: 	# 
1.6       foxr      910: 	
1.41      albertel  911:     } elsif ($State eq "SendingRequest") {
1.1       foxr      912: 	#  We need to be writable for this and probably don't belong
                    913: 	#  here inthe first place.
                    914: 
1.41      albertel  915: 	Deubg(6, "SendingRequest state encountered in readable");
                    916: 	$Watcher->poll("w");
                    917: 	$Watcher->cb(\&LondWritable);
1.1       foxr      918: 
1.41      albertel  919:     } elsif ($State eq "ReceivingReply") {
1.1       foxr      920: 
                    921: 
1.41      albertel  922:     } else {
                    923: 	# Invalid state.
                    924: 	Debug(4, "Invalid state in LondReadable");
                    925:     }
1.1       foxr      926: }
1.3       albertel  927: 
1.1       foxr      928: =pod
1.3       albertel  929: 
1.1       foxr      930: =head2 LondWritable
1.3       albertel  931: 
1.1       foxr      932: This function is called whenever a lond connection
                    933: becomes writable while there is a writeable monitoring
                    934: event.  The action taken is very state dependent:
1.3       albertel  935: 
1.1       foxr      936: =head3 State = Connected 
1.3       albertel  937: 
                    938: The connection is in the process of sending the 'init' hailing to the
                    939: lond on the remote end.  The connection object''s Writable member is
                    940: called.  On error, ConnectionError is called to destroy the connection
                    941: and remove it from the ActiveConnections hash
                    942: 
1.1       foxr      943: =head3 Initialized
1.3       albertel  944: 
                    945: 'init' has been sent, writability monitoring is removed and
                    946: readability monitoring is started with LondReadable as the callback.
                    947: 
1.1       foxr      948: =head3 ChallengeReceived
1.3       albertel  949: 
                    950: The connection has received the who are you challenge from the remote
                    951: system, and is in the process of sending the challenge
                    952: response. Writable is called.
                    953: 
1.1       foxr      954: =head3 ChallengeReplied
1.3       albertel  955: 
                    956: The connection has replied to the initial challenge The we switch to
                    957: monitoring readability looking for the server to reply with 'ok'.
                    958: 
1.1       foxr      959: =head3 RequestingKey
1.3       albertel  960: 
                    961: The connection is in the process of requesting its encryption key.
                    962: Writable is called.
                    963: 
1.1       foxr      964: =head3 ReceivingKey
1.3       albertel  965: 
                    966: The connection has sent the request for a key.  Switch to readability
                    967: monitoring to accept the key
                    968: 
1.1       foxr      969: =head3 SendingRequest
1.3       albertel  970: 
                    971: The connection is in the process of sending a request to the server.
                    972: This request is part of a client transaction.  All the states until
                    973: now represent the client setup protocol. Writable is called.
                    974: 
1.1       foxr      975: =head3 ReceivingReply
                    976: 
1.3       albertel  977: The connection has sent a request.  Now it must receive a reply.
                    978: Readability monitoring is requested.
                    979: 
                    980: This function is an event handler and therefore receives as
1.1       foxr      981: a parameter the event that has fired.  The data for the watcher
                    982: of this event is a reference to a list of one or two elements,
                    983: depending on state. The first (and possibly only) element is the
                    984: socket.  The second (present only if a request is in progress)
                    985: is the socket on which to return a reply to the caller.
                    986: 
                    987: =cut
1.3       albertel  988: 
1.1       foxr      989: sub LondWritable {
                    990:     my $Event   = shift;
                    991:     my $Watcher = $Event->w;
1.8       foxr      992:     my $Socket  = $Watcher->data;
                    993:     my $State   = $Socket->GetState();
1.1       foxr      994: 
1.8       foxr      995:     Debug(6,"LondWritable State = ".$State."\n");
1.1       foxr      996: 
1.8       foxr      997:  
1.1       foxr      998:     #  Figure out what to do depending on the state of the socket:
                    999:     
                   1000: 
                   1001: 
                   1002: 
                   1003:     SocketDump(6,$Socket);
                   1004: 
1.42      foxr     1005:     #  If the socket is writable, we must always write.
                   1006:     # Only by writing will we undergo state transitions.
                   1007:     # Old logic wrote in state specific code below, however
                   1008:     # That forces us at least through another invocation of
                   1009:     # this function after writability is possible again.
                   1010:     # This logic also factors out common code for handling
                   1011:     # write failures... in all cases, write failures 
                   1012:     # Kill the socket.
                   1013:     #  This logic makes the branches of the >big< if below
                   1014:     # so that the writing states are actually NO-OPs.
                   1015: 
                   1016:     if ($Socket->Writable() != 0) {
1.43      albertel 1017: 	#  The write resulted in an error.
                   1018: 	# We'll treat this as if the socket got disconnected:
                   1019: 	Log("WARNING", "Connection to ".$RemoteHost.
                   1020: 	    " has been disconnected");
                   1021: 	if(exists($ActiveTransactions{$Socket})) {
                   1022: 	    FailTransaction($ActiveTransactions{$Socket});
1.56      foxr     1023: 	} else {
                   1024: 	    #  In the process of conneting, so need to turn that off.
                   1025: 	    
                   1026: 	    $LondConnecting = 0;
1.43      albertel 1027: 	}
                   1028: 	$Watcher->cancel();
                   1029: 	KillSocket($Socket);
                   1030: 	return;
1.42      foxr     1031:     }
                   1032: 
                   1033: 
                   1034: 
1.41      albertel 1035:     if      ($State eq "Connected")         {
1.1       foxr     1036: 
1.41      albertel 1037: 	#  "init" is being sent...
1.42      foxr     1038:  
1.41      albertel 1039:     } elsif ($State eq "Initialized")       {
1.4       foxr     1040: 
1.41      albertel 1041: 	# Now that init was sent, we switch 
                   1042: 	# to watching for readability:
1.1       foxr     1043: 
1.41      albertel 1044: 	$Watcher->cb(\&LondReadable);
                   1045: 	$Watcher->poll("r");
                   1046: 	
                   1047:     } elsif ($State eq "ChallengeReceived") {
                   1048: 	# We received the challenge, now we 
                   1049: 	# are echoing it back. This is a no-op,
                   1050: 	# we're waiting for the state to change
1.1       foxr     1051: 	
1.41      albertel 1052:     } elsif ($State eq "ChallengeReplied")  {
                   1053: 	# The echo was sent back, so we switch
                   1054: 	# to watching readability.
                   1055: 
                   1056: 	$Watcher->cb(\&LondReadable);
                   1057: 	$Watcher->poll("r");
                   1058:     } elsif ($State eq "RequestingVersion") {
                   1059: 	# Sending the peer a version request...
1.42      foxr     1060: 
1.41      albertel 1061:     } elsif ($State eq "ReadingVersionString") {
                   1062: 	# Transition to read since we have sent the
                   1063: 	# version command and now just need to read the
                   1064: 	# version string from the peer:
1.40      foxr     1065:       
1.41      albertel 1066: 	$Watcher->cb(\&LondReadable);
                   1067: 	$Watcher->poll("r");
1.40      foxr     1068:       
1.41      albertel 1069:     } elsif ($State eq "SetHost") {
                   1070: 	#  Setting the remote domain...
1.42      foxr     1071: 
1.41      albertel 1072:     } elsif ($State eq "HostSet") {
                   1073: 	# Back to readable to get the ok.
1.40      foxr     1074:       
1.41      albertel 1075: 	$Watcher->cb(\&LondReadable);
                   1076: 	$Watcher->poll("r");
1.40      foxr     1077:       
                   1078: 
1.41      albertel 1079:     } elsif ($State eq "RequestingKey")     {
                   1080: 	# At this time we're requesting the key.
                   1081: 	# again, this is essentially a no-op.
                   1082: 
                   1083:     } elsif ($State eq "ReceivingKey")      {
                   1084: 	# Now we need to wait for the key
                   1085: 	# to come back from the peer:
                   1086: 
                   1087: 	$Watcher->cb(\&LondReadable);
                   1088: 	$Watcher->poll("r");
                   1089: 
                   1090:     } elsif ($State eq "SendingRequest")    {
1.40      foxr     1091:  
1.41      albertel 1092: 	# At this time we are sending a request to the
1.1       foxr     1093: 	# peer... write the next chunk:
                   1094: 
1.41      albertel 1095: 
                   1096:     } elsif ($State eq "ReceivingReply")    {
                   1097: 	# The send has completed.  Wait for the
                   1098: 	# data to come in for a reply.
                   1099: 	Debug(8,"Writable sent request/receiving reply");
                   1100: 	$Watcher->cb(\&LondReadable);
                   1101: 	$Watcher->poll("r");
1.1       foxr     1102: 
1.41      albertel 1103:     } else {
                   1104: 	#  Control only passes here on an error: 
                   1105: 	#  the socket state does not match any
                   1106: 	#  of the known states... so an error
                   1107: 	#  must be logged.
1.1       foxr     1108: 
1.41      albertel 1109: 	&Debug(4, "Invalid socket state ".$State."\n");
                   1110:     }
1.1       foxr     1111:     
                   1112: }
1.6       foxr     1113: =pod
                   1114:     
                   1115: =cut
                   1116: sub QueueDelayed {
1.8       foxr     1117:     Debug(3,"QueueDelayed called");
                   1118: 
1.6       foxr     1119:     my $path = "$perlvar{'lonSockDir'}/delayed";
1.8       foxr     1120: 
                   1121:     Debug(4, "Delayed path: ".$path);
1.6       foxr     1122:     opendir(DIRHANDLE, $path);
1.8       foxr     1123:     
1.23      foxr     1124:     my @alldelayed = grep /\.$RemoteHost$/, readdir DIRHANDLE;
1.6       foxr     1125:     closedir(DIRHANDLE);
                   1126:     my $dfname;
1.8       foxr     1127:     my $reqfile;
                   1128:     foreach $dfname (sort  @alldelayed) {
                   1129: 	$reqfile = "$path/$dfname";
                   1130: 	Debug(4, "queueing ".$reqfile);
1.6       foxr     1131: 	my $Handle = IO::File->new($reqfile);
                   1132: 	my $cmd    = <$Handle>;
1.8       foxr     1133: 	chomp $cmd;		# There may or may not be a newline...
1.12      foxr     1134: 	$cmd = $cmd."\n";	# now for sure there's exactly one newline.
1.7       foxr     1135: 	my $Transaction = LondTransaction->new($cmd);
                   1136: 	$Transaction->SetDeferred($reqfile);
                   1137: 	QueueTransaction($Transaction);
1.6       foxr     1138:     }
                   1139:     
                   1140: }
1.1       foxr     1141: 
                   1142: =pod
1.3       albertel 1143: 
1.1       foxr     1144: =head2 MakeLondConnection
1.3       albertel 1145: 
                   1146: Create a new lond connection object, and start it towards its initial
                   1147: idleness.  Once idle, it becomes elligible to receive transactions
                   1148: from the work queue.  If the work queue is not empty when the
                   1149: connection is completed and becomes idle, it will dequeue an entry and
                   1150: start off on it.
                   1151: 
1.1       foxr     1152: =cut
1.3       albertel 1153: 
1.1       foxr     1154: sub MakeLondConnection {     
                   1155:     Debug(4,"MakeLondConnection to ".GetServerHost()." on port "
                   1156: 	  .GetServerPort());
                   1157: 
                   1158:     my $Connection = LondConnection->new(&GetServerHost(),
                   1159: 					 &GetServerPort());
                   1160: 
1.30      foxr     1161:     if($Connection eq undef) {	# Needs to be more robust later.
1.9       foxr     1162: 	Log("CRITICAL","Failed to make a connection with lond.");
1.10      foxr     1163: 	$ConnectionRetriesLeft--;
                   1164: 	return 0;		# Failure.
1.5       foxr     1165:     }  else {
1.22      foxr     1166: 
1.5       foxr     1167: 	# The connection needs to have writability 
                   1168: 	# monitored in order to send the init sequence
                   1169: 	# that starts the whole authentication/key
                   1170: 	# exchange underway.
                   1171: 	#
                   1172: 	my $Socket = $Connection->GetSocket();
1.30      foxr     1173: 	if($Socket eq undef) {
1.5       foxr     1174: 	    die "did not get a socket from the connection";
                   1175: 	} else {
                   1176: 	    &Debug(9,"MakeLondConnection got socket: ".$Socket);
                   1177: 	}
1.1       foxr     1178: 	
1.21      foxr     1179: 	$Connection->SetTimeoutCallback(\&SocketTimeout);
                   1180: 
1.23      foxr     1181: 	my $event = Event->io(fd       => $Socket,
1.5       foxr     1182: 			   poll     => 'w',
                   1183: 			   cb       => \&LondWritable,
1.8       foxr     1184: 			   data     => $Connection,
1.5       foxr     1185: 			   desc => 'Connection to lond server');
                   1186: 	$ActiveConnections{$Connection} = $event;
1.52      foxr     1187: 	if ($ConnectionCount == 0) {
                   1188: 	    &SetupTimer;	# Need to handle timeouts with connections...
                   1189: 	}
1.5       foxr     1190: 	$ConnectionCount++;
1.8       foxr     1191: 	Debug(4, "Connection count = ".$ConnectionCount);
1.6       foxr     1192: 	if($ConnectionCount == 1) { # First Connection:
                   1193: 	    QueueDelayed;
                   1194: 	}
1.9       foxr     1195: 	Log("SUCESS", "Created connection ".$ConnectionCount
                   1196: 	    ." to host ".GetServerHost());
1.54      foxr     1197: 	$LondConnecting = 1;	# Connection in progress.
1.10      foxr     1198: 	return 1;		# Return success.
1.1       foxr     1199:     }
                   1200:     
                   1201: }
1.3       albertel 1202: 
1.1       foxr     1203: =pod
1.3       albertel 1204: 
1.1       foxr     1205: =head2 StartRequest
1.3       albertel 1206: 
                   1207: Starts a lond request going on a specified lond connection.
                   1208: parameters are:
                   1209: 
                   1210: =item $Lond
                   1211: 
                   1212: Connection to the lond that will send the transaction and receive the
                   1213: reply.
                   1214: 
                   1215: =item $Client
                   1216: 
                   1217: Connection to the client that is making this request We got the
                   1218: request from this socket, and when the request has been relayed to
                   1219: lond and we get a reply back from lond it will get sent to this
                   1220: socket.
                   1221: 
                   1222: =item $Request
                   1223: 
                   1224: The text of the request to send.
                   1225: 
1.1       foxr     1226: =cut
                   1227: 
                   1228: sub StartRequest {
1.47      foxr     1229: 
                   1230:     my ($Lond, $Request) = @_;
1.1       foxr     1231:     
1.7       foxr     1232:     Debug(6, "StartRequest: ".$Request->getRequest());
1.1       foxr     1233: 
                   1234:     my $Socket = $Lond->GetSocket();
                   1235:     
1.7       foxr     1236:     $Request->Activate($Lond);
                   1237:     $ActiveTransactions{$Lond} = $Request;
1.1       foxr     1238: 
1.7       foxr     1239:     $Lond->InitiateTransaction($Request->getRequest());
1.23      foxr     1240:     my $event = Event->io(fd      => $Socket,
1.1       foxr     1241: 		       poll    => "w",
                   1242: 		       cb      => \&LondWritable,
                   1243: 		       data    => $Lond,
                   1244: 		       desc    => "lond transaction connection");
                   1245:     $ActiveConnections{$Lond} = $event;
                   1246:     Debug(8," Start Request made watcher data with ".$event->data."\n");
                   1247: }
                   1248: 
                   1249: =pod
1.3       albertel 1250: 
1.1       foxr     1251: =head2 QueueTransaction
1.3       albertel 1252: 
                   1253: If there is an idle lond connection, it is put to work doing this
                   1254: transaction.  Otherwise, the transaction is placed in the work queue.
                   1255: If placed in the work queue and the maximum number of connections has
                   1256: not yet been created, a new connection will be started.  Our goal is
                   1257: to eventually have a sufficient number of connections that the work
                   1258: queue will typically be empty.  parameters are:
                   1259: 
                   1260: =item Socket
                   1261: 
                   1262: open on the lonc client.
                   1263: 
                   1264: =item Request
                   1265: 
                   1266: data to send to the lond.
1.1       foxr     1267: 
                   1268: =cut
1.3       albertel 1269: 
1.1       foxr     1270: sub QueueTransaction {
                   1271: 
1.7       foxr     1272:     my $requestData   = shift;	# This is a LondTransaction.
                   1273:     my $cmd           = $requestData->getRequest();
                   1274: 
                   1275:     Debug(6,"QueueTransaction: ".$cmd);
1.1       foxr     1276: 
                   1277:     my $LondSocket    = $IdleConnections->pop();
                   1278:     if(!defined $LondSocket) {	# Need to queue request.
1.29      foxr     1279: 	Debug(5,"Must queue...");
1.1       foxr     1280: 	$WorkQueue->enqueue($requestData);
1.56      foxr     1281: 	Debug(5, "Queue Transaction startnew $ConnectionCount $LondConnecting");
                   1282: 	if(($ConnectionCount < $MaxConnectionCount)   && (! $LondConnecting)) {
                   1283: 
1.22      foxr     1284: 	    if($ConnectionRetriesLeft > 0) {
1.29      foxr     1285: 		Debug(5,"Starting additional lond connection");
1.56      foxr     1286: 		if(&MakeLondConnection() == 0) {
1.22      foxr     1287: 		    EmptyQueue();	# Fail transactions, can't make connection.
1.42      foxr     1288: 		    CloseAllLondConnections; # Should all be closed but...
1.22      foxr     1289: 		}
                   1290: 	    } else {
                   1291: 		ShowStatus(GetServerHost()." >>> DEAD !!!! <<<");
1.56      foxr     1292: 		$LondConnecting = 0;
1.22      foxr     1293: 		EmptyQueue();	# It's worse than that ... he's dead Jim.
1.42      foxr     1294: 		CloseAllLondConnections; # Should all be closed but..
1.17      foxr     1295: 	    }
1.1       foxr     1296: 	}
                   1297:     } else {			# Can start the request:
                   1298: 	Debug(8,"Can start...");
1.7       foxr     1299: 	StartRequest($LondSocket,  $requestData);
1.1       foxr     1300:     }
                   1301: }
                   1302: 
                   1303: #-------------------------- Lonc UNIX socket handling ---------------------
1.3       albertel 1304: 
1.1       foxr     1305: =pod
1.3       albertel 1306: 
1.1       foxr     1307: =head2 ClientRequest
1.3       albertel 1308: Callback that is called when data can be read from the UNIX domain
                   1309: socket connecting us with an apache server process.
1.1       foxr     1310: 
                   1311: =cut
                   1312: 
                   1313: sub ClientRequest {
                   1314:     Debug(6, "ClientRequest");
                   1315:     my $event   = shift;
                   1316:     my $watcher = $event->w;
                   1317:     my $socket  = $watcher->fd;
                   1318:     my $data    = $watcher->data;
                   1319:     my $thisread;
                   1320: 
                   1321:     Debug(9, "  Watcher named: ".$watcher->desc);
                   1322: 
                   1323:     my $rv = $socket->recv($thisread, POSIX::BUFSIZ, 0);
                   1324:     Debug(8, "rcv:  data length = ".length($thisread)
                   1325: 	  ." read =".$thisread);
1.29      foxr     1326:     unless (defined $rv  && length($thisread)) {
1.1       foxr     1327: 	 # Likely eof on socket.
                   1328: 	Debug(5,"Client Socket closed on lonc for ".$RemoteHost);
                   1329: 	close($socket);
                   1330: 	$watcher->cancel();
                   1331: 	delete($ActiveClients{$socket});
1.10      foxr     1332: 	return;
1.1       foxr     1333:     }
                   1334:     Debug(8,"Data: ".$data." this read: ".$thisread);
                   1335:     $data = $data.$thisread;	# Append new data.
                   1336:     $watcher->data($data);
1.44      albertel 1337:     if($data =~ /\n$/) {	# Request entirely read.
1.10      foxr     1338: 	if($data eq "close_connection_exit\n") {
1.9       foxr     1339: 	    Log("CRITICAL",
                   1340: 		"Request Close Connection ... exiting");
                   1341: 	    CloseAllLondConnections();
                   1342: 	    exit;
                   1343: 	}
1.1       foxr     1344: 	Debug(8, "Complete transaction received: ".$data);
1.39      foxr     1345: 	if($LogTransactions) {
                   1346: 	    Log("SUCCESS", "Transaction: '$data'"); # Transaction has \n.
                   1347: 	}
1.8       foxr     1348: 	my $Transaction = LondTransaction->new($data);
1.7       foxr     1349: 	$Transaction->SetClient($socket);
                   1350: 	QueueTransaction($Transaction);
1.1       foxr     1351: 	$watcher->cancel();	# Done looking for input data.
                   1352:     }
                   1353: 
                   1354: }
                   1355: 
                   1356: 
                   1357: =pod
1.3       albertel 1358: 
1.1       foxr     1359: =head2  NewClient
1.3       albertel 1360: 
                   1361: Callback that is called when a connection is received on the unix
                   1362: socket for a new client of lonc.  The callback is parameterized by the
                   1363: event.. which is a-priori assumed to be an io event, and therefore has
                   1364: an fd member that is the Listener socket.  We Accept the connection
                   1365: and register a new event on the readability of that socket:
                   1366: 
1.1       foxr     1367: =cut
1.3       albertel 1368: 
1.1       foxr     1369: sub NewClient {
                   1370:     Debug(6, "NewClient");
                   1371:     my $event      = shift;		# Get the event parameters.
                   1372:     my $watcher    = $event->w; 
                   1373:     my $socket     = $watcher->fd;	# Get the event' socket.
                   1374:     my $connection = $socket->accept();	# Accept the client connection.
                   1375:     Debug(8,"Connection request accepted from "
                   1376: 	  .GetPeername($connection, AF_UNIX));
                   1377: 
                   1378: 
                   1379:     my $description = sprintf("Connection to lonc client %d",
                   1380: 			      $ClientConnection);
                   1381:     Debug(9, "Creating event named: ".$description);
                   1382:     Event->io(cb      => \&ClientRequest,
                   1383: 	      poll    => 'r',
                   1384: 	      desc    => $description,
                   1385: 	      data    => "",
                   1386: 	      fd      => $connection);
                   1387:     $ActiveClients{$connection} = $ClientConnection;
                   1388:     $ClientConnection++;
                   1389: }
1.3       albertel 1390: 
                   1391: =pod
                   1392: 
                   1393: =head2 GetLoncSocketPath
                   1394: 
                   1395: Returns the name of the UNIX socket on which to listen for client
                   1396: connections.
1.1       foxr     1397: 
1.58    ! foxr     1398: =head2 Parameters:
        !          1399: 
        !          1400:     host (optional)  - Name of the host socket to return.. defaults to
        !          1401:                        the return from GetServerHost().
        !          1402: 
1.1       foxr     1403: =cut
1.3       albertel 1404: 
1.1       foxr     1405: sub GetLoncSocketPath {
1.58    ! foxr     1406: 
        !          1407:     my $host = GetServerHost();	# Default host.
        !          1408:     if (@_) {
        !          1409: 	($host)  = @_;		# Override if supplied.
        !          1410:     }
        !          1411:     return $UnixSocketDir."/".$host;
1.1       foxr     1412: }
                   1413: 
1.3       albertel 1414: =pod
                   1415: 
                   1416: =head2 GetServerHost
                   1417: 
                   1418: Returns the host whose lond we talk with.
                   1419: 
1.1       foxr     1420: =cut
1.3       albertel 1421: 
1.7       foxr     1422: sub GetServerHost {
1.1       foxr     1423:     return $RemoteHost;		# Setup by the fork.
                   1424: }
1.3       albertel 1425: 
                   1426: =pod
                   1427: 
                   1428: =head2 GetServerPort
                   1429: 
                   1430: Returns the lond port number.
                   1431: 
1.1       foxr     1432: =cut
1.3       albertel 1433: 
1.7       foxr     1434: sub GetServerPort {
1.1       foxr     1435:     return $perlvar{londPort};
                   1436: }
1.3       albertel 1437: 
                   1438: =pod
                   1439: 
                   1440: =head2 SetupLoncListener
                   1441: 
                   1442: Setup a lonc listener event.  The event is called when the socket
                   1443: becomes readable.. that corresponds to the receipt of a new
                   1444: connection.  The event handler established will accept the connection
                   1445: (creating a communcations channel), that int turn will establish
                   1446: another event handler to subess requests.
1.1       foxr     1447: 
1.58    ! foxr     1448: =head2  Parameters:
        !          1449: 
        !          1450:    host (optional)   Name of the host to set up a unix socket to.
        !          1451: 
1.1       foxr     1452: =cut
1.3       albertel 1453: 
1.1       foxr     1454: sub SetupLoncListener {
                   1455: 
1.58    ! foxr     1456:     my $host       = GetServerHost(); # Default host.
        !          1457:     if (@_) {
        !          1458: 	($host)    = @_		# Override host with parameter.
        !          1459:     }
        !          1460: 
1.1       foxr     1461:     my $socket;
1.58    ! foxr     1462:     my $SocketName = GetLoncSocketPath($host);
1.1       foxr     1463:     unlink($SocketName);
1.7       foxr     1464:     unless ($socket =IO::Socket::UNIX->new(Local  => $SocketName,
1.55      albertel 1465: 					    Listen => 250, 
1.1       foxr     1466: 					    Type   => SOCK_STREAM)) {
                   1467: 	die "Failed to create a lonc listner socket";
                   1468:     }
                   1469:     Event->io(cb     => \&NewClient,
                   1470: 	      poll   => 'r',
                   1471: 	      desc   => 'Lonc listener Unix Socket',
                   1472: 	      fd     => $socket);
                   1473: }
                   1474: 
1.39      foxr     1475: #
                   1476: #   Toggle transaction logging.
                   1477: #  Implicit inputs:  
                   1478: #     LogTransactions
                   1479: #  Implicit Outputs:
                   1480: #     LogTransactions
                   1481: sub ToggleTransactionLogging {
                   1482:     print STDERR "Toggle transaction logging...\n";
                   1483:     if(!$LogTransactions) {
                   1484: 	$LogTransactions = 1;
                   1485:     } else {
                   1486: 	$LogTransactions = 0;
                   1487:     }
                   1488: 
                   1489: 
                   1490:     Log("SUCCESS", "Toggled transaction logging: $LogTransactions \n");
                   1491: }
                   1492: 
1.14      foxr     1493: =pod 
                   1494: 
                   1495: =head2 ChildStatus
                   1496:  
                   1497: Child USR1 signal handler to report the most recent status
                   1498: into the status file.
                   1499: 
1.22      foxr     1500: We also use this to reset the retries count in order to allow the
                   1501: client to retry connections with a previously dead server.
1.14      foxr     1502: =cut
1.46      albertel 1503: 
1.14      foxr     1504: sub ChildStatus {
                   1505:     my $event = shift;
                   1506:     my $watcher = $event->w;
                   1507: 
                   1508:     Debug(2, "Reporting child status because : ".$watcher->data);
                   1509:     my $docdir = $perlvar{'lonDocRoot'};
                   1510:     my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt");
                   1511:     print $fh $$."\t".$RemoteHost."\t".$Status."\t".
                   1512: 	$RecentLogEntry."\n";
1.38      foxr     1513:     #
                   1514:     #  Write out information about each of the connections:
                   1515:     #
1.46      albertel 1516:     if ($DebugLevel > 2) {
                   1517: 	print $fh "Active connection statuses: \n";
                   1518: 	my $i = 1;
                   1519: 	print STDERR  "================================= Socket Status Dump:\n";
                   1520: 	foreach my $item (keys %ActiveConnections) {
                   1521: 	    my $Socket = $ActiveConnections{$item}->data;
                   1522: 	    my $state  = $Socket->GetState();
                   1523: 	    print $fh "Connection $i State: $state\n";
                   1524: 	    print STDERR "---------------------- Connection $i \n";
1.48      foxr     1525: 	    $Socket->Dump(-1);	# Ensure it gets dumped..
1.46      albertel 1526: 	    $i++;	
                   1527: 	}
1.38      foxr     1528:     }
1.22      foxr     1529:     $ConnectionRetriesLeft = $ConnectionRetries;
1.14      foxr     1530: }
                   1531: 
1.1       foxr     1532: =pod
1.3       albertel 1533: 
1.10      foxr     1534: =head2 SignalledToDeath
                   1535: 
                   1536: Called in response to a signal that causes a chid process to die.
                   1537: 
                   1538: =cut
                   1539: 
                   1540: 
                   1541: sub SignalledToDeath {
1.14      foxr     1542:     my $event  = shift;
                   1543:     my $watcher= $event->w;
                   1544: 
                   1545:     Debug(2,"Signalled to death! via ".$watcher->data);
1.17      foxr     1546:     my ($signal) = $watcher->data;
1.10      foxr     1547:     chomp($signal);
                   1548:     Log("CRITICAL", "Abnormal exit.  Child $$ for $RemoteHost "
                   1549: 	."died through "."\"$signal\"");
                   1550:     LogPerm("F:lonc: $$ on $RemoteHost signalled to death: "
                   1551: 	    ."\"$signal\"");
1.12      foxr     1552:     exit 0;
1.10      foxr     1553: 
                   1554: }
1.16      foxr     1555: 
                   1556: =head2 ToggleDebug
                   1557: 
                   1558: This sub toggles trace debugging on and off.
                   1559: 
                   1560: =cut
                   1561: 
                   1562: sub ToggleDebug {
                   1563:     my $Current    = $DebugLevel;
                   1564:        $DebugLevel = $NextDebugLevel;
                   1565:        $NextDebugLevel = $Current;
                   1566: 
                   1567:     Log("SUCCESS", "New debugging level for $RemoteHost now $DebugLevel");
                   1568: 
                   1569: }
                   1570: 
1.1       foxr     1571: =head2 ChildProcess
                   1572: 
                   1573: This sub implements a child process for a single lonc daemon.
                   1574: 
                   1575: =cut
                   1576: 
                   1577: sub ChildProcess {
                   1578: 
                   1579: 
1.14      foxr     1580:     #
                   1581:     #  Signals must be handled by the Event framework...
                   1582: #
                   1583: 
                   1584:     Event->signal(signal   => "QUIT",
                   1585: 		  cb       => \&SignalledToDeath,
                   1586: 		  data     => "QUIT");
                   1587:     Event->signal(signal   => "HUP",
                   1588: 		  cb       => \&ChildStatus,
                   1589: 		  data     => "HUP");
                   1590:     Event->signal(signal   => "USR1",
                   1591: 		  cb       => \&ChildStatus,
                   1592: 		  data     => "USR1");
1.39      foxr     1593:     Event->signal(signal   => "USR2",
                   1594: 		  cb       => \&ToggleTransactionLogging);
1.16      foxr     1595:     Event->signal(signal   => "INT",
                   1596: 		  cb       => \&ToggleDebug,
                   1597: 		  data     => "INT");
1.1       foxr     1598: 
                   1599:     
                   1600:     SetupLoncListener();
                   1601:     
                   1602:     $Event::Debuglevel = $DebugLevel;
                   1603:     
                   1604:     Debug(9, "Making initial lond connection for ".$RemoteHost);
                   1605: 
                   1606: # Setup the initial server connection:
                   1607:     
1.14      foxr     1608:      # &MakeLondConnection(); // let first work requirest do it.
1.10      foxr     1609: 
1.5       foxr     1610: 
1.1       foxr     1611:     Debug(9,"Entering event loop");
                   1612:     my $ret = Event::loop();		#  Start the main event loop.
                   1613:     
                   1614:     
                   1615:     die "Main event loop exited!!!";
                   1616: }
                   1617: 
                   1618: #  Create a new child for host passed in:
                   1619: 
                   1620: sub CreateChild {
1.52      foxr     1621:     my $host = shift;
                   1622: 
1.12      foxr     1623:     my $sigset = POSIX::SigSet->new(SIGINT);
                   1624:     sigprocmask(SIG_BLOCK, $sigset);
1.1       foxr     1625:     $RemoteHost = $host;
1.9       foxr     1626:     Log("CRITICAL", "Forking server for ".$host);
1.23      foxr     1627:     my $pid          = fork;
1.1       foxr     1628:     if($pid) {			# Parent
1.17      foxr     1629: 	$RemoteHost = "Parent";
1.27      foxr     1630: 	$ChildHash{$pid} = $host;
1.26      foxr     1631: 	$HostToPid{$host}= $pid;
1.12      foxr     1632: 	sigprocmask(SIG_UNBLOCK, $sigset);
                   1633: 
1.1       foxr     1634:     } else {			# child.
1.5       foxr     1635: 	ShowStatus("Connected to ".$RemoteHost);
1.23      foxr     1636: 	$SIG{INT} = 'DEFAULT';
1.12      foxr     1637: 	sigprocmask(SIG_UNBLOCK, $sigset);
                   1638: 	ChildProcess;		# Does not return.
1.1       foxr     1639:     }
                   1640: 
                   1641: }
                   1642: #
                   1643: #  Parent process logic pass 1:
                   1644: #   For each entry in the hosts table, we will
                   1645: #  fork off an instance of ChildProcess to service the transactions
                   1646: #  to that host.  Each pid will be entered in a global hash
                   1647: #  with the value of the key, the host.
                   1648: #  The parent will then enter a loop to wait for process exits.
                   1649: #  Each exit gets logged and the child gets restarted.
                   1650: #
                   1651: 
1.5       foxr     1652: #
                   1653: #   Fork and start in new session so hang-up isn't going to 
                   1654: #   happen without intent.
                   1655: #
                   1656: 
                   1657: 
1.6       foxr     1658: 
                   1659: 
1.8       foxr     1660: 
1.6       foxr     1661: 
                   1662: ShowStatus("Forming new session");
                   1663: my $childpid = fork;
                   1664: if ($childpid != 0) {
                   1665:     sleep 4;			# Give child a chacne to break to
                   1666:     exit 0;			# a new sesion.
                   1667: }
1.8       foxr     1668: #
                   1669: #   Write my pid into the pid file so I can be located
                   1670: #
                   1671: 
                   1672: ShowStatus("Parent writing pid file:");
1.23      foxr     1673: my $execdir = $perlvar{'lonDaemons'};
1.8       foxr     1674: open (PIDSAVE, ">$execdir/logs/lonc.pid");
                   1675: print PIDSAVE "$$\n";
                   1676: close(PIDSAVE);
1.6       foxr     1677: 
1.17      foxr     1678: 
                   1679: 
1.6       foxr     1680: if (POSIX::setsid() < 0) {
                   1681:     print "Could not create new session\n";
                   1682:     exit -1;
                   1683: }
1.5       foxr     1684: 
                   1685: ShowStatus("Forking node servers");
                   1686: 
1.9       foxr     1687: Log("CRITICAL", "--------------- Starting children ---------------");
                   1688: 
1.31      foxr     1689: LondConnection::ReadConfig;               # Read standard config files.
1.1       foxr     1690: my $HostIterator = LondConnection::GetHostIterator;
                   1691: while (! $HostIterator->end()) {
                   1692: 
1.23      foxr     1693:     my $hostentryref = $HostIterator->get();
1.1       foxr     1694:     CreateChild($hostentryref->[0]);
1.26      foxr     1695:     $HostHash{$hostentryref->[0]} = $hostentryref->[4];
1.1       foxr     1696:     $HostIterator->next();
                   1697: }
1.12      foxr     1698: $RemoteHost = "Parent Server";
1.1       foxr     1699: 
                   1700: # Maintain the population:
1.5       foxr     1701: 
                   1702: ShowStatus("Parent keeping the flock");
1.1       foxr     1703: 
1.10      foxr     1704: #
                   1705: #   Set up parent signals:
                   1706: #
1.12      foxr     1707: 
1.14      foxr     1708: $SIG{INT}  = \&Terminate;
                   1709: $SIG{TERM} = \&Terminate; 
1.13      foxr     1710: $SIG{HUP}  = \&Restart;
1.14      foxr     1711: $SIG{USR1} = \&CheckKids; 
1.24      foxr     1712: $SIG{USR2} = \&UpdateKids;	# LonManage update request.
1.10      foxr     1713: 
1.1       foxr     1714: while(1) {
1.23      foxr     1715:     my $deadchild = wait();
1.1       foxr     1716:     if(exists $ChildHash{$deadchild}) {	# need to restart.
1.23      foxr     1717: 	my $deadhost = $ChildHash{$deadchild};
1.26      foxr     1718: 	delete($HostToPid{$deadhost});
1.1       foxr     1719: 	delete($ChildHash{$deadchild});
1.9       foxr     1720: 	Log("WARNING","Lost child pid= ".$deadchild.
1.1       foxr     1721: 	      "Connected to host ".$deadhost);
1.9       foxr     1722: 	Log("INFO", "Restarting child procesing ".$deadhost);
1.1       foxr     1723: 	CreateChild($deadhost);
                   1724:     }
1.13      foxr     1725: }
                   1726: 
1.14      foxr     1727: 
                   1728: 
                   1729: =pod
                   1730: 
                   1731: =head1 CheckKids
                   1732: 
                   1733:   Since kids do not die as easily in this implementation
                   1734: as the previous one, there  is no need to restart the
                   1735: dead ones (all dead kids get restarted when they die!!)
                   1736: The only thing this function does is to pass USR1 to the
                   1737: kids so that they report their status.
                   1738: 
                   1739: =cut
                   1740: 
                   1741: sub CheckKids {
                   1742:     Debug(2, "Checking status of children");
                   1743:     my $docdir = $perlvar{'lonDocRoot'};
                   1744:     my $fh = IO::File->new(">$docdir/lon-status/loncstatus.txt");
                   1745:     my $now=time;
                   1746:     my $local=localtime($now);
                   1747:     print $fh "LONC status $local - parent $$ \n\n";
1.23      foxr     1748:     foreach my $pid (keys %ChildHash) {
1.14      foxr     1749: 	Debug(2, "Sending USR1 -> $pid");
                   1750: 	kill 'USR1' => $pid;	# Tell Child to report status.
                   1751: 	sleep 1;		# Wait so file doesn't intermix.
                   1752:     }
                   1753: }
1.24      foxr     1754: 
                   1755: =pod
                   1756: 
                   1757: =head1  UpdateKids
                   1758: 
1.25      foxr     1759: parent's SIGUSR2 handler.  This handler:
1.24      foxr     1760: 
                   1761: =item
                   1762: 
                   1763: Rereads the hosts file.
                   1764: 
                   1765: =item
                   1766:  
                   1767: Kills off (via sigint) children for hosts that have disappeared.
                   1768: 
                   1769: =item
                   1770: 
1.27      foxr     1771: QUITs  children for hosts that already exist (this just forces a status display
1.24      foxr     1772: and resets the connection retry count for that host.
                   1773: 
                   1774: =item
                   1775: 
                   1776: Starts new children for hosts that have been added to the hosts.tab file since
                   1777: the start of the master program and maintains them.
                   1778: 
                   1779: =cut
                   1780: 
                   1781: sub UpdateKids {
1.27      foxr     1782: 
1.25      foxr     1783:     Log("INFO", "Updating connections via SIGUSR2");
1.27      foxr     1784: 
                   1785:     #  Just in case we need to kill our own lonc, we wait a few seconds to
                   1786:     #  give it a chance to receive and relay lond's response to the 
                   1787:     #  re-init command.
                   1788:     #
                   1789: 
                   1790:     sleep(2);			# Wait a couple of seconds.
                   1791: 
                   1792:     my %hosts;                   # Indexed by loncapa hostname, value=ip.
                   1793:     
                   1794:     # Need to re-read  the host table:
                   1795:     
                   1796:     
                   1797:     LondConnection::ReadConfig();
                   1798:     my $I = LondConnection::GetHostIterator;
                   1799:     while (! $I->end()) {
                   1800: 	my $item = $I->get();
                   1801: 	$hosts{$item->[0]} = $item->[4];
                   1802: 	$I->next();
                   1803:     }
                   1804: 
                   1805:     #  The logic below is written for clarity not for efficiency.
                   1806:     #  Since I anticipate that this function is only rarely called, that's
                   1807:     #  appropriate.  There are certainly ways to combine the loops below,
                   1808:     #  and anyone wishing to obscure the logic is welcome to go for it.
                   1809:     #  Note that we don't re-direct sigchild.  Instead we do what's needed
                   1810:     #  to the data structures that keep track of children to ensure that
                   1811:     #  when sigchild is honored, no new child is born.
                   1812:     #
                   1813: 
                   1814:     #  For each existing child; if it's host doesn't exist, kill the child.
                   1815: 
                   1816:     foreach my $child (keys %ChildHash) {
                   1817: 	my $oldhost = $ChildHash{$child};
                   1818: 	if (!(exists $hosts{$oldhost})) {
                   1819: 	    Log("CRITICAL", "Killing child for $oldhost  host no longer exists");
                   1820: 	    delete $ChildHash{$child};
                   1821: 	    delete $HostToPid{$oldhost};
                   1822: 	    kill 'QUIT' => $child;
                   1823: 	}
                   1824:     }
                   1825:     # For each remaining existing child; if it's host's ip has changed,
                   1826:     # Restart the child on the new IP.
                   1827: 
                   1828:     foreach my $child (keys %ChildHash) {
                   1829: 	my $oldhost = $ChildHash{$child};
                   1830: 	my $oldip   = $HostHash{$oldhost};
                   1831: 	if ($hosts{$oldhost} ne $oldip) {
                   1832: 
                   1833: 	    # kill the old child.
                   1834: 
                   1835: 	    Log("CRITICAL", "Killing child for $oldhost host ip has changed...");
                   1836: 	    delete $ChildHash{$child};
                   1837: 	    delete $HostToPid{$oldhost};
                   1838: 	    kill 'QUIT' => $child;
                   1839: 
                   1840: 	    # Do the book-keeping needed to start a new child on the
                   1841: 	    # new ip.
                   1842: 
                   1843: 	    $HostHash{$oldhost} = $hosts{$oldhost};
                   1844: 	    CreateChild($oldhost);
                   1845: 	}
                   1846:     }
                   1847:     # Finally, for each new host, not in the host hash, create a
                   1848:     # enter the host and create a new child.
                   1849:     # Force a status display of any existing process.
                   1850: 
                   1851:     foreach my $host (keys %hosts) {
                   1852: 	if(!(exists $HostHash{$host})) {
                   1853: 	    Log("INFO", "New host $host discovered in hosts.tab...");
                   1854: 	    $HostHash{$host} = $hosts{$host};
                   1855: 	    CreateChild($host);
                   1856: 	} else {
                   1857: 	    kill 'HUP' => $HostToPid{$host};    # status display.
                   1858: 	}
                   1859:     }
1.24      foxr     1860: }
                   1861: 
1.14      foxr     1862: 
1.13      foxr     1863: =pod
                   1864: 
                   1865: =head1 Restart
                   1866: 
                   1867: Signal handler for HUP... all children are killed and
                   1868: we self restart.  This is an el-cheapo way to re read
                   1869: the config file.
                   1870: 
                   1871: =cut
                   1872: 
                   1873: sub Restart {
1.23      foxr     1874:     &KillThemAll;		# First kill all the children.
1.13      foxr     1875:     Log("CRITICAL", "Restarting");
                   1876:     my $execdir = $perlvar{'lonDaemons'};
                   1877:     unlink("$execdir/logs/lonc.pid");
1.28      albertel 1878:     exec("$execdir/loncnew");
1.10      foxr     1879: }
1.12      foxr     1880: 
                   1881: =pod
                   1882: 
                   1883: =head1 KillThemAll
                   1884: 
                   1885: Signal handler that kills all children by sending them a 
1.17      foxr     1886: SIGHUP.  Responds to sigint and sigterm.
1.12      foxr     1887: 
                   1888: =cut
                   1889: 
1.10      foxr     1890: sub KillThemAll {
1.12      foxr     1891:     Debug(2, "Kill them all!!");
                   1892:     local($SIG{CHLD}) = 'IGNORE';      # Our children >will< die.
1.23      foxr     1893:     foreach my $pid (keys %ChildHash) {
1.12      foxr     1894: 	my $serving = $ChildHash{$pid};
1.52      foxr     1895: 	ShowStatus("Nicely Killing lonc for $serving pid = $pid");
                   1896: 	Log("CRITICAL", "Nicely Killing lonc for $serving pid = $pid");
1.17      foxr     1897: 	kill 'QUIT' => $pid;
1.12      foxr     1898:     }
1.52      foxr     1899: 
1.17      foxr     1900: 
1.1       foxr     1901: }
1.12      foxr     1902: 
1.52      foxr     1903: 
                   1904: #
                   1905: #  Kill all children via KILL.  Just in case the
                   1906: #  first shot didn't get them.
                   1907: 
                   1908: sub really_kill_them_all_dammit
                   1909: {
                   1910:     Debug(2, "Kill them all Dammit");
                   1911:     local($SIG{CHLD} = 'IGNORE'); # In case some purist reenabled them.
                   1912:     foreach my $pid (keys %ChildHash) {
                   1913: 	my $serving = $ChildHash{$pid};
                   1914: 	&ShowStatus("Nastily killing lonc for $serving pid = $pid");
                   1915: 	Log("CRITICAL", "Nastily killing lonc for $serving pid = $pid");
                   1916: 	kill 'KILL' => $pid;
                   1917: 	delete($ChildHash{$pid});
                   1918: 	my $execdir = $perlvar{'lonDaemons'};
                   1919: 	unlink("$execdir/logs/lonc.pid");
                   1920:     }
                   1921: }
1.14      foxr     1922: =pod
                   1923: 
                   1924: =head1 Terminate
                   1925:  
                   1926: Terminate the system.
                   1927: 
                   1928: =cut
                   1929: 
                   1930: sub Terminate {
1.52      foxr     1931:     &Log("CRITICAL", "Asked to kill children.. first be nice...");
                   1932:     &KillThemAll;
                   1933:     #
                   1934:     #  By now they really should all be dead.. but just in case 
                   1935:     #  send them all SIGKILL's after a bit of waiting:
                   1936: 
                   1937:     sleep(4);
                   1938:     &Log("CRITICAL", "Now kill children nasty");
                   1939:     &really_kill_them_all_dammit;
1.17      foxr     1940:     Log("CRITICAL","Master process exiting");
                   1941:     exit 0;
1.14      foxr     1942: 
                   1943: }
1.12      foxr     1944: =pod
1.1       foxr     1945: 
                   1946: =head1 Theory
1.3       albertel 1947: 
                   1948: The event class is used to build this as a single process with an
                   1949: event driven model.  The following events are handled:
1.1       foxr     1950: 
                   1951: =item UNIX Socket connection Received
                   1952: 
                   1953: =item Request data arrives on UNIX data transfer socket.
                   1954: 
                   1955: =item lond connection becomes writable.
                   1956: 
                   1957: =item timer fires at 1 second intervals.
                   1958: 
                   1959: All sockets are run in non-blocking mode.  Timeouts managed by the timer
                   1960: handler prevents hung connections.
                   1961: 
                   1962: Key data structures:
                   1963: 
1.3       albertel 1964: =item RequestQueue
                   1965: 
                   1966: A queue of requests received from UNIX sockets that are
                   1967: waiting for a chance to be forwarded on a lond connection socket.
                   1968: 
                   1969: =item ActiveConnections
                   1970: 
                   1971: A hash of lond connections that have transactions in process that are
                   1972: available to be timed out.
                   1973: 
                   1974: =item ActiveTransactions
                   1975: 
                   1976: A hash indexed by lond connections that contain the client reply
                   1977: socket for each connection that has an active transaction on it.
                   1978: 
                   1979: =item IdleConnections
                   1980: 
                   1981: A hash of lond connections that have no work to do.  These connections
                   1982: can be closed if they are idle for a long enough time.
1.1       foxr     1983: 
                   1984: =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>