Annotation of loncom/loncnew, revision 1.59

1.1       foxr        1: #!/usr/bin/perl
1.2       albertel    2: # The LearningOnline Network with CAPA
                      3: # lonc maintains the connections to remote computers
                      4: #
1.57      foxr        5: # $Id: loncnew,v 1.56 2004/09/21 10:54:43 foxr Exp $
1.2       albertel    6: #
                      7: # Copyright Michigan State University Board of Trustees
                      8: #
                      9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
1.17      foxr       10: ## LON-CAPA is free software; you can redistribute it and/or modify
1.2       albertel   11: # it under the terms of the GNU General Public License as published by
                     12: # the Free Software Foundation; either version 2 of the License, or
                     13: # (at your option) any later version.
                     14: #
                     15: # LON-CAPA is distributed in the hope that it will be useful,
                     16: # but WITHOUT ANY WARRANTY; without even the implied warranty of
                     17: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
                     18: # GNU General Public License for more details.
                     19: #
                     20: # You should have received a copy of the GNU General Public License
                     21: # along with LON-CAPA; if not, write to the Free Software
                     22: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
                     23: #
                     24: # /home/httpd/html/adm/gpl.txt
                     25: #
                     26: # http://www.lon-capa.org/
                     27: #
1.1       foxr       28: #
1.15      foxr       29: # new lonc handles n request out bver m connections to londs.
1.1       foxr       30: # This module is based on the Event class.
                     31: #   Development iterations:
                     32: #    - Setup basic event loop.   (done)
                     33: #    - Add timer dispatch.       (done)
                     34: #    - Add ability to accept lonc UNIX domain sockets.  (done)
                     35: #    - Add ability to create/negotiate lond connections (done).
1.7       foxr       36: #    - Add general logic for dispatching requests and timeouts. (done).
                     37: #    - Add support for the lonc/lond requests.          (done).
1.38      foxr       38: #    - Add logging/status monitoring.                    (done)
                     39: #    - Add Signal handling - HUP restarts. USR1 status report. (done)
1.7       foxr       40: #    - Add Configuration file I/O                       (done).
1.38      foxr       41: #    - Add management/status request interface.         (done)
1.8       foxr       42: #    - Add deferred request capability.                  (done)
1.38      foxr       43: #    - Detect transmission timeouts.                     (done)
1.7       foxr       44: #
                     45: 
1.23      foxr       46: use strict;
1.1       foxr       47: use lib "/home/httpd/lib/perl/";
                     48: use Event qw(:DEFAULT );
                     49: use POSIX qw(:signal_h);
1.12      foxr       50: use POSIX;
1.1       foxr       51: use IO::Socket;
                     52: use IO::Socket::INET;
                     53: use IO::Socket::UNIX;
1.9       foxr       54: use IO::File;
1.6       foxr       55: use IO::Handle;
1.1       foxr       56: use Socket;
                     57: use Crypt::IDEA;
                     58: use LONCAPA::Queue;
                     59: use LONCAPA::Stack;
                     60: use LONCAPA::LondConnection;
1.7       foxr       61: use LONCAPA::LondTransaction;
1.1       foxr       62: use LONCAPA::Configuration;
                     63: use LONCAPA::HashIterator;
                     64: 
                     65: 
                     66: # Read the httpd configuration file to get perl variables
                     67: # normally set in apache modules:
                     68: 
                     69: my $perlvarref = LONCAPA::Configuration::read_conf('loncapa.conf');
                     70: my %perlvar    = %{$perlvarref};
                     71: 
                     72: #
                     73: #  parent and shared variables.
                     74: 
                     75: my %ChildHash;			# by pid -> host.
1.26      foxr       76: my %HostToPid;			# By host -> pid.
                     77: my %HostHash;			# by loncapaname -> IP.
1.1       foxr       78: 
                     79: 
1.9       foxr       80: my $MaxConnectionCount = 10;	# Will get from config later.
1.1       foxr       81: my $ClientConnection = 0;	# Uniquifier for client events.
                     82: 
1.9       foxr       83: my $DebugLevel = 0;
1.29      foxr       84: my $NextDebugLevel= 2;		# So Sigint can toggle this.
1.50      albertel   85: my $IdleTimeout= 600;		# Wait 10 minutes before pruning connections.
1.1       foxr       86: 
1.39      foxr       87: my $LogTransactions = 0;	# When True, all transactions/replies get logged.
                     88: 
1.1       foxr       89: #
                     90: #  The variables below are only used by the child processes.
                     91: #
                     92: my $RemoteHost;			# Name of host child is talking to.
1.20      albertel   93: my $UnixSocketDir= $perlvar{'lonSockDir'};
1.1       foxr       94: my $IdleConnections = Stack->new(); # Set of idle connections
                     95: my %ActiveConnections;		# Connections to the remote lond.
1.7       foxr       96: my %ActiveTransactions;		# LondTransactions in flight.
1.1       foxr       97: my %ActiveClients;		# Serial numbers of active clients by socket.
                     98: my $WorkQueue       = Queue->new(); # Queue of pending transactions.
                     99: my $ConnectionCount = 0;
1.4       foxr      100: my $IdleSeconds     = 0;	# Number of seconds idle.
1.9       foxr      101: my $Status          = "";	# Current status string.
1.14      foxr      102: my $RecentLogEntry  = "";
1.30      foxr      103: my $ConnectionRetries=2;	# Number of connection retries allowed.
                    104: my $ConnectionRetriesLeft=2;	# Number of connection retries remaining.
1.40      foxr      105: my $LondVersion     = "unknown"; # Version of lond we talk with.
1.49      foxr      106: my $KeyMode         = "";       # e.g. ssl, local, insecure from last connect.
1.54      foxr      107: my $LondConnecting  = 0;       # True when a connection is being built.
1.1       foxr      108: 
1.57      foxr      109: my $DieWhenIdle     = 0;	# When true children die when trimmed -> 0.
                    110: 
1.1       foxr      111: #
1.9       foxr      112: #   The hash below gives the HTML format for log messages
                    113: #   given a severity.
                    114: #    
                    115: my %LogFormats;
                    116: 
1.45      albertel  117: $LogFormats{"CRITICAL"} = "<font color='red'>CRITICAL: %s</font>";
                    118: $LogFormats{"SUCCESS"}  = "<font color='green'>SUCCESS: %s</font>";
                    119: $LogFormats{"INFO"}     = "<font color='yellow'>INFO: %s</font>";
                    120: $LogFormats{"WARNING"}  = "<font color='blue'>WARNING: %s</font>";
1.9       foxr      121: $LogFormats{"DEFAULT"}  = " %s ";
                    122: 
1.10      foxr      123: 
1.57      foxr      124: #  UpdateStatus;
                    125: #    Update the idle status display to show how many connections
                    126: #    are left, retries and other stuff.
                    127: #
                    128: sub UpdateStatus {
                    129:     if ($ConnectionRetriesLeft > 0) {
                    130: 	ShowStatus(GetServerHost()." Connection count: ".$ConnectionCount
                    131: 		   ." Retries remaining: ".$ConnectionRetriesLeft
                    132: 		   ." ($KeyMode)");
                    133:     } else {
                    134: 	ShowStatus(GetServerHost()." >> DEAD <<");
                    135:     }
                    136: }
                    137: 
1.10      foxr      138: 
                    139: =pod
                    140: 
                    141: =head2 LogPerm
                    142: 
                    143: Makes an entry into the permanent log file.
                    144: 
                    145: =cut
                    146: sub LogPerm {
                    147:     my $message=shift;
                    148:     my $execdir=$perlvar{'lonDaemons'};
                    149:     my $now=time;
                    150:     my $local=localtime($now);
                    151:     my $fh=IO::File->new(">>$execdir/logs/lonnet.perm.log");
                    152:     print $fh "$now:$message:$local\n";
                    153: }
1.9       foxr      154: 
                    155: =pod
                    156: 
                    157: =head2 Log
                    158: 
                    159: Logs a message to the log file.
                    160: Parameters:
                    161: 
                    162: =item severity
                    163: 
                    164: One of CRITICAL, WARNING, INFO, SUCCESS used to select the
                    165: format string used to format the message.  if the severity is
                    166: not a defined severity the Default format string is used.
                    167: 
                    168: =item message
                    169: 
                    170: The base message.  In addtion to the format string, the message
                    171: will be appended to a string containing the name of our remote
                    172: host and the time will be formatted into the message.
                    173: 
                    174: =cut
                    175: 
                    176: sub Log {
1.47      foxr      177: 
                    178:     my ($severity, $message) = @_;
                    179: 
1.9       foxr      180:     if(!$LogFormats{$severity}) {
                    181: 	$severity = "DEFAULT";
                    182:     }
                    183: 
                    184:     my $format = $LogFormats{$severity};
                    185:     
                    186:     #  Put the window dressing in in front of the message format:
                    187: 
                    188:     my $now   = time;
                    189:     my $local = localtime($now);
                    190:     my $finalformat = "$local ($$) [$RemoteHost] [$Status] ";
                    191:     my $finalformat = $finalformat.$format."\n";
                    192: 
                    193:     # open the file and put the result.
                    194: 
                    195:     my $execdir = $perlvar{'lonDaemons'};
                    196:     my $fh      = IO::File->new(">>$execdir/logs/lonc.log");
                    197:     my $msg = sprintf($finalformat, $message);
1.14      foxr      198:     $RecentLogEntry = $msg;
1.9       foxr      199:     print $fh $msg;
                    200:     
1.10      foxr      201:     
1.9       foxr      202: }
1.6       foxr      203: 
1.3       albertel  204: 
1.1       foxr      205: =pod
1.3       albertel  206: 
                    207: =head2 GetPeerName
                    208: 
                    209: Returns the name of the host that a socket object is connected to.
                    210: 
1.1       foxr      211: =cut
                    212: 
                    213: sub GetPeername {
1.47      foxr      214: 
                    215: 
                    216:     my ($connection, $AdrFamily) = @_;
                    217: 
1.1       foxr      218:     my $peer       = $connection->peername();
                    219:     my $peerport;
                    220:     my $peerip;
                    221:     if($AdrFamily == AF_INET) {
                    222: 	($peerport, $peerip) = sockaddr_in($peer);
1.23      foxr      223: 	my $peername    = gethostbyaddr($peerip, $AdrFamily);
1.1       foxr      224: 	return $peername;
                    225:     } elsif ($AdrFamily == AF_UNIX) {
                    226: 	my $peerfile;
                    227: 	($peerfile) = sockaddr_un($peer);
                    228: 	return $peerfile;
                    229:     }
                    230: }
                    231: =pod
1.3       albertel  232: 
1.1       foxr      233: =head2 Debug
1.3       albertel  234: 
                    235: Invoked to issue a debug message.
                    236: 
1.1       foxr      237: =cut
1.3       albertel  238: 
1.1       foxr      239: sub Debug {
1.47      foxr      240: 
                    241:     my ($level, $message) = @_;
                    242: 
1.1       foxr      243:     if ($level <= $DebugLevel) {
1.23      foxr      244: 	Log("INFO", "-Debug- $message host = $RemoteHost");
1.1       foxr      245:     }
                    246: }
                    247: 
                    248: sub SocketDump {
1.47      foxr      249: 
                    250:     my ($level, $socket) = @_;
                    251: 
1.1       foxr      252:     if($level <= $DebugLevel) {
1.48      foxr      253: 	$socket->Dump(-1);	# Ensure it will get dumped.
1.1       foxr      254:     }
                    255: }
1.3       albertel  256: 
1.1       foxr      257: =pod
1.3       albertel  258: 
1.5       foxr      259: =head2 ShowStatus
                    260: 
                    261:  Place some text as our pid status.
1.10      foxr      262:  and as what we return in a SIGUSR1
1.5       foxr      263: 
                    264: =cut
                    265: sub ShowStatus {
1.10      foxr      266:     my $state = shift;
                    267:     my $now = time;
                    268:     my $local = localtime($now);
                    269:     $Status   = $local.": ".$state;
                    270:     $0='lonc: '.$state.' '.$local;
1.5       foxr      271: }
                    272: 
                    273: =pod
                    274: 
1.15      foxr      275: =head 2 SocketTimeout
                    276: 
                    277:     Called when an action on the socket times out.  The socket is 
                    278:    destroyed and any active transaction is failed.
                    279: 
                    280: 
                    281: =cut
                    282: sub SocketTimeout {
                    283:     my $Socket = shift;
1.38      foxr      284:     Log("WARNING", "A socket timeout was detected");
1.52      foxr      285:     Debug(5, " SocketTimeout called: ");
1.48      foxr      286:     $Socket->Dump(0);
1.42      foxr      287:     if(exists($ActiveTransactions{$Socket})) {
1.43      albertel  288: 	FailTransaction($ActiveTransactions{$Socket});
1.42      foxr      289:     }
1.22      foxr      290:     KillSocket($Socket);	# A transaction timeout also counts as
                    291:                                 # a connection failure:
                    292:     $ConnectionRetriesLeft--;
1.42      foxr      293:     if($ConnectionRetriesLeft <= 0) {
1.52      foxr      294: 	Log("CRITICAL", "Host marked DEAD: ".GetServerHost());
1.56      foxr      295: 	$LondConnecting = 0;
1.42      foxr      296:     }
                    297: 
1.15      foxr      298: }
1.35      foxr      299: #----------------------------- Timer management ------------------------
1.15      foxr      300: 
                    301: =pod
                    302: 
1.1       foxr      303: =head2 Tick
1.3       albertel  304: 
                    305: Invoked  each timer tick.
                    306: 
1.1       foxr      307: =cut
                    308: 
1.5       foxr      309: 
1.1       foxr      310: sub Tick {
1.52      foxr      311:     my ($Event)       = @_;
                    312:     my $clock_watcher = $Event->w;
                    313: 
1.1       foxr      314:     my $client;
1.57      foxr      315:     UpdateStatus();
                    316: 
1.4       foxr      317:     # Is it time to prune connection count:
                    318: 
                    319: 
                    320:     if($IdleConnections->Count()  && 
                    321:        ($WorkQueue->Count() == 0)) { # Idle connections and nothing to do?
1.52      foxr      322: 	$IdleSeconds++;
1.4       foxr      323: 	if($IdleSeconds > $IdleTimeout) { # Prune a connection...
1.23      foxr      324: 	    my $Socket = $IdleConnections->pop();
1.6       foxr      325: 	    KillSocket($Socket);
1.54      foxr      326: 	    $IdleSeconds = 0;	# Otherwise all connections get trimmed to fast.
1.57      foxr      327: 	    UpdateStatus();
                    328: 	    if(($ConnectionCount == 0) && $DieWhenIdle) {
                    329: 		#
                    330: 		#  Create a lock file since there will be a time window
                    331: 		#  between our exit and the parent's picking up the listen
                    332: 		#  during which no listens will be done on the
                    333: 		#  lonnet client socket.
                    334: 		#
                    335: 		my $lock_file = GetLoncSocketPath().".lock";
                    336: 		open(LOCK,">$lock_file");
                    337: 		print LOCK "Contents not important";
                    338: 		close(LOCK);
                    339: 		
                    340: 		exit(0);
                    341: 	    }
1.4       foxr      342: 	}
                    343:     } else {
                    344: 	$IdleSeconds = 0;	# Reset idle count if not idle.
                    345:     }
1.15      foxr      346:     #
                    347:     #  For each inflight transaction, tick down its timeout counter.
                    348:     #
1.35      foxr      349: 
1.34      albertel  350:     foreach my $item (keys %ActiveConnections) {
                    351: 	my $State = $ActiveConnections{$item}->data->GetState();
1.35      foxr      352: 	if ($State ne 'Idle') {
1.34      albertel  353: 	    Debug(5,"Ticking Socket $State $item");
                    354: 	    $ActiveConnections{$item}->data->Tick();
                    355: 	}
1.15      foxr      356:     }
1.5       foxr      357:     # Do we have work in the queue, but no connections to service them?
                    358:     # If so, try to make some new connections to get things going again.
                    359:     #
1.57      foxr      360:     #   Note this code is dead now...
                    361:     #
1.5       foxr      362:     my $Requests = $WorkQueue->Count();
1.56      foxr      363:     if (($ConnectionCount == 0)  && ($Requests > 0) && (!$LondConnecting)) { 
1.10      foxr      364: 	if ($ConnectionRetriesLeft > 0) {
1.56      foxr      365: 	    Debug(5,"Work but no connections, Make a new one");
                    366: 	    my $success;
                    367: 	    $success    = &MakeLondConnection;
                    368: 	    if($success == 0) { # All connections failed:
1.29      foxr      369: 		Debug(5,"Work in queue failed to make any connectiouns\n");
1.22      foxr      370: 		EmptyQueue();	# Fail pending transactions with con_lost.
1.42      foxr      371: 		CloseAllLondConnections(); # Should all be closed but....
1.10      foxr      372: 	    }
                    373: 	} else {
1.56      foxr      374: 	    $LondConnecting = 0;
1.22      foxr      375: 	    ShowStatus(GetServerHost()." >>> DEAD!!! <<<");
1.29      foxr      376: 	    Debug(5,"Work in queue, but gave up on connections..flushing\n");
1.10      foxr      377: 	    EmptyQueue();	# Connections can't be established.
1.42      foxr      378: 	    CloseAllLondConnections(); # Should all already be closed but...
1.5       foxr      379: 	}
                    380:        
                    381:     }
1.49      foxr      382:     if ($ConnectionCount == 0) {
                    383: 	$KeyMode = ""; 
1.52      foxr      384: 	$clock_watcher->cancel();
1.49      foxr      385:     }
1.1       foxr      386: }
                    387: 
                    388: =pod
1.3       albertel  389: 
1.1       foxr      390: =head2 SetupTimer
                    391: 
1.3       albertel  392: Sets up a 1 per sec recurring timer event.  The event handler is used to:
1.1       foxr      393: 
1.3       albertel  394: =item
                    395: 
                    396: Trigger timeouts on communications along active sockets.
                    397: 
                    398: =item
                    399: 
                    400: Trigger disconnections of idle sockets.
1.1       foxr      401: 
                    402: =cut
                    403: 
                    404: sub SetupTimer {
1.52      foxr      405:     Debug(6, "SetupTimer");
                    406:     Event->timer(interval => 1, cb => \&Tick );
1.1       foxr      407: }
1.3       albertel  408: 
1.1       foxr      409: =pod
1.3       albertel  410: 
1.1       foxr      411: =head2 ServerToIdle
1.3       albertel  412: 
                    413: This function is called when a connection to the server is
                    414: ready for more work.
                    415: 
                    416: If there is work in the Work queue the top element is dequeued
1.1       foxr      417: and the connection will start to work on it.  If the work queue is
                    418: empty, the connection is pushed on the idle connection stack where
                    419: it will either get another work unit, or alternatively, if it sits there
                    420: long enough, it will be shut down and released.
                    421: 
1.3       albertel  422: =cut
1.1       foxr      423: 
                    424: sub ServerToIdle {
                    425:     my $Socket   = shift;	# Get the socket.
1.49      foxr      426:     $KeyMode = $Socket->{AuthenticationMode};
1.7       foxr      427:     delete($ActiveTransactions{$Socket}); # Server has no transaction
1.1       foxr      428: 
1.29      foxr      429:     &Debug(5, "Server to idle");
1.1       foxr      430: 
                    431:     #  If there's work to do, start the transaction:
                    432: 
1.23      foxr      433:     my $reqdata = $WorkQueue->dequeue(); # This is a LondTransaction
1.29      foxr      434:     if ($reqdata ne undef)  {
                    435: 	Debug(5, "Queue gave request data: ".$reqdata->getRequest());
1.7       foxr      436: 	&StartRequest($Socket,  $reqdata);
1.8       foxr      437: 
1.1       foxr      438:     } else {
                    439: 	
                    440:     #  There's no work waiting, so push the server to idle list.
1.29      foxr      441: 	&Debug(5, "No new work requests, server connection going idle");
1.1       foxr      442: 	$IdleConnections->push($Socket);
                    443:     }
                    444: }
1.3       albertel  445: 
1.1       foxr      446: =pod
1.3       albertel  447: 
1.1       foxr      448: =head2 ClientWritable
1.3       albertel  449: 
                    450: Event callback for when a client socket is writable.
                    451: 
                    452: This callback is established when a transaction reponse is
                    453: avaiable from lond.  The response is forwarded to the unix socket
                    454: as it becomes writable in this sub.
                    455: 
1.1       foxr      456: Parameters:
                    457: 
1.3       albertel  458: =item Event
                    459: 
                    460: The event that has been triggered. Event->w->data is
                    461: the data and Event->w->fd is the socket to write.
1.1       foxr      462: 
                    463: =cut
1.3       albertel  464: 
1.1       foxr      465: sub ClientWritable {
                    466:     my $Event    = shift;
                    467:     my $Watcher  = $Event->w;
                    468:     my $Data     = $Watcher->data;
                    469:     my $Socket   = $Watcher->fd;
                    470: 
                    471:     # Try to send the data:
                    472: 
                    473:     &Debug(6, "ClientWritable writing".$Data);
                    474:     &Debug(9, "Socket is: ".$Socket);
                    475: 
1.6       foxr      476:     if($Socket->connected) {
                    477: 	my $result = $Socket->send($Data, 0);
                    478: 	
                    479: 	# $result undefined: the write failed.
                    480: 	# otherwise $result is the number of bytes written.
                    481: 	# Remove that preceding string from the data.
                    482: 	# If the resulting data is empty, destroy the watcher
                    483: 	# and set up a read event handler to accept the next
                    484: 	# request.
                    485: 	
                    486: 	&Debug(9,"Send result is ".$result." Defined: ".defined($result));
1.29      foxr      487: 	if($result ne undef) {
1.6       foxr      488: 	    &Debug(9, "send result was defined");
                    489: 	    if($result == length($Data)) { # Entire string sent.
                    490: 		&Debug(9, "ClientWritable data all written");
                    491: 		$Watcher->cancel();
                    492: 		#
                    493: 		#  Set up to read next request from socket:
                    494: 		
                    495: 		my $descr     = sprintf("Connection to lonc client %d",
                    496: 					$ActiveClients{$Socket});
                    497: 		Event->io(cb    => \&ClientRequest,
                    498: 			  poll  => 'r',
                    499: 			  desc  => $descr,
                    500: 			  data  => "",
                    501: 			  fd    => $Socket);
                    502: 		
                    503: 	    } else {		# Partial string sent.
                    504: 		$Watcher->data(substr($Data, $result));
1.15      foxr      505: 		if($result == 0) {    # client hung up on us!!
1.52      foxr      506: 		    # Log("INFO", "lonc pipe client hung up on us!");
1.15      foxr      507: 		    $Watcher->cancel;
                    508: 		    $Socket->shutdown(2);
                    509: 		    $Socket->close();
                    510: 		}
1.6       foxr      511: 	    }
                    512: 	    
                    513: 	} else {			# Error of some sort...
                    514: 	    
                    515: 	    # Some errnos are possible:
                    516: 	    my $errno = $!;
                    517: 	    if($errno == POSIX::EWOULDBLOCK   ||
                    518: 	       $errno == POSIX::EAGAIN        ||
                    519: 	       $errno == POSIX::EINTR) {
                    520: 		# No action taken?
                    521: 	    } else {		# Unanticipated errno.
                    522: 		&Debug(5,"ClientWritable error or peer shutdown".$RemoteHost);
                    523: 		$Watcher->cancel;	# Stop the watcher.
                    524: 		$Socket->shutdown(2); # Kill connection
                    525: 		$Socket->close();	# Close the socket.
                    526: 	    }
1.1       foxr      527: 	    
                    528: 	}
1.6       foxr      529:     } else {
                    530: 	$Watcher->cancel();	# A delayed request...just cancel.
1.1       foxr      531:     }
                    532: }
                    533: 
                    534: =pod
1.3       albertel  535: 
1.1       foxr      536: =head2 CompleteTransaction
1.3       albertel  537: 
                    538: Called when the reply data has been received for a lond 
1.1       foxr      539: transaction.   The reply data must now be sent to the
                    540: ultimate client on the other end of the Unix socket.  This is
                    541: done by setting up a writable event for the socket with the
                    542: data the reply data.
1.3       albertel  543: 
1.1       foxr      544: Parameters:
1.3       albertel  545: 
                    546: =item Socket
                    547: 
                    548: Socket on which the lond transaction occured.  This is a
                    549: LondConnection. The data received is in the TransactionReply member.
                    550: 
1.7       foxr      551: =item Transaction
1.3       albertel  552: 
1.7       foxr      553: The transaction that is being completed.
1.1       foxr      554: 
                    555: =cut
1.3       albertel  556: 
1.1       foxr      557: sub CompleteTransaction {
1.29      foxr      558:     &Debug(5,"Complete transaction");
1.47      foxr      559: 
                    560:     my ($Socket, $Transaction) = @_;
1.1       foxr      561: 
1.7       foxr      562:     if (!$Transaction->isDeferred()) { # Normal transaction
                    563: 	my $data   = $Socket->GetReply(); # Data to send.
1.39      foxr      564: 	if($LogTransactions) {
                    565: 	    Log("SUCCESS", "Reply from lond: '$data'");
                    566: 	}
1.7       foxr      567: 	StartClientReply($Transaction, $data);
                    568:     } else {			# Delete deferred transaction file.
1.9       foxr      569: 	Log("SUCCESS", "A delayed transaction was completed");
1.23      foxr      570: 	LogPerm("S:$Transaction->getClient() :".$Transaction->getRequest());
1.7       foxr      571: 	unlink $Transaction->getFile();
                    572:     }
1.6       foxr      573: }
1.42      foxr      574: 
1.6       foxr      575: =pod
1.42      foxr      576: 
1.6       foxr      577: =head1 StartClientReply
                    578: 
                    579:    Initiates a reply to a client where the reply data is a parameter.
                    580: 
1.7       foxr      581: =head2  parameters:
                    582: 
                    583: =item Transaction
                    584: 
                    585:     The transaction for which we are responding to the client.
                    586: 
                    587: =item data
                    588: 
                    589:     The data to send to apached client.
                    590: 
1.6       foxr      591: =cut
1.42      foxr      592: 
1.6       foxr      593: sub StartClientReply {
1.1       foxr      594: 
1.47      foxr      595:     my ($Transaction, $data) = @_;
1.12      foxr      596: 
1.7       foxr      597:     my $Client   = $Transaction->getClient();
                    598: 
1.1       foxr      599:     &Debug(8," Reply was: ".$data);
                    600:     my $Serial         = $ActiveClients{$Client};
                    601:     my $desc           = sprintf("Connection to lonc client %d",
                    602: 				 $Serial);
                    603:     Event->io(fd       => $Client,
                    604: 	      poll     => "w",
                    605: 	      desc     => $desc,
                    606: 	      cb       => \&ClientWritable,
                    607: 	      data     => $data);
                    608: }
1.42      foxr      609: 
1.4       foxr      610: =pod
1.42      foxr      611: 
1.4       foxr      612: =head2 FailTransaction
                    613: 
                    614:   Finishes a transaction with failure because the associated lond socket
1.7       foxr      615:   disconnected.  There are two possibilities:
                    616:   - The transaction is deferred: in which case we just quietly
                    617:     delete the transaction since there is no client connection.
                    618:   - The transaction is 'live' in which case we initiate the sending
                    619:     of "con_lost" to the client.
                    620: 
1.42      foxr      621: Deleting the transaction means killing it from the %ActiveTransactions hash.
1.4       foxr      622: 
                    623: Parameters:
                    624: 
                    625: =item client  
                    626:  
1.7       foxr      627:    The LondTransaction we are failing.
                    628:  
1.42      foxr      629: 
1.4       foxr      630: =cut
                    631: 
                    632: sub FailTransaction {
1.7       foxr      633:     my $transaction = shift;
1.52      foxr      634:     
                    635:     #  If the socket is dead, that's already logged.
                    636: 
                    637:     if ($ConnectionRetriesLeft > 0) {
                    638: 	Log("WARNING", "Failing transaction "
                    639: 	    .$transaction->getRequest());
                    640:     }
1.30      foxr      641:     Debug(1, "Failing transaction: ".$transaction->getRequest());
1.10      foxr      642:     if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it.
1.11      foxr      643: 	my $client  = $transaction->getClient();
1.30      foxr      644: 	Debug(1," Replying con_lost to ".$transaction->getRequest());
1.11      foxr      645: 	StartClientReply($transaction, "con_lost\n");
1.7       foxr      646:     }
1.4       foxr      647: 
                    648: }
                    649: 
                    650: =pod
1.6       foxr      651: =head1  EmptyQueue
1.7       foxr      652: 
1.6       foxr      653:   Fails all items in the work queue with con_lost.
1.7       foxr      654:   Note that each item in the work queue is a transaction.
                    655: 
1.6       foxr      656: =cut
                    657: sub EmptyQueue {
1.22      foxr      658:     $ConnectionRetriesLeft--;	# Counts as connection failure too.
1.6       foxr      659:     while($WorkQueue->Count()) {
1.10      foxr      660: 	my $request = $WorkQueue->dequeue(); # This is a transaction
1.7       foxr      661: 	FailTransaction($request);
1.6       foxr      662:     }
                    663: }
                    664: 
                    665: =pod
1.4       foxr      666: 
1.9       foxr      667: =head2 CloseAllLondConnections
                    668: 
                    669: Close all connections open on lond prior to exit e.g.
                    670: 
                    671: =cut
                    672: sub CloseAllLondConnections {
1.23      foxr      673:     foreach my $Socket (keys %ActiveConnections) {
1.42      foxr      674:       if(exists($ActiveTransactions{$Socket})) {
                    675: 	FailTransaction($ActiveTransactions{$Socket});
                    676:       }
                    677:       KillSocket($Socket);
1.9       foxr      678:     }
                    679: }
                    680: =cut
                    681: 
                    682: =pod
                    683: 
1.4       foxr      684: =head2 KillSocket
                    685:  
                    686: Destroys a socket.  This function can be called either when a socket
                    687: has died of 'natural' causes or because a socket needs to be pruned due to
                    688: idleness.  If the socket has died naturally, if there are no longer any 
                    689: live connections a new connection is created (in case there are transactions
                    690: in the queue).  If the socket has been pruned, it is never re-created.
                    691: 
                    692: Parameters:
1.1       foxr      693: 
1.4       foxr      694: =item Socket
                    695:  
                    696:   The socket to kill off.
                    697: 
                    698: =item Restart
                    699: 
                    700: nonzero if we are allowed to create a new connection.
                    701: 
                    702: 
                    703: =cut
                    704: sub KillSocket {
                    705:     my $Socket = shift;
                    706: 
1.17      foxr      707:     Log("WARNING", "Shutting down a socket");
1.9       foxr      708:     $Socket->Shutdown();
                    709: 
1.7       foxr      710:     #  If the socket came from the active connection set,
                    711:     #  delete its transaction... note that FailTransaction should
                    712:     #  already have been called!!!
                    713:     #  otherwise it came from the idle set.
                    714:     #  
1.4       foxr      715:     
                    716:     if(exists($ActiveTransactions{$Socket})) {
                    717: 	delete ($ActiveTransactions{$Socket});
                    718:     }
                    719:     if(exists($ActiveConnections{$Socket})) {
                    720: 	delete($ActiveConnections{$Socket});
1.37      albertel  721: 	$ConnectionCount--;
                    722: 	if ($ConnectionCount < 0) { $ConnectionCount = 0; }
1.4       foxr      723:     }
1.6       foxr      724:     #  If the connection count has gone to zero and there is work in the
                    725:     #  work queue, the work all gets failed with con_lost.
                    726:     #
                    727:     if($ConnectionCount == 0) {
1.22      foxr      728: 	EmptyQueue();
1.42      foxr      729: 	CloseAllLondConnections; # Should all already be closed but...
1.4       foxr      730:     }
                    731: }
1.1       foxr      732: 
                    733: =pod
1.3       albertel  734: 
1.1       foxr      735: =head2 LondReadable
1.3       albertel  736: 
1.1       foxr      737: This function is called whenever a lond connection
                    738: is readable.  The action is state dependent:
                    739: 
1.3       albertel  740: =head3 State=Initialized
                    741: 
                    742: We''re waiting for the challenge, this is a no-op until the
1.1       foxr      743: state changes.
1.3       albertel  744: 
1.1       foxr      745: =head3 State=Challenged 
1.3       albertel  746: 
                    747: The challenge has arrived we need to transition to Writable.
1.1       foxr      748: The connection must echo the challenge back.
1.3       albertel  749: 
1.1       foxr      750: =head3 State=ChallengeReplied
1.3       albertel  751: 
                    752: The challenge has been replied to.  The we are receiveing the 
1.1       foxr      753: 'ok' from the partner.
1.3       albertel  754: 
1.40      foxr      755: =head3  State=ReadingVersionString
                    756: 
                    757: We have requested the lond version and are reading the
                    758: version back.  Upon completion, we'll store the version away
                    759: for future use(?).
                    760: 
                    761: =head3 State=HostSet
                    762: 
                    763: We have selected the domain name of our peer (multhomed hosts)
                    764: and are getting the reply (presumably ok) back.
                    765: 
1.1       foxr      766: =head3 State=RequestingKey
1.3       albertel  767: 
                    768: The ok has been received and we need to send the request for
1.1       foxr      769: an encryption key.  Transition to writable for that.
1.3       albertel  770: 
1.1       foxr      771: =head3 State=ReceivingKey
1.3       albertel  772: 
                    773: The the key has been requested, now we are reading the new key.
                    774: 
1.1       foxr      775: =head3 State=Idle 
1.3       albertel  776: 
                    777: The encryption key has been negotiated or we have finished 
1.1       foxr      778: reading data from the a transaction.   If the callback data has
                    779: a client as well as the socket iformation, then we are 
                    780: doing a transaction and the data received is relayed to the client
                    781: before the socket is put on the idle list.
1.3       albertel  782: 
1.1       foxr      783: =head3 State=SendingRequest
1.3       albertel  784: 
                    785: I do not think this state can be received here, but if it is,
1.1       foxr      786: the appropriate thing to do is to transition to writable, and send
                    787: the request.
1.3       albertel  788: 
1.1       foxr      789: =head3 State=ReceivingReply
1.3       albertel  790: 
                    791: We finished sending the request to the server and now transition
1.1       foxr      792: to readable to receive the reply. 
                    793: 
                    794: The parameter to this function are:
1.3       albertel  795: 
1.1       foxr      796: The event. Implicit in this is the watcher and its data.  The data 
                    797: contains at least the lond connection object and, if a 
                    798: transaction is in progress, the socket attached to the local client.
                    799: 
1.3       albertel  800: =cut
1.1       foxr      801: 
                    802: sub LondReadable {
1.8       foxr      803: 
1.41      albertel  804:     my $Event      = shift;
                    805:     my $Watcher    = $Event->w;
                    806:     my $Socket     = $Watcher->data;
                    807:     my $client     = undef;
1.40      foxr      808: 
1.41      albertel  809:     &Debug(6,"LondReadable called state = ".$Socket->GetState());
1.40      foxr      810: 
                    811: 
1.41      albertel  812:     my $State = $Socket->GetState(); # All action depends on the state.
1.40      foxr      813: 
1.41      albertel  814:     SocketDump(6, $Socket);
                    815:     my $status = $Socket->Readable();
1.40      foxr      816: 
1.41      albertel  817:     &Debug(2, "Socket->Readable returned: $status");
1.40      foxr      818: 
1.41      albertel  819:     if($status != 0) {
                    820: 	# bad return from socket read. Currently this means that
                    821: 	# The socket has become disconnected. We fail the transaction.
1.40      foxr      822: 
1.41      albertel  823: 	Log("WARNING",
                    824: 	    "Lond connection lost.");
                    825: 	if(exists($ActiveTransactions{$Socket})) {
                    826: 	    FailTransaction($ActiveTransactions{$Socket});
1.56      foxr      827: 	} else {
                    828: 	    #  Socket is connecting and failed... need to mark
                    829: 	    #  no longer connecting.
                    830: 	   
                    831: 	    $LondConnecting = 0;
1.41      albertel  832: 	}
                    833: 	$Watcher->cancel();
                    834: 	KillSocket($Socket);
                    835: 	$ConnectionRetriesLeft--;       # Counts as connection failure
                    836: 	return;
                    837:     }
                    838:     SocketDump(6,$Socket);
1.17      foxr      839: 
1.41      albertel  840:     $State = $Socket->GetState(); # Update in case of transition.
                    841:     &Debug(6, "After read, state is ".$State);
1.1       foxr      842: 
1.41      albertel  843:     if($State eq "Initialized") {
1.1       foxr      844: 
                    845: 
1.41      albertel  846:     } elsif ($State eq "ChallengeReceived") {
1.1       foxr      847: 	#  The challenge must be echoed back;  The state machine
                    848: 	# in the connection takes care of setting that up.  Just
                    849: 	# need to transition to writable:
1.41      albertel  850: 	
                    851: 	$Watcher->cb(\&LondWritable);
                    852: 	$Watcher->poll("w");
1.1       foxr      853: 
1.41      albertel  854:     } elsif ($State eq "ChallengeReplied") {
1.1       foxr      855: 
1.41      albertel  856:     } elsif ($State eq "RequestingVersion") {
                    857: 	# Need to ask for the version... that is writiability:
1.1       foxr      858: 
1.41      albertel  859: 	$Watcher->cb(\&LondWritable);
                    860: 	$Watcher->poll("w");
                    861: 
                    862:     } elsif ($State eq "ReadingVersionString") {
                    863: 	# Read the rest of the version string... 
                    864:     } elsif ($State eq "SetHost") {
                    865: 	# Need to request the actual domain get set...
                    866: 
                    867: 	$Watcher->cb(\&LondWritable);
                    868: 	$Watcher->poll("w");
                    869:     } elsif ($State eq "HostSet") {
                    870: 	# Reading the 'ok' from the peer.
                    871: 
                    872:     } elsif ($State eq "RequestingKey") {
1.1       foxr      873: 	#  The ok was received.  Now we need to request the key
                    874: 	#  That requires us to be writable:
                    875: 
1.41      albertel  876: 	$Watcher->cb(\&LondWritable);
                    877: 	$Watcher->poll("w");
1.1       foxr      878: 
1.41      albertel  879:     } elsif ($State eq "ReceivingKey") {
1.1       foxr      880: 
1.41      albertel  881:     } elsif ($State eq "Idle") {
1.40      foxr      882:    
1.41      albertel  883: 	# This is as good a spot as any to get the peer version
                    884: 	# string:
1.40      foxr      885:    
1.41      albertel  886: 	if($LondVersion eq "unknown") {
                    887: 	    $LondVersion = $Socket->PeerVersion();
                    888: 	    Log("INFO", "Connected to lond version: $LondVersion");
                    889: 	}
1.1       foxr      890: 	# If necessary, complete a transaction and then go into the
                    891: 	# idle queue.
1.22      foxr      892: 	#  Note that a trasition to idle indicates a live lond
                    893: 	# on the other end so reset the connection retries.
                    894: 	#
1.41      albertel  895: 	$ConnectionRetriesLeft = $ConnectionRetries; # success resets the count
                    896: 	$Watcher->cancel();
                    897: 	if(exists($ActiveTransactions{$Socket})) {
                    898: 	    Debug(5,"Completing transaction!!");
                    899: 	    CompleteTransaction($Socket, 
                    900: 				$ActiveTransactions{$Socket});
                    901: 	} else {
                    902: 	    Log("SUCCESS", "Connection ".$ConnectionCount." to "
                    903: 		.$RemoteHost." now ready for action");
                    904: 	}
                    905: 	ServerToIdle($Socket);	# Next work unit or idle.
1.54      foxr      906: 
                    907: 	#
                    908: 	$LondConnecting = 0;	# Best spot I can think of for this.
                    909: 	# 
1.6       foxr      910: 	
1.41      albertel  911:     } elsif ($State eq "SendingRequest") {
1.1       foxr      912: 	#  We need to be writable for this and probably don't belong
                    913: 	#  here inthe first place.
                    914: 
1.41      albertel  915: 	Deubg(6, "SendingRequest state encountered in readable");
                    916: 	$Watcher->poll("w");
                    917: 	$Watcher->cb(\&LondWritable);
1.1       foxr      918: 
1.41      albertel  919:     } elsif ($State eq "ReceivingReply") {
1.1       foxr      920: 
                    921: 
1.41      albertel  922:     } else {
                    923: 	# Invalid state.
                    924: 	Debug(4, "Invalid state in LondReadable");
                    925:     }
1.1       foxr      926: }
1.3       albertel  927: 
1.1       foxr      928: =pod
1.3       albertel  929: 
1.1       foxr      930: =head2 LondWritable
1.3       albertel  931: 
1.1       foxr      932: This function is called whenever a lond connection
                    933: becomes writable while there is a writeable monitoring
                    934: event.  The action taken is very state dependent:
1.3       albertel  935: 
1.1       foxr      936: =head3 State = Connected 
1.3       albertel  937: 
                    938: The connection is in the process of sending the 'init' hailing to the
                    939: lond on the remote end.  The connection object''s Writable member is
                    940: called.  On error, ConnectionError is called to destroy the connection
                    941: and remove it from the ActiveConnections hash
                    942: 
1.1       foxr      943: =head3 Initialized
1.3       albertel  944: 
                    945: 'init' has been sent, writability monitoring is removed and
                    946: readability monitoring is started with LondReadable as the callback.
                    947: 
1.1       foxr      948: =head3 ChallengeReceived
1.3       albertel  949: 
                    950: The connection has received the who are you challenge from the remote
                    951: system, and is in the process of sending the challenge
                    952: response. Writable is called.
                    953: 
1.1       foxr      954: =head3 ChallengeReplied
1.3       albertel  955: 
                    956: The connection has replied to the initial challenge The we switch to
                    957: monitoring readability looking for the server to reply with 'ok'.
                    958: 
1.1       foxr      959: =head3 RequestingKey
1.3       albertel  960: 
                    961: The connection is in the process of requesting its encryption key.
                    962: Writable is called.
                    963: 
1.1       foxr      964: =head3 ReceivingKey
1.3       albertel  965: 
                    966: The connection has sent the request for a key.  Switch to readability
                    967: monitoring to accept the key
                    968: 
1.1       foxr      969: =head3 SendingRequest
1.3       albertel  970: 
                    971: The connection is in the process of sending a request to the server.
                    972: This request is part of a client transaction.  All the states until
                    973: now represent the client setup protocol. Writable is called.
                    974: 
1.1       foxr      975: =head3 ReceivingReply
                    976: 
1.3       albertel  977: The connection has sent a request.  Now it must receive a reply.
                    978: Readability monitoring is requested.
                    979: 
                    980: This function is an event handler and therefore receives as
1.1       foxr      981: a parameter the event that has fired.  The data for the watcher
                    982: of this event is a reference to a list of one or two elements,
                    983: depending on state. The first (and possibly only) element is the
                    984: socket.  The second (present only if a request is in progress)
                    985: is the socket on which to return a reply to the caller.
                    986: 
                    987: =cut
1.3       albertel  988: 
1.1       foxr      989: sub LondWritable {
                    990:     my $Event   = shift;
                    991:     my $Watcher = $Event->w;
1.8       foxr      992:     my $Socket  = $Watcher->data;
                    993:     my $State   = $Socket->GetState();
1.1       foxr      994: 
1.8       foxr      995:     Debug(6,"LondWritable State = ".$State."\n");
1.1       foxr      996: 
1.8       foxr      997:  
1.1       foxr      998:     #  Figure out what to do depending on the state of the socket:
                    999:     
                   1000: 
                   1001: 
                   1002: 
                   1003:     SocketDump(6,$Socket);
                   1004: 
1.42      foxr     1005:     #  If the socket is writable, we must always write.
                   1006:     # Only by writing will we undergo state transitions.
                   1007:     # Old logic wrote in state specific code below, however
                   1008:     # That forces us at least through another invocation of
                   1009:     # this function after writability is possible again.
                   1010:     # This logic also factors out common code for handling
                   1011:     # write failures... in all cases, write failures 
                   1012:     # Kill the socket.
                   1013:     #  This logic makes the branches of the >big< if below
                   1014:     # so that the writing states are actually NO-OPs.
                   1015: 
                   1016:     if ($Socket->Writable() != 0) {
1.43      albertel 1017: 	#  The write resulted in an error.
                   1018: 	# We'll treat this as if the socket got disconnected:
                   1019: 	Log("WARNING", "Connection to ".$RemoteHost.
                   1020: 	    " has been disconnected");
                   1021: 	if(exists($ActiveTransactions{$Socket})) {
                   1022: 	    FailTransaction($ActiveTransactions{$Socket});
1.56      foxr     1023: 	} else {
                   1024: 	    #  In the process of conneting, so need to turn that off.
                   1025: 	    
                   1026: 	    $LondConnecting = 0;
1.43      albertel 1027: 	}
                   1028: 	$Watcher->cancel();
                   1029: 	KillSocket($Socket);
                   1030: 	return;
1.42      foxr     1031:     }
                   1032: 
                   1033: 
                   1034: 
1.41      albertel 1035:     if      ($State eq "Connected")         {
1.1       foxr     1036: 
1.41      albertel 1037: 	#  "init" is being sent...
1.42      foxr     1038:  
1.41      albertel 1039:     } elsif ($State eq "Initialized")       {
1.4       foxr     1040: 
1.41      albertel 1041: 	# Now that init was sent, we switch 
                   1042: 	# to watching for readability:
1.1       foxr     1043: 
1.41      albertel 1044: 	$Watcher->cb(\&LondReadable);
                   1045: 	$Watcher->poll("r");
                   1046: 	
                   1047:     } elsif ($State eq "ChallengeReceived") {
                   1048: 	# We received the challenge, now we 
                   1049: 	# are echoing it back. This is a no-op,
                   1050: 	# we're waiting for the state to change
1.1       foxr     1051: 	
1.41      albertel 1052:     } elsif ($State eq "ChallengeReplied")  {
                   1053: 	# The echo was sent back, so we switch
                   1054: 	# to watching readability.
                   1055: 
                   1056: 	$Watcher->cb(\&LondReadable);
                   1057: 	$Watcher->poll("r");
                   1058:     } elsif ($State eq "RequestingVersion") {
                   1059: 	# Sending the peer a version request...
1.42      foxr     1060: 
1.41      albertel 1061:     } elsif ($State eq "ReadingVersionString") {
                   1062: 	# Transition to read since we have sent the
                   1063: 	# version command and now just need to read the
                   1064: 	# version string from the peer:
1.40      foxr     1065:       
1.41      albertel 1066: 	$Watcher->cb(\&LondReadable);
                   1067: 	$Watcher->poll("r");
1.40      foxr     1068:       
1.41      albertel 1069:     } elsif ($State eq "SetHost") {
                   1070: 	#  Setting the remote domain...
1.42      foxr     1071: 
1.41      albertel 1072:     } elsif ($State eq "HostSet") {
                   1073: 	# Back to readable to get the ok.
1.40      foxr     1074:       
1.41      albertel 1075: 	$Watcher->cb(\&LondReadable);
                   1076: 	$Watcher->poll("r");
1.40      foxr     1077:       
                   1078: 
1.41      albertel 1079:     } elsif ($State eq "RequestingKey")     {
                   1080: 	# At this time we're requesting the key.
                   1081: 	# again, this is essentially a no-op.
                   1082: 
                   1083:     } elsif ($State eq "ReceivingKey")      {
                   1084: 	# Now we need to wait for the key
                   1085: 	# to come back from the peer:
                   1086: 
                   1087: 	$Watcher->cb(\&LondReadable);
                   1088: 	$Watcher->poll("r");
                   1089: 
                   1090:     } elsif ($State eq "SendingRequest")    {
1.40      foxr     1091:  
1.41      albertel 1092: 	# At this time we are sending a request to the
1.1       foxr     1093: 	# peer... write the next chunk:
                   1094: 
1.41      albertel 1095: 
                   1096:     } elsif ($State eq "ReceivingReply")    {
                   1097: 	# The send has completed.  Wait for the
                   1098: 	# data to come in for a reply.
                   1099: 	Debug(8,"Writable sent request/receiving reply");
                   1100: 	$Watcher->cb(\&LondReadable);
                   1101: 	$Watcher->poll("r");
1.1       foxr     1102: 
1.41      albertel 1103:     } else {
                   1104: 	#  Control only passes here on an error: 
                   1105: 	#  the socket state does not match any
                   1106: 	#  of the known states... so an error
                   1107: 	#  must be logged.
1.1       foxr     1108: 
1.41      albertel 1109: 	&Debug(4, "Invalid socket state ".$State."\n");
                   1110:     }
1.1       foxr     1111:     
                   1112: }
1.6       foxr     1113: =pod
                   1114:     
                   1115: =cut
                   1116: sub QueueDelayed {
1.8       foxr     1117:     Debug(3,"QueueDelayed called");
                   1118: 
1.6       foxr     1119:     my $path = "$perlvar{'lonSockDir'}/delayed";
1.8       foxr     1120: 
                   1121:     Debug(4, "Delayed path: ".$path);
1.6       foxr     1122:     opendir(DIRHANDLE, $path);
1.8       foxr     1123:     
1.23      foxr     1124:     my @alldelayed = grep /\.$RemoteHost$/, readdir DIRHANDLE;
1.6       foxr     1125:     closedir(DIRHANDLE);
                   1126:     my $dfname;
1.8       foxr     1127:     my $reqfile;
                   1128:     foreach $dfname (sort  @alldelayed) {
                   1129: 	$reqfile = "$path/$dfname";
                   1130: 	Debug(4, "queueing ".$reqfile);
1.6       foxr     1131: 	my $Handle = IO::File->new($reqfile);
                   1132: 	my $cmd    = <$Handle>;
1.8       foxr     1133: 	chomp $cmd;		# There may or may not be a newline...
1.12      foxr     1134: 	$cmd = $cmd."\n";	# now for sure there's exactly one newline.
1.7       foxr     1135: 	my $Transaction = LondTransaction->new($cmd);
                   1136: 	$Transaction->SetDeferred($reqfile);
                   1137: 	QueueTransaction($Transaction);
1.6       foxr     1138:     }
                   1139:     
                   1140: }
1.1       foxr     1141: 
                   1142: =pod
1.3       albertel 1143: 
1.1       foxr     1144: =head2 MakeLondConnection
1.3       albertel 1145: 
                   1146: Create a new lond connection object, and start it towards its initial
                   1147: idleness.  Once idle, it becomes elligible to receive transactions
                   1148: from the work queue.  If the work queue is not empty when the
                   1149: connection is completed and becomes idle, it will dequeue an entry and
                   1150: start off on it.
                   1151: 
1.1       foxr     1152: =cut
1.3       albertel 1153: 
1.1       foxr     1154: sub MakeLondConnection {     
                   1155:     Debug(4,"MakeLondConnection to ".GetServerHost()." on port "
                   1156: 	  .GetServerPort());
                   1157: 
                   1158:     my $Connection = LondConnection->new(&GetServerHost(),
                   1159: 					 &GetServerPort());
                   1160: 
1.30      foxr     1161:     if($Connection eq undef) {	# Needs to be more robust later.
1.9       foxr     1162: 	Log("CRITICAL","Failed to make a connection with lond.");
1.10      foxr     1163: 	$ConnectionRetriesLeft--;
                   1164: 	return 0;		# Failure.
1.5       foxr     1165:     }  else {
1.22      foxr     1166: 
1.5       foxr     1167: 	# The connection needs to have writability 
                   1168: 	# monitored in order to send the init sequence
                   1169: 	# that starts the whole authentication/key
                   1170: 	# exchange underway.
                   1171: 	#
                   1172: 	my $Socket = $Connection->GetSocket();
1.30      foxr     1173: 	if($Socket eq undef) {
1.5       foxr     1174: 	    die "did not get a socket from the connection";
                   1175: 	} else {
                   1176: 	    &Debug(9,"MakeLondConnection got socket: ".$Socket);
                   1177: 	}
1.1       foxr     1178: 	
1.21      foxr     1179: 	$Connection->SetTimeoutCallback(\&SocketTimeout);
                   1180: 
1.23      foxr     1181: 	my $event = Event->io(fd       => $Socket,
1.5       foxr     1182: 			   poll     => 'w',
                   1183: 			   cb       => \&LondWritable,
1.8       foxr     1184: 			   data     => $Connection,
1.5       foxr     1185: 			   desc => 'Connection to lond server');
                   1186: 	$ActiveConnections{$Connection} = $event;
1.52      foxr     1187: 	if ($ConnectionCount == 0) {
                   1188: 	    &SetupTimer;	# Need to handle timeouts with connections...
                   1189: 	}
1.5       foxr     1190: 	$ConnectionCount++;
1.8       foxr     1191: 	Debug(4, "Connection count = ".$ConnectionCount);
1.6       foxr     1192: 	if($ConnectionCount == 1) { # First Connection:
                   1193: 	    QueueDelayed;
                   1194: 	}
1.9       foxr     1195: 	Log("SUCESS", "Created connection ".$ConnectionCount
                   1196: 	    ." to host ".GetServerHost());
1.54      foxr     1197: 	$LondConnecting = 1;	# Connection in progress.
1.10      foxr     1198: 	return 1;		# Return success.
1.1       foxr     1199:     }
                   1200:     
                   1201: }
1.3       albertel 1202: 
1.1       foxr     1203: =pod
1.3       albertel 1204: 
1.1       foxr     1205: =head2 StartRequest
1.3       albertel 1206: 
                   1207: Starts a lond request going on a specified lond connection.
                   1208: parameters are:
                   1209: 
                   1210: =item $Lond
                   1211: 
                   1212: Connection to the lond that will send the transaction and receive the
                   1213: reply.
                   1214: 
                   1215: =item $Client
                   1216: 
                   1217: Connection to the client that is making this request We got the
                   1218: request from this socket, and when the request has been relayed to
                   1219: lond and we get a reply back from lond it will get sent to this
                   1220: socket.
                   1221: 
                   1222: =item $Request
                   1223: 
                   1224: The text of the request to send.
                   1225: 
1.1       foxr     1226: =cut
                   1227: 
                   1228: sub StartRequest {
1.47      foxr     1229: 
                   1230:     my ($Lond, $Request) = @_;
1.1       foxr     1231:     
1.7       foxr     1232:     Debug(6, "StartRequest: ".$Request->getRequest());
1.1       foxr     1233: 
                   1234:     my $Socket = $Lond->GetSocket();
                   1235:     
1.7       foxr     1236:     $Request->Activate($Lond);
                   1237:     $ActiveTransactions{$Lond} = $Request;
1.1       foxr     1238: 
1.7       foxr     1239:     $Lond->InitiateTransaction($Request->getRequest());
1.23      foxr     1240:     my $event = Event->io(fd      => $Socket,
1.1       foxr     1241: 		       poll    => "w",
                   1242: 		       cb      => \&LondWritable,
                   1243: 		       data    => $Lond,
                   1244: 		       desc    => "lond transaction connection");
                   1245:     $ActiveConnections{$Lond} = $event;
                   1246:     Debug(8," Start Request made watcher data with ".$event->data."\n");
                   1247: }
                   1248: 
                   1249: =pod
1.3       albertel 1250: 
1.1       foxr     1251: =head2 QueueTransaction
1.3       albertel 1252: 
                   1253: If there is an idle lond connection, it is put to work doing this
                   1254: transaction.  Otherwise, the transaction is placed in the work queue.
                   1255: If placed in the work queue and the maximum number of connections has
                   1256: not yet been created, a new connection will be started.  Our goal is
                   1257: to eventually have a sufficient number of connections that the work
                   1258: queue will typically be empty.  parameters are:
                   1259: 
                   1260: =item Socket
                   1261: 
                   1262: open on the lonc client.
                   1263: 
                   1264: =item Request
                   1265: 
                   1266: data to send to the lond.
1.1       foxr     1267: 
                   1268: =cut
1.3       albertel 1269: 
1.1       foxr     1270: sub QueueTransaction {
                   1271: 
1.7       foxr     1272:     my $requestData   = shift;	# This is a LondTransaction.
                   1273:     my $cmd           = $requestData->getRequest();
                   1274: 
                   1275:     Debug(6,"QueueTransaction: ".$cmd);
1.1       foxr     1276: 
                   1277:     my $LondSocket    = $IdleConnections->pop();
                   1278:     if(!defined $LondSocket) {	# Need to queue request.
1.29      foxr     1279: 	Debug(5,"Must queue...");
1.1       foxr     1280: 	$WorkQueue->enqueue($requestData);
1.56      foxr     1281: 	Debug(5, "Queue Transaction startnew $ConnectionCount $LondConnecting");
                   1282: 	if(($ConnectionCount < $MaxConnectionCount)   && (! $LondConnecting)) {
                   1283: 
1.22      foxr     1284: 	    if($ConnectionRetriesLeft > 0) {
1.29      foxr     1285: 		Debug(5,"Starting additional lond connection");
1.56      foxr     1286: 		if(&MakeLondConnection() == 0) {
1.22      foxr     1287: 		    EmptyQueue();	# Fail transactions, can't make connection.
1.42      foxr     1288: 		    CloseAllLondConnections; # Should all be closed but...
1.22      foxr     1289: 		}
                   1290: 	    } else {
                   1291: 		ShowStatus(GetServerHost()." >>> DEAD !!!! <<<");
1.56      foxr     1292: 		$LondConnecting = 0;
1.22      foxr     1293: 		EmptyQueue();	# It's worse than that ... he's dead Jim.
1.42      foxr     1294: 		CloseAllLondConnections; # Should all be closed but..
1.17      foxr     1295: 	    }
1.1       foxr     1296: 	}
                   1297:     } else {			# Can start the request:
                   1298: 	Debug(8,"Can start...");
1.7       foxr     1299: 	StartRequest($LondSocket,  $requestData);
1.1       foxr     1300:     }
                   1301: }
                   1302: 
                   1303: #-------------------------- Lonc UNIX socket handling ---------------------
1.3       albertel 1304: 
1.1       foxr     1305: =pod
1.3       albertel 1306: 
1.1       foxr     1307: =head2 ClientRequest
1.3       albertel 1308: Callback that is called when data can be read from the UNIX domain
                   1309: socket connecting us with an apache server process.
1.1       foxr     1310: 
                   1311: =cut
                   1312: 
                   1313: sub ClientRequest {
                   1314:     Debug(6, "ClientRequest");
                   1315:     my $event   = shift;
                   1316:     my $watcher = $event->w;
                   1317:     my $socket  = $watcher->fd;
                   1318:     my $data    = $watcher->data;
                   1319:     my $thisread;
                   1320: 
                   1321:     Debug(9, "  Watcher named: ".$watcher->desc);
                   1322: 
                   1323:     my $rv = $socket->recv($thisread, POSIX::BUFSIZ, 0);
                   1324:     Debug(8, "rcv:  data length = ".length($thisread)
                   1325: 	  ." read =".$thisread);
1.29      foxr     1326:     unless (defined $rv  && length($thisread)) {
1.1       foxr     1327: 	 # Likely eof on socket.
                   1328: 	Debug(5,"Client Socket closed on lonc for ".$RemoteHost);
                   1329: 	close($socket);
                   1330: 	$watcher->cancel();
                   1331: 	delete($ActiveClients{$socket});
1.10      foxr     1332: 	return;
1.1       foxr     1333:     }
                   1334:     Debug(8,"Data: ".$data." this read: ".$thisread);
                   1335:     $data = $data.$thisread;	# Append new data.
                   1336:     $watcher->data($data);
1.44      albertel 1337:     if($data =~ /\n$/) {	# Request entirely read.
1.10      foxr     1338: 	if($data eq "close_connection_exit\n") {
1.9       foxr     1339: 	    Log("CRITICAL",
                   1340: 		"Request Close Connection ... exiting");
                   1341: 	    CloseAllLondConnections();
                   1342: 	    exit;
                   1343: 	}
1.1       foxr     1344: 	Debug(8, "Complete transaction received: ".$data);
1.39      foxr     1345: 	if($LogTransactions) {
                   1346: 	    Log("SUCCESS", "Transaction: '$data'"); # Transaction has \n.
                   1347: 	}
1.8       foxr     1348: 	my $Transaction = LondTransaction->new($data);
1.7       foxr     1349: 	$Transaction->SetClient($socket);
                   1350: 	QueueTransaction($Transaction);
1.1       foxr     1351: 	$watcher->cancel();	# Done looking for input data.
                   1352:     }
                   1353: 
                   1354: }
                   1355: 
                   1356: 
                   1357: =pod
1.3       albertel 1358: 
1.1       foxr     1359: =head2  NewClient
1.3       albertel 1360: 
                   1361: Callback that is called when a connection is received on the unix
                   1362: socket for a new client of lonc.  The callback is parameterized by the
                   1363: event.. which is a-priori assumed to be an io event, and therefore has
                   1364: an fd member that is the Listener socket.  We Accept the connection
                   1365: and register a new event on the readability of that socket:
                   1366: 
1.1       foxr     1367: =cut
1.3       albertel 1368: 
1.1       foxr     1369: sub NewClient {
                   1370:     Debug(6, "NewClient");
                   1371:     my $event      = shift;		# Get the event parameters.
                   1372:     my $watcher    = $event->w; 
                   1373:     my $socket     = $watcher->fd;	# Get the event' socket.
                   1374:     my $connection = $socket->accept();	# Accept the client connection.
                   1375:     Debug(8,"Connection request accepted from "
                   1376: 	  .GetPeername($connection, AF_UNIX));
                   1377: 
                   1378: 
                   1379:     my $description = sprintf("Connection to lonc client %d",
                   1380: 			      $ClientConnection);
                   1381:     Debug(9, "Creating event named: ".$description);
                   1382:     Event->io(cb      => \&ClientRequest,
                   1383: 	      poll    => 'r',
                   1384: 	      desc    => $description,
                   1385: 	      data    => "",
                   1386: 	      fd      => $connection);
                   1387:     $ActiveClients{$connection} = $ClientConnection;
                   1388:     $ClientConnection++;
                   1389: }
1.3       albertel 1390: 
                   1391: =pod
                   1392: 
                   1393: =head2 GetLoncSocketPath
                   1394: 
                   1395: Returns the name of the UNIX socket on which to listen for client
                   1396: connections.
1.1       foxr     1397: 
1.58      foxr     1398: =head2 Parameters:
                   1399: 
                   1400:     host (optional)  - Name of the host socket to return.. defaults to
                   1401:                        the return from GetServerHost().
                   1402: 
1.1       foxr     1403: =cut
1.3       albertel 1404: 
1.1       foxr     1405: sub GetLoncSocketPath {
1.58      foxr     1406: 
                   1407:     my $host = GetServerHost();	# Default host.
                   1408:     if (@_) {
                   1409: 	($host)  = @_;		# Override if supplied.
                   1410:     }
                   1411:     return $UnixSocketDir."/".$host;
1.1       foxr     1412: }
                   1413: 
1.3       albertel 1414: =pod
                   1415: 
                   1416: =head2 GetServerHost
                   1417: 
                   1418: Returns the host whose lond we talk with.
                   1419: 
1.1       foxr     1420: =cut
1.3       albertel 1421: 
1.7       foxr     1422: sub GetServerHost {
1.1       foxr     1423:     return $RemoteHost;		# Setup by the fork.
                   1424: }
1.3       albertel 1425: 
                   1426: =pod
                   1427: 
                   1428: =head2 GetServerPort
                   1429: 
                   1430: Returns the lond port number.
                   1431: 
1.1       foxr     1432: =cut
1.3       albertel 1433: 
1.7       foxr     1434: sub GetServerPort {
1.1       foxr     1435:     return $perlvar{londPort};
                   1436: }
1.3       albertel 1437: 
                   1438: =pod
                   1439: 
                   1440: =head2 SetupLoncListener
                   1441: 
                   1442: Setup a lonc listener event.  The event is called when the socket
                   1443: becomes readable.. that corresponds to the receipt of a new
                   1444: connection.  The event handler established will accept the connection
                   1445: (creating a communcations channel), that int turn will establish
                   1446: another event handler to subess requests.
1.1       foxr     1447: 
1.58      foxr     1448: =head2  Parameters:
                   1449: 
                   1450:    host (optional)   Name of the host to set up a unix socket to.
                   1451: 
1.1       foxr     1452: =cut
1.3       albertel 1453: 
1.1       foxr     1454: sub SetupLoncListener {
                   1455: 
1.58      foxr     1456:     my $host       = GetServerHost(); # Default host.
                   1457:     if (@_) {
                   1458: 	($host)    = @_		# Override host with parameter.
                   1459:     }
                   1460: 
1.1       foxr     1461:     my $socket;
1.58      foxr     1462:     my $SocketName = GetLoncSocketPath($host);
1.1       foxr     1463:     unlink($SocketName);
1.7       foxr     1464:     unless ($socket =IO::Socket::UNIX->new(Local  => $SocketName,
1.55      albertel 1465: 					    Listen => 250, 
1.1       foxr     1466: 					    Type   => SOCK_STREAM)) {
                   1467: 	die "Failed to create a lonc listner socket";
                   1468:     }
1.59    ! foxr     1469:     return $socket;
1.1       foxr     1470: }
                   1471: 
1.39      foxr     1472: #
                   1473: #   Toggle transaction logging.
                   1474: #  Implicit inputs:  
                   1475: #     LogTransactions
                   1476: #  Implicit Outputs:
                   1477: #     LogTransactions
                   1478: sub ToggleTransactionLogging {
                   1479:     print STDERR "Toggle transaction logging...\n";
                   1480:     if(!$LogTransactions) {
                   1481: 	$LogTransactions = 1;
                   1482:     } else {
                   1483: 	$LogTransactions = 0;
                   1484:     }
                   1485: 
                   1486: 
                   1487:     Log("SUCCESS", "Toggled transaction logging: $LogTransactions \n");
                   1488: }
                   1489: 
1.14      foxr     1490: =pod 
                   1491: 
                   1492: =head2 ChildStatus
                   1493:  
                   1494: Child USR1 signal handler to report the most recent status
                   1495: into the status file.
                   1496: 
1.22      foxr     1497: We also use this to reset the retries count in order to allow the
                   1498: client to retry connections with a previously dead server.
1.14      foxr     1499: =cut
1.46      albertel 1500: 
1.14      foxr     1501: sub ChildStatus {
                   1502:     my $event = shift;
                   1503:     my $watcher = $event->w;
                   1504: 
                   1505:     Debug(2, "Reporting child status because : ".$watcher->data);
                   1506:     my $docdir = $perlvar{'lonDocRoot'};
                   1507:     my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt");
                   1508:     print $fh $$."\t".$RemoteHost."\t".$Status."\t".
                   1509: 	$RecentLogEntry."\n";
1.38      foxr     1510:     #
                   1511:     #  Write out information about each of the connections:
                   1512:     #
1.46      albertel 1513:     if ($DebugLevel > 2) {
                   1514: 	print $fh "Active connection statuses: \n";
                   1515: 	my $i = 1;
                   1516: 	print STDERR  "================================= Socket Status Dump:\n";
                   1517: 	foreach my $item (keys %ActiveConnections) {
                   1518: 	    my $Socket = $ActiveConnections{$item}->data;
                   1519: 	    my $state  = $Socket->GetState();
                   1520: 	    print $fh "Connection $i State: $state\n";
                   1521: 	    print STDERR "---------------------- Connection $i \n";
1.48      foxr     1522: 	    $Socket->Dump(-1);	# Ensure it gets dumped..
1.46      albertel 1523: 	    $i++;	
                   1524: 	}
1.38      foxr     1525:     }
1.22      foxr     1526:     $ConnectionRetriesLeft = $ConnectionRetries;
1.14      foxr     1527: }
                   1528: 
1.1       foxr     1529: =pod
1.3       albertel 1530: 
1.10      foxr     1531: =head2 SignalledToDeath
                   1532: 
                   1533: Called in response to a signal that causes a chid process to die.
                   1534: 
                   1535: =cut
                   1536: 
                   1537: 
                   1538: sub SignalledToDeath {
1.14      foxr     1539:     my $event  = shift;
                   1540:     my $watcher= $event->w;
                   1541: 
                   1542:     Debug(2,"Signalled to death! via ".$watcher->data);
1.17      foxr     1543:     my ($signal) = $watcher->data;
1.10      foxr     1544:     chomp($signal);
                   1545:     Log("CRITICAL", "Abnormal exit.  Child $$ for $RemoteHost "
                   1546: 	."died through "."\"$signal\"");
                   1547:     LogPerm("F:lonc: $$ on $RemoteHost signalled to death: "
                   1548: 	    ."\"$signal\"");
1.12      foxr     1549:     exit 0;
1.10      foxr     1550: 
                   1551: }
1.16      foxr     1552: 
                   1553: =head2 ToggleDebug
                   1554: 
                   1555: This sub toggles trace debugging on and off.
                   1556: 
                   1557: =cut
                   1558: 
                   1559: sub ToggleDebug {
                   1560:     my $Current    = $DebugLevel;
                   1561:        $DebugLevel = $NextDebugLevel;
                   1562:        $NextDebugLevel = $Current;
                   1563: 
                   1564:     Log("SUCCESS", "New debugging level for $RemoteHost now $DebugLevel");
                   1565: 
                   1566: }
                   1567: 
1.1       foxr     1568: =head2 ChildProcess
                   1569: 
                   1570: This sub implements a child process for a single lonc daemon.
                   1571: 
                   1572: =cut
                   1573: 
                   1574: sub ChildProcess {
                   1575: 
                   1576: 
1.14      foxr     1577:     #
                   1578:     #  Signals must be handled by the Event framework...
                   1579: #
                   1580: 
                   1581:     Event->signal(signal   => "QUIT",
                   1582: 		  cb       => \&SignalledToDeath,
                   1583: 		  data     => "QUIT");
                   1584:     Event->signal(signal   => "HUP",
                   1585: 		  cb       => \&ChildStatus,
                   1586: 		  data     => "HUP");
                   1587:     Event->signal(signal   => "USR1",
                   1588: 		  cb       => \&ChildStatus,
                   1589: 		  data     => "USR1");
1.39      foxr     1590:     Event->signal(signal   => "USR2",
                   1591: 		  cb       => \&ToggleTransactionLogging);
1.16      foxr     1592:     Event->signal(signal   => "INT",
                   1593: 		  cb       => \&ToggleDebug,
                   1594: 		  data     => "INT");
1.1       foxr     1595: 
                   1596:     
1.59    ! foxr     1597:     my $socket =  SetupLoncListener();
        !          1598:     Event->io(cb   => \&NewClient,
        !          1599: 	      poll => 'r',
        !          1600: 	      desc => 'Lonc Listener Unix Socket',
        !          1601: 	      fd   => $socket);
1.1       foxr     1602:     
                   1603:     $Event::Debuglevel = $DebugLevel;
                   1604:     
                   1605:     Debug(9, "Making initial lond connection for ".$RemoteHost);
                   1606: 
                   1607: # Setup the initial server connection:
                   1608:     
1.14      foxr     1609:      # &MakeLondConnection(); // let first work requirest do it.
1.10      foxr     1610: 
1.5       foxr     1611: 
1.1       foxr     1612:     Debug(9,"Entering event loop");
                   1613:     my $ret = Event::loop();		#  Start the main event loop.
                   1614:     
                   1615:     
                   1616:     die "Main event loop exited!!!";
                   1617: }
                   1618: 
                   1619: #  Create a new child for host passed in:
                   1620: 
                   1621: sub CreateChild {
1.52      foxr     1622:     my $host = shift;
                   1623: 
1.12      foxr     1624:     my $sigset = POSIX::SigSet->new(SIGINT);
                   1625:     sigprocmask(SIG_BLOCK, $sigset);
1.1       foxr     1626:     $RemoteHost = $host;
1.9       foxr     1627:     Log("CRITICAL", "Forking server for ".$host);
1.23      foxr     1628:     my $pid          = fork;
1.1       foxr     1629:     if($pid) {			# Parent
1.17      foxr     1630: 	$RemoteHost = "Parent";
1.27      foxr     1631: 	$ChildHash{$pid} = $host;
1.26      foxr     1632: 	$HostToPid{$host}= $pid;
1.12      foxr     1633: 	sigprocmask(SIG_UNBLOCK, $sigset);
                   1634: 
1.1       foxr     1635:     } else {			# child.
1.5       foxr     1636: 	ShowStatus("Connected to ".$RemoteHost);
1.23      foxr     1637: 	$SIG{INT} = 'DEFAULT';
1.12      foxr     1638: 	sigprocmask(SIG_UNBLOCK, $sigset);
                   1639: 	ChildProcess;		# Does not return.
1.1       foxr     1640:     }
                   1641: 
                   1642: }
                   1643: #
                   1644: #  Parent process logic pass 1:
                   1645: #   For each entry in the hosts table, we will
                   1646: #  fork off an instance of ChildProcess to service the transactions
                   1647: #  to that host.  Each pid will be entered in a global hash
                   1648: #  with the value of the key, the host.
                   1649: #  The parent will then enter a loop to wait for process exits.
                   1650: #  Each exit gets logged and the child gets restarted.
                   1651: #
                   1652: 
1.5       foxr     1653: #
                   1654: #   Fork and start in new session so hang-up isn't going to 
                   1655: #   happen without intent.
                   1656: #
                   1657: 
                   1658: 
1.6       foxr     1659: 
                   1660: 
1.8       foxr     1661: 
1.6       foxr     1662: 
                   1663: ShowStatus("Forming new session");
                   1664: my $childpid = fork;
                   1665: if ($childpid != 0) {
                   1666:     sleep 4;			# Give child a chacne to break to
                   1667:     exit 0;			# a new sesion.
                   1668: }
1.8       foxr     1669: #
                   1670: #   Write my pid into the pid file so I can be located
                   1671: #
                   1672: 
                   1673: ShowStatus("Parent writing pid file:");
1.23      foxr     1674: my $execdir = $perlvar{'lonDaemons'};
1.8       foxr     1675: open (PIDSAVE, ">$execdir/logs/lonc.pid");
                   1676: print PIDSAVE "$$\n";
                   1677: close(PIDSAVE);
1.6       foxr     1678: 
1.17      foxr     1679: 
                   1680: 
1.6       foxr     1681: if (POSIX::setsid() < 0) {
                   1682:     print "Could not create new session\n";
                   1683:     exit -1;
                   1684: }
1.5       foxr     1685: 
                   1686: ShowStatus("Forking node servers");
                   1687: 
1.9       foxr     1688: Log("CRITICAL", "--------------- Starting children ---------------");
                   1689: 
1.31      foxr     1690: LondConnection::ReadConfig;               # Read standard config files.
1.1       foxr     1691: my $HostIterator = LondConnection::GetHostIterator;
                   1692: while (! $HostIterator->end()) {
                   1693: 
1.23      foxr     1694:     my $hostentryref = $HostIterator->get();
1.1       foxr     1695:     CreateChild($hostentryref->[0]);
1.26      foxr     1696:     $HostHash{$hostentryref->[0]} = $hostentryref->[4];
1.1       foxr     1697:     $HostIterator->next();
                   1698: }
1.12      foxr     1699: $RemoteHost = "Parent Server";
1.1       foxr     1700: 
                   1701: # Maintain the population:
1.5       foxr     1702: 
                   1703: ShowStatus("Parent keeping the flock");
1.1       foxr     1704: 
1.10      foxr     1705: #
                   1706: #   Set up parent signals:
                   1707: #
1.12      foxr     1708: 
1.14      foxr     1709: $SIG{INT}  = \&Terminate;
                   1710: $SIG{TERM} = \&Terminate; 
1.13      foxr     1711: $SIG{HUP}  = \&Restart;
1.14      foxr     1712: $SIG{USR1} = \&CheckKids; 
1.24      foxr     1713: $SIG{USR2} = \&UpdateKids;	# LonManage update request.
1.10      foxr     1714: 
1.1       foxr     1715: while(1) {
1.23      foxr     1716:     my $deadchild = wait();
1.1       foxr     1717:     if(exists $ChildHash{$deadchild}) {	# need to restart.
1.23      foxr     1718: 	my $deadhost = $ChildHash{$deadchild};
1.26      foxr     1719: 	delete($HostToPid{$deadhost});
1.1       foxr     1720: 	delete($ChildHash{$deadchild});
1.9       foxr     1721: 	Log("WARNING","Lost child pid= ".$deadchild.
1.1       foxr     1722: 	      "Connected to host ".$deadhost);
1.9       foxr     1723: 	Log("INFO", "Restarting child procesing ".$deadhost);
1.1       foxr     1724: 	CreateChild($deadhost);
                   1725:     }
1.13      foxr     1726: }
                   1727: 
1.14      foxr     1728: 
                   1729: 
                   1730: =pod
                   1731: 
                   1732: =head1 CheckKids
                   1733: 
                   1734:   Since kids do not die as easily in this implementation
                   1735: as the previous one, there  is no need to restart the
                   1736: dead ones (all dead kids get restarted when they die!!)
                   1737: The only thing this function does is to pass USR1 to the
                   1738: kids so that they report their status.
                   1739: 
                   1740: =cut
                   1741: 
                   1742: sub CheckKids {
                   1743:     Debug(2, "Checking status of children");
                   1744:     my $docdir = $perlvar{'lonDocRoot'};
                   1745:     my $fh = IO::File->new(">$docdir/lon-status/loncstatus.txt");
                   1746:     my $now=time;
                   1747:     my $local=localtime($now);
                   1748:     print $fh "LONC status $local - parent $$ \n\n";
1.23      foxr     1749:     foreach my $pid (keys %ChildHash) {
1.14      foxr     1750: 	Debug(2, "Sending USR1 -> $pid");
                   1751: 	kill 'USR1' => $pid;	# Tell Child to report status.
                   1752: 	sleep 1;		# Wait so file doesn't intermix.
                   1753:     }
                   1754: }
1.24      foxr     1755: 
                   1756: =pod
                   1757: 
                   1758: =head1  UpdateKids
                   1759: 
1.25      foxr     1760: parent's SIGUSR2 handler.  This handler:
1.24      foxr     1761: 
                   1762: =item
                   1763: 
                   1764: Rereads the hosts file.
                   1765: 
                   1766: =item
                   1767:  
                   1768: Kills off (via sigint) children for hosts that have disappeared.
                   1769: 
                   1770: =item
                   1771: 
1.27      foxr     1772: QUITs  children for hosts that already exist (this just forces a status display
1.24      foxr     1773: and resets the connection retry count for that host.
                   1774: 
                   1775: =item
                   1776: 
                   1777: Starts new children for hosts that have been added to the hosts.tab file since
                   1778: the start of the master program and maintains them.
                   1779: 
                   1780: =cut
                   1781: 
                   1782: sub UpdateKids {
1.27      foxr     1783: 
1.25      foxr     1784:     Log("INFO", "Updating connections via SIGUSR2");
1.27      foxr     1785: 
                   1786:     #  Just in case we need to kill our own lonc, we wait a few seconds to
                   1787:     #  give it a chance to receive and relay lond's response to the 
                   1788:     #  re-init command.
                   1789:     #
                   1790: 
                   1791:     sleep(2);			# Wait a couple of seconds.
                   1792: 
                   1793:     my %hosts;                   # Indexed by loncapa hostname, value=ip.
                   1794:     
                   1795:     # Need to re-read  the host table:
                   1796:     
                   1797:     
                   1798:     LondConnection::ReadConfig();
                   1799:     my $I = LondConnection::GetHostIterator;
                   1800:     while (! $I->end()) {
                   1801: 	my $item = $I->get();
                   1802: 	$hosts{$item->[0]} = $item->[4];
                   1803: 	$I->next();
                   1804:     }
                   1805: 
                   1806:     #  The logic below is written for clarity not for efficiency.
                   1807:     #  Since I anticipate that this function is only rarely called, that's
                   1808:     #  appropriate.  There are certainly ways to combine the loops below,
                   1809:     #  and anyone wishing to obscure the logic is welcome to go for it.
                   1810:     #  Note that we don't re-direct sigchild.  Instead we do what's needed
                   1811:     #  to the data structures that keep track of children to ensure that
                   1812:     #  when sigchild is honored, no new child is born.
                   1813:     #
                   1814: 
                   1815:     #  For each existing child; if it's host doesn't exist, kill the child.
                   1816: 
                   1817:     foreach my $child (keys %ChildHash) {
                   1818: 	my $oldhost = $ChildHash{$child};
                   1819: 	if (!(exists $hosts{$oldhost})) {
                   1820: 	    Log("CRITICAL", "Killing child for $oldhost  host no longer exists");
                   1821: 	    delete $ChildHash{$child};
                   1822: 	    delete $HostToPid{$oldhost};
                   1823: 	    kill 'QUIT' => $child;
                   1824: 	}
                   1825:     }
                   1826:     # For each remaining existing child; if it's host's ip has changed,
                   1827:     # Restart the child on the new IP.
                   1828: 
                   1829:     foreach my $child (keys %ChildHash) {
                   1830: 	my $oldhost = $ChildHash{$child};
                   1831: 	my $oldip   = $HostHash{$oldhost};
                   1832: 	if ($hosts{$oldhost} ne $oldip) {
                   1833: 
                   1834: 	    # kill the old child.
                   1835: 
                   1836: 	    Log("CRITICAL", "Killing child for $oldhost host ip has changed...");
                   1837: 	    delete $ChildHash{$child};
                   1838: 	    delete $HostToPid{$oldhost};
                   1839: 	    kill 'QUIT' => $child;
                   1840: 
                   1841: 	    # Do the book-keeping needed to start a new child on the
                   1842: 	    # new ip.
                   1843: 
                   1844: 	    $HostHash{$oldhost} = $hosts{$oldhost};
                   1845: 	    CreateChild($oldhost);
                   1846: 	}
                   1847:     }
                   1848:     # Finally, for each new host, not in the host hash, create a
                   1849:     # enter the host and create a new child.
                   1850:     # Force a status display of any existing process.
                   1851: 
                   1852:     foreach my $host (keys %hosts) {
                   1853: 	if(!(exists $HostHash{$host})) {
                   1854: 	    Log("INFO", "New host $host discovered in hosts.tab...");
                   1855: 	    $HostHash{$host} = $hosts{$host};
                   1856: 	    CreateChild($host);
                   1857: 	} else {
                   1858: 	    kill 'HUP' => $HostToPid{$host};    # status display.
                   1859: 	}
                   1860:     }
1.24      foxr     1861: }
                   1862: 
1.14      foxr     1863: 
1.13      foxr     1864: =pod
                   1865: 
                   1866: =head1 Restart
                   1867: 
                   1868: Signal handler for HUP... all children are killed and
                   1869: we self restart.  This is an el-cheapo way to re read
                   1870: the config file.
                   1871: 
                   1872: =cut
                   1873: 
                   1874: sub Restart {
1.23      foxr     1875:     &KillThemAll;		# First kill all the children.
1.13      foxr     1876:     Log("CRITICAL", "Restarting");
                   1877:     my $execdir = $perlvar{'lonDaemons'};
                   1878:     unlink("$execdir/logs/lonc.pid");
1.28      albertel 1879:     exec("$execdir/loncnew");
1.10      foxr     1880: }
1.12      foxr     1881: 
                   1882: =pod
                   1883: 
                   1884: =head1 KillThemAll
                   1885: 
                   1886: Signal handler that kills all children by sending them a 
1.17      foxr     1887: SIGHUP.  Responds to sigint and sigterm.
1.12      foxr     1888: 
                   1889: =cut
                   1890: 
1.10      foxr     1891: sub KillThemAll {
1.12      foxr     1892:     Debug(2, "Kill them all!!");
                   1893:     local($SIG{CHLD}) = 'IGNORE';      # Our children >will< die.
1.23      foxr     1894:     foreach my $pid (keys %ChildHash) {
1.12      foxr     1895: 	my $serving = $ChildHash{$pid};
1.52      foxr     1896: 	ShowStatus("Nicely Killing lonc for $serving pid = $pid");
                   1897: 	Log("CRITICAL", "Nicely Killing lonc for $serving pid = $pid");
1.17      foxr     1898: 	kill 'QUIT' => $pid;
1.12      foxr     1899:     }
1.52      foxr     1900: 
1.17      foxr     1901: 
1.1       foxr     1902: }
1.12      foxr     1903: 
1.52      foxr     1904: 
                   1905: #
                   1906: #  Kill all children via KILL.  Just in case the
                   1907: #  first shot didn't get them.
                   1908: 
                   1909: sub really_kill_them_all_dammit
                   1910: {
                   1911:     Debug(2, "Kill them all Dammit");
                   1912:     local($SIG{CHLD} = 'IGNORE'); # In case some purist reenabled them.
                   1913:     foreach my $pid (keys %ChildHash) {
                   1914: 	my $serving = $ChildHash{$pid};
                   1915: 	&ShowStatus("Nastily killing lonc for $serving pid = $pid");
                   1916: 	Log("CRITICAL", "Nastily killing lonc for $serving pid = $pid");
                   1917: 	kill 'KILL' => $pid;
                   1918: 	delete($ChildHash{$pid});
                   1919: 	my $execdir = $perlvar{'lonDaemons'};
                   1920: 	unlink("$execdir/logs/lonc.pid");
                   1921:     }
                   1922: }
1.14      foxr     1923: =pod
                   1924: 
                   1925: =head1 Terminate
                   1926:  
                   1927: Terminate the system.
                   1928: 
                   1929: =cut
                   1930: 
                   1931: sub Terminate {
1.52      foxr     1932:     &Log("CRITICAL", "Asked to kill children.. first be nice...");
                   1933:     &KillThemAll;
                   1934:     #
                   1935:     #  By now they really should all be dead.. but just in case 
                   1936:     #  send them all SIGKILL's after a bit of waiting:
                   1937: 
                   1938:     sleep(4);
                   1939:     &Log("CRITICAL", "Now kill children nasty");
                   1940:     &really_kill_them_all_dammit;
1.17      foxr     1941:     Log("CRITICAL","Master process exiting");
                   1942:     exit 0;
1.14      foxr     1943: 
                   1944: }
1.12      foxr     1945: =pod
1.1       foxr     1946: 
                   1947: =head1 Theory
1.3       albertel 1948: 
                   1949: The event class is used to build this as a single process with an
                   1950: event driven model.  The following events are handled:
1.1       foxr     1951: 
                   1952: =item UNIX Socket connection Received
                   1953: 
                   1954: =item Request data arrives on UNIX data transfer socket.
                   1955: 
                   1956: =item lond connection becomes writable.
                   1957: 
                   1958: =item timer fires at 1 second intervals.
                   1959: 
                   1960: All sockets are run in non-blocking mode.  Timeouts managed by the timer
                   1961: handler prevents hung connections.
                   1962: 
                   1963: Key data structures:
                   1964: 
1.3       albertel 1965: =item RequestQueue
                   1966: 
                   1967: A queue of requests received from UNIX sockets that are
                   1968: waiting for a chance to be forwarded on a lond connection socket.
                   1969: 
                   1970: =item ActiveConnections
                   1971: 
                   1972: A hash of lond connections that have transactions in process that are
                   1973: available to be timed out.
                   1974: 
                   1975: =item ActiveTransactions
                   1976: 
                   1977: A hash indexed by lond connections that contain the client reply
                   1978: socket for each connection that has an active transaction on it.
                   1979: 
                   1980: =item IdleConnections
                   1981: 
                   1982: A hash of lond connections that have no work to do.  These connections
                   1983: can be closed if they are idle for a long enough time.
1.1       foxr     1984: 
                   1985: =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
500 Internal Server Error

Internal Server Error

The server encountered an internal error or misconfiguration and was unable to complete your request.

Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

More information about this error may be available in the server error log.