--- loncom/loncnew 2004/09/22 10:34:44 1.58 +++ loncom/loncnew 2007/05/01 01:04:23 1.85 @@ -2,7 +2,7 @@ # The LearningOnline Network with CAPA # lonc maintains the connections to remote computers # -# $Id: loncnew,v 1.58 2004/09/22 10:34:44 foxr Exp $ +# $Id: loncnew,v 1.85 2007/05/01 01:04:23 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -60,7 +60,7 @@ use LONCAPA::Stack; use LONCAPA::LondConnection; use LONCAPA::LondTransaction; use LONCAPA::Configuration; -use LONCAPA::HashIterator; +use Fcntl qw(:flock); # Read the httpd configuration file to get perl variables @@ -72,10 +72,13 @@ my %perlvar = %{$perlvarref}; # # parent and shared variables. -my %ChildHash; # by pid -> host. -my %HostToPid; # By host -> pid. -my %HostHash; # by loncapaname -> IP. +my %ChildPid; # by pid -> host. +my %ChildHost; # by host. +my %listening_to; # Socket->host table for who the parent + # is listening to. +my %parent_dispatchers; # host-> listener watcher events. +my %parent_handlers; # Parent signal handlers... my $MaxConnectionCount = 10; # Will get from config later. my $ClientConnection = 0; # Uniquifier for client events. @@ -85,11 +88,14 @@ my $NextDebugLevel= 2; # So Sigint can my $IdleTimeout= 600; # Wait 10 minutes before pruning connections. my $LogTransactions = 0; # When True, all transactions/replies get logged. +my $executable = $0; # Get the full path to me. # # The variables below are only used by the child processes. # my $RemoteHost; # Name of host child is talking to. +my $RemoteHostId; # default lonid of host child is talking to. +my @all_host_ids; my $UnixSocketDir= $perlvar{'lonSockDir'}; my $IdleConnections = Stack->new(); # Set of idle connections my %ActiveConnections; # Connections to the remote lond. @@ -100,13 +106,15 @@ my $ConnectionCount = 0; my $IdleSeconds = 0; # Number of seconds idle. my $Status = ""; # Current status string. my $RecentLogEntry = ""; -my $ConnectionRetries=2; # Number of connection retries allowed. -my $ConnectionRetriesLeft=2; # Number of connection retries remaining. +my $ConnectionRetries=5; # Number of connection retries allowed. +my $ConnectionRetriesLeft=5; # Number of connection retries remaining. my $LondVersion = "unknown"; # Version of lond we talk with. my $KeyMode = ""; # e.g. ssl, local, insecure from last connect. my $LondConnecting = 0; # True when a connection is being built. -my $DieWhenIdle = 0; # When true children die when trimmed -> 0. + + +my $I_am_child = 0; # True if this is the child process. # # The hash below gives the HTML format for log messages @@ -143,6 +151,7 @@ sub UpdateStatus { Makes an entry into the permanent log file. =cut + sub LogPerm { my $message=shift; my $execdir=$perlvar{'lonDaemons'}; @@ -188,7 +197,7 @@ sub Log { my $now = time; my $local = localtime($now); my $finalformat = "$local ($$) [$RemoteHost] [$Status] "; - my $finalformat = $finalformat.$format."\n"; + $finalformat = $finalformat.$format."\n"; # open the file and put the result. @@ -262,6 +271,7 @@ sub SocketDump { and as what we return in a SIGUSR1 =cut + sub ShowStatus { my $state = shift; my $now = time; @@ -272,13 +282,14 @@ sub ShowStatus { =pod -=head 2 SocketTimeout +=head2 SocketTimeout Called when an action on the socket times out. The socket is destroyed and any active transaction is failed. =cut + sub SocketTimeout { my $Socket = shift; Log("WARNING", "A socket timeout was detected"); @@ -296,6 +307,41 @@ sub SocketTimeout { } } + +# +# This function should be called by the child in all cases where it must +# exit. The child process must create a lock file for the AF_UNIX socket +# in order to prevent connection requests from lonnet in the time between +# process exit and the parent picking up the listen again. +# +# Parameters: +# exit_code - Exit status value, however see the next parameter. +# message - If this optional parameter is supplied, the exit +# is via a die with this message. +# +sub child_exit { + my ($exit_code, $message) = @_; + + # Regardless of how we exit, we may need to do the lock thing: + + # + # Create a lock file since there will be a time window + # between our exit and the parent's picking up the listen + # during which no listens will be done on the + # lonnet client socket. + # + my $lock_file = &GetLoncSocketPath().".lock"; + open(LOCK,">$lock_file"); + print LOCK "Contents not important"; + close(LOCK); + unlink(&GetLoncSocketPath()); + + if ($message) { + die($message); + } else { + exit($exit_code); + } +} #----------------------------- Timer management ------------------------ =pod @@ -325,19 +371,9 @@ sub Tick { KillSocket($Socket); $IdleSeconds = 0; # Otherwise all connections get trimmed to fast. UpdateStatus(); - if(($ConnectionCount == 0) && $DieWhenIdle) { - # - # Create a lock file since there will be a time window - # between our exit and the parent's picking up the listen - # during which no listens will be done on the - # lonnet client socket. - # - my $lock_file = GetLoncSocketPath().".lock"; - open(LOCK,">$lock_file"); - print LOCK "Contents not important"; - close(LOCK); - - exit(0); + if(($ConnectionCount == 0)) { + &child_exit(0); + } } } else { @@ -383,6 +419,7 @@ sub Tick { $KeyMode = ""; $clock_watcher->cancel(); } + &UpdateStatus(); } =pod @@ -465,6 +502,9 @@ the data and Event->w->fd is the socket sub ClientWritable { my $Event = shift; my $Watcher = $Event->w; + if (!defined($Watcher)) { + &child_exit(-1,'No watcher for event in ClientWritable'); + } my $Data = $Watcher->data; my $Socket = $Watcher->fd; @@ -528,6 +568,7 @@ sub ClientWritable { } } else { $Watcher->cancel(); # A delayed request...just cancel. + return; } } @@ -636,9 +677,9 @@ sub FailTransaction { if ($ConnectionRetriesLeft > 0) { Log("WARNING", "Failing transaction " - .$transaction->getRequest()); + .$transaction->getLoggableRequest()); } - Debug(1, "Failing transaction: ".$transaction->getRequest()); + Debug(1, "Failing transaction: ".$transaction->getLoggableRequest()); if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it. my $client = $transaction->getClient(); Debug(1," Replying con_lost to ".$transaction->getRequest()); @@ -648,12 +689,14 @@ sub FailTransaction { } =pod + =head1 EmptyQueue Fails all items in the work queue with con_lost. Note that each item in the work queue is a transaction. =cut + sub EmptyQueue { $ConnectionRetriesLeft--; # Counts as connection failure too. while($WorkQueue->Count()) { @@ -669,6 +712,7 @@ sub EmptyQueue { Close all connections open on lond prior to exit e.g. =cut + sub CloseAllLondConnections { foreach my $Socket (keys %ActiveConnections) { if(exists($ActiveTransactions{$Socket})) { @@ -677,7 +721,6 @@ sub CloseAllLondConnections { KillSocket($Socket); } } -=cut =pod @@ -699,8 +742,8 @@ Parameters: nonzero if we are allowed to create a new connection. - =cut + sub KillSocket { my $Socket = shift; @@ -912,7 +955,7 @@ sub LondReadable { # We need to be writable for this and probably don't belong # here inthe first place. - Deubg(6, "SendingRequest state encountered in readable"); + Debug(6, "SendingRequest state encountered in readable"); $Watcher->poll("w"); $Watcher->cb(\&LondWritable); @@ -1110,9 +1153,12 @@ sub LondWritable { } } + =pod =cut + + sub QueueDelayed { Debug(3,"QueueDelayed called"); @@ -1120,19 +1166,19 @@ sub QueueDelayed { Debug(4, "Delayed path: ".$path); opendir(DIRHANDLE, $path); - - my @alldelayed = grep /\.$RemoteHost$/, readdir DIRHANDLE; + + my $host_id_re = '(?:'.join('|',map {quotemeta($_)} (@all_host_ids)).')'; + my @alldelayed = grep(/\.$host_id_re$/, readdir(DIRHANDLE)); closedir(DIRHANDLE); - my $dfname; - my $reqfile; - foreach $dfname (sort @alldelayed) { - $reqfile = "$path/$dfname"; - Debug(4, "queueing ".$reqfile); + foreach my $dfname (sort(@alldelayed)) { + my $reqfile = "$path/$dfname"; + my ($host_id) = ($dfname =~ /\.([^.]*)$/); + Debug(4, "queueing ".$reqfile." for $host_id"); my $Handle = IO::File->new($reqfile); my $cmd = <$Handle>; chomp $cmd; # There may or may not be a newline... $cmd = $cmd."\n"; # now for sure there's exactly one newline. - my $Transaction = LondTransaction->new($cmd); + my $Transaction = LondTransaction->new("sethost:$host_id:$cmd"); $Transaction->SetDeferred($reqfile); QueueTransaction($Transaction); } @@ -1156,7 +1202,8 @@ sub MakeLondConnection { .GetServerPort()); my $Connection = LondConnection->new(&GetServerHost(), - &GetServerPort()); + &GetServerPort(), + &GetHostId()); if($Connection eq undef) { # Needs to be more robust later. Log("CRITICAL","Failed to make a connection with lond."); @@ -1164,6 +1211,7 @@ sub MakeLondConnection { return 0; # Failure. } else { + $LondConnecting = 1; # Connection in progress. # The connection needs to have writability # monitored in order to send the init sequence # that starts the whole authentication/key @@ -1171,7 +1219,7 @@ sub MakeLondConnection { # my $Socket = $Connection->GetSocket(); if($Socket eq undef) { - die "did not get a socket from the connection"; + &child_exit(-1, "did not get a socket from the connection"); } else { &Debug(9,"MakeLondConnection got socket: ".$Socket); } @@ -1194,7 +1242,6 @@ sub MakeLondConnection { } Log("SUCESS", "Created connection ".$ConnectionCount ." to host ".GetServerHost()); - $LondConnecting = 1; # Connection in progress. return 1; # Return success. } @@ -1353,6 +1400,40 @@ sub ClientRequest { } +# +# Accept a connection request for a client (lonc child) and +# start up an event watcher to keep an eye on input from that +# Event. This can be called both from NewClient and from +# ChildProcess. +# Parameters: +# $socket - The listener socket. +# Returns: +# NONE +# Side Effects: +# An event is made to watch the accepted connection. +# Active clients hash is updated to reflect the new connection. +# The client connection count is incremented. +# +sub accept_client { + my ($socket) = @_; + + Debug(8, "Entering accept for lonc UNIX socket\n"); + my $connection = $socket->accept(); # Accept the client connection. + Debug(8,"Connection request accepted from " + .GetPeername($connection, AF_UNIX)); + + + my $description = sprintf("Connection to lonc client %d", + $ClientConnection); + Debug(9, "Creating event named: ".$description); + Event->io(cb => \&ClientRequest, + poll => 'r', + desc => $description, + data => "", + fd => $connection); + $ActiveClients{$connection} = $ClientConnection; + $ClientConnection++; +} =pod @@ -1371,21 +1452,8 @@ sub NewClient { my $event = shift; # Get the event parameters. my $watcher = $event->w; my $socket = $watcher->fd; # Get the event' socket. - my $connection = $socket->accept(); # Accept the client connection. - Debug(8,"Connection request accepted from " - .GetPeername($connection, AF_UNIX)); - - my $description = sprintf("Connection to lonc client %d", - $ClientConnection); - Debug(9, "Creating event named: ".$description); - Event->io(cb => \&ClientRequest, - poll => 'r', - desc => $description, - data => "", - fd => $connection); - $ActiveClients{$connection} = $ClientConnection; - $ClientConnection++; + &accept_client($socket); } =pod @@ -1425,6 +1493,18 @@ sub GetServerHost { =pod +=head2 GetServerId + +Returns the hostid whose lond we talk with. + +=cut + +sub GetHostId { + return $RemoteHostId; # Setup by the fork. +} + +=pod + =head2 GetServerPort Returns the lond port number. @@ -1452,24 +1532,24 @@ another event handler to subess requests =cut sub SetupLoncListener { + my ($host,$SocketName) = @_; + if (!$host) { $host = &GetServerHost(); } + if (!$SocketName) { $SocketName = &GetLoncSocketPath($host); } - my $host = GetServerHost(); # Default host. - if (@_) { - ($host) = @_ # Override host with parameter. - } - my $socket; - my $SocketName = GetLoncSocketPath($host); unlink($SocketName); + + my $socket; unless ($socket =IO::Socket::UNIX->new(Local => $SocketName, Listen => 250, Type => SOCK_STREAM)) { - die "Failed to create a lonc listner socket"; + if($I_am_child) { + &child_exit(-1, "Failed to create a lonc listener socket"); + } else { + die "Failed to create a lonc listner socket"; + } } - Event->io(cb => \&NewClient, - poll => 'r', - desc => 'Lonc listener Unix Socket', - fd => $socket); + return $socket; } # @@ -1499,6 +1579,7 @@ into the status file. We also use this to reset the retries count in order to allow the client to retry connections with a previously dead server. + =cut sub ChildStatus { @@ -1507,26 +1588,31 @@ sub ChildStatus { Debug(2, "Reporting child status because : ".$watcher->data); my $docdir = $perlvar{'lonDocRoot'}; - my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt"); - print $fh $$."\t".$RemoteHost."\t".$Status."\t". + + open(LOG,">>$docdir/lon-status/loncstatus.txt"); + flock(LOG,LOCK_EX); + print LOG $$."\t".$RemoteHost."\t".$Status."\t". $RecentLogEntry."\n"; # # Write out information about each of the connections: # if ($DebugLevel > 2) { - print $fh "Active connection statuses: \n"; + print LOG "Active connection statuses: \n"; my $i = 1; print STDERR "================================= Socket Status Dump:\n"; foreach my $item (keys %ActiveConnections) { my $Socket = $ActiveConnections{$item}->data; my $state = $Socket->GetState(); - print $fh "Connection $i State: $state\n"; + print LOG "Connection $i State: $state\n"; print STDERR "---------------------- Connection $i \n"; $Socket->Dump(-1); # Ensure it gets dumped.. $i++; } } + flock(LOG,LOCK_UN); + close(LOG); $ConnectionRetriesLeft = $ConnectionRetries; + UpdateStatus(); } =pod @@ -1547,12 +1633,14 @@ sub SignalledToDeath { chomp($signal); Log("CRITICAL", "Abnormal exit. Child $$ for $RemoteHost " ."died through "."\"$signal\""); - LogPerm("F:lonc: $$ on $RemoteHost signalled to death: " - ."\"$signal\""); + #LogPerm("F:lonc: $$ on $RemoteHost signalled to death: " +# ."\"$signal\""); exit 0; } +=pod + =head2 ToggleDebug This sub toggles trace debugging on and off. @@ -1568,18 +1656,54 @@ sub ToggleDebug { } +=pod + =head2 ChildProcess This sub implements a child process for a single lonc daemon. +Optional parameter: + $socket - if provided, this is a socket already open for listen + on the client socket. Otherwise, a new listen is set up. =cut sub ChildProcess { + # We've inherited all the + # events of our parent and those have to be cancelled or else + # all holy bloody chaos will result.. trust me, I already made + # >that< mistake. + + my $host = GetServerHost(); + foreach my $listener (keys %parent_dispatchers) { + my $watcher = $parent_dispatchers{$listener}; + my $s = $watcher->fd; + if ($listener ne $host) { # Close everyone but me. + Debug(5, "Closing listen socket for $listener"); + $s->close(); + } + Debug(5, "Killing watcher for $listener"); + + $watcher->cancel(); + delete($parent_dispatchers{$listener}); + + } + + # kill off the parent's signal handlers too! + # + + for my $handler (keys %parent_handlers) { + my $watcher = $parent_handlers{$handler}; + $watcher->cancel(); + delete($parent_handlers{$handler}); + } + + $I_am_child = 1; # Seems like in spite of it all I may still getting + # parent event dispatches.. flag I'm a child. # # Signals must be handled by the Event framework... -# + # Event->signal(signal => "QUIT", cb => \&SignalledToDeath, @@ -1596,29 +1720,45 @@ sub ChildProcess { cb => \&ToggleDebug, data => "INT"); + # Figure out if we got passed a socket or need to open one to listen for + # client requests. + + my ($socket) = @_; + if (!$socket) { + + $socket = SetupLoncListener(); + } + # Establish an event to listen for client connection requests. + + + Event->io(cb => \&NewClient, + poll => 'r', + desc => 'Lonc Listener Unix Socket', + fd => $socket); - SetupLoncListener(); - - $Event::Debuglevel = $DebugLevel; + $Event::DebugLevel = $DebugLevel; Debug(9, "Making initial lond connection for ".$RemoteHost); # Setup the initial server connection: - # &MakeLondConnection(); // let first work requirest do it. + # &MakeLondConnection(); // let first work request do it. + + # need to accept the connection since the event may not fire. + &accept_client($socket); Debug(9,"Entering event loop"); my $ret = Event::loop(); # Start the main event loop. - die "Main event loop exited!!!"; + &child_exit (-1,"Main event loop exited!!!"); } # Create a new child for host passed in: sub CreateChild { - my $host = shift; + my ($host, $hostid) = @_; my $sigset = POSIX::SigSet->new(SIGINT); sigprocmask(SIG_BLOCK, $sigset); @@ -1627,18 +1767,197 @@ sub CreateChild { my $pid = fork; if($pid) { # Parent $RemoteHost = "Parent"; - $ChildHash{$pid} = $host; - $HostToPid{$host}= $pid; + $ChildPid{$pid} = $host; sigprocmask(SIG_UNBLOCK, $sigset); - + undef(@all_host_ids); } else { # child. + $RemoteHostId = $hostid; ShowStatus("Connected to ".$RemoteHost); $SIG{INT} = 'DEFAULT'; sigprocmask(SIG_UNBLOCK, $sigset); - ChildProcess; # Does not return. + &ChildProcess(); # Does not return. } +} +# parent_client_connection: +# Event handler that processes client connections for the parent process. +# This sub is called when the parent is listening on a socket and +# a connection request arrives. We must: +# Start a child process to accept the connection request. +# Kill our listen on the socket. +# Parameter: +# event - The event object that was created to monitor this socket. +# event->w->fd is the socket. +# Returns: +# NONE +# +sub parent_client_connection { + if ($I_am_child) { + # Should not get here, but seem to anyway: + &Debug(5," Child caught parent client connection event!!"); + my ($event) = @_; + my $watcher = $event->w; + $watcher->cancel(); # Try to kill it off again!! + } else { + &Debug(9, "parent_client_connection"); + my ($event) = @_; + my $watcher = $event->w; + my $socket = $watcher->fd; + my $connection = $socket->accept(); # Accept the client connection. + Event->io(cb => \&get_remote_hostname, + poll => 'r', + data => "", + fd => $connection); + } } + +sub get_remote_hostname { + my ($event) = @_; + my $watcher = $event->w; + my $socket = $watcher->fd; + + my $thisread; + my $rv = $socket->recv($thisread, POSIX::BUFSIZ, 0); + Debug(8, "rcv: data length = ".length($thisread)." read =".$thisread); + if (!defined($rv) || length($thisread) == 0) { + # Likely eof on socket. + Debug(5,"Client Socket closed on lonc for p_c_c"); + close($socket); + $watcher->cancel(); + return; + } + + my $data = $watcher->data().$thisread; + $watcher->data($data); + if($data =~ /\n$/) { # Request entirely read. + chomp($data); + } else { + return; + } + + &Debug(5,"Creating child for $data (parent_client_connection)"); + (my $hostname,my $lonid,@all_host_ids) = split(':',$data); + $ChildHost{$hostname}++; + if ($ChildHost{$hostname} == 1) { + &CreateChild($hostname,$lonid); + } else { + &Log('WARNING',"Request for a second child on $hostname"); + } + # Clean up the listen since now the child takes over until it exits. + $watcher->cancel(); # Nolonger listening to this event + $socket->send("done\n"); + $socket->close(); +} + +# parent_listen: +# Opens a socket and starts a listen for the parent process on a client UNIX +# domain socket. +# +# This involves: +# Creating a socket for listen. +# Removing any socket lock file +# Adding an event handler for this socket becoming readable +# To the parent's event dispatcher. +# Parameters: +# loncapa_host - LonCAPA cluster name of the host represented by the client +# socket. +# Returns: +# NONE +# +sub parent_listen { + my ($loncapa_host) = @_; + Debug(5, "parent_listen: $loncapa_host"); + + my ($socket,$file); + if (!$loncapa_host) { + $loncapa_host = 'common_parent'; + $file = $perlvar{'lonSockCreate'}; + } else { + $file = &GetLoncSocketPath($loncapa_host); + } + $socket = &SetupLoncListener($loncapa_host,$file); + + $listening_to{$socket} = $loncapa_host; + if (!$socket) { + die "Unable to create a listen socket for $loncapa_host"; + } + + my $lock_file = $file.".lock"; + unlink($lock_file); # No problem if it doesn't exist yet [startup e.g.] + + my $watcher = + Event->io(cb => \&parent_client_connection, + poll => 'r', + desc => "Parent listener unix socket ($loncapa_host)", + data => "", + fd => $socket); + $parent_dispatchers{$loncapa_host} = $watcher; + +} + +sub parent_clean_up { + my ($loncapa_host) = @_; + Debug(-1, "parent_clean_up: $loncapa_host"); + + my $socket_file = &GetLoncSocketPath($loncapa_host); + unlink($socket_file); # No problem if it doesn't exist yet [startup e.g.] + my $lock_file = $socket_file.".lock"; + unlink($lock_file); # No problem if it doesn't exist yet [startup e.g.] +} + + + +# This sub initiates a listen on the common unix domain lonc client socket. +# loncnew starts up with no children, and only spawns off children when a +# connection request occurs on the common client unix socket. The spawned +# child continues to run until it has been idle a while at which point it +# eventually exits and once more the parent picks up the listen. +# +# Parameters: +# NONE +# Implicit Inputs: +# The configuration file that has been read in by LondConnection. +# Returns: +# NONE +# +sub listen_on_common_socket { + Debug(5, "listen_on_common_socket"); + &parent_listen(); +} + +# server_died is called whenever a child process exits. +# Since this is dispatched via a signal, we must process all +# dead children until there are no more left. The action +# is to: +# - Remove the child from the bookeeping hashes +# - Re-establish a listen on the unix domain socket associated +# with that host. +# Parameters: +# The event, but we don't actually care about it. +sub server_died { + &Debug(9, "server_died called..."); + + while(1) { # Loop until waitpid nowait fails. + my $pid = waitpid(-1, WNOHANG); + if($pid <= 0) { + return; # Nothing left to wait for. + } + # need the host to restart: + + my $host = $ChildPid{$pid}; + if($host) { # It's for real... + &Debug(9, "Caught sigchild for $host"); + delete($ChildPid{$pid}); + delete($ChildHost{$host}); + &parent_clean_up($host); + + } else { + &Debug(5, "Caught sigchild for pid not in hosts hash: $pid"); + } + } + +} + # # Parent process logic pass 1: # For each entry in the hosts table, we will @@ -1687,44 +2006,49 @@ ShowStatus("Forking node servers"); Log("CRITICAL", "--------------- Starting children ---------------"); LondConnection::ReadConfig; # Read standard config files. -my $HostIterator = LondConnection::GetHostIterator; -while (! $HostIterator->end()) { - my $hostentryref = $HostIterator->get(); - CreateChild($hostentryref->[0]); - $HostHash{$hostentryref->[0]} = $hostentryref->[4]; - $HostIterator->next(); -} +$RemoteHost = "[parent]"; +&listen_on_common_socket(); + $RemoteHost = "Parent Server"; # Maintain the population: ShowStatus("Parent keeping the flock"); -# -# Set up parent signals: -# - -$SIG{INT} = \&Terminate; -$SIG{TERM} = \&Terminate; -$SIG{HUP} = \&Restart; -$SIG{USR1} = \&CheckKids; -$SIG{USR2} = \&UpdateKids; # LonManage update request. - -while(1) { - my $deadchild = wait(); - if(exists $ChildHash{$deadchild}) { # need to restart. - my $deadhost = $ChildHash{$deadchild}; - delete($HostToPid{$deadhost}); - delete($ChildHash{$deadchild}); - Log("WARNING","Lost child pid= ".$deadchild. - "Connected to host ".$deadhost); - Log("INFO", "Restarting child procesing ".$deadhost); - CreateChild($deadhost); - } -} +# We need to setup a SIGChild event to handle the exit (natural or otherwise) +# of the children. +Event->signal(cb => \&server_died, + desc => "Child exit handler", + signal => "CHLD"); + + +# Set up all the other signals we set up. + +$parent_handlers{INT} = Event->signal(cb => \&Terminate, + desc => "Parent INT handler", + signal => "INT"); +$parent_handlers{TERM} = Event->signal(cb => \&Terminate, + desc => "Parent TERM handler", + signal => "TERM"); +$parent_handlers{HUP} = Event->signal(cb => \&KillThemAll, + desc => "Parent HUP handler.", + signal => "HUP"); +$parent_handlers{USR1} = Event->signal(cb => \&CheckKids, + desc => "Parent USR1 handler", + signal => "USR1"); +$parent_handlers{USR2} = Event->signal(cb => \&UpdateKids, + desc => "Parent USR2 handler.", + signal => "USR2"); + +# Start procdesing events. + +$Event::DebugLevel = $DebugLevel; +Debug(9, "Parent entering event loop"); +my $ret = Event::loop(); +die "Main Event loop exited: $ret"; =pod @@ -1745,11 +2069,14 @@ sub CheckKids { my $now=time; my $local=localtime($now); print $fh "LONC status $local - parent $$ \n\n"; - foreach my $pid (keys %ChildHash) { + foreach my $host (keys %parent_dispatchers) { + print $fh "LONC Parent process listening for $host\n"; + } + foreach my $pid (keys %ChildPid) { Debug(2, "Sending USR1 -> $pid"); kill 'USR1' => $pid; # Tell Child to report status. - sleep 1; # Wait so file doesn't intermix. } + } =pod @@ -1782,81 +2109,14 @@ sub UpdateKids { Log("INFO", "Updating connections via SIGUSR2"); - # Just in case we need to kill our own lonc, we wait a few seconds to - # give it a chance to receive and relay lond's response to the - # re-init command. - # - - sleep(2); # Wait a couple of seconds. - - my %hosts; # Indexed by loncapa hostname, value=ip. - - # Need to re-read the host table: - - - LondConnection::ReadConfig(); - my $I = LondConnection::GetHostIterator; - while (! $I->end()) { - my $item = $I->get(); - $hosts{$item->[0]} = $item->[4]; - $I->next(); - } - - # The logic below is written for clarity not for efficiency. - # Since I anticipate that this function is only rarely called, that's - # appropriate. There are certainly ways to combine the loops below, - # and anyone wishing to obscure the logic is welcome to go for it. - # Note that we don't re-direct sigchild. Instead we do what's needed - # to the data structures that keep track of children to ensure that - # when sigchild is honored, no new child is born. - # - - # For each existing child; if it's host doesn't exist, kill the child. - - foreach my $child (keys %ChildHash) { - my $oldhost = $ChildHash{$child}; - if (!(exists $hosts{$oldhost})) { - Log("CRITICAL", "Killing child for $oldhost host no longer exists"); - delete $ChildHash{$child}; - delete $HostToPid{$oldhost}; - kill 'QUIT' => $child; - } - } - # For each remaining existing child; if it's host's ip has changed, - # Restart the child on the new IP. - - foreach my $child (keys %ChildHash) { - my $oldhost = $ChildHash{$child}; - my $oldip = $HostHash{$oldhost}; - if ($hosts{$oldhost} ne $oldip) { - - # kill the old child. - - Log("CRITICAL", "Killing child for $oldhost host ip has changed..."); - delete $ChildHash{$child}; - delete $HostToPid{$oldhost}; - kill 'QUIT' => $child; - - # Do the book-keeping needed to start a new child on the - # new ip. + # I'm not sure what I was thinking in the first implementation. + # someone will have to work hard to convince me the effect is any + # different than Restart, especially now that we don't start up + # per host servers automatically, may as well just restart. + # The down side is transactions that are in flight will get timed out + # (lost unless they are critical). - $HostHash{$oldhost} = $hosts{$oldhost}; - CreateChild($oldhost); - } - } - # Finally, for each new host, not in the host hash, create a - # enter the host and create a new child. - # Force a status display of any existing process. - - foreach my $host (keys %hosts) { - if(!(exists $HostHash{$host})) { - Log("INFO", "New host $host discovered in hosts.tab..."); - $HostHash{$host} = $hosts{$host}; - CreateChild($host); - } else { - kill 'HUP' => $HostToPid{$host}; # status display. - } - } + &KillThemAll(); } @@ -1875,7 +2135,7 @@ sub Restart { Log("CRITICAL", "Restarting"); my $execdir = $perlvar{'lonDaemons'}; unlink("$execdir/logs/lonc.pid"); - exec("$execdir/loncnew"); + exec("$executable"); } =pod @@ -1889,15 +2149,19 @@ SIGHUP. Responds to sigint and sigterm. sub KillThemAll { Debug(2, "Kill them all!!"); - local($SIG{CHLD}) = 'IGNORE'; # Our children >will< die. - foreach my $pid (keys %ChildHash) { - my $serving = $ChildHash{$pid}; + + #local($SIG{CHLD}) = 'IGNORE'; + # Our children >will< die. + # but we need to catch their death and cleanup after them in case this is + # a restart set of kills + my @allpids = keys(%ChildPid); + foreach my $pid (@allpids) { + my $serving = $ChildPid{$pid}; ShowStatus("Nicely Killing lonc for $serving pid = $pid"); Log("CRITICAL", "Nicely Killing lonc for $serving pid = $pid"); kill 'QUIT' => $pid; } - - + ShowStatus("Finished killing child processes off."); } @@ -1909,16 +2173,17 @@ sub really_kill_them_all_dammit { Debug(2, "Kill them all Dammit"); local($SIG{CHLD} = 'IGNORE'); # In case some purist reenabled them. - foreach my $pid (keys %ChildHash) { - my $serving = $ChildHash{$pid}; + foreach my $pid (keys %ChildPid) { + my $serving = $ChildPid{$pid}; &ShowStatus("Nastily killing lonc for $serving pid = $pid"); Log("CRITICAL", "Nastily killing lonc for $serving pid = $pid"); kill 'KILL' => $pid; - delete($ChildHash{$pid}); + delete($ChildPid{$pid}); my $execdir = $perlvar{'lonDaemons'}; unlink("$execdir/logs/lonc.pid"); } } + =pod =head1 Terminate @@ -1941,6 +2206,14 @@ sub Terminate { exit 0; } + +sub my_hostname { + use Sys::Hostname; + my $name = &hostname(); + &Debug(9,"Name is $name"); + return $name; +} + =pod =head1 Theory