--- loncom/metadata_database/parse_activity_log.pl 2004/09/15 21:11:06 1.5 +++ loncom/metadata_database/parse_activity_log.pl 2004/12/20 19:53:36 1.8 @@ -2,7 +2,7 @@ # # The LearningOnline Network # -# $Id: parse_activity_log.pl,v 1.5 2004/09/15 21:11:06 matthew Exp $ +# $Id: parse_activity_log.pl,v 1.8 2004/12/20 19:53:36 matthew Exp $ # # Copyright Michigan State University Board of Trustees # @@ -37,14 +37,29 @@ # 5 Unspecified error? # +# +# Notes: +# +# Logging is done via the $logthis variable, which may be the result of +# overcleverness. log via $logthis->('logtext'); Those are parentheses, +# not curly braces. If the -log command line parameter is set, the $logthis +# routine is set to a routine which writes to a file. If the command line +# parameter is not set $logthis is set to ¬hing, which does what you +# would expect. +# + use strict; use DBI; -use lib '/home/httpd/lib/perl/Apache'; +use lib '/home/httpd/lib/perl/'; +use LONCAPA::Configuration(); +use Apache::lonmysql(); use lonmysql(); use Time::HiRes(); use Getopt::Long(); use IO::File; use File::Copy; +use Fcntl qw(:flock); + # # Determine parameters my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup); @@ -91,8 +106,8 @@ my $initial_time = Time::HiRes::time; ## ## Read in configuration parameters ## -my %perlvar; -&initialize_configuration(); +my %perlvar = %{&LONCAPA::Configuration::read_conf('loncapa.conf')}; + if (! defined($domain) || $domain eq '') { $domain = $perlvar{'lonDefDomain'}; } @@ -117,26 +132,34 @@ if ($log) { ## my $sourcefilename; # activity log data my $newfilename; # $sourcefilename will be renamed to this -my $sql_filename; # the mysql backup data file name. +my $gz_sql_filename; # the gzipped mysql backup data file name. my $error_filename; # Errors in parsing the activity log will be written here if ($file) { $sourcefilename = $file; } else { $sourcefilename = &get_filename($course,$domain); } -$sql_filename = $sourcefilename; +my $sql_filename = $sourcefilename; $sql_filename =~ s|[^/]*$|activity.log.sql|; +$gz_sql_filename = $sql_filename.'.gz'; $error_filename = $sourcefilename; $error_filename =~ s|[^/]*$|activity.log.errors|; $logthis->('Beginning logging '.time); + +# +# Wait for a lock on the lockfile to avoid collisions +my $lockfilename = $sourcefilename.'.lock'; +open(LOCKFILE,'>'.$lockfilename); +flock(LOCKFILE,LOCK_EX) || die("Unable to lock $lockfilename. Aborting".$/); + ## ## There will only be a $newfilename file if a copy of this program is already ## running. my $newfilename = $sourcefilename.'.processing'; if (-e $newfilename) { warn "$newfilename exists"; - $logthis->($newfilename.' exists'); + $logthis->($newfilename.' exists, so I cannot work on it.'); exit 2; } @@ -152,6 +175,8 @@ if (-e $sourcefilename) { $logthis->('touch was completed'); } +close(LOCKFILE); + ## ## Table definitions ## @@ -248,36 +273,33 @@ my $activity_table_def = 'KEY' => [{columns => ['student_id']}, {columns => ['time']},], }; -my @Activity_Table = ($activity_table_def); - -#my @ID_Tables = ($student_table_def,$res_table_def, -# $action_table_def,$machine_table_def); +my @Activity_Table = ($activity_table_def); my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def); - - ## ## End of table definitions ## -# $logthis->('Connectiong to mysql'); -&Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'}, +&Apache::lonmysql::set_mysql_user_and_password('www', $perlvar{'lonSqlAccess'}); if (!&Apache::lonmysql::verify_sql_connection()) { warn "Unable to connect to MySQL database."; $logthis->("Unable to connect to MySQL database."); exit 3; } - $logthis->('SQL connection is up'); if ($drop) { &drop_tables(); $logthis->('dropped tables'); } -if (-s $sql_filename) { - # if ANY one of the tables does not exist, load the tables from the - # backup. +if (-s $gz_sql_filename) { + my $backup_modification_time = (stat($gz_sql_filename))[9]; + $logthis->($gz_sql_filename.' was last modified '. + localtime($backup_modification_time). + '('.$backup_modification_time.')'); + # Check for missing tables my @Current_Tables = &Apache::lonmysql::tables_in_db(); + $logthis->(join(',',@Current_Tables)); my %Found; foreach my $tablename (@Current_Tables) { foreach my $table (@Activity_Table,@ID_Tables) { @@ -286,14 +308,37 @@ if (-s $sql_filename) { } } } + $logthis->('Found tables '.join(',',keys(%Found))); + my $missing_a_table = 0; foreach my $table (@Activity_Table,@ID_Tables) { + # Hmmm, should I dump the tables? if (! $Found{$table->{'id'}}) { - $time_this->(); - &load_backup_tables($sql_filename); - $time_this->('load backup tables'); + $logthis->('Missing table '.$table->{'id'}); + $missing_a_table = 1; last; } } + if ($missing_a_table) { + my $table_modification_time = $backup_modification_time; + # If the backup happened prior to the last table modification, + foreach my $table (@Activity_Table,@ID_Tables) { + my %tabledata = &Apache::lonmysql::table_information($table->{'id'}); + next if (! scalar(keys(%tabledata))); # table does not exist + if ($table_modification_time < $tabledata{'Update_time'}) { + $table_modification_time = $tabledata{'Update_time'}; + } + } + $logthis->("Table modification time = ".$table_modification_time); + if ($table_modification_time > $backup_modification_time) { + # Save the current tables in case we need them another time. + my $backup_name = $gz_sql_filename.'.'.time; + $logthis->('Backing existing tables up in '.$backup_name); + &backup_tables($backup_name); + } + $time_this->(); + &load_backup_tables($gz_sql_filename); + $time_this->('load backup tables'); + } } ## @@ -327,8 +372,8 @@ if (-s $newfilename) { exit 5; } elsif ($result > 0) { $time_this->(); - $logthis->('process_courselog returned '.$result.' backup up tables'); - &backup_tables($sql_filename); + $logthis->('process_courselog returned '.$result.' backing up tables'); + &backup_tables($gz_sql_filename); $time_this->('write backup tables'); } } @@ -579,14 +624,14 @@ sub outputtimes { ## Use mysqldump to store backups of the tables ## sub backup_tables { - my ($sql_filename) = @_; + my ($gz_sql_filename) = @_; my $command = qq{mysqldump --opt loncapa }; foreach my $table (@ID_Tables,@Activity_Table) { my $tablename = $table->{'id'}; $command .= $tablename.' '; } - $command .= '>'.$sql_filename; + $command .= '| gzip >'.$gz_sql_filename; $logthis->($command); system($command); } @@ -595,26 +640,20 @@ sub backup_tables { ## Load in mysqldumped files ## sub load_backup_tables { - my ($sql_filename) = @_; - return undef if (! -e $sql_filename); - # Check for .my.cnf - my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa'; - $logthis->('loading previously saved sql table'.$/.$command); - system($command); - $logthis->('finished loading old data'); + my ($gz_sql_filename) = @_; + if (-s $gz_sql_filename) { + $logthis->('loading data from gzipped sql file'); + my $command='gzip -dc '.$gz_sql_filename.' | mysql --database=loncapa'; + system($command); + $logthis->('finished loading gzipped data');; + } else { + return undef; + } } ## ## ## -sub initialize_configuration { - # Fake it for now: - $perlvar{'lonSqlUser'} = 'www'; - $perlvar{'lonSqlAccess'} = 'localhostkey'; - $perlvar{'lonUsersDir'} = '/home/httpd/lonUsers'; - $perlvar{'lonDefDomain'} = '103'; -} - sub update_process_name { my ($text) = @_; $0 = 'parse_activity_log.pl: '.$text;