Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.6 and 1.10

version 1.6, 2004/12/16 15:03:29 version 1.10, 2004/12/22 19:25:42
Line 34 Line 34
 #   2     Activity log does not exist  #   2     Activity log does not exist
 #   3     Unable to connect to database  #   3     Unable to connect to database
 #   4     Unable to create database tables  #   4     Unable to create database tables
 #   5     Unspecified error?  #   5     Unable to open log file
   #   6     Unable to get lock on activity log
   #
   
   #
   # Notes:
   #
   # Logging is done via the $logthis variable, which may be the result of 
   # overcleverness.  log via $logthis->('logtext');  Those are parentheses,
   # not curly braces.  If the -log command line parameter is set, the $logthis
   # routine is set to a routine which writes to a file.  If the command line
   # parameter is not set $logthis is set to &nothing, which does what you
   # would expect.
 #  #
   
 use strict;  use strict;
 use DBI;  use DBI;
 use lib '/home/httpd/lib/perl/Apache';  use lib '/home/httpd/lib/perl/Apache';
   use lib '/home/httpd/lib/perl/';
   use LONCAPA::Configuration();
   use Apache::lonmysql();
 use lonmysql();  use lonmysql();
 use Time::HiRes();  use Time::HiRes();
 use Getopt::Long();  use Getopt::Long();
 use IO::File;  use IO::File;
 use File::Copy;  use File::Copy;
   use Fcntl qw(:flock);
   
 #  #
 # Determine parameters  # Determine parameters
 my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);  my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);
Line 91  my $initial_time = Time::HiRes::time; Line 108  my $initial_time = Time::HiRes::time;
 ##  ##
 ## Read in configuration parameters  ## Read in configuration parameters
 ##  ##
 my %perlvar;  my %perlvar = %{&LONCAPA::Configuration::read_conf('loncapa.conf')};
 &initialize_configuration();  
 if (! defined($domain) || $domain eq '') {  if (! defined($domain) || $domain eq '') {
     $domain = $perlvar{'lonDefDomain'};      $domain = $perlvar{'lonDefDomain'};
 }  }
Line 105  if ($log) { Line 122  if ($log) {
     my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;      my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
     print STDERR "$0: logging to $logfile".$/;      print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {      if (! open(LOGFILE,">$logfile")) {
         die "Unable to open $logfile for writing.  Run aborted.";          warn("Unable to open $logfile for writing.  Run aborted.");
           exit 5;
     } else {      } else {
         $logthis = \&log_to_file;          $logthis = \&log_to_file;
     }      }
Line 131  $error_filename = $sourcefilename; Line 149  $error_filename = $sourcefilename;
 $error_filename =~ s|[^/]*$|activity.log.errors|;  $error_filename =~ s|[^/]*$|activity.log.errors|;
 $logthis->('Beginning logging '.time);  $logthis->('Beginning logging '.time);
   
   
   #
   # Wait for a lock on the lockfile to avoid collisions
   my $lockfilename = $sourcefilename.'.lock';
   open(LOCKFILE,'>'.$lockfilename);
   if (!flock(LOCKFILE,LOCK_EX)) {
       warn("Unable to lock $lockfilename.  Aborting".$/);
       exit 6;
   }
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
 ## running.  ## running.
 my $newfilename = $sourcefilename.'.processing';  my $newfilename = $sourcefilename.'.processing';
 if (-e $newfilename) {  if (-e $newfilename) {
     warn "$newfilename exists";      warn "$newfilename exists";
     $logthis->($newfilename.' exists');      $logthis->($newfilename.' exists, so I cannot work on it.');
     exit 2;      exit 2;
 }  }
   
Line 153  if (-e $sourcefilename) { Line 181  if (-e $sourcefilename) {
     $logthis->('touch was completed');      $logthis->('touch was completed');
 }  }
   
   close(LOCKFILE);
   
 ##  ##
 ## Table definitions  ## Table definitions
 ##  ##
Line 249  my $activity_table_def = Line 279  my $activity_table_def =
       'KEY' => [{columns => ['student_id']},        'KEY' => [{columns => ['student_id']},
                 {columns => ['time']},],                  {columns => ['time']},],
 };  };
 my @Activity_Table = ($activity_table_def);   
   
 #my @ID_Tables = ($student_table_def,$res_table_def,  
 #                 $action_table_def,$machine_table_def);  
   
   my @Activity_Table = ($activity_table_def);
 my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);  my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);
                         
   
 ##  ##
 ## End of table definitions  ## End of table definitions
 ##  ##
   
 #   
 $logthis->('Connectiong to mysql');  $logthis->('Connectiong to mysql');
 &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},  &Apache::lonmysql::set_mysql_user_and_password('www',
                                                $perlvar{'lonSqlAccess'});                                                 $perlvar{'lonSqlAccess'});
 if (!&Apache::lonmysql::verify_sql_connection()) {  if (!&Apache::lonmysql::verify_sql_connection()) {
     warn "Unable to connect to MySQL database.";      warn "Unable to connect to MySQL database.";
     $logthis->("Unable to connect to MySQL database.");      $logthis->("Unable to connect to MySQL database.");
     exit 3;      exit 3;
 }  }
   
 $logthis->('SQL connection is up');  $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
   
 if (-s $gz_sql_filename) {  if (-s $gz_sql_filename) {
     # if ANY one of the tables does not exist, load the tables from the      my $backup_modification_time = (stat($gz_sql_filename))[9];
     # backup.      $logthis->($gz_sql_filename.' was last modified '.
                  localtime($backup_modification_time).
                  '('.$backup_modification_time.')');
       # Check for missing tables
     my @Current_Tables = &Apache::lonmysql::tables_in_db();      my @Current_Tables = &Apache::lonmysql::tables_in_db();
       $logthis->(join(',',@Current_Tables));
     my %Found;      my %Found;
     foreach my $tablename (@Current_Tables) {      foreach my $tablename (@Current_Tables) {
         foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {
Line 287  if (-s $gz_sql_filename) { Line 314  if (-s $gz_sql_filename) {
             }              }
         }          }
     }      }
       $logthis->('Found tables '.join(',',keys(%Found)));
       my $missing_a_table = 0;
     foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {    
           # Hmmm, should I dump the tables?
         if (! $Found{$table->{'id'}}) {          if (! $Found{$table->{'id'}}) {
             $time_this->();              $logthis->('Missing table '.$table->{'id'});
             &load_backup_tables($gz_sql_filename);              $missing_a_table = 1;
             $time_this->('load backup tables');  
             last;              last;
         }          }
     }      }
       if ($missing_a_table) {
           my $table_modification_time = $backup_modification_time;
           # If the backup happened prior to the last table modification,
           foreach my $table (@Activity_Table,@ID_Tables) {    
               my %tabledata = &Apache::lonmysql::table_information($table->{'id'});
               next if (! scalar(keys(%tabledata))); # table does not exist
               if ($table_modification_time < $tabledata{'Update_time'}) {
                   $table_modification_time = $tabledata{'Update_time'};
               }
           }
           $logthis->("Table modification time = ".$table_modification_time);
           if ($table_modification_time > $backup_modification_time) {
               # Save the current tables in case we need them another time.
               my $backup_name = $gz_sql_filename.'.'.time;
               $logthis->('Backing existing tables up in '.$backup_name);
               &backup_tables($backup_name);
           }
           $time_this->();
           &load_backup_tables($gz_sql_filename);
           $time_this->('load backup tables');
       }
 }  }
   
 ##  ##
Line 328  if (-s $newfilename) { Line 378  if (-s $newfilename) {
         exit 5;          exit 5;
     } elsif ($result > 0) {      } elsif ($result > 0) {
         $time_this->();          $time_this->();
         $logthis->('process_courselog returned '.$result.' backup up tables');          $logthis->('process_courselog returned '.$result.' backing up tables');
         &backup_tables($gz_sql_filename);          &backup_tables($gz_sql_filename);
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
Line 474  sub process_courselog { Line 524  sub process_courselog {
                 next; # skip this chunk                  next; # skip this chunk
             }              }
             #              #
             my $values = $dbh->quote(join('',@values));              my $values = $dbh->quote(join(':',map { &escape($_); } @values));
             $time_this->('get_ids');              $time_this->('get_ids');
             #              #
             my $row = [$res_id,              my $row = [$res_id,
Line 598  sub backup_tables { Line 648  sub backup_tables {
 sub load_backup_tables {  sub load_backup_tables {
     my ($gz_sql_filename) = @_;      my ($gz_sql_filename) = @_;
     if (-s $gz_sql_filename) {      if (-s $gz_sql_filename) {
         &logthis('loading data from gzipped sql file');          $logthis->('loading data from gzipped sql file');
         my $command='gzip -dc activity.log.sql.gz | mysql --database=loncapa';          my $command='gzip -dc '.$gz_sql_filename.' | mysql --database=loncapa';
         system($command);          system($command);
         $logthis->('finished loading gzipped data');;          $logthis->('finished loading gzipped data');;
     } else {      } else {
Line 610  sub load_backup_tables { Line 660  sub load_backup_tables {
 ##  ##
 ##   ## 
 ##  ##
 sub initialize_configuration {  
     # Fake it for now:  
     $perlvar{'lonSqlUser'} = 'www';  
     $perlvar{'lonSqlAccess'} = 'localhostkey';  
     $perlvar{'lonUsersDir'} = '/home/httpd/lonUsers';  
     $perlvar{'lonDefDomain'} = '103';  
 }  
   
 sub update_process_name {  sub update_process_name {
     my ($text) = @_;      my ($text) = @_;
     $0 = 'parse_activity_log.pl: '.$text;      $0 = 'parse_activity_log.pl: '.$text;

Removed from v.1.6  
changed lines
  Added in v.1.10


FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>