[BACK]Return to parse_activity_log.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.4 and 1.10

version 1.4, 2004/08/25 15:55:17 version 1.10, 2004/12/22 19:25:42
Line 34 Line 34
 #   2     Activity log does not exist  #   2     Activity log does not exist
 #   3     Unable to connect to database  #   3     Unable to connect to database
 #   4     Unable to create database tables  #   4     Unable to create database tables
 #   5     Unspecified error?  #   5     Unable to open log file
   #   6     Unable to get lock on activity log
   #
   
   #
   # Notes:
   #
   # Logging is done via the $logthis variable, which may be the result of 
   # overcleverness.  log via $logthis->('logtext');  Those are parentheses,
   # not curly braces.  If the -log command line parameter is set, the $logthis
   # routine is set to a routine which writes to a file.  If the command line
   # parameter is not set $logthis is set to &nothing, which does what you
   # would expect.
 #  #
   
 use strict;  use strict;
 use DBI;  use DBI;
 use lib '/home/httpd/lib/perl/Apache';  use lib '/home/httpd/lib/perl/Apache';
   use lib '/home/httpd/lib/perl/';
   use LONCAPA::Configuration();
   use Apache::lonmysql();
 use lonmysql();  use lonmysql();
 use Time::HiRes();  use Time::HiRes();
 use Getopt::Long();  use Getopt::Long();
 use IO::File;  use IO::File;
   use File::Copy;
   use Fcntl qw(:flock);
   
 #  #
 # Determine parameters  # Determine parameters
 my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log);  my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);
 &Getopt::Long::GetOptions( "course=s"  => \$course,  &Getopt::Long::GetOptions( "course=s"  => \$course,
                            "domain=s"  => \$domain,                             "domain=s"  => \$domain,
                              "backup"    => \$backup,
                            "help"      => \$help,                             "help"      => \$help,
                            "logfile=s" => \$file,                             "logfile=s" => \$file,
                            "timerun"   => \$time_run,                             "timerun"   => \$time_run,
Line 64  Process a lon-capa activity log into a d Line 82  Process a lon-capa activity log into a d
 Parameters:  Parameters:
    course             Required     course             Required
    domain             Optional     domain             Optional
      backup             optional   if present, backup the activity log file
                                    before processing it
    drop               optional   if present, drop all course      drop               optional   if present, drop all course 
                                  specific activity log tables.                                   specific activity log tables.
    file               optional   Specify the file to parse, including path     file               optional   Specify the file to parse, including path
Line 88  my $initial_time = Time::HiRes::time; Line 108  my $initial_time = Time::HiRes::time;
 ##  ##
 ## Read in configuration parameters  ## Read in configuration parameters
 ##  ##
 my %perlvar;  my %perlvar = %{&LONCAPA::Configuration::read_conf('loncapa.conf')};
 &initialize_configuration();  
 if (! defined($domain) || $domain eq '') {  if (! defined($domain) || $domain eq '') {
     $domain = $perlvar{'lonDefDomain'};      $domain = $perlvar{'lonDefDomain'};
 }  }
Line 102  if ($log) { Line 122  if ($log) {
     my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;      my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
     print STDERR "$0: logging to $logfile".$/;      print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {      if (! open(LOGFILE,">$logfile")) {
         die "Unable to open $logfile for writing.  Run aborted.";          warn("Unable to open $logfile for writing.  Run aborted.");
           exit 5;
     } else {      } else {
         $logthis = \&log_to_file;          $logthis = \&log_to_file;
     }      }
Line 114  if ($log) { Line 135  if ($log) {
 ##  ##
 my $sourcefilename;   # activity log data  my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this  my $newfilename;      # $sourcefilename will be renamed to this
 my $sql_filename;     # the mysql backup data file name.  my $gz_sql_filename;  # the gzipped mysql backup data file name.
 my $error_filename;   # Errors in parsing the activity log will be written here  my $error_filename;   # Errors in parsing the activity log will be written here
 if ($file) {  if ($file) {
     $sourcefilename = $file;      $sourcefilename = $file;
 } else {  } else {
     $sourcefilename = &get_filename($course,$domain);      $sourcefilename = &get_filename($course,$domain);
 }  }
 $sql_filename = $sourcefilename;  my $sql_filename = $sourcefilename;
 $sql_filename =~ s|[^/]*$|activity.log.sql|;  $sql_filename =~ s|[^/]*$|activity.log.sql|;
   $gz_sql_filename = $sql_filename.'.gz';
 $error_filename = $sourcefilename;  $error_filename = $sourcefilename;
 $error_filename =~ s|[^/]*$|activity.log.errors|;  $error_filename =~ s|[^/]*$|activity.log.errors|;
 $logthis->('Beginning logging '.time);  $logthis->('Beginning logging '.time);
   
   
   #
   # Wait for a lock on the lockfile to avoid collisions
   my $lockfilename = $sourcefilename.'.lock';
   open(LOCKFILE,'>'.$lockfilename);
   if (!flock(LOCKFILE,LOCK_EX)) {
       warn("Unable to lock $lockfilename.  Aborting".$/);
       exit 6;
   }
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
 ## running.  ## running.
 my $newfilename = $sourcefilename.'.processing';  my $newfilename = $sourcefilename.'.processing';
 if (-e $newfilename) {  if (-e $newfilename) {
     warn "$newfilename exists";      warn "$newfilename exists";
     $logthis->($newfilename.' exists');      $logthis->($newfilename.' exists, so I cannot work on it.');
     exit 2;      exit 2;
 }  }
   
 if (-e $sourcefilename) {  if (-e $sourcefilename) {
     $logthis->('renaming '.$sourcefilename.' to '.$newfilename);      $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
     rename($sourcefilename,$newfilename);      rename($sourcefilename,$newfilename);
       Copy($newfilename,$newfilename.'.'.time) if ($backup);
     $logthis->("renamed $sourcefilename to $newfilename");      $logthis->("renamed $sourcefilename to $newfilename");
 } else {  } else {
     my $command = 'touch '.$newfilename;      my $command = 'touch '.$newfilename;
Line 148  if (-e $sourcefilename) { Line 181  if (-e $sourcefilename) {
     $logthis->('touch was completed');      $logthis->('touch was completed');
 }  }
   
   close(LOCKFILE);
   
 ##  ##
 ## Table definitions  ## Table definitions
 ##  ##
Line 244  my $activity_table_def = Line 279  my $activity_table_def =
       'KEY' => [{columns => ['student_id']},        'KEY' => [{columns => ['student_id']},
                 {columns => ['time']},],                  {columns => ['time']},],
 };  };
 my @Activity_Table = ($activity_table_def);   
   
 #my @ID_Tables = ($student_table_def,$res_table_def,  
 #                 $action_table_def,$machine_table_def);  
   
   my @Activity_Table = ($activity_table_def);
 my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);  my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);
                         
   
 ##  ##
 ## End of table definitions  ## End of table definitions
 ##  ##
   
 #   
 $logthis->('Connectiong to mysql');  $logthis->('Connectiong to mysql');
 &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},  &Apache::lonmysql::set_mysql_user_and_password('www',
                                                $perlvar{'lonSqlAccess'});                                                 $perlvar{'lonSqlAccess'});
 if (!&Apache::lonmysql::verify_sql_connection()) {  if (!&Apache::lonmysql::verify_sql_connection()) {
     warn "Unable to connect to MySQL database.";      warn "Unable to connect to MySQL database.";
     $logthis->("Unable to connect to MySQL database.");      $logthis->("Unable to connect to MySQL database.");
     exit 3;      exit 3;
 }  }
   
 $logthis->('SQL connection is up');  $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
   
 if (-s $sql_filename) {  if (-s $gz_sql_filename) {
     # if ANY one of the tables does not exist, load the tables from the      my $backup_modification_time = (stat($gz_sql_filename))[9];
     # backup.      $logthis->($gz_sql_filename.' was last modified '.
                  localtime($backup_modification_time).
                  '('.$backup_modification_time.')');
       # Check for missing tables
     my @Current_Tables = &Apache::lonmysql::tables_in_db();      my @Current_Tables = &Apache::lonmysql::tables_in_db();
       $logthis->(join(',',@Current_Tables));
     my %Found;      my %Found;
     foreach my $tablename (@Current_Tables) {      foreach my $tablename (@Current_Tables) {
         foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {
Line 282  if (-s $sql_filename) { Line 314  if (-s $sql_filename) {
             }              }
         }          }
     }      }
       $logthis->('Found tables '.join(',',keys(%Found)));
       my $missing_a_table = 0;
     foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {    
           # Hmmm, should I dump the tables?
         if (! $Found{$table->{'id'}}) {          if (! $Found{$table->{'id'}}) {
             $time_this->();              $logthis->('Missing table '.$table->{'id'});
             &load_backup_tables($sql_filename);              $missing_a_table = 1;
             $time_this->('load backup tables');  
             last;              last;
         }          }
     }      }
       if ($missing_a_table) {
           my $table_modification_time = $backup_modification_time;
           # If the backup happened prior to the last table modification,
           foreach my $table (@Activity_Table,@ID_Tables) {    
               my %tabledata = &Apache::lonmysql::table_information($table->{'id'});
               next if (! scalar(keys(%tabledata))); # table does not exist
               if ($table_modification_time < $tabledata{'Update_time'}) {
                   $table_modification_time = $tabledata{'Update_time'};
               }
           }
           $logthis->("Table modification time = ".$table_modification_time);
           if ($table_modification_time > $backup_modification_time) {
               # Save the current tables in case we need them another time.
               my $backup_name = $gz_sql_filename.'.'.time;
               $logthis->('Backing existing tables up in '.$backup_name);
               &backup_tables($backup_name);
           }
           $time_this->();
           &load_backup_tables($gz_sql_filename);
           $time_this->('load backup tables');
       }
 }  }
   
 ##  ##
Line 323  if (-s $newfilename) { Line 378  if (-s $newfilename) {
         exit 5;          exit 5;
     } elsif ($result > 0) {      } elsif ($result > 0) {
         $time_this->();          $time_this->();
         $logthis->('process_courselog returned '.$result.' backup up tables');          $logthis->('process_courselog returned '.$result.' backing up tables');
         &backup_tables($sql_filename);          &backup_tables($gz_sql_filename);
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
 }  }
Line 469  sub process_courselog { Line 524  sub process_courselog {
                 next; # skip this chunk                  next; # skip this chunk
             }              }
             #              #
             my $values = $dbh->quote(join('',@values));              my $values = $dbh->quote(join(':',map { &escape($_); } @values));
             $time_this->('get_ids');              $time_this->('get_ids');
             #              #
             my $row = [$res_id,              my $row = [$res_id,
Line 575  sub outputtimes { Line 630  sub outputtimes {
 ## Use mysqldump to store backups of the tables  ## Use mysqldump to store backups of the tables
 ##  ##
 sub backup_tables {  sub backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     my $command = qq{mysqldump --opt loncapa };      my $command = qq{mysqldump --opt loncapa };
                                                             
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $tablename = $table->{'id'};          my $tablename = $table->{'id'};
         $command .= $tablename.' ';          $command .= $tablename.' ';
     }      }
     $command .= '>'.$sql_filename;      $command .= '| gzip >'.$gz_sql_filename;
     $logthis->($command);      $logthis->($command);
     system($command);      system($command);
 }  }
Line 591  sub backup_tables { Line 646  sub backup_tables {
 ## Load in mysqldumped files  ## Load in mysqldumped files
 ##  ##
 sub load_backup_tables {  sub load_backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     return undef if (! -e $sql_filename);      if (-s $gz_sql_filename) {
     # Check for .my.cnf          $logthis->('loading data from gzipped sql file');
     my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';          my $command='gzip -dc '.$gz_sql_filename.' | mysql --database=loncapa';
     $logthis->('loading previously saved sql table'.$/.$command);          system($command);
     system($command);          $logthis->('finished loading gzipped data');;
     $logthis->('finished loading old data');      } else {
           return undef;
       }
 }  }
   
 ##  ##
 ##   ## 
 ##  ##
 sub initialize_configuration {  
     # Fake it for now:  
     $perlvar{'lonSqlUser'} = 'www';  
     $perlvar{'lonSqlAccess'} = 'localhostkey';  
     $perlvar{'lonUsersDir'} = '/home/httpd/lonUsers';  
     $perlvar{'lonDefDomain'} = '103';  
 }  
   
 sub update_process_name {  sub update_process_name {
     my ($text) = @_;      my ($text) = @_;
     $0 = 'parse_activity_log.pl: '.$text;      $0 = 'parse_activity_log.pl: '.$text;
Removed from v.1.4  
changed lines
  Added in v.1.10

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>