[BACK]Return to parse_activity_log.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.3 and 1.8

version 1.3, 2004/08/19 21:08:46 version 1.8, 2004/12/20 19:53:36
Line 37 Line 37
 #   5     Unspecified error?  #   5     Unspecified error?
 #  #
   
   #
   # Notes:
   #
   # Logging is done via the $logthis variable, which may be the result of 
   # overcleverness.  log via $logthis->('logtext');  Those are parentheses,
   # not curly braces.  If the -log command line parameter is set, the $logthis
   # routine is set to a routine which writes to a file.  If the command line
   # parameter is not set $logthis is set to &nothing, which does what you
   # would expect.
   #
   
 use strict;  use strict;
 use DBI;  use DBI;
 use lib '/home/httpd/lib/perl/Apache';  use lib '/home/httpd/lib/perl/';
   use LONCAPA::Configuration();
   use Apache::lonmysql();
 use lonmysql();  use lonmysql();
 use Time::HiRes();  use Time::HiRes();
 use Getopt::Long();  use Getopt::Long();
 use IO::File;  use IO::File;
   use File::Copy;
   use Fcntl qw(:flock);
   
 #  #
 # Determine parameters  # Determine parameters
 my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log);  my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);
 &Getopt::Long::GetOptions( "course=s"  => \$course,  &Getopt::Long::GetOptions( "course=s"  => \$course,
                            "domain=s"  => \$domain,                             "domain=s"  => \$domain,
                              "backup"    => \$backup,
                            "help"      => \$help,                             "help"      => \$help,
                            "logfile=s" => \$file,                             "logfile=s" => \$file,
                            "timerun"   => \$time_run,                             "timerun"   => \$time_run,
Line 64  Process a lon-capa activity log into a d Line 80  Process a lon-capa activity log into a d
 Parameters:  Parameters:
    course             Required     course             Required
    domain             Optional     domain             Optional
      backup             optional   if present, backup the activity log file
                                    before processing it
    drop               optional   if present, drop all course      drop               optional   if present, drop all course 
                                  specific activity log tables.                                   specific activity log tables.
    file               optional   Specify the file to parse, including path     file               optional   Specify the file to parse, including path
Line 88  my $initial_time = Time::HiRes::time; Line 106  my $initial_time = Time::HiRes::time;
 ##  ##
 ## Read in configuration parameters  ## Read in configuration parameters
 ##  ##
 my %perlvar;  my %perlvar = %{&LONCAPA::Configuration::read_conf('loncapa.conf')};
 &initialize_configuration();  
 if (! defined($domain) || $domain eq '') {  if (! defined($domain) || $domain eq '') {
     $domain = $perlvar{'lonDefDomain'};      $domain = $perlvar{'lonDefDomain'};
 }  }
Line 99  if (! defined($domain) || $domain eq '') Line 117  if (! defined($domain) || $domain eq '')
 ## Set up logging code  ## Set up logging code
 my $logthis = \&nothing;  my $logthis = \&nothing;
 if ($log) {  if ($log) {
     my $logfile = "/tmp/parse_activity_log.log.".time;      my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
     print STDERR "$0: logging to $logfile".$/;      print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {      if (! open(LOGFILE,">$logfile")) {
         die "Unable to open $logfile for writing.  Run aborted.";          die "Unable to open $logfile for writing.  Run aborted.";
Line 114  if ($log) { Line 132  if ($log) {
 ##  ##
 my $sourcefilename;   # activity log data  my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this  my $newfilename;      # $sourcefilename will be renamed to this
 my $sql_filename;     # the mysql backup data file name.  my $gz_sql_filename;  # the gzipped mysql backup data file name.
 my $error_filename;   # Errors in parsing the activity log will be written here  my $error_filename;   # Errors in parsing the activity log will be written here
 if ($file) {  if ($file) {
     $sourcefilename = $file;      $sourcefilename = $file;
 } else {  } else {
     $sourcefilename = &get_filename($course,$domain);      $sourcefilename = &get_filename($course,$domain);
 }  }
 $sql_filename = $sourcefilename;  my $sql_filename = $sourcefilename;
 $sql_filename =~ s|[^/]*$|activity.log.sql|;  $sql_filename =~ s|[^/]*$|activity.log.sql|;
   $gz_sql_filename = $sql_filename.'.gz';
 $error_filename = $sourcefilename;  $error_filename = $sourcefilename;
 $error_filename =~ s|[^/]*$|activity.log.errors|;  $error_filename =~ s|[^/]*$|activity.log.errors|;
 $logthis->('Beginning logging '.time);  $logthis->('Beginning logging '.time);
   
   
   #
   # Wait for a lock on the lockfile to avoid collisions
   my $lockfilename = $sourcefilename.'.lock';
   open(LOCKFILE,'>'.$lockfilename);
   flock(LOCKFILE,LOCK_EX) || die("Unable to lock $lockfilename.  Aborting".$/);
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
 ## running.  ## running.
 my $newfilename = $sourcefilename.'.processing';  my $newfilename = $sourcefilename.'.processing';
 if (-e $newfilename) {  if (-e $newfilename) {
     warn "$newfilename exists";      warn "$newfilename exists";
     $logthis->($newfilename.' exists');      $logthis->($newfilename.' exists, so I cannot work on it.');
     exit 2;      exit 2;
 }  }
   
 if (-e $sourcefilename) {  if (-e $sourcefilename) {
     $logthis->('renaming '.$sourcefilename.' to '.$newfilename);      $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
     rename($sourcefilename,$newfilename);      rename($sourcefilename,$newfilename);
       Copy($newfilename,$newfilename.'.'.time) if ($backup);
     $logthis->("renamed $sourcefilename to $newfilename");      $logthis->("renamed $sourcefilename to $newfilename");
 } else {  } else {
     my $command = 'touch '.$newfilename;      my $command = 'touch '.$newfilename;
Line 148  if (-e $sourcefilename) { Line 175  if (-e $sourcefilename) {
     $logthis->('touch was completed');      $logthis->('touch was completed');
 }  }
   
   close(LOCKFILE);
   
 ##  ##
 ## Table definitions  ## Table definitions
 ##  ##
Line 183  my $res_table_def = Line 212  my $res_table_def =
   'PRIMARY KEY' => ['res_id'],    'PRIMARY KEY' => ['res_id'],
 };  };
   
 my $action_table = $prefix.'actions';  #my $action_table = $prefix.'actions';
 my $action_table_def =  #my $action_table_def =
 { id => $action_table,  #{ id => $action_table,
   permanent => 'no',  #  permanent => 'no',
   columns => [{ name => 'action_id',  #  columns => [{ name => 'action_id',
                 type => 'MEDIUMINT UNSIGNED',  #                type => 'MEDIUMINT UNSIGNED',
                 restrictions => 'NOT NULL',  #                restrictions => 'NOT NULL',
                 auto_inc     => 'yes', },  #                auto_inc     => 'yes', },
               { name => 'action',  #              { name => 'action',
                 type => 'VARCHAR(100)',  #                type => 'VARCHAR(100)',
                 restrictions => 'NOT NULL'},  #                restrictions => 'NOT NULL'},
               ],  #              ],
   'PRIMARY KEY' => ['action_id',],   #  'PRIMARY KEY' => ['action_id',], 
 };  #};
   
 my $machine_table = $prefix.'machine_table';  my $machine_table = $prefix.'machine_table';
 my $machine_table_def =  my $machine_table_def =
Line 227  my $activity_table_def = Line 256  my $activity_table_def =
               { name => 'student_id',                { name => 'student_id',
                 type => 'MEDIUMINT UNSIGNED',                  type => 'MEDIUMINT UNSIGNED',
                 restrictions => 'NOT NULL',},                  restrictions => 'NOT NULL',},
               { name => 'action_id',                { name => 'action',
                 type => 'MEDIUMINT UNSIGNED',                  type => 'VARCHAR(10)',
                 restrictions => 'NOT NULL',},                  restrictions => 'NOT NULL',},
               { name => 'idx',                # This is here in case a student                { name => 'idx',                # This is here in case a student
                 type => 'MEDIUMINT UNSIGNED', # has multiple submissions during                  type => 'MEDIUMINT UNSIGNED', # has multiple submissions during
Line 240  my $activity_table_def = Line 269  my $activity_table_def =
               { name => 'action_values',                { name => 'action_values',
                 type => 'MEDIUMTEXT', },                  type => 'MEDIUMTEXT', },
               ],                 ], 
       'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'],        'PRIMARY KEY' => ['time','student_id','res_id','idx'],
         'KEY' => [{columns => ['student_id']},
                   {columns => ['time']},],
 };  };
 my @Activity_Table = ($activity_table_def);  
 my @ID_Tables = ($student_table_def,$res_table_def,  
                  $action_table_def,$machine_table_def);  
                          
   
   my @Activity_Table = ($activity_table_def);
   my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);
 ##  ##
 ## End of table definitions  ## End of table definitions
 ##  ##
   
 #   
 $logthis->('Connectiong to mysql');  $logthis->('Connectiong to mysql');
 &Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},  &Apache::lonmysql::set_mysql_user_and_password('www',
                                                $perlvar{'lonSqlAccess'});                                                 $perlvar{'lonSqlAccess'});
 if (!&Apache::lonmysql::verify_sql_connection()) {  if (!&Apache::lonmysql::verify_sql_connection()) {
     warn "Unable to connect to MySQL database.";      warn "Unable to connect to MySQL database.";
     $logthis->("Unable to connect to MySQL database.");      $logthis->("Unable to connect to MySQL database.");
     exit 3;      exit 3;
 }  }
   
 $logthis->('SQL connection is up');  $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
   
 if (-s $sql_filename) {  if (-s $gz_sql_filename) {
     # if ANY one of the tables does not exist, load the tables from the      my $backup_modification_time = (stat($gz_sql_filename))[9];
     # backup.      $logthis->($gz_sql_filename.' was last modified '.
                  localtime($backup_modification_time).
                  '('.$backup_modification_time.')');
       # Check for missing tables
     my @Current_Tables = &Apache::lonmysql::tables_in_db();      my @Current_Tables = &Apache::lonmysql::tables_in_db();
       $logthis->(join(',',@Current_Tables));
     my %Found;      my %Found;
     foreach my $tablename (@Current_Tables) {      foreach my $tablename (@Current_Tables) {
         foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {
Line 277  if (-s $sql_filename) { Line 308  if (-s $sql_filename) {
             }              }
         }          }
     }      }
       $logthis->('Found tables '.join(',',keys(%Found)));
       my $missing_a_table = 0;
     foreach my $table (@Activity_Table,@ID_Tables) {          foreach my $table (@Activity_Table,@ID_Tables) {    
           # Hmmm, should I dump the tables?
         if (! $Found{$table->{'id'}}) {          if (! $Found{$table->{'id'}}) {
             $time_this->();              $logthis->('Missing table '.$table->{'id'});
             &load_backup_tables($sql_filename);              $missing_a_table = 1;
             $time_this->('load backup tables');  
             last;              last;
         }          }
     }      }
       if ($missing_a_table) {
           my $table_modification_time = $backup_modification_time;
           # If the backup happened prior to the last table modification,
           foreach my $table (@Activity_Table,@ID_Tables) {    
               my %tabledata = &Apache::lonmysql::table_information($table->{'id'});
               next if (! scalar(keys(%tabledata))); # table does not exist
               if ($table_modification_time < $tabledata{'Update_time'}) {
                   $table_modification_time = $tabledata{'Update_time'};
               }
           }
           $logthis->("Table modification time = ".$table_modification_time);
           if ($table_modification_time > $backup_modification_time) {
               # Save the current tables in case we need them another time.
               my $backup_name = $gz_sql_filename.'.'.time;
               $logthis->('Backing existing tables up in '.$backup_name);
               &backup_tables($backup_name);
           }
           $time_this->();
           &load_backup_tables($gz_sql_filename);
           $time_this->('load backup tables');
       }
 }  }
   
 ##  ##
Line 318  if (-s $newfilename) { Line 372  if (-s $newfilename) {
         exit 5;          exit 5;
     } elsif ($result > 0) {      } elsif ($result > 0) {
         $time_this->();          $time_this->();
         $logthis->('process_courselog returned '.$result.' backup up tables');          $logthis->('process_courselog returned '.$result.' backing up tables');
         &backup_tables($sql_filename);          &backup_tables($gz_sql_filename);
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
 }  }
Line 443  sub process_courselog { Line 497  sub process_courselog {
             $time_this->('split_and_error_check');              $time_this->('split_and_error_check');
             my $student_id = &get_id($student_table,'student',$student);              my $student_id = &get_id($student_table,'student',$student);
             my $res_id     = &get_id($res_table,'resource',$res);              my $res_id     = &get_id($res_table,'resource',$res);
             my $action_id  = &get_id($action_table,'action',$action);  #            my $action_id  = &get_id($action_table,'action',$action);
             my $sql_time   = &Apache::lonmysql::sqltime($time);              my $sql_time   = &Apache::lonmysql::sqltime($time);
             #              #
             if (! defined($student_id) || $student_id eq '') {               if (! defined($student_id) || $student_id eq '') { 
Line 452  sub process_courselog { Line 506  sub process_courselog {
             if (! defined($res_id) || $res_id eq '') {               if (! defined($res_id) || $res_id eq '') { 
                 $warningflag.='res_id';                   $warningflag.='res_id'; 
             }              }
             if (! defined($action_id) || $action_id eq '') {   #            if (! defined($action_id) || $action_id eq '') { 
                 $warningflag.='action_id';   #                $warningflag.='action_id'; 
             }  #            }
             if ($warningflag ne '') {              if ($warningflag ne '') {
                 print $error_fh 'full log entry:'.$log.$/;                  print $error_fh 'full log entry:'.$log.$/;
                 print $error_fh 'error on chunk:'.$chunk.$/;                  print $error_fh 'error on chunk:'.$chunk.$/;
Line 470  sub process_courselog { Line 524  sub process_courselog {
             my $row = [$res_id,              my $row = [$res_id,
                        qq{'$sql_time'},                         qq{'$sql_time'},
                        $student_id,                         $student_id,
                        $action_id,                         "'".$action."'",
   #                       $action_id,
                        qq{''},        # idx                         qq{''},        # idx
                        $machine_id,                         $machine_id,
                        $values];                         $values];
Line 569  sub outputtimes { Line 624  sub outputtimes {
 ## Use mysqldump to store backups of the tables  ## Use mysqldump to store backups of the tables
 ##  ##
 sub backup_tables {  sub backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     my $command = qq{mysqldump --opt loncapa };      my $command = qq{mysqldump --opt loncapa };
                                                             
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $tablename = $table->{'id'};          my $tablename = $table->{'id'};
         $command .= $tablename.' ';          $command .= $tablename.' ';
     }      }
     $command .= '>'.$sql_filename;      $command .= '| gzip >'.$gz_sql_filename;
     $logthis->($command);      $logthis->($command);
     system($command);      system($command);
 }  }
Line 585  sub backup_tables { Line 640  sub backup_tables {
 ## Load in mysqldumped files  ## Load in mysqldumped files
 ##  ##
 sub load_backup_tables {  sub load_backup_tables {
     my ($sql_filename) = @_;      my ($gz_sql_filename) = @_;
     return undef if (! -e $sql_filename);      if (-s $gz_sql_filename) {
     # Check for .my.cnf          $logthis->('loading data from gzipped sql file');
     my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';          my $command='gzip -dc '.$gz_sql_filename.' | mysql --database=loncapa';
     $logthis->('loading previously saved sql table'.$/.$command);          system($command);
     system($command);          $logthis->('finished loading gzipped data');;
     $logthis->('finished loading old data');      } else {
           return undef;
       }
 }  }
   
 ##  ##
 ##   ## 
 ##  ##
 sub initialize_configuration {  
     # Fake it for now:  
     $perlvar{'lonSqlUser'} = 'www';  
     $perlvar{'lonSqlAccess'} = 'localhostkey';  
     $perlvar{'lonUsersDir'} = '/home/httpd/lonUsers';  
     $perlvar{'lonDefDomain'} = '103';  
 }  
   
 sub update_process_name {  sub update_process_name {
     my ($text) = @_;      my ($text) = @_;
     $0 = 'parse_activity_log.pl: '.$text;      $0 = 'parse_activity_log.pl: '.$text;
Line 619  sub get_filename { Line 668  sub get_filename {
 sub create_tables {  sub create_tables {
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = &Apache::lonmysql::create_table($table);          my $table_id = &Apache::lonmysql::create_table($table);
   #        print STDERR "Unable to create table ".$table->{'id'}.$/;
   #        print STDERR join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
         if (! defined($table_id)) {          if (! defined($table_id)) {
             warn "Unable to create table ".$table->{'id'}.$/;              warn "Unable to create table ".$table->{'id'}.$/;
             warn &Apache::lonmysql::build_table_creation_request($table).$/;              warn join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
             return 0;              return 0;
         }          }
     }      }
Removed from v.1.3  
changed lines
  Added in v.1.8

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>