--- loncom/metadata_database/parse_activity_log.pl 2004/08/18 19:33:27 1.2 +++ loncom/metadata_database/parse_activity_log.pl 2005/11/15 20:52:02 1.21 @@ -2,7 +2,7 @@ # # The LearningOnline Network # -# $Id: parse_activity_log.pl,v 1.2 2004/08/18 19:33:27 matthew Exp $ +# $Id: parse_activity_log.pl,v 1.21 2005/11/15 20:52:02 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -26,16 +26,7 @@ # # http://www.lon-capa.org/ # -############################################################################### -# -# Expects -# -# ../key/$class.key - key file $username:$keynumber -# ../rawdata/$class.log - log file -# ../rawdata/$class.seq - sequence file -# ../data writable -# ------------------------------------------------------------------ Course log - +#-------------------------------------------------------------------- # # Exit codes # 0 Everything is okay @@ -43,26 +34,50 @@ # 2 Activity log does not exist # 3 Unable to connect to database # 4 Unable to create database tables -# 5 Unspecified error? +# 5 Unable to open log file +# 6 Unable to get lock on activity log # +# +# Notes: +# +# Logging is done via the $logthis variable, which may be the result of +# overcleverness. log via $logthis->('logtext'); Those are parentheses, +# not curly braces. If the -log command line parameter is set, the $logthis +# routine is set to a routine which writes to a file. If the command line +# parameter is not set $logthis is set to ¬hing, which does what you +# would expect. +# +BEGIN { + eval "use Apache2::compat();"; +}; use strict; use DBI; use lib '/home/httpd/lib/perl/Apache'; +use lib '/home/httpd/lib/perl/'; +use LONCAPA::Configuration(); +use Apache::lonmysql(); use lonmysql(); use Time::HiRes(); use Getopt::Long(); +use IO::File; +use File::Copy; +use Fcntl qw(:flock); +use HTML::TokeParser; # # Determine parameters -my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log); +my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup,$xmlfile); &Getopt::Long::GetOptions( "course=s" => \$course, "domain=s" => \$domain, + "backup" => \$backup, "help" => \$help, - "logfile=s" => \$file, + "logfile=s" => \$logfile, + "srcfile=s" => \$srcfile, + "justloadxml=s" => \$xmlfile, "timerun" => \$time_run, "nocleanup" => \$nocleanup, - "drop" => \$drop, + "dropwhendone" => \$drop_when_done, "log" => \$log); if (! defined($course) || $help) { print<$logfile")) { - die "Unable to open $logfile for writing. Run aborted."; + warn("Unable to open $logfile for writing. Run aborted."); + &clean_up_and_exit(5); } else { $logthis = \&log_to_file; } } -## -## Read in configuration parameters -## -my %perlvar; -&initialize_configuration(); -if (! defined($domain) || $domain eq '') { - $domain = $perlvar{'lonDefDomain'}; -} -&update_process_name($course.'@'.$domain); + ## ## Determine filenames ## my $sourcefilename; # activity log data my $newfilename; # $sourcefilename will be renamed to this -my $sql_filename; # the mysql backup data file name. -if ($file) { - $sourcefilename = $file; +my $error_filename; # Errors in parsing the activity log will be written here +my $chunk_filename; # where we save data we are not going to write to db +if ($srcfile) { + $sourcefilename = $srcfile; } else { $sourcefilename = &get_filename($course,$domain); } -$sql_filename = $sourcefilename; +my $sql_filename = $sourcefilename; $sql_filename =~ s|[^/]*$|activity.log.sql|; - -## -## There will only be a $newfilename file if a copy of this program is already -## running. -my $newfilename = $sourcefilename.'.processing'; -if (-e $newfilename) { - warn "$newfilename exists"; - $logthis->($newfilename.' exists'); - exit 2; -} - -if (-e $sourcefilename) { - rename($sourcefilename,$newfilename); - $logthis->("renamed $sourcefilename to $newfilename"); +my $gz_sql_filename = $sql_filename.'.gz'; +# +$chunk_filename = $sourcefilename.".unprocessed_chunks"; +# +my $xml_filename = $sourcefilename; +my $gz_xml_filename = $xml_filename.'.gz'; +if (defined($xmlfile)) { + $xml_filename = $xmlfile; + if ($xml_filename =~ /\.gz$/) { + $gz_xml_filename = $xml_filename; + } else { + $gz_xml_filename = $xml_filename.'.gz'; + } +} else { + my $xml_filename = $sourcefilename; + $xml_filename =~ s|[^/]*$|activity.log.xml|; + $gz_xml_filename = $xml_filename.'.gz'; +} +# +$error_filename = $sourcefilename; +$error_filename =~ s|[^/]*$|activity.log.errors|; +$logthis->('Beginning logging '.time); + +# +# Wait for a lock on the lockfile to avoid collisions +my $lockfilename = $sourcefilename.'.lock'; +$newfilename = $sourcefilename.'.processing'; +if (! defined($xmlfile)) { + open(LOCKFILE,'>'.$lockfilename); + if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) { + warn("Unable to lock $lockfilename. Aborting".$/); + # don't call clean_up_and_exit another instance is running and + # we don't want to 'cleanup' there files + exit 6; + } + + if (! -e $newfilename && -e $sourcefilename) { + $logthis->('renaming '.$sourcefilename.' to '.$newfilename); + rename($sourcefilename,$newfilename); + Copy($newfilename,$newfilename.'.'.time) if ($backup); + $logthis->("renamed $sourcefilename to $newfilename"); + } elsif (! -e $newfilename) { + utime(undef,undef,$newfilename); + } } ## ## Table definitions ## -my $prefix = $course.'_'.$domain.'_'; -my $student_table = $prefix.'students'; +my %tables = &table_names($course,$domain); my $student_table_def = -{ id => $student_table, +{ id => $tables{'student'}, permanent => 'no', columns => [ { name => 'student_id', @@ -166,9 +219,8 @@ my $student_table_def = 'PRIMARY KEY' => ['student_id',], }; -my $res_table = $prefix.'resource'; my $res_table_def = -{ id => $res_table, +{ id => $tables{'res'}, permanent => 'no', columns => [{ name => 'res_id', type => 'MEDIUMINT UNSIGNED', @@ -181,24 +233,22 @@ my $res_table_def = 'PRIMARY KEY' => ['res_id'], }; -my $action_table = $prefix.'actions'; -my $action_table_def = -{ id => $action_table, - permanent => 'no', - columns => [{ name => 'action_id', - type => 'MEDIUMINT UNSIGNED', - restrictions => 'NOT NULL', - auto_inc => 'yes', }, - { name => 'action', - type => 'VARCHAR(100)', - restrictions => 'NOT NULL'}, - ], - 'PRIMARY KEY' => ['action_id',], -}; +#my $action_table_def = +#{ id => $action_table, +# permanent => 'no', +# columns => [{ name => 'action_id', +# type => 'MEDIUMINT UNSIGNED', +# restrictions => 'NOT NULL', +# auto_inc => 'yes', }, +# { name => 'action', +# type => 'VARCHAR(100)', +# restrictions => 'NOT NULL'}, +# ], +# 'PRIMARY KEY' => ['action_id',], +#}; -my $machine_table = $prefix.'machine_table'; my $machine_table_def = -{ id => $machine_table, +{ id => $tables{'machine'}, permanent => 'no', columns => [{ name => 'machine_id', type => 'MEDIUMINT UNSIGNED', @@ -211,9 +261,8 @@ my $machine_table_def = 'PRIMARY KEY' => ['machine_id',], }; -my $activity_table = $prefix.'activity'; my $activity_table_def = -{ id => $activity_table, +{ id => $tables{'activity'}, permanent => 'no', columns => [ { name => 'res_id', @@ -225,8 +274,8 @@ my $activity_table_def = { name => 'student_id', type => 'MEDIUMINT UNSIGNED', restrictions => 'NOT NULL',}, - { name => 'action_id', - type => 'MEDIUMINT UNSIGNED', + { name => 'action', + type => 'VARCHAR(10)', restrictions => 'NOT NULL',}, { name => 'idx', # This is here in case a student type => 'MEDIUMINT UNSIGNED', # has multiple submissions during @@ -238,83 +287,123 @@ my $activity_table_def = { name => 'action_values', type => 'MEDIUMTEXT', }, ], - 'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'], + 'PRIMARY KEY' => ['time','student_id','res_id','idx'], + 'KEY' => [{columns => ['student_id']}, + {columns => ['time']},], }; -my @Activity_Tables = ($student_table_def,$res_table_def, - $action_table_def,$machine_table_def, - $activity_table_def); - +my @Activity_Table = ($activity_table_def); +my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def); + ## ## End of table definitions ## +$logthis->('tables = '.join(',',keys(%tables))); -# -&Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'}, +$logthis->('Connectiong to mysql'); +&Apache::lonmysql::set_mysql_user_and_password('www', $perlvar{'lonSqlAccess'}); if (!&Apache::lonmysql::verify_sql_connection()) { warn "Unable to connect to MySQL database."; $logthis->("Unable to connect to MySQL database."); - exit 3; + &clean_up_and_exit(3); } +$logthis->('SQL connection is up'); -if ($drop) { &drop_tables(); $logthis->('dropped tables'); } -if (-e $sql_filename) { - $logthis->('reading in from '.$sql_filename); - # if ANY one of the tables does not exist, load the tables from the - # backup. - my @Current_Tables = &Apache::lonmysql::tables_in_db(); - my %Found; - foreach my $tablename (@Current_Tables) { - foreach my $table (@Activity_Tables) { - if ($tablename eq $table->{'id'}) { - $Found{$tablename}++; - } +my $missing_table = &check_for_missing_tables(values(%tables)); +if (-s $gz_sql_filename && ! -s $gz_xml_filename) { + my $backup_modification_time = (stat($gz_sql_filename))[9]; + $logthis->($gz_sql_filename.' was last modified '. + localtime($backup_modification_time). + '('.$backup_modification_time.')'); + if ($missing_table) { + # If the backup happened prior to the last table modification, + # we need to save the tables. + if (&latest_table_modification_time() > $backup_modification_time) { + # Save the current tables in case we need them another time. + $logthis->('Backing existing tables up'); + &backup_tables_as_xml($gz_xml_filename.'.save_'.time,\%tables); } - } - foreach my $table (@Activity_Tables) { - if (! $Found{$table->{'id'}}) { - $time_this->(); - &load_backup_tables($sql_filename); - $time_this->('load backup tables'); - last; + $time_this->(); + &load_backup_sql_tables($gz_sql_filename); + &backup_tables_as_xml($gz_xml_filename,\%tables); + $time_this->('load backup tables'); + } +} elsif (-s $gz_xml_filename) { + my $backup_modification_time = (stat($gz_xml_filename))[9]; + $logthis->($gz_xml_filename.' was last modified '. + localtime($backup_modification_time). + '('.$backup_modification_time.')'); + if ($missing_table) { + my $table_modification_time = $backup_modification_time; + # If the backup happened prior to the last table modification, + # we need to save the tables. + if (&latest_table_modification_time() > $backup_modification_time) { + # Save the current tables in case we need them another time. + $logthis->('Backing existing tables up'); + &backup_tables_as_xml($gz_xml_filename.'.save_'.time,\%tables); } - } + $time_this->(); + # We have to make our own tables for the xml format + &drop_tables(); + &create_tables(); + &load_backup_xml_tables($gz_xml_filename,\%tables); + $time_this->('load backup tables'); + } +} + +if (defined($xmlfile)) { + &clean_up_and_exit(0); } +## +## Ensure the tables we need exist # create_tables does not complain if the tables already exist +$logthis->('creating tables'); if (! &create_tables()) { warn "Unable to create tables"; $logthis->('Unable to create tables'); - exit 4; + &clean_up_and_exit(4); } +## +## Read the ids used for various tables $logthis->('reading id tables'); &read_id_tables(); $logthis->('finished reading id tables'); ## -## Do the main bit of work -if (-e $newfilename) { - my $result = &process_courselog($newfilename); +## Set up the errors file +my $error_fh = IO::File->new(">>$error_filename"); + +## +## Parse the course log +$logthis->('processing course log'); +if (-s $newfilename) { + my $result = &process_courselog($newfilename,$error_fh,\%tables); if (! defined($result)) { # Something went wrong along the way... $logthis->('process_courselog returned undef'); - exit 5; + &clean_up_and_exit(5); } elsif ($result > 0) { $time_this->(); - $logthis->('process_courselog returned '.$result.' backup up tables'); - &backup_tables($sql_filename); + $logthis->('process_courselog returned '.$result.'.'.$/. + 'Backing up tables'); + &backup_tables_as_xml($gz_xml_filename,\%tables); $time_this->('write backup tables'); } + if ($drop_when_done) { &drop_tables(); $logthis->('dropped tables'); } } +close($error_fh); ## ## Clean up the filesystem -## &Apache::lonmysql::disconnect_from_db(); -unlink($newfilename) if (! $nocleanup); +unlink($newfilename) if (-e $newfilename && ! $nocleanup); +## +## Print timing data +$logthis->('printing timing data'); if ($time_run) { my $elapsed_time = Time::HiRes::time - $initial_time; print "Overall time: ".$elapsed_time.$/; @@ -323,11 +412,40 @@ if ($time_run) { $logthis->(&outputtimes()); } -if ($log) { - close LOGFILE; +&clean_up_and_exit(0); + +######################################################## +######################################################## + +sub clean_up_and_exit { + my ($exit_code) = @_; + # Close files + close(LOCKFILE); + close(LOGFILE); + # Remove zero length files + foreach my $file ($lockfilename, $error_filename,$logfile) { + if (defined($file) && -z $file) { + unlink($file); + } + } + + exit $exit_code; } -exit 0; # Everything is okay, so end here before it gets worse. +######################################################## +######################################################## +sub table_names { + my ($course,$domain) = @_; + my $prefix = $course.'_'.$domain.'_'; + # + my %tables = + ( student =>&Apache::lonmysql::fix_table_name($prefix.'students'), + res =>&Apache::lonmysql::fix_table_name($prefix.'resource'), + machine =>&Apache::lonmysql::fix_table_name($prefix.'machine_table'), + activity=>&Apache::lonmysql::fix_table_name($prefix.'activity'), + ); + return %tables; +} ######################################################## ######################################################## @@ -339,7 +457,7 @@ exit 0; # Everything is okay, so end h # # Returns the number of lines in the activity.log file that were processed. sub process_courselog { - my ($inputfile) = @_; + my ($inputfile,$error_fh,$tables) = @_; if (! open(IN,$inputfile)) { warn "Unable to open '$inputfile' for reading"; $logthis->("Unable to open '$inputfile' for reading"); @@ -348,8 +466,7 @@ sub process_courselog { my ($linecount,$insertcount); my $dbh = &Apache::lonmysql::get_dbh(); # - # Timing variables - my @RowData; + &store_entry(); while (my $line=){ # last if ($linecount > 1000); # @@ -359,7 +476,6 @@ sub process_courselog { $linecount++; # print $linecount++.$/; my ($timestamp,$host,$log)=split(/\:/,$line,3); - $time_this->('splitline'); # # $log has the actual log entries; currently still escaped, and # %26(timestamp)%3a(url)%3a(user)%3a(domain) @@ -371,130 +487,102 @@ sub process_courselog { # get delimiter between timestamped entries to be &&& $log=~s/\%26(\d{9,10})\%3a/\&\&\&$1\%3a/g; $log = &unescape($log); - $time_this->('translate_and_unescape'); # now go over all log entries if (! defined($host)) { $host = 'unknown'; } - my $machine_id = &get_id($machine_table,'machine',$host); my $prevchunk = 'none'; foreach my $chunk (split(/\&\&\&/,$log)) { + if (length($chunk) > 20000) { + # avoid putting too much data into the database + # (usually an uploaded file or something similar) + if (! &savechunk(\$chunk,$timestamp,$host)) { + close(IN); + return undef; + } + next; + } my $warningflag = ''; - $time_this->(); my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk); - my $student = $uname.':'.$udom; + # if (! defined($res) || $res =~ /^\s*$/) { $res = '/adm/roles'; $action = 'LOGIN'; -# $warningflag .= 'res'; } if ($res =~ m|^/prtspool/|) { $res = '/prtspool/'; } if (! defined($action) || $action eq '') { $action = 'VIEW'; -# $warningflag .= 'action'; } if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) { $warningflag .= 'action'; - } - if (! defined($student) || $student eq ':') { - $student = 'unknown'; - $warningflag .= 'student'; - } - if (! defined($res) || $res =~ /^\s*$/) { - $res = 'unknown'; - $warningflag .= 'res'; - } - if (! defined($action) || $action =~ /^\s*$/) { - $action = 'unknown'; - $warningflag .= 'action'; - } - if (! defined($time) || $time !~ /^\d+$/) { - $time = 0; - $warningflag .= 'time'; + print $error_fh 'full log entry:'.$log.$/; + print $error_fh 'error on chunk (saving)'.$/; + if (! &savechunk(\$chunk,$timestamp,$host)) { + close(IN); + return undef; + } + $logthis->('(action) Unable to parse chunk'.$/. + 'got '. + 'time = '.$time.$/. + 'res = '.$res.$/. + 'uname= '.$uname.$/. + 'udom = '.$udom.$/. + 'action='.$action.$/. + '@values = '.join('&',@values)); + next; #skip it if we cannot understand what is happening. } # - $time_this->('split_and_error_check'); - my $student_id = &get_id($student_table,'student',$student); - my $res_id = &get_id($res_table,'resource',$res); - my $action_id = &get_id($action_table,'action',$action); - my $sql_time = &Apache::lonmysql::sqltime($time); - # - if (! defined($student_id) || $student_id eq '') { - $warningflag.='student_id'; - } - if (! defined($res_id) || $res_id eq '') { - $warningflag.='res_id'; - } - if (! defined($action_id) || $action_id eq '') { - $warningflag.='action_id'; + my %data = (student => $uname.':'.$udom, + resource => $res, + machine => $host, + action => $action, + time => &Apache::lonmysql::sqltime($time)); + if ($action eq 'POST') { + $data{'action_values'} = + $dbh->quote(join('&',map { &escape($_); } @values)); + } else { + $data{'action_values'} = $dbh->quote(join('&',@values)); + } + my $error = &store_entry($dbh,$tables,\%data); + if ($error) { + $logthis->('error store_entry:'.$error." on %data"); } - if ($warningflag ne '') { - $logthis->('warningflag ('.$warningflag.') on chunk '. - $/.$chunk.$/.'prevchunk = '.$/.$prevchunk); - $prevchunk .= $chunk; - next; # skip this chunk - } - # - my $values = $dbh->quote(join('',@values)); - $time_this->('get_ids'); - # - my $row = [$res_id, - qq{'$sql_time'}, - $student_id, - $action_id, - qq{''}, # idx - $machine_id, - $values]; - push(@RowData,$row); - $time_this->('push_row'); $prevchunk = $chunk; - # - } - $time_this->(); - if ((scalar(@RowData) > 0) && ($linecount % 100 == 0)) { - my $result = &Apache::lonmysql::bulk_store_rows($activity_table, - undef, - \@RowData); - # $logthis->('result = '.$result); - $time_this->('bulk_store_rows'); - if (! defined($result)) { - my $error = &Apache::lonmysql::get_error(); - warn "Error occured during insert.".$error; - $logthis->('error = '.$error); - } - undef(@RowData); } } - if (@RowData) { - $time_this->(); - $logthis->('storing '.$linecount); - my $result = &Apache::lonmysql::bulk_store_rows($activity_table, - undef, - \@RowData); - $logthis->('result = '.$result); - $time_this->('bulk_store_rows'); - if (! defined($result)) { - my $error = &Apache::lonmysql::get_error(); - warn "Error occured during insert.".$error; - $logthis->('error = '.$error); - } - undef(@RowData); + my $result = &store_entry($dbh,$tables); + if (! defined($result)) { + my $error = &Apache::lonmysql::get_error(); + warn "Error occured during insert.".$error; + $logthis->('error = '.$error); } close IN; -# print "Number of lines: ".$linecount.$/; -# print "Number of inserts: ".$insertcount.$/; return $linecount; + ## + ## + sub savechunk { + my ($chunkref,$timestamp,$host) = @_; + my $chunk = &escape(${$chunkref}); + if (! open(CHUNKFILE,">>$chunk_filename") || + ! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) { + # abort + close(CHUNKFILE); + return 0; + } + close(CHUNKFILE); + return 1; + } } ## -## Somtimes, instead of doing something, doing nothing is appropriate. +## default value for $logthis and $time_this sub nothing { return; } ## -## Logging routine +## Logging routine (look for $log) ## sub log_to_file { my ($input)=@_; @@ -535,19 +623,55 @@ sub outputtimes { } +sub latest_table_modification_time { + my $latest_time; + foreach my $table (@Activity_Table,@ID_Tables) { + my %tabledata = &Apache::lonmysql::table_information($table->{'id'}); + next if (! scalar(keys(%tabledata))); # table does not exist + if (! defined($latest_time) || + $latest_time < $tabledata{'Update_time'}) { + $latest_time = $tabledata{'Update_time'}; + } + } + return $latest_time; +} + +sub check_for_missing_tables { + my @wanted_tables = @_; + # Check for missing tables + my @Current_Tables = &Apache::lonmysql::tables_in_db(); + my %Found; + foreach my $tablename (@Current_Tables) { + foreach my $table (@wanted_tables) { + if ($tablename eq $table) { + $Found{$tablename}++; + } + } + } + $logthis->('Found tables '.join(',',keys(%Found))); + my $missing_a_table = 0; + foreach my $table (@wanted_tables) { + if (! $Found{$table}) { + $logthis->('Missing table '.$table); + $missing_a_table = 1; + last; + } + } + return $missing_a_table; +} ## ## Use mysqldump to store backups of the tables ## -sub backup_tables { - my ($sql_filename) = @_; - my $command = qq{mysqldump --opt loncapa }; - - foreach my $table (@Activity_Tables) { +sub backup_tables_as_sql { + my ($gz_sql_filename) = @_; + my $command = qq{mysqldump --quote-names --opt loncapa }; + foreach my $table (@ID_Tables,@Activity_Table) { my $tablename = $table->{'id'}; + $tablename =~ s/\`//g; $command .= $tablename.' '; } - $command .= '>'.$sql_filename; + $command .= '| gzip >'.$gz_sql_filename; $logthis->($command); system($command); } @@ -555,26 +679,21 @@ sub backup_tables { ## ## Load in mysqldumped files ## -sub load_backup_tables { - my ($sql_filename) = @_; - return undef if (! -e $sql_filename); - # Check for .my.cnf - my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa'; - $logthis->('loading previously saved sql table'.$/.$command); - system($command); +sub load_backup_sql_tables { + my ($gz_sql_filename) = @_; + if (-s $gz_sql_filename) { + $logthis->('loading data from gzipped sql file'); + my $command='gzip -dc '.$gz_sql_filename.' | mysql --database=loncapa'; + system($command); + $logthis->('finished loading gzipped data');; + } else { + return undef; + } } ## ## ## -sub initialize_configuration { - # Fake it for now: - $perlvar{'lonSqlUser'} = 'www'; - $perlvar{'lonSqlAccess'} = 'localhostkey'; - $perlvar{'lonUsersDir'} = '/home/httpd/lonUsers'; - $perlvar{'lonDefDomain'} = '103'; -} - sub update_process_name { my ($text) = @_; $0 = 'parse_activity_log.pl: '.$text; @@ -587,11 +706,12 @@ sub get_filename { } sub create_tables { - foreach my $table (@Activity_Tables) { + foreach my $table (@ID_Tables,@Activity_Table) { my $table_id = &Apache::lonmysql::create_table($table); if (! defined($table_id)) { warn "Unable to create table ".$table->{'id'}.$/; - warn &Apache::lonmysql::build_table_creation_request($table).$/; + $logthis->('Unable to create table '.$table->{'id'}); + $logthis->(join($/,&Apache::lonmysql::build_table_creation_request($table))); return 0; } } @@ -599,7 +719,7 @@ sub create_tables { } sub drop_tables { - foreach my $table (@Activity_Tables) { + foreach my $table (@ID_Tables,@Activity_Table) { my $table_id = $table->{'id'}; &Apache::lonmysql::drop_table($table_id); } @@ -616,31 +736,32 @@ sub drop_tables { my %IDs; sub read_id_tables { - foreach my $table (@Activity_Tables) { + foreach my $table (@ID_Tables) { my @Data = &Apache::lonmysql::get_rows($table->{'id'}); + my $count = 0; foreach my $row (@Data) { $IDs{$table->{'id'}}->{$row->[1]} = $row->[0]; } } + return; } sub get_id { my ($table,$fieldname,$value) = @_; - if (exists($IDs{$table}->{$value})) { + if (exists($IDs{$table}->{$value}) && $IDs{$table}->{$value} =~ /^\d+$/) { return $IDs{$table}->{$value}; } else { # insert into the table - if the item already exists, that is # okay. my $result = &Apache::lonmysql::store_row($table,[undef,$value]); if (! defined($result)) { - warn("Got error on id insert for $value\n".&Apache::lonmysql::get_error()); + warn("Got error on id insert for $value\n". + &Apache::lonmysql::get_error()); } # get the id - my @Data = - &Apache::lonmysql::get_rows($table,qq{$fieldname='$value'}); - if (@Data) { - $IDs{$table}->{$value}=$Data[0]->[0]; - return $IDs{$table}->{$value}; + my $id = &Apache::lonmysql::get_dbh()->{'mysql_insertid'}; + if (defined($id)) { + $IDs{$table}->{$value}=$id; } else { $logthis->("Unable to retrieve id for $table $fieldname $value"); return undef; @@ -650,6 +771,235 @@ sub get_id { } # End of ID scoping +############################################################### +############################################################### +## +## Save as XML +## +############################################################### +############################################################### +sub backup_tables_as_xml { + my ($filename,$tables) = @_; + open(XMLFILE,"|gzip - > $filename") || return ('error:unable to write '.$filename); + my $query = qq{ + SELECT B.resource, + A.time, + A.idx, + C.student, + A.action, + E.machine, + A.action_values + FROM $tables->{'activity'} AS A + LEFT JOIN $tables->{'res'} AS B ON B.res_id=A.res_id + LEFT JOIN $tables->{'student'} AS C ON C.student_id=A.student_id + LEFT JOIN $tables->{'machine'} AS E ON E.machine_id=A.machine_id + ORDER BY A.time DESC + }; + $query =~ s/\s+/ /g; + my $dbh = &Apache::lonmysql::get_dbh(); + my $sth = $dbh->prepare($query); + if (! $sth->execute()) { + $logthis->(''. + 'WARNING: Could not retrieve from database:'. + $sth->errstr().''); + return undef; + } else { + my ($res,$sqltime,$idx,$student,$action,$machine,$action_values); + if ($sth->bind_columns(\$res,\$sqltime,\$idx,\$student,\$action, + \$machine,\$action_values)) { + + while ($sth->fetch) { + print XMLFILE ''. + qq{$res}. + qq{}. + qq{$idx}. + qq{$student}. + qq{$action}. + qq{$machine}. + qq{$action_values}. + ''.$/; + } + } else { + warn "Unable to bind to columns.\n"; + return undef; + } + } + close XMLFILE; + return; +} + +############################################################### +############################################################### +## +## load as xml +## +############################################################### +############################################################### +{ + my @fields = ('resource','time', + 'student','action','idx','machine','action_values'); + my %ids = (); +sub load_backup_xml_tables { + my ($filename,$tables) = @_; + my $dbh = &Apache::lonmysql::get_dbh(); + my $xmlfh; + open($xmlfh,"cat $filename | gzip -d - |"); + if (! defined($xmlfh)) { + return ('error:unable to read '.$filename); + } + # + %ids = (resource=> {"\0count"=>1}, + student=> {"\0count"=>1}, + machine=> {"\0count"=>1}); + # + my %data; + while (my $inputline = <$xmlfh>) { + my ($resource,$time,undef,$student,$action,$machine,$action_values) = + ($inputline =~ m{ + (.*) + + (.*) + (.*) + (.*) + (.*) + (.*) + $ + }x + ); + my $resource_id = &xml_get_id('resource',$resource); + my $student_id = &xml_get_id('student',$student); + my $machine_id = &xml_get_id('machine',$machine); + &xml_store_activity_row(map { defined($_)?$dbh->quote($_):'' + } ($resource_id, + $time, + $student_id, + $action, + 'NULL', + $machine_id, + $action_values)); + } + &xml_store_activity_row(); + close($xmlfh); + # Store id tables + while (my ($id_name,$id_data) = each(%ids)) { + if ($id_name eq 'resource') { $id_name = 'res'; } + delete($id_data->{"\0count"}); + &xml_store_id_table($id_name,$id_data); + } + return; +} + +sub xml_get_id { + my ($table,$element) = @_; + if (! exists($ids{$table}->{$element})) { + $ids{$table}->{$element} = $ids{$table}->{"\0count"}++; + } + return $ids{$table}->{$element}; +} + +{ + my @data_rows; +sub xml_store_activity_row { + my @data = @_; + if (scalar(@data)) { + push(@data_rows,[@data]); + } + if (! scalar(@data) || scalar(@data_rows) > 500) { + if (! &Apache::lonmysql::bulk_store_rows($tables{'activity'}, + scalar(@{$data_rows[0]}), + \@data_rows)) { + $logthis->("Error:".&Apache::lonmysql::get_error()); + warn("Error:".&Apache::lonmysql::get_error()); + } else { + undef(@data_rows); + } + } + return; +} + +} + +sub xml_store_id_table { + my ($table,$tabledata) =@_; + my $dbh = &Apache::lonmysql::get_dbh(); + if (! &Apache::lonmysql::bulk_store_rows + ($tables{$table},2, + [map{[$tabledata->{$_},$dbh->quote($_)]} keys(%$tabledata)])) { + $logthis->("Error:".&Apache::lonmysql::get_error()); + warn "Error:".&Apache::lonmysql::get_error().$/; + } +} + +} # End of load xml scoping + +####################################################################### +####################################################################### +## +## store_entry - accumulate data to be inserted into the database +## +## Pass no values in to clear accumulator +## Pass ($dbh,\%tables) to initiate storage of values +## Pass ($dbh,\%tables,\%data) to use normally +## +####################################################################### +####################################################################### +{ + my @rows; + my $max_row_count = 100; + +sub store_entry { + if (! @_) { + undef(@rows); + return ''; + } + my ($dbh,$tables,$data) = @_; + return if (! defined($tables)); + if (defined($data)) { + my $error; + foreach my $field ('student','resource','action','time') { + if (! defined($data->{$field}) || $data->{$field} eq ':' || + $data->{$field}=~ /^\s*$/) { + $error.=$field.','; + } + } + if ($error) { $error=~s/,$//; return $error; } + # + my $student_id = &get_id($tables->{'student'},'student', + $data->{'student'}); + my $res_id = &get_id($tables->{'res'}, + 'resource',$data->{'resource'}); + my $machine_id = &get_id($tables->{'machine'}, + 'machine',$data->{'machine'}); + my $idx = $data->{'idx'}; if (! $idx) { $idx = "''"; } + # + push(@rows,[$res_id, + qq{'$data->{'time'}'}, + $student_id, + qq{'$data->{'action'}'}, + $idx, + $machine_id, + $data->{'action_values'}]); + } + if (defined($tables) && + ( (! defined($data) && scalar(@rows)) || scalar(@rows)>$max_row_count) + ){ + # Store the rows + my $result = + &Apache::lonmysql::bulk_store_rows($tables->{'activity'}, + undef, + \@rows); + if (! defined($result)) { + my $error = &Apache::lonmysql::get_error(); + warn "Error occured during insert.".$error; + return $error; + } + undef(@rows); + return $result if (! defined($data)); + } + return ''; +} + +} # end of scope for &store_entry ############################################################### ###############################################################