--- loncom/metadata_database/parse_activity_log.pl 2005/03/31 22:09:40 1.14 +++ loncom/metadata_database/parse_activity_log.pl 2005/07/11 05:16:19 1.15 @@ -2,7 +2,7 @@ # # The LearningOnline Network # -# $Id: parse_activity_log.pl,v 1.14 2005/03/31 22:09:40 matthew Exp $ +# $Id: parse_activity_log.pl,v 1.15 2005/07/11 05:16:19 matthew Exp $ # # Copyright Michigan State University Board of Trustees # @@ -65,13 +65,14 @@ use HTML::TokeParser; # # Determine parameters -my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup); +my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup,$xmlfile); &Getopt::Long::GetOptions( "course=s" => \$course, "domain=s" => \$domain, "backup" => \$backup, "help" => \$help, "logfile=s" => \$logfile, "srcfile=s" => \$srcfile, + "justloadxml=s" => \$xmlfile, "timerun" => \$time_run, "nocleanup" => \$nocleanup, "dropwhendone" => \$drop_when_done, @@ -153,8 +154,19 @@ $sql_filename =~ s|[^/]*$|activity.log.s my $gz_sql_filename = $sql_filename.'.gz'; # my $xml_filename = $sourcefilename; -$xml_filename =~ s|[^/]*$|activity.log.xml|; my $gz_xml_filename = $xml_filename.'.gz'; +if (defined($xmlfile)) { + $xml_filename = $xmlfile; + if ($xml_filename =~ /\.gz$/) { + $gz_xml_filename = $xml_filename; + } else { + $gz_xml_filename = $xml_filename.'.gz'; + } +} else { + my $xml_filename = $sourcefilename; + $xml_filename =~ s|[^/]*$|activity.log.xml|; + $gz_xml_filename = $xml_filename.'.gz'; +} # $error_filename = $sourcefilename; $error_filename =~ s|[^/]*$|activity.log.errors|; @@ -163,6 +175,7 @@ $logthis->('Beginning logging '.time); # # Wait for a lock on the lockfile to avoid collisions my $lockfilename = $sourcefilename.'.lock'; +if (! defined($xmlfile)) { open(LOCKFILE,'>'.$lockfilename); if (!flock(LOCKFILE,LOCK_EX)) { warn("Unable to lock $lockfilename. Aborting".$/); @@ -192,7 +205,7 @@ if (-e $sourcefilename) { } close(LOCKFILE); - +} ## ## Table definitions ## @@ -345,6 +358,10 @@ if (-s $gz_sql_filename && ! -s $gz_xml_ } } +if (defined($xmlfile)) { + exit(0); +} + ## ## Ensure the tables we need exist # create_tables does not complain if the tables already exist @@ -376,7 +393,8 @@ if (-s $newfilename) { exit 5; } elsif ($result > 0) { $time_this->(); - $logthis->('process_courselog returned '.$result.' backing up tables'); + $logthis->('process_courselog returned '.$result.'.'.$/. + 'Backing up tables'); &backup_tables_as_xml($gz_xml_filename,\%tables); $time_this->('write backup tables'); } @@ -405,7 +423,7 @@ if ($log) { } foreach my $file ($lockfilename, $error_filename,$logfile) { - if (-z $file) { + if (defined($file) && -z $file) { unlink($file); } } @@ -788,6 +806,10 @@ sub backup_tables_as_xml { ## ############################################################### ############################################################### +{ + my @fields = ('resource','time', + 'student','action','idx','machine','action_values'); + my %ids = (); sub load_backup_xml_tables { my ($filename,$tables) = @_; my $xmlfh; @@ -795,44 +817,101 @@ sub load_backup_xml_tables { if (! defined($xmlfh)) { return ('error:unable to read '.$filename); } - my $dbh = &Apache::lonmysql::get_dbh(); - my $parser = HTML::TokeParser->new($xmlfh); - $parser->xml_mode('1'); - &store_entry(); + # + %ids = (resource=> {"\0count"=>1}, + student=> {"\0count"=>1}, + machine=> {"\0count"=>1}); + # my %data; - while (my $token = $parser->get_token()) { - if ($token->[0] eq 'S' && $token->[1] eq 'row') { - undef(%data); - } - foreach my $tag ('resource','time','idx', - 'student','action','machine','action_values') { - if ($token->[0] eq 'S' && $token->[1] eq $tag) { - my $text = $parser->get_text("/$tag"); - $data{$tag} = $text; - } - } - if ($token->[0] eq 'E' && $token->[1] eq 'row') { - $data{'action_values'} =qq{'$data{'action_values'}'}; - my $error = &store_entry($dbh,$tables,\%data); + while (my $inputline = <$xmlfh>) { + my ($resource,$time,undef,$student,$action,$machine,$action_values) = + ($inputline =~ m{ + (.*) + + (.*) + (.*) + (.*) + (.*) + (.*) + $ + }x + ); + my $resource_id = &xml_get_id('resource',$resource); + my $student_id = &xml_get_id('student',$student); + my $machine_id = &xml_get_id('machine',$machine); + &xml_store_activity_row(map { defined($_)?qq{'$_'}:'' + } ($resource_id, + $time, + $student_id, + $action, + 'NULL', + $machine_id, + $action_values)); + } + &xml_store_activity_row(); + close($xmlfh); + # Store id tables + while (my ($id_name,$id_data) = each(%ids)) { + if ($id_name eq 'resource') { $id_name = 'res'; } + delete($id_data->{"\0count"}); + &xml_store_id_table($id_name,$id_data); + } + return; +} + +sub xml_get_id { + my ($table,$element) = @_; + if (! exists($ids{$table}->{$element})) { + $ids{$table}->{$element} = $ids{$table}->{"\0count"}++; + } + return $ids{$table}->{$element}; +} + +{ + my @data_rows; +sub xml_store_activity_row { + my @data = @_; + if (scalar(@data)) { + push(@data_rows,[@data]); + } + if (! scalar(@data) || scalar(@data_rows) > 500) { + if (! &Apache::lonmysql::bulk_store_rows($tables{'activity'}, + scalar(@{$data_rows[0]}), + \@data_rows)) { + $logthis->("Error:".&Apache::lonmysql::get_error()); + warn("Error:".&Apache::lonmysql::get_error()); + } else { + undef(@data_rows); } } - &store_entry($dbh,$tables); return; } +} + +sub xml_store_id_table { + my ($table,$tabledata) =@_; + if (! &Apache::lonmysql::bulk_store_rows + ($tables{$table},2, + [map{[$tabledata->{$_},qq{"$_"}]} keys(%$tabledata)])) { + $logthis->("Error:".&Apache::lonmysql::get_error()); + warn "Error:".&Apache::lonmysql::get_error().$/; + } +} + +} # End of load xml scoping ####################################################################### ####################################################################### ## ## store_entry - accumulate data to be inserted into the database -## +## ## Pass no values in to clear accumulator ## Pass ($dbh,\%tables) to initiate storage of values ## Pass ($dbh,\%tables,\%data) to use normally ## ####################################################################### ####################################################################### - { my @rows; my $max_row_count = 100; @@ -870,11 +949,11 @@ sub store_entry { $machine_id, $data->{'action_values'}]); } - if (defined($tables) && + if (defined($tables) && ( (! defined($data) && scalar(@rows)) || scalar(@rows)>$max_row_count) ){ # Store the rows - my $result = + my $result = &Apache::lonmysql::bulk_store_rows($tables->{'activity'}, undef, \@rows); @@ -886,7 +965,6 @@ sub store_entry { undef(@rows); return $result if (! defined($data)); } - return ''; }