--- loncom/metadata_database/parse_activity_log.pl	2005/03/31 22:09:40	1.14
+++ loncom/metadata_database/parse_activity_log.pl	2005/09/19 18:31:57	1.16
@@ -2,7 +2,7 @@
 #
 # The LearningOnline Network
 #
-# $Id: parse_activity_log.pl,v 1.14 2005/03/31 22:09:40 matthew Exp $
+# $Id: parse_activity_log.pl,v 1.16 2005/09/19 18:31:57 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -65,13 +65,14 @@ use HTML::TokeParser;
 
 #
 # Determine parameters
-my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup);
+my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup,$xmlfile);
 &Getopt::Long::GetOptions( "course=s"  => \$course,
                            "domain=s"  => \$domain,
                            "backup"    => \$backup,
                            "help"      => \$help,
                            "logfile=s" => \$logfile,
                            "srcfile=s" => \$srcfile,
+                           "justloadxml=s" => \$xmlfile,
                            "timerun"   => \$time_run,
                            "nocleanup" => \$nocleanup,
                            "dropwhendone" => \$drop_when_done,
@@ -130,7 +131,7 @@ if ($log) {
     print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {
         warn("Unable to open $logfile for writing.  Run aborted.");
-        exit 5;
+        &cleanup_and_exit(5);
     } else {
         $logthis = \&log_to_file;
     }
@@ -153,8 +154,19 @@ $sql_filename =~ s|[^/]*$|activity.log.s
 my $gz_sql_filename = $sql_filename.'.gz';
 #
 my $xml_filename = $sourcefilename;
-$xml_filename =~ s|[^/]*$|activity.log.xml|;
 my $gz_xml_filename = $xml_filename.'.gz';
+if (defined($xmlfile)) {
+    $xml_filename = $xmlfile;
+    if ($xml_filename =~ /\.gz$/) {
+        $gz_xml_filename = $xml_filename;
+    } else {
+        $gz_xml_filename = $xml_filename.'.gz';
+    }
+} else {
+    my $xml_filename = $sourcefilename;
+    $xml_filename =~ s|[^/]*$|activity.log.xml|;
+    $gz_xml_filename = $xml_filename.'.gz';
+}
 #
 $error_filename = $sourcefilename;
 $error_filename =~ s|[^/]*$|activity.log.errors|;
@@ -163,36 +175,24 @@ $logthis->('Beginning logging '.time);
 #
 # Wait for a lock on the lockfile to avoid collisions
 my $lockfilename = $sourcefilename.'.lock';
-open(LOCKFILE,'>'.$lockfilename);
-if (!flock(LOCKFILE,LOCK_EX)) {
-    warn("Unable to lock $lockfilename.  Aborting".$/);
-    exit 6;
-}
+$newfilename = $sourcefilename.'.processing';
+if (! defined($xmlfile)) {
+    open(LOCKFILE,'>'.$lockfilename);
+    if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) {
+        warn("Unable to lock $lockfilename.  Aborting".$/);
+        &clean_up_and_exit(6);
+    }
 
-##
-## There will only be a $newfilename file if a copy of this program is already
-## running.
-my $newfilename = $sourcefilename.'.processing';
-if (-e $newfilename) {
-    warn "$newfilename exists";
-    $logthis->($newfilename.' exists, so I cannot work on it.');
-    exit 2;
-}
-
-if (-e $sourcefilename) {
-    $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
-    rename($sourcefilename,$newfilename);
-    Copy($newfilename,$newfilename.'.'.time) if ($backup);
-    $logthis->("renamed $sourcefilename to $newfilename");
-} else {
-    my $command = 'touch '.$newfilename;
-    $logthis->($command);
-    system($command);
-    $logthis->('touch was completed');
+    if (! -e $newfilename && -e $sourcefilename) {
+        $logthis->('renaming '.$sourcefilename.' to '.$newfilename);
+        rename($sourcefilename,$newfilename);
+        Copy($newfilename,$newfilename.'.'.time) if ($backup);
+        $logthis->("renamed $sourcefilename to $newfilename");
+    } elsif (! -e $newfilename) {
+        utime(undef,undef,$newfilename);
+    }
 }
 
-close(LOCKFILE);
-
 ##
 ## Table definitions
 ##
@@ -299,7 +299,7 @@ $logthis->('Connectiong to mysql');
 if (!&Apache::lonmysql::verify_sql_connection()) {
     warn "Unable to connect to MySQL database.";
     $logthis->("Unable to connect to MySQL database.");
-    exit 3;
+    &clean_up_and_exit(3);
 }
 $logthis->('SQL connection is up');
 
@@ -345,6 +345,10 @@ if (-s $gz_sql_filename && ! -s $gz_xml_
     }    
 }
 
+if (defined($xmlfile)) {
+    &clean_up_and_exit(0);
+}
+
 ##
 ## Ensure the tables we need exist
 # create_tables does not complain if the tables already exist
@@ -352,7 +356,7 @@ $logthis->('creating tables');
 if (! &create_tables()) {
     warn "Unable to create tables";
     $logthis->('Unable to create tables');
-    exit 4;
+    &clean_up_and_exit(4);
 }
 
 ##
@@ -373,10 +377,11 @@ if (-s $newfilename) {
     if (! defined($result)) {
         # Something went wrong along the way...
         $logthis->('process_courselog returned undef');
-        exit 5;
+        &clean_up_and_exit(5);
     } elsif ($result > 0) {
         $time_this->();
-        $logthis->('process_courselog returned '.$result.' backing up tables');
+        $logthis->('process_courselog returned '.$result.'.'.$/.
+                   'Backing up tables');
         &backup_tables_as_xml($gz_xml_filename,\%tables);
         $time_this->('write backup tables');
     }
@@ -400,17 +405,25 @@ if ($time_run) {
     $logthis->(&outputtimes());
 }
 
-if ($log) {
-    close LOGFILE;
-}
+&clean_up_and_exit(0);
+
+########################################################
+########################################################
 
-foreach my $file ($lockfilename, $error_filename,$logfile) {
-    if (-z $file) { 
-        unlink($file); 
+sub clean_up_and_exit {
+    my ($exit_code) = @_;
+    # Close files
+    close(LOCKFILE);
+    close(LOGFILE);
+    # Remove zero length files
+    foreach my $file ($lockfilename, $error_filename,$logfile) {
+        if (defined($file) && -z $file) { 
+            unlink($file); 
+        }
     }
-}
 
-exit 0;   # Everything is okay, so end here before it gets worse.
+    exit $exit_code;
+}
 
 ########################################################
 ########################################################
@@ -700,21 +713,20 @@ sub read_id_tables {
 
 sub get_id {
     my ($table,$fieldname,$value) = @_;
-    if (exists($IDs{$table}->{$value})) {
+    if (exists($IDs{$table}->{$value}) && $IDs{$table}->{$value} =~ /^\d+$/) {
         return $IDs{$table}->{$value};
     } else {
         # insert into the table - if the item already exists, that is
         # okay.
         my $result = &Apache::lonmysql::store_row($table,[undef,$value]);
         if (! defined($result)) {
-            warn("Got error on id insert for $value\n".&Apache::lonmysql::get_error());
+            warn("Got error on id insert for $value\n".
+                 &Apache::lonmysql::get_error());
         }
         # get the id
-        my @Data = 
-            &Apache::lonmysql::get_rows($table,qq{$fieldname='$value'});
-        if (@Data) {
-            $IDs{$table}->{$value}=$Data[0]->[0];
-            return $IDs{$table}->{$value};
+        my $id = &Apache::lonmysql::get_dbh()->{'mysql_insertid'};
+        if (defined($id)) {
+            $IDs{$table}->{$value}=$id;
         } else {
             $logthis->("Unable to retrieve id for $table $fieldname $value");
             return undef;
@@ -788,51 +800,114 @@ sub backup_tables_as_xml {
 ##
 ###############################################################
 ###############################################################
+{
+    my @fields = ('resource','time',
+                  'student','action','idx','machine','action_values');
+    my %ids = ();
 sub load_backup_xml_tables {
     my ($filename,$tables) = @_;
+    my $dbh = &Apache::lonmysql::get_dbh();
     my $xmlfh;
     open($xmlfh,"cat $filename | gzip -d - |");
     if (! defined($xmlfh)) {
         return ('error:unable to read '.$filename);
     }
-    my $dbh = &Apache::lonmysql::get_dbh();
-    my $parser = HTML::TokeParser->new($xmlfh);
-    $parser->xml_mode('1');
-    &store_entry();
+    #
+    %ids = (resource=> {"\0count"=>1},
+            student=> {"\0count"=>1},
+            machine=> {"\0count"=>1});
+    #
     my %data;
-    while (my $token = $parser->get_token()) {
-        if ($token->[0] eq 'S' && $token->[1] eq 'row') {
-            undef(%data);
-        }
-        foreach my $tag ('resource','time','idx',
-                         'student','action','machine','action_values') {
-            if ($token->[0] eq 'S' && $token->[1] eq $tag) {
-                my $text = $parser->get_text("/$tag");
-                $data{$tag} = $text;
-            }
-        }
-        if ($token->[0] eq 'E' && $token->[1] eq 'row') {
-            $data{'action_values'} =qq{'$data{'action_values'}'};
-            my $error = &store_entry($dbh,$tables,\%data);
+    while (my $inputline = <$xmlfh>) {
+        my ($resource,$time,undef,$student,$action,$machine,$action_values) = 
+            ($inputline =~ m{<row>
+                                 <resource>(.*)</resource>
+                                 <time>(.*)</time>
+                                 <idx>(.*)</idx>
+                                 <student>(.*)</student>
+                                 <action>(.*)</action>
+                                 <machine>(.*)</machine>
+                                 <action_values>(.*)</action_values>
+                                 </row>$
+                             }x
+             );
+        my $resource_id = &xml_get_id('resource',$resource);
+        my $student_id  = &xml_get_id('student',$student);
+        my $machine_id  = &xml_get_id('machine',$machine);
+        &xml_store_activity_row(map { defined($_)?$dbh->quote($_):'' 
+                                  } ($resource_id,
+                                     $time,
+                                     $student_id,
+                                     $action,
+                                     'NULL',
+                                     $machine_id,
+                                     $action_values));
+    }
+    &xml_store_activity_row();
+    close($xmlfh);
+    # Store id tables
+    while (my ($id_name,$id_data) = each(%ids)) {
+        if ($id_name eq 'resource') { $id_name = 'res'; }
+        delete($id_data->{"\0count"});
+        &xml_store_id_table($id_name,$id_data);
+    }
+    return;
+}
+
+sub xml_get_id {
+    my ($table,$element) = @_;
+    if (! exists($ids{$table}->{$element})) {
+        $ids{$table}->{$element} = $ids{$table}->{"\0count"}++;
+    }
+    return $ids{$table}->{$element};
+}
+
+{
+    my @data_rows;
+sub xml_store_activity_row {
+    my @data = @_;
+    if (scalar(@data)) {
+        push(@data_rows,[@data]);
+    }
+    if (! scalar(@data) || scalar(@data_rows) > 500) {
+        if (! &Apache::lonmysql::bulk_store_rows($tables{'activity'},
+                                                 scalar(@{$data_rows[0]}),
+                                                 \@data_rows)) {
+            $logthis->("Error:".&Apache::lonmysql::get_error());
+            warn("Error:".&Apache::lonmysql::get_error());
+        } else {
+            undef(@data_rows);
         }
     }
-    &store_entry($dbh,$tables);
     return;
 }
 
+}
+
+sub xml_store_id_table {
+    my ($table,$tabledata) =@_;
+    my $dbh = &Apache::lonmysql::get_dbh();
+    if (! &Apache::lonmysql::bulk_store_rows
+        ($tables{$table},2,
+         [map{[$tabledata->{$_},$dbh->quote($_)]} keys(%$tabledata)])) {
+        $logthis->("Error:".&Apache::lonmysql::get_error());
+        warn "Error:".&Apache::lonmysql::get_error().$/;
+    }
+}
+
+} # End of load xml scoping
 
 #######################################################################
 #######################################################################
 ##
 ## store_entry - accumulate data to be inserted into the database
-## 
+##
 ## Pass no values in to clear accumulator
 ## Pass ($dbh,\%tables) to initiate storage of values
 ## Pass ($dbh,\%tables,\%data) to use normally
 ##
 #######################################################################
 #######################################################################
-
 {
     my @rows;
     my $max_row_count = 100;
@@ -870,11 +945,11 @@ sub store_entry {
                     $machine_id,
                     $data->{'action_values'}]);
     }
-    if (defined($tables) && 
+    if (defined($tables) &&
         ( (! defined($data) && scalar(@rows)) || scalar(@rows)>$max_row_count)
         ){
         # Store the rows
-        my $result = 
+        my $result =
             &Apache::lonmysql::bulk_store_rows($tables->{'activity'},
                                                undef,
                                                \@rows);
@@ -886,7 +961,6 @@ sub store_entry {
         undef(@rows);
         return $result if (! defined($data));
     }
-
     return '';
 }