--- loncom/metadata_database/parse_activity_log.pl	2005/09/20 17:49:51	1.19
+++ loncom/metadata_database/parse_activity_log.pl	2006/04/08 07:10:10	1.24
@@ -2,7 +2,7 @@
 #
 # The LearningOnline Network
 #
-# $Id: parse_activity_log.pl,v 1.19 2005/09/20 17:49:51 matthew Exp $
+# $Id: parse_activity_log.pl,v 1.24 2006/04/08 07:10:10 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -48,14 +48,11 @@
 # parameter is not set $logthis is set to &nothing, which does what you
 # would expect.
 #
-
 use strict;
 use DBI;
-use lib '/home/httpd/lib/perl/Apache';
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration();
 use Apache::lonmysql();
-use lonmysql();
 use Time::HiRes();
 use Getopt::Long();
 use IO::File;
@@ -184,7 +181,7 @@ if (! defined($xmlfile)) {
     if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) {
         warn("Unable to lock $lockfilename.  Aborting".$/);
         # don't call clean_up_and_exit another instance is running and
-        # we don't want to 'cleanup' there files
+        # we don't want to 'cleanup' their files
         exit 6;
     }
 
@@ -308,6 +305,7 @@ if (!&Apache::lonmysql::verify_sql_conne
 }
 $logthis->('SQL connection is up');
 
+&update_process_name($course.'@'.$domain." loading existing data");
 my $missing_table = &check_for_missing_tables(values(%tables));
 if (-s $gz_sql_filename && ! -s $gz_xml_filename) {
     my $backup_modification_time = (stat($gz_sql_filename))[9];
@@ -377,8 +375,10 @@ my $error_fh = IO::File->new(">>$error_f
 ##
 ## Parse the course log
 $logthis->('processing course log');
+&update_process_name($course.'@'.$domain." processing new data");
 if (-s $newfilename) {
     my $result = &process_courselog($newfilename,$error_fh,\%tables);
+    &update_process_name($course.'@'.$domain." backing up new data");
     if (! defined($result)) {
         # Something went wrong along the way...
         $logthis->('process_courselog returned undef');
@@ -492,15 +492,10 @@ sub process_courselog {
             if (length($chunk) > 20000) {
                 # avoid putting too much data into the database
                 # (usually an uploaded file or something similar)
-                $chunk = &escape($chunk);
-                if (! open(CHUNKFILE,">>$chunk_filename") ||
-                    ! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) {
-                    # abort
-                    close(CHUNKFILE);
+                if (! &savechunk(\$chunk,$timestamp,$host)) {
                     close(IN);
                     return undef;
                 }
-                close(CHUNKFILE);
                 next;
             }
             my $warningflag = '';
@@ -519,8 +514,12 @@ sub process_courselog {
             if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) {
                 $warningflag .= 'action';
                 print $error_fh 'full log entry:'.$log.$/;
-                print $error_fh 'error on chunk:'.$chunk.$/;
-                $logthis->('(action) Unable to parse '.$/.$chunk.$/.
+                print $error_fh 'error on chunk (saving)'.$/;
+                if (! &savechunk(\$chunk,$timestamp,$host)) {
+                    close(IN);
+                    return undef;
+                }
+                $logthis->('(action) Unable to parse chunk'.$/.
                          'got '.
                          'time = '.$time.$/.
                          'res  = '.$res.$/.
@@ -557,6 +556,20 @@ sub process_courselog {
     }
     close IN;
     return $linecount;
+    ##
+    ##
+    sub savechunk {
+        my ($chunkref,$timestamp,$host) = @_;
+        my $chunk = &escape(${$chunkref});
+        if (! open(CHUNKFILE,">>$chunk_filename") ||
+            ! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) {
+            # abort
+            close(CHUNKFILE);
+            return 0;
+        }
+        close(CHUNKFILE);
+        return 1;
+    }
 }
 
 
@@ -930,9 +943,9 @@ sub xml_store_id_table {
 #######################################################################
 {
     my @rows;
-    my $max_row_count = 100;
 
 sub store_entry {
+    my $max_row_count = 100;
     if (! @_) {
         undef(@rows);
         return '';