--- loncom/metadata_database/parse_activity_log.pl	2005/09/20 16:50:40	1.18
+++ loncom/metadata_database/parse_activity_log.pl	2006/03/30 06:57:50	1.23
@@ -2,7 +2,7 @@
 #
 # The LearningOnline Network
 #
-# $Id: parse_activity_log.pl,v 1.18 2005/09/20 16:50:40 matthew Exp $
+# $Id: parse_activity_log.pl,v 1.23 2006/03/30 06:57:50 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -48,7 +48,9 @@
 # parameter is not set $logthis is set to &nothing, which does what you
 # would expect.
 #
-
+BEGIN {
+    eval "use Apache2::compat();";
+};
 use strict;
 use DBI;
 use lib '/home/httpd/lib/perl/Apache';
@@ -144,6 +146,7 @@ if ($log) {
 my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this
 my $error_filename;   # Errors in parsing the activity log will be written here
+my $chunk_filename;   # where we save data we are not going to write to db
 if ($srcfile) {
     $sourcefilename = $srcfile;
 } else {
@@ -153,6 +156,8 @@ my $sql_filename = $sourcefilename;
 $sql_filename =~ s|[^/]*$|activity.log.sql|;
 my $gz_sql_filename = $sql_filename.'.gz';
 #
+$chunk_filename = $sourcefilename.".unprocessed_chunks";
+#
 my $xml_filename = $sourcefilename;
 my $gz_xml_filename = $xml_filename.'.gz';
 if (defined($xmlfile)) {
@@ -181,7 +186,7 @@ if (! defined($xmlfile)) {
     if (!flock(LOCKFILE,LOCK_EX|LOCK_NB)) {
         warn("Unable to lock $lockfilename.  Aborting".$/);
         # don't call clean_up_and_exit another instance is running and
-        # we don't want to 'cleanup' there files
+        # we don't want to 'cleanup' their files
         exit 6;
     }
 
@@ -486,20 +491,17 @@ sub process_courselog {
         if (! defined($host)) { $host = 'unknown'; }
         my $prevchunk = 'none';
         foreach my $chunk (split(/\&\&\&/,$log)) {
-            my $warningflag = '';
-	    my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk);
-            # 
-            # Sometimes we get a file pasted into the activity.log from
-            # an upload form.  Here we try to detect it and avoid inserting
-            # it into the database to avoid the quiet death of the database
-            # connection
-            my $i;
-            for ($i=0;$i<$#values;$i++) {
-                if ($values[$i] =~ /^HWVAL/) {
-                    $#values = $i;
-                    last;
+            if (length($chunk) > 20000) {
+                # avoid putting too much data into the database
+                # (usually an uploaded file or something similar)
+                if (! &savechunk(\$chunk,$timestamp,$host)) {
+                    close(IN);
+                    return undef;
                 }
+                next;
             }
+            my $warningflag = '';
+	    my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk);
             #
             if (! defined($res) || $res =~ /^\s*$/) {
                 $res = '/adm/roles';
@@ -514,8 +516,12 @@ sub process_courselog {
             if ($action !~ /^(LOGIN|VIEW|POST|CSTORE|STORE)$/) {
                 $warningflag .= 'action';
                 print $error_fh 'full log entry:'.$log.$/;
-                print $error_fh 'error on chunk:'.$chunk.$/;
-                $logthis->('(action) Unable to parse '.$/.$chunk.$/.
+                print $error_fh 'error on chunk (saving)'.$/;
+                if (! &savechunk(\$chunk,$timestamp,$host)) {
+                    close(IN);
+                    return undef;
+                }
+                $logthis->('(action) Unable to parse chunk'.$/.
                          'got '.
                          'time = '.$time.$/.
                          'res  = '.$res.$/.
@@ -552,6 +558,20 @@ sub process_courselog {
     }
     close IN;
     return $linecount;
+    ##
+    ##
+    sub savechunk {
+        my ($chunkref,$timestamp,$host) = @_;
+        my $chunk = &escape(${$chunkref});
+        if (! open(CHUNKFILE,">>$chunk_filename") ||
+            ! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) {
+            # abort
+            close(CHUNKFILE);
+            return 0;
+        }
+        close(CHUNKFILE);
+        return 1;
+    }
 }
 
 
@@ -925,9 +945,9 @@ sub xml_store_id_table {
 #######################################################################
 {
     my @rows;
-    my $max_row_count = 100;
 
 sub store_entry {
+    my $max_row_count = 100;
     if (! @_) {
         undef(@rows);
         return '';