--- loncom/interface/loncoursedata.pm	2016/08/16 00:46:58	1.201
+++ loncom/interface/loncoursedata.pm	2021/09/16 16:20:36	1.201.2.6
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: loncoursedata.pm,v 1.201 2016/08/16 00:46:58 raeburn Exp $
+# $Id: loncoursedata.pm,v 1.201.2.6 2021/09/16 16:20:36 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -777,6 +777,8 @@ sub symb_is_for_task {
     return ($symb =~ /\.task$/);
 }
 
+my $requested_max_packet = 0;
+my $max_allowed_packet;
 
 sub update_full_student_data {
     my ($sname,$sdom,$courseid) = @_;
@@ -964,9 +966,26 @@ sub update_full_student_data {
     }
     ##
     ## Store the response data
-    $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table.
+    my $store_prefix = 'INSERT IGNORE INTO '.$fulldump_response_table.
         ' VALUES '."\n";
     $store_rows = 0;
+    unless ($requested_max_packet) {
+        (undef,$max_allowed_packet) = $dbh->selectrow_array(
+                                             qq{show variables LIKE ? },
+                                             undef,
+                                             "max_allowed_packet");
+        if ($max_allowed_packet !~ /^\d+$/) {
+            $max_allowed_packet = '';
+        }
+        $requested_max_packet = 1;
+    }
+    my @store_values = ();
+    my $curr_values = '';
+    my $curr_length = 0;
+    my ($max_values);
+    if ($max_allowed_packet) {
+        $max_values = $max_allowed_packet - length($store_prefix);
+    }
     while (my ($symb_id,$hash1) = each (%$respdata)) {
         while (my ($part_id,$hash2) = each (%$hash1)) {
             while (my ($resp_id,$hash3) = each (%$hash2)) {
@@ -978,7 +997,7 @@ sub update_full_student_data {
                     # and odd number of '\' cause insert errors to occur.  
                     # Best trap this somehow...
                     $submission = $dbh->quote($submission);
-                    $store_command .= "('".
+                    my $sql_values = "('".
                         join("','",$symb_id,$part_id,
                              $resp_id,$student_id,
                              $transaction,
@@ -986,20 +1005,51 @@ sub update_full_student_data {
                              $data->{'response_specific'},
                              $data->{'response_specific_value'},
                              $data->{'response_specific_2'},
-                             $data->{'response_specific_value_2'}).
-                             "',".$submission."),";
+                             $data->{'response_specific_value_2'})."',";
+                    if ($max_values) {
+                        my $length = length($sql_values) + length($submission."),");
+                        if ($length > $max_values) {
+                            &Apache::lonnet::logthis("SQL responsedata insert for student: $sname would exceed max_allowed_packet size");
+                            &Apache::lonnet::logthis("symb_id: $symb_id, part_id: $part_id, resp_id: $resp_id");
+                            &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                            $sql_values .= $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),";
+                            $length = length($sql_values);
+                            &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission");
+                            &Apache::lonnet::logthis("See grading interface for the actual submission");
+                        } else {
+                            $sql_values .= $submission."),";
+                        }
+                        if ($length + $curr_length > $max_values) {
+                            push(@store_values,$curr_values);
+                            $curr_values = $sql_values;
+                            $curr_length = $length;
+                        } else {
+                            $curr_values .= $sql_values;
+                            $curr_length += $length;
+                        }
+                    } else {
+                        $curr_values .= $sql_values.$submission."),";
+                    }
                     $store_rows++;
                 }
             }
         }
     }
     if ($store_rows) {
-        chop($store_command);
-        $dbh->do($store_command);
-        if ($dbh->err) {
-            $returnstatus = 'error saving response data';
-            &Apache::lonnet::logthis('insert error '.$dbh->errstr());
-            &Apache::lonnet::logthis("While attempting\n".$store_command);
+        if ($curr_values ne '') {
+            push(@store_values,$curr_values);
+        }
+        foreach my $item (@store_values) {
+            chop($item);
+            if ($item ne '') {
+                $dbh->do($store_prefix.$item);
+                if ($dbh->err) {
+                    $returnstatus = 'error saving response data';
+                    &Apache::lonnet::logthis('insert error '.$dbh->errstr());
+                    &Apache::lonnet::logthis("While attempting\n".$store_prefix.$item);
+                    last;
+                }
+            }
         }
     }
     ##
@@ -1012,7 +1062,7 @@ sub update_full_student_data {
         $returnstatus = 'error saving current data:'.$status;
     } elsif ($status ne 'okay') {
         $returnstatus .= ' error saving current data:'.$status;
-    }        
+    }
     ##
     ## Update the students time......
     if ($returnstatus eq 'okay') {
@@ -1102,12 +1152,33 @@ sub store_student_data {
     my $starttime = Time::HiRes::time;
     my $elapsed = 0;
     my $rows_stored;
-    my $store_parameters_command  = 'INSERT IGNORE INTO '.$parameters_table.
+    my $store_parameters_prefix  = 'INSERT IGNORE INTO '.$parameters_table.
         ' VALUES '."\n";
     my $num_parameters = 0;
-    my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table.
+    my $store_performance_prefix = 'INSERT IGNORE INTO '.$performance_table.
         ' VALUES '."\n";
     return ('error',undef) if (! defined($dbh));
+    unless ($requested_max_packet) {
+        (undef,$max_allowed_packet) = $dbh->selectrow_array(
+                                             qq{show variables LIKE ? },
+                                             undef,
+                                             "max_allowed_packet");
+        if ($max_allowed_packet !~ /^\d+$/) {
+            $max_allowed_packet = '';
+        }
+        $requested_max_packet = 1;
+    }
+    my @store_parameters_values = ();
+    my $curr_params_values = '';
+    my $curr_params_length = 0;
+    my @store_performance_values = ();
+    my $curr_perf_values = '';
+    my $curr_perf_length = 0;
+    my ($max_param,$max_perf);
+    if ($max_allowed_packet) {
+        $max_param = $max_allowed_packet - length($store_parameters_prefix);
+        $max_perf = $max_allowed_packet - length($store_performance_prefix);
+    }
     while (my ($current_symb,$param_hash) = each(%{$student_data})) {
         #
         # make sure the symb is set up properly
@@ -1120,10 +1191,39 @@ sub store_student_data {
                                               $symb_id,$student_id,
                                               $parameter)."',".
                                                   $dbh->quote($value)."),\n";
-                $num_parameters ++;
                 if ($sql_parameter !~ /''/) {
-                    $store_parameters_command .= $sql_parameter;
+                    if ($max_param) {
+                        my $length = length($sql_parameter);
+                        if ($length > $max_param) {
+                            &Apache::lonnet::logthis("SQL parameter insert for student: $sname for parameter: $parameter would exceed max_allowed_packet size");
+                            &Apache::lonnet::logthis("symb_id: $symb_id");
+                            &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                            if ($parameter =~ /\.submission$/) {
+                                $sql_parameter = "('".join("','",
+                                              $symb_id,$student_id,
+                                              $parameter)."',".
+                                                  $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),\n";
+                                $length = length($sql_parameter);
+                                &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission");
+                                &Apache::lonnet::logthis("See grading interface for the actual submission");
+                            } else {
+                                &Apache::lonnet::logthis("Skipping this item");
+                                next;
+                            }
+                        }
+                        if ($length + $curr_params_length > $max_param) {
+                            push(@store_parameters_values,$curr_params_values);
+                            $curr_params_values = $sql_parameter;
+                            $curr_params_length = $length;
+                        } else {
+                            $curr_params_values .= $sql_parameter;
+                            $curr_params_length += $length;
+                        }
+                    } else {
+                        $curr_params_values .= $sql_parameter;
+                    }
                     #$rows_stored++;
+                    $num_parameters ++;
                 }
             }
         }
@@ -1165,31 +1265,66 @@ sub store_student_data {
                 "('".join("','",$symb_id,$student_id,$part_id,$part,
                                 $solved,$tries,$awarded,$award,
                                 $awarddetail,$timestamp)."'),\n";
-            $store_performance_command .= $sql_performance;
+            if ($max_perf) {
+                my $length = length($sql_performance);
+                if ($length > $max_perf) {
+                            &Apache::lonnet::logthis("SQL performance insert for student: $sname would exceed max_allowed_packet size");
+                            &Apache::lonnet::logthis("symb_id: $symb_id");
+                            &Apache::lonnet::logthis("Skipping this item.  You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                            next;
+                } else {
+                    if ($length + $curr_perf_length > $max_perf) {
+                        push(@store_performance_values,$curr_perf_values);
+                        $curr_perf_values = $sql_performance;
+                        $curr_perf_length = $length;
+                    } else {
+                        $curr_perf_values .= $sql_performance;
+                        $curr_perf_length += $length;
+                    }
+                }
+            } else {
+                $curr_perf_values .= $sql_performance;
+            }
             $rows_stored++;
         }
     }
+    if ($curr_params_values ne '') {
+        push(@store_parameters_values,$curr_params_values);
+    }
+    if ($curr_perf_values ne '') {
+        push(@store_performance_values,$curr_perf_values);
+    }
     if (! $rows_stored) { return ($returnstatus, undef); }
-    $store_parameters_command =~ s|,\n$||;
-    $store_performance_command =~ s|,\n$||;
     my $start = Time::HiRes::time;
-    $dbh->do($store_performance_command);
-    if ($dbh->err()) {
-        &Apache::lonnet::logthis('performance bigass insert error:'.
-                                 $dbh->errstr());
-        &Apache::lonnet::logthis('command = '.$/.$store_performance_command);
-        $returnstatus = 'error: unable to insert performance into database';
-        return ($returnstatus,$student_data);
+    foreach my $item (@store_performance_values) {
+        $item =~ s|,\n$||;
+        if ($item ne '') {
+            $dbh->do($store_performance_prefix.$item);
+            if ($dbh->err()) {
+                &Apache::lonnet::logthis('performance insert error:'.
+                                         $dbh->errstr());
+                &Apache::lonnet::logthis('command = '.$/.$store_performance_prefix.$item);
+                $returnstatus = 'error: unable to insert performance into database';
+                return ($returnstatus,$student_data);
+            }
+        }
     }
-    $dbh->do($store_parameters_command) if ($num_parameters>0);
-    if ($dbh->err()) {
-        &Apache::lonnet::logthis('parameters bigass insert error:'.
-                                 $dbh->errstr());
-        &Apache::lonnet::logthis('command = '.$/.$store_parameters_command);
-        &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
-        &Apache::lonnet::logthis('student_id = '.$student_id);
-        $returnstatus = 'error: unable to insert parameters into database';
-        return ($returnstatus,$student_data);
+    if ($num_parameters > 0) {
+        foreach my $item (@store_parameters_values) {
+            $item =~ s|,\n$||;
+            if ($item ne '') {
+                $dbh->do($store_parameters_prefix.$item);
+                if ($dbh->err()) {
+                     &Apache::lonnet::logthis('parameters insert error:'.
+                                              $dbh->errstr());
+                     &Apache::lonnet::logthis('command = '.$/.$store_parameters_prefix.$item);
+                     &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
+                     &Apache::lonnet::logthis('student_id = '.$student_id);
+                     $returnstatus = 'error: unable to insert parameters into database';
+                     return ($returnstatus,$student_data);
+                }
+            }
+        }
     }
     $elapsed += Time::HiRes::time - $start;
     return ($returnstatus,$student_data);
@@ -1677,22 +1812,13 @@ sub get_problem_statistics {
     #
     $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table.
         ' WHERE awarded>0';
-    if (defined($time_requirements)) {
-        $request .= ' AND '.$time_requirements;
-    }
     my ($max,$min) = &execute_SQL_request($dbh,$request);
     #
     $request = 'SELECT SUM(awarded) FROM '.$stats_table;
-    if (defined($time_requirements)) {
-        $request .= ' AND '.$time_requirements;
-    }
     my ($Solved) = &execute_SQL_request($dbh,$request);
     #
     $request = 'SELECT SUM(awarded) FROM '.$stats_table.
         " WHERE solved='correct_by_override'";
-    if (defined($time_requirements)) {
-        $request .= ' AND '.$time_requirements;
-    }
     my ($solved) = &execute_SQL_request($dbh,$request);
     #
     $Solved -= $solved;
@@ -1933,7 +2059,7 @@ sub rank_students_by_scores_on_resources
         $limits =~ s/( AND )$//;   # Remove extra conjunction
         $request .= "WHERE $limits";
     } 
-    $request .= " $award_clause GROUP BY a.student_id ORDER BY score";
+    $request .= " $award_clause GROUP BY a.student_id ORDER BY score, b.student";
     #&Apache::lonnet::logthis('request = '.$/.$request);
     my $sth = $dbh->prepare($request) or die "Can't prepare $request";
     $sth->execute();
@@ -2394,7 +2520,7 @@ sub get_student_scores {
         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
         return undef;
     }
-    $request = 'SELECT score,COUNT(*) FROM '.$tmptable.' GROUP BY score';
+    $request = 'SELECT score,COUNT(*) FROM '.$tmptable.' GROUP BY score ORDER BY score';
 #    &Apache::lonnet::logthis("request = \n".$request);
     $sth = $dbh->prepare($request);
     $sth->execute();