--- loncom/interface/loncoursedata.pm 2011/10/03 13:11:39 1.192 +++ loncom/interface/loncoursedata.pm 2021/03/05 17:18:40 1.201.2.5 @@ -1,6 +1,6 @@ # The LearningOnline Network with CAPA # -# $Id: loncoursedata.pm,v 1.192 2011/10/03 13:11:39 raeburn Exp $ +# $Id: loncoursedata.pm,v 1.201.2.5 2021/03/05 17:18:40 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -58,7 +58,7 @@ use Digest::MD5(); =pod -=head 2 make_into_hash +=head2 make_into_hash Turn a colon separated string into a hash and return a reference to it. Numbering from 0 even elements are keys and odd elements @@ -777,6 +777,8 @@ sub symb_is_for_task { return ($symb =~ /\.task$/); } +my $requested_max_packet = 0; +my $max_allowed_packet; sub update_full_student_data { my ($sname,$sdom,$courseid) = @_; @@ -964,9 +966,26 @@ sub update_full_student_data { } ## ## Store the response data - $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table. + my $store_prefix = 'INSERT IGNORE INTO '.$fulldump_response_table. ' VALUES '."\n"; $store_rows = 0; + unless ($requested_max_packet) { + (undef,$max_allowed_packet) = $dbh->selectrow_array( + qq{show variables LIKE ? }, + undef, + "max_allowed_packet"); + if ($max_allowed_packet !~ /^\d+$/) { + $max_allowed_packet = ''; + } + $requested_max_packet = 1; + } + my @store_values = (); + my $curr_values = ''; + my $curr_length = 0; + my ($max_values); + if ($max_allowed_packet) { + $max_values = $max_allowed_packet - length($store_prefix); + } while (my ($symb_id,$hash1) = each (%$respdata)) { while (my ($part_id,$hash2) = each (%$hash1)) { while (my ($resp_id,$hash3) = each (%$hash2)) { @@ -978,7 +997,7 @@ sub update_full_student_data { # and odd number of '\' cause insert errors to occur. # Best trap this somehow... $submission = $dbh->quote($submission); - $store_command .= "('". + my $sql_values = "('". join("','",$symb_id,$part_id, $resp_id,$student_id, $transaction, @@ -986,20 +1005,51 @@ sub update_full_student_data { $data->{'response_specific'}, $data->{'response_specific_value'}, $data->{'response_specific_2'}, - $data->{'response_specific_value_2'}). - "',".$submission."),"; + $data->{'response_specific_value_2'})."',"; + if ($max_values) { + my $length = length($sql_values) + length($submission."),"); + if ($length > $max_values) { + &Apache::lonnet::logthis("SQL responsedata insert for student: $sname would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id, part_id: $part_id, resp_id: $resp_id"); + &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + $sql_values .= $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),"; + $length = length($sql_values); + &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission"); + &Apache::lonnet::logthis("See grading interface for the actual submission"); + } else { + $sql_values .= $submission."),"; + } + if ($length + $curr_length > $max_values) { + push(@store_values,$curr_values); + $curr_values = $sql_values; + $curr_length = $length; + } else { + $curr_values .= $sql_values; + $curr_length += $length; + } + } else { + $curr_values .= $sql_values.$submission."),"; + } $store_rows++; } } } } if ($store_rows) { - chop($store_command); - $dbh->do($store_command); - if ($dbh->err) { - $returnstatus = 'error saving response data'; - &Apache::lonnet::logthis('insert error '.$dbh->errstr()); - &Apache::lonnet::logthis("While attempting\n".$store_command); + if ($curr_values ne '') { + push(@store_values,$curr_values); + } + foreach my $item (@store_values) { + chop($item); + if ($item ne '') { + $dbh->do($store_prefix.$item); + if ($dbh->err) { + $returnstatus = 'error saving response data'; + &Apache::lonnet::logthis('insert error '.$dbh->errstr()); + &Apache::lonnet::logthis("While attempting\n".$store_prefix.$item); + last; + } + } } } ## @@ -1012,7 +1062,7 @@ sub update_full_student_data { $returnstatus = 'error saving current data:'.$status; } elsif ($status ne 'okay') { $returnstatus .= ' error saving current data:'.$status; - } + } ## ## Update the students time...... if ($returnstatus eq 'okay') { @@ -1102,12 +1152,33 @@ sub store_student_data { my $starttime = Time::HiRes::time; my $elapsed = 0; my $rows_stored; - my $store_parameters_command = 'INSERT IGNORE INTO '.$parameters_table. + my $store_parameters_prefix = 'INSERT IGNORE INTO '.$parameters_table. ' VALUES '."\n"; my $num_parameters = 0; - my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table. + my $store_performance_prefix = 'INSERT IGNORE INTO '.$performance_table. ' VALUES '."\n"; return ('error',undef) if (! defined($dbh)); + unless ($requested_max_packet) { + (undef,$max_allowed_packet) = $dbh->selectrow_array( + qq{show variables LIKE ? }, + undef, + "max_allowed_packet"); + if ($max_allowed_packet !~ /^\d+$/) { + $max_allowed_packet = ''; + } + $requested_max_packet = 1; + } + my @store_parameters_values = (); + my $curr_params_values = ''; + my $curr_params_length = 0; + my @store_performance_values = (); + my $curr_perf_values = ''; + my $curr_perf_length = 0; + my ($max_param,$max_perf); + if ($max_allowed_packet) { + $max_param = $max_allowed_packet - length($store_parameters_prefix); + $max_perf = $max_allowed_packet - length($store_performance_prefix); + } while (my ($current_symb,$param_hash) = each(%{$student_data})) { # # make sure the symb is set up properly @@ -1120,10 +1191,39 @@ sub store_student_data { $symb_id,$student_id, $parameter)."',". $dbh->quote($value)."),\n"; - $num_parameters ++; if ($sql_parameter !~ /''/) { - $store_parameters_command .= $sql_parameter; + if ($max_param) { + my $length = length($sql_parameter); + if ($length > $max_param) { + &Apache::lonnet::logthis("SQL parameter insert for student: $sname for parameter: $parameter would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id"); + &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + if ($parameter =~ /\.submission$/) { + $sql_parameter = "('".join("','", + $symb_id,$student_id, + $parameter)."',". + $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),\n"; + $length = length($sql_parameter); + &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission"); + &Apache::lonnet::logthis("See grading interface for the actual submission"); + } else { + &Apache::lonnet::logthis("Skipping this item"); + next; + } + } + if ($length + $curr_params_length > $max_param) { + push(@store_parameters_values,$curr_params_values); + $curr_params_values = $sql_parameter; + $curr_params_length = $length; + } else { + $curr_params_values .= $sql_parameter; + $curr_params_length += $length; + } + } else { + $curr_params_values .= $sql_parameter; + } #$rows_stored++; + $num_parameters ++; } } } @@ -1165,31 +1265,66 @@ sub store_student_data { "('".join("','",$symb_id,$student_id,$part_id,$part, $solved,$tries,$awarded,$award, $awarddetail,$timestamp)."'),\n"; - $store_performance_command .= $sql_performance; + if ($max_perf) { + my $length = length($sql_performance); + if ($length > $max_perf) { + &Apache::lonnet::logthis("SQL performance insert for student: $sname would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id"); + &Apache::lonnet::logthis("Skipping this item. You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + next; + } else { + if ($length + $curr_perf_length > $max_perf) { + push(@store_performance_values,$curr_perf_values); + $curr_perf_values = $sql_performance; + $curr_perf_length = $length; + } else { + $curr_perf_values .= $sql_performance; + $curr_perf_length += $length; + } + } + } else { + $curr_perf_values .= $sql_performance; + } $rows_stored++; } } + if ($curr_params_values ne '') { + push(@store_parameters_values,$curr_params_values); + } + if ($curr_perf_values ne '') { + push(@store_performance_values,$curr_perf_values); + } if (! $rows_stored) { return ($returnstatus, undef); } - $store_parameters_command =~ s|,\n$||; - $store_performance_command =~ s|,\n$||; my $start = Time::HiRes::time; - $dbh->do($store_performance_command); - if ($dbh->err()) { - &Apache::lonnet::logthis('performance bigass insert error:'. - $dbh->errstr()); - &Apache::lonnet::logthis('command = '.$/.$store_performance_command); - $returnstatus = 'error: unable to insert performance into database'; - return ($returnstatus,$student_data); + foreach my $item (@store_performance_values) { + $item =~ s|,\n$||; + if ($item ne '') { + $dbh->do($store_performance_prefix.$item); + if ($dbh->err()) { + &Apache::lonnet::logthis('performance insert error:'. + $dbh->errstr()); + &Apache::lonnet::logthis('command = '.$/.$store_performance_prefix.$item); + $returnstatus = 'error: unable to insert performance into database'; + return ($returnstatus,$student_data); + } + } } - $dbh->do($store_parameters_command) if ($num_parameters>0); - if ($dbh->err()) { - &Apache::lonnet::logthis('parameters bigass insert error:'. - $dbh->errstr()); - &Apache::lonnet::logthis('command = '.$/.$store_parameters_command); - &Apache::lonnet::logthis('rows_stored = '.$rows_stored); - &Apache::lonnet::logthis('student_id = '.$student_id); - $returnstatus = 'error: unable to insert parameters into database'; - return ($returnstatus,$student_data); + if ($num_parameters > 0) { + foreach my $item (@store_parameters_values) { + $item =~ s|,\n$||; + if ($item ne '') { + $dbh->do($store_parameters_prefix.$item); + if ($dbh->err()) { + &Apache::lonnet::logthis('parameters insert error:'. + $dbh->errstr()); + &Apache::lonnet::logthis('command = '.$/.$store_parameters_prefix.$item); + &Apache::lonnet::logthis('rows_stored = '.$rows_stored); + &Apache::lonnet::logthis('student_id = '.$student_id); + $returnstatus = 'error: unable to insert parameters into database'; + return ($returnstatus,$student_data); + } + } + } } $elapsed += Time::HiRes::time - $start; return ($returnstatus,$student_data); @@ -1251,6 +1386,7 @@ sub ensure_current_data { } my $student_id = &get_student_id($sname,$sdom); + &get_students_groupids($student_id); my @Result = &Apache::lonmysql::get_rows($student_table, "student_id ='$student_id'"); my $data = undef; @@ -1277,6 +1413,7 @@ sub ensure_current_full_data { ($sdom,$sname,$courseid.'.db',$getuserdir); # my $student_id = &get_student_id($sname,$sdom); + &get_students_groupids($student_id); my @Result = &Apache::lonmysql::get_rows($student_table, "student_id ='$student_id'"); my $updatetime; @@ -1289,6 +1426,196 @@ sub ensure_current_full_data { return $status; } +sub ensure_current_groups { + my ($courseid) = @_; + my ($cdom,$cnum); + if (defined($courseid)) { + my %coursehash = &Apache::lonnet::coursedescription($courseid); + $cdom = $coursehash{'domain'}; + $cnum = $coursehash{'num'}; + } elsif ($env{'request.course.id'}) { + $courseid = $env{'request.course.id'}; + $cdom = $env{'course.'.$courseid.'.domain'}; + $cnum = $env{'course.'.$courseid.'.num'}; + } + if ($cdom eq '' || $cnum eq '') { + return 'error: invalid course'; + } + &setup_table_names($courseid); + my @CurrentTables = &Apache::lonmysql::tables_in_db(); + unless (grep(/^\Q$groupnames_table\E$/,@CurrentTables)) { + return; + } + # Get the update time for the groupnames table + my $getuserdir = 1; + my $modifiedtime = &Apache::lonnet::GetFileTimestamp + ($cdom,$cnum,'coursegroups.db',$getuserdir); + my %tableinfo = &Apache::lonmysql::table_information($groupnames_table); + my $updatetime; + if ($tableinfo{'Update_time'}) { + $updatetime = $tableinfo{'Update_time'}; + } + if (! defined($updatetime) || $modifiedtime > $updatetime) { + my (%groups_in_sql,%removegroups,$addgroup); + my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum); + my @Result = &Apache::lonmysql::get_rows($groupnames_table); + foreach my $row (@Result) { + my ($id,$name) = @{$row}; + unless (exists($curr_groups{$name})) { + $groups_in_sql{$name}=$id; + } elsif ($id) { + $removegroups{$id} = $name; + } + } + foreach my $group (keys(%curr_groups)) { + unless (exists($groups_in_sql{$group})) { + $addgroup = 1; + last; + } + } + if (keys(%removegroups)) { + my $dbh = &Apache::lonmysql::get_dbh(); + foreach my $group_id (keys(%removegroups)) { + my $command = 'DELETE FROM '.$groupnames_table.' WHERE group_id='. + $group_id; + $dbh->do($command); + if ($dbh->err()) { + &Apache::lonnet::logthis("error ".$dbh->errstr(). + " occurred executing \n". + "SQL command: $command"); + } + } + } + if ($addgroup) { + &populate_groupnames_table($courseid); + } + } + return; +} + +sub ensure_current_students_groups { + my ($courseid) = @_; + my ($cdom,$cnum); + if (defined($courseid)) { + my %coursehash = &Apache::lonnet::coursedescription($courseid); + $cdom = $coursehash{'domain'}; + $cnum = $coursehash{'num'}; + } elsif ($env{'request.course.id'}) { + $courseid = $env{'request.course.id'}; + $cdom = $env{'course.'.$courseid.'.domain'}; + $cnum = $env{'course.'.$courseid.'.num'}; + } + &setup_table_names($courseid); + my @CurrentTables = &Apache::lonmysql::tables_in_db(); + unless (grep(/^\Q$students_groups_table\E$/,@CurrentTables)) { + return; + } + # Get the update time for the groupnames table + my $getuserdir = 1; + my $modifiedtime = &Apache::lonnet::GetFileTimestamp + ($cdom,$cnum,'groupmembership.db',$getuserdir); + my %tableinfo = &Apache::lonmysql::table_information($students_groups_table); + my $updatetime; + if ($tableinfo{'Update_time'}) { + $updatetime = $tableinfo{'Update_time'}; + } + if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) { + if (&Apache::lonmysql::drop_table($students_groups_table)) { + if (&init_dbs($courseid)) { + return "error creating $students_groups_table\n"; + } else { + &populate_students_groups_table($courseid); + } + } + } + return; +} + +sub ensure_current_sections { + my ($courseid) = @_; + my ($cdom,$cnum); + if (defined($courseid)) { + my %coursehash = &Apache::lonnet::coursedescription($courseid); + $cdom = $coursehash{'domain'}; + $cnum = $coursehash{'num'}; + } elsif ($env{'request.course.id'}) { + $courseid = $env{'request.course.id'}; + $cdom = $env{'course.'.$courseid.'.domain'}; + $cnum = $env{'course.'.$courseid.'.num'}; + } + &setup_table_names($courseid); + my @CurrentTables = &Apache::lonmysql::tables_in_db(); + unless (grep(/^\Q$student_table\E$/,@CurrentTables)) { + return; + } + # Get the update time for the student table + my $getuserdir = 1; + my $modifiedtime = &Apache::lonnet::GetFileTimestamp + ($cdom,$cnum,'classlist.db',$getuserdir); + my %tableinfo = &Apache::lonmysql::table_information($student_table); + my $updatetime; + if ($tableinfo{'Update_time'}) { + $updatetime = $tableinfo{'Update_time'}; + } + if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) { + &update_student_table($cdom,$cnum); + } + return; +} + +sub update_student_table { + my ($cdom,$cnum) = @_; + return unless (($cdom ne '') && ($cnum ne '')); + my (%roster,%sqldata); + my $classlist = &get_classlist($cdom,$cnum); + while (my ($student,$data) = each (%$classlist)) { + my ($section,$start,$end) = ($data->[&CL_SECTION()], + $data->[&CL_START()], + $data->[&CL_END()]); + if ($section eq '' || $section =~ /^\s*$/) { + $section = 'none'; + } + if ($start eq '') { $start = 0; } + if ($end eq '') { $end = 0; } + $roster{$student}{'section'} = $section; + $roster{$student}{'start'} = $start; + $roster{$student}{'end'} = $end; + } + my $dbh = &Apache::lonmysql::get_dbh(); + my $statement = "SELECT student_id,student,section,start,end FROM $student_table"; + my $sth = $dbh->prepare($statement); + $sth->execute(); + if ($sth->err()) { + &Apache::lonnet::logthis("Unable to execute MySQL request:"); + &Apache::lonnet::logthis("\n".$statement."\n"); + &Apache::lonnet::logthis("error is:".$sth->errstr()); + return undef; + } + foreach my $row (@{$sth->fetchall_arrayref}) { + my ($id,$student,$section,$start,$end) = (@$row); + if (ref($roster{$student}) eq 'HASH') { + if (($roster{$student}{'section'} ne $section) || + ($roster{$student}{'start'} ne $start) || + ($roster{$student}{'end'} ne $end)) { + $sqldata{$id} = { + section => $roster{$student}{'section'}, + start => $roster{$student}{'start'}, + end => $roster{$student}{'end'}, + }; + } + } + } + $sth->finish(); + if (keys(%sqldata)) { + foreach my $id (sort { $a <=> $b } keys(%sqldata)) { + my $request = "UPDATE $student_table SET section='$sqldata{$id}{section}'". + ", start='$sqldata{$id}{start}'". + ", end='$sqldata{$id}{end}' WHERE student_id='$id'"; + $dbh->do($request); + } + } + return; +} sub get_student_data_from_performance_cache { my ($sname,$sdom,$symb,$courseid)=@_; @@ -1485,22 +1812,13 @@ sub get_problem_statistics { # $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table. ' WHERE awarded>0'; - if (defined($time_requirements)) { - $request .= ' AND '.$time_requirements; - } my ($max,$min) = &execute_SQL_request($dbh,$request); # $request = 'SELECT SUM(awarded) FROM '.$stats_table; - if (defined($time_requirements)) { - $request .= ' AND '.$time_requirements; - } my ($Solved) = &execute_SQL_request($dbh,$request); # $request = 'SELECT SUM(awarded) FROM '.$stats_table. " WHERE solved='correct_by_override'"; - if (defined($time_requirements)) { - $request .= ' AND '.$time_requirements; - } my ($solved) = &execute_SQL_request($dbh,$request); # $Solved -= $solved; @@ -1741,7 +2059,7 @@ sub rank_students_by_scores_on_resources $limits =~ s/( AND )$//; # Remove extra conjunction $request .= "WHERE $limits"; } - $request .= " $award_clause GROUP BY a.student_id ORDER BY score"; + $request .= " $award_clause GROUP BY a.student_id ORDER BY score, b.student"; #&Apache::lonnet::logthis('request = '.$/.$request); my $sth = $dbh->prepare($request) or die "Can't prepare $request"; $sth->execute(); @@ -2291,12 +2609,16 @@ sub CL_FULLNAME { return 6; } sub CL_STATUS { return 7; } sub CL_TYPE { return 8; } sub CL_LOCKEDTYPE { return 9; } -sub CL_GROUP { return 10; } -sub CL_PERMANENTEMAIL { return 11; } -sub CL_ROLE { return 12; } -sub CL_EXTENT { return 13; } -sub CL_PHOTO { return 14; } -sub CL_THUMBNAIL { return 15; } +sub CL_CREDITS { return 10; } +sub CL_INSTSEC { return 11; } +sub CL_GROUP { return 12; } +sub CL_PERMANENTEMAIL { return 13; } +sub CL_ROLE { return 14; } +sub CL_EXTENT { return 15; } +sub CL_PHOTO { return 16; } +sub CL_THUMBNAIL { return 17; } +sub CL_AUTHORQUOTA { return 18; } +sub CL_AUTHORUSAGE { return 19; } sub get_classlist { my ($cdom,$cnum) = @_; @@ -2316,9 +2638,9 @@ sub get_classlist { } my ($sname,$sdom) = split(/:/,$student); my @Values = split(/:/,$info); - my ($end,$start,$id,$section,$fullname,$type,$lockedtype); + my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec); if (@Values > 2) { - ($end,$start,$id,$section,$fullname,$type,$lockedtype) = @Values; + ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec) = @Values; } else { # We have to get the data ourselves ($end,$start) = @Values; $section = &Apache::lonnet::getsection($sdom,$sname,$cid); @@ -2357,11 +2679,13 @@ sub get_classlist { $status='Future'; } $classlist{$student} = - [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,$lockedtype]; + [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type, + $lockedtype,$credits,$instsec]; } if (wantarray()) { return (\%classlist,['domain','username','end','start','id', - 'section','fullname','status','type','lockedtype']); + 'section','fullname','status','type', + 'lockedtype','credits','instsec']); } else { return \%classlist; } @@ -2982,7 +3306,7 @@ Inputs: $starttime, $endtime, $table Returns: $time_limits -=item C<&limit_by_section_and_status()C< +=item C<&limit_by_section_and_status()C> Build SQL WHERE condition which limits the data collected by section and student status. @@ -3091,7 +3415,8 @@ $env{'course.'.$cid.'.domain'}, and $env Returns a reference to a hash which contains: keys '$sname:$sdom' - values [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,$lockedtype] + values [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type, + $lockedtype,$credits,$instsec] The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used as indices into the returned list to future-proof clients against