--- loncom/interface/loncoursedata.pm 2016/04/25 23:00:58 1.197 +++ loncom/interface/loncoursedata.pm 2021/03/02 19:51:02 1.205 @@ -1,6 +1,6 @@ # The LearningOnline Network with CAPA # -# $Id: loncoursedata.pm,v 1.197 2016/04/25 23:00:58 raeburn Exp $ +# $Id: loncoursedata.pm,v 1.205 2021/03/02 19:51:02 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -1082,6 +1082,9 @@ sub store_updatetime { $dbh->do($request); } +my $requested_max_packet = 0; +my $max_allowed_packet; + sub store_student_data { my ($sname,$sdom,$courseid,$student_data) = @_; # @@ -1102,12 +1105,33 @@ sub store_student_data { my $starttime = Time::HiRes::time; my $elapsed = 0; my $rows_stored; - my $store_parameters_command = 'INSERT IGNORE INTO '.$parameters_table. + my $store_parameters_prefix = 'INSERT IGNORE INTO '.$parameters_table. ' VALUES '."\n"; my $num_parameters = 0; - my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table. + my $store_performance_prefix = 'INSERT IGNORE INTO '.$performance_table. ' VALUES '."\n"; return ('error',undef) if (! defined($dbh)); + unless ($requested_max_packet) { + (undef,$max_allowed_packet) = $dbh->selectrow_array( + qq{show variables LIKE ? }, + undef, + "max_allowed_packet"); + if ($max_allowed_packet !~ /^\d+$/) { + $max_allowed_packet = ''; + } + $requested_max_packet = 1; + } + my @store_parameters_values = (); + my $curr_params_values = ''; + my $curr_params_length = 0; + my @store_performance_values = (); + my $curr_perf_values = ''; + my $curr_perf_length = 0; + my ($max_param,$max_perf); + if ($max_allowed_packet) { + $max_param = $max_allowed_packet - length($store_parameters_prefix); + $max_perf = $max_allowed_packet - length($store_performance_prefix); + } while (my ($current_symb,$param_hash) = each(%{$student_data})) { # # make sure the symb is set up properly @@ -1120,10 +1144,29 @@ sub store_student_data { $symb_id,$student_id, $parameter)."',". $dbh->quote($value)."),\n"; - $num_parameters ++; if ($sql_parameter !~ /''/) { - $store_parameters_command .= $sql_parameter; + if ($max_param) { + my $length = length($sql_parameter); + if ($length > $max_param) { + &Apache::lonnet::logthis("SQL parameter insert for student: $sname for parameter: $parameter would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id"); + &Apache::lonnet::logthis("Skipping this item. You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + next; + } else { + if ($length + $curr_params_length > $max_param) { + push(@store_parameters_values,$curr_params_values); + $curr_params_values = $sql_parameter; + $curr_params_length = $length; + } else { + $curr_params_values .= $sql_parameter; + $curr_params_length += $length; + } + } + } else { + $curr_params_values .= $sql_parameter; + } #$rows_stored++; + $num_parameters ++; } } } @@ -1165,31 +1208,66 @@ sub store_student_data { "('".join("','",$symb_id,$student_id,$part_id,$part, $solved,$tries,$awarded,$award, $awarddetail,$timestamp)."'),\n"; - $store_performance_command .= $sql_performance; + if ($max_perf) { + my $length = length($sql_performance); + if ($length > $max_perf) { + &Apache::lonnet::logthis("SQL performance insert for student: $sname would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id"); + &Apache::lonnet::logthis("Skipping this item. You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + next; + } else { + if ($length + $curr_perf_length > $max_perf) { + push(@store_performance_values,$curr_perf_values); + $curr_perf_values = $sql_performance; + $curr_perf_length = $length; + } else { + $curr_perf_values .= $sql_performance; + $curr_perf_length += $length; + } + } + } else { + $curr_perf_values .= $sql_performance; + } $rows_stored++; } } + if ($curr_params_values ne '') { + push(@store_parameters_values,$curr_params_values); + } + if ($curr_perf_values ne '') { + push(@store_performance_values,$curr_perf_values); + } if (! $rows_stored) { return ($returnstatus, undef); } - $store_parameters_command =~ s|,\n$||; - $store_performance_command =~ s|,\n$||; my $start = Time::HiRes::time; - $dbh->do($store_performance_command); - if ($dbh->err()) { - &Apache::lonnet::logthis('performance bigass insert error:'. - $dbh->errstr()); - &Apache::lonnet::logthis('command = '.$/.$store_performance_command); - $returnstatus = 'error: unable to insert performance into database'; - return ($returnstatus,$student_data); + foreach my $item (@store_performance_values) { + $item =~ s|,\n$||; + if ($item ne '') { + $dbh->do($store_performance_prefix.$item); + if ($dbh->err()) { + &Apache::lonnet::logthis('performance insert error:'. + $dbh->errstr()); + &Apache::lonnet::logthis('command = '.$/.$store_performance_prefix.$item); + $returnstatus = 'error: unable to insert performance into database'; + return ($returnstatus,$student_data); + } + } } - $dbh->do($store_parameters_command) if ($num_parameters>0); - if ($dbh->err()) { - &Apache::lonnet::logthis('parameters bigass insert error:'. - $dbh->errstr()); - &Apache::lonnet::logthis('command = '.$/.$store_parameters_command); - &Apache::lonnet::logthis('rows_stored = '.$rows_stored); - &Apache::lonnet::logthis('student_id = '.$student_id); - $returnstatus = 'error: unable to insert parameters into database'; - return ($returnstatus,$student_data); + if ($num_parameters > 0) { + foreach my $item (@store_parameters_values) { + $item =~ s|,\n$||; + if ($item ne '') { + $dbh->do($store_parameters_prefix.$item); + if ($dbh->err()) { + &Apache::lonnet::logthis('parameters insert error:'. + $dbh->errstr()); + &Apache::lonnet::logthis('command = '.$/.$store_parameters_prefix.$item); + &Apache::lonnet::logthis('rows_stored = '.$rows_stored); + &Apache::lonnet::logthis('student_id = '.$student_id); + $returnstatus = 'error: unable to insert parameters into database'; + return ($returnstatus,$student_data); + } + } + } } $elapsed += Time::HiRes::time - $start; return ($returnstatus,$student_data); @@ -1306,7 +1384,11 @@ sub ensure_current_groups { if ($cdom eq '' || $cnum eq '') { return 'error: invalid course'; } - &ensure_tables_are_set_up($courseid); + &setup_table_names($courseid); + my @CurrentTables = &Apache::lonmysql::tables_in_db(); + unless (grep(/^\Q$groupnames_table\E$/,@CurrentTables)) { + return; + } # Get the update time for the groupnames table my $getuserdir = 1; my $modifiedtime = &Apache::lonnet::GetFileTimestamp @@ -1316,7 +1398,6 @@ sub ensure_current_groups { if ($tableinfo{'Update_time'}) { $updatetime = $tableinfo{'Update_time'}; } - print STDERR "1. updatetime is ||$updatetime|| and modifidtime is ||$modifiedtime||\n"; if (! defined($updatetime) || $modifiedtime > $updatetime) { my (%groups_in_sql,%removegroups,$addgroup); my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum); @@ -1367,7 +1448,11 @@ sub ensure_current_students_groups { $cdom = $env{'course.'.$courseid.'.domain'}; $cnum = $env{'course.'.$courseid.'.num'}; } - &ensure_tables_are_set_up($courseid); + &setup_table_names($courseid); + my @CurrentTables = &Apache::lonmysql::tables_in_db(); + unless (grep(/^\Q$students_groups_table\E$/,@CurrentTables)) { + return; + } # Get the update time for the groupnames table my $getuserdir = 1; my $modifiedtime = &Apache::lonnet::GetFileTimestamp @@ -1377,7 +1462,6 @@ sub ensure_current_students_groups { if ($tableinfo{'Update_time'}) { $updatetime = $tableinfo{'Update_time'}; } - print STDERR "2. updatetime is ||$updatetime|| and modifidtime is ||$modifiedtime||\n"; if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) { if (&Apache::lonmysql::drop_table($students_groups_table)) { if (&init_dbs($courseid)) { @@ -1390,6 +1474,92 @@ sub ensure_current_students_groups { return; } +sub ensure_current_sections { + my ($courseid) = @_; + my ($cdom,$cnum); + if (defined($courseid)) { + my %coursehash = &Apache::lonnet::coursedescription($courseid); + $cdom = $coursehash{'domain'}; + $cnum = $coursehash{'num'}; + } elsif ($env{'request.course.id'}) { + $courseid = $env{'request.course.id'}; + $cdom = $env{'course.'.$courseid.'.domain'}; + $cnum = $env{'course.'.$courseid.'.num'}; + } + &setup_table_names($courseid); + my @CurrentTables = &Apache::lonmysql::tables_in_db(); + unless (grep(/^\Q$student_table\E$/,@CurrentTables)) { + return; + } + # Get the update time for the student table + my $getuserdir = 1; + my $modifiedtime = &Apache::lonnet::GetFileTimestamp + ($cdom,$cnum,'classlist.db',$getuserdir); + my %tableinfo = &Apache::lonmysql::table_information($student_table); + my $updatetime; + if ($tableinfo{'Update_time'}) { + $updatetime = $tableinfo{'Update_time'}; + } + if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) { + &update_student_table($cdom,$cnum); + } + return; +} + +sub update_student_table { + my ($cdom,$cnum) = @_; + return unless (($cdom ne '') && ($cnum ne '')); + my (%roster,%sqldata); + my $classlist = &get_classlist($cdom,$cnum); + while (my ($student,$data) = each (%$classlist)) { + my ($section,$start,$end) = ($data->[&CL_SECTION()], + $data->[&CL_START()], + $data->[&CL_END()]); + if ($section eq '' || $section =~ /^\s*$/) { + $section = 'none'; + } + if ($start eq '') { $start = 0; } + if ($end eq '') { $end = 0; } + $roster{$student}{'section'} = $section; + $roster{$student}{'start'} = $start; + $roster{$student}{'end'} = $end; + } + my $dbh = &Apache::lonmysql::get_dbh(); + my $statement = "SELECT student_id,student,section,start,end FROM $student_table"; + my $sth = $dbh->prepare($statement); + $sth->execute(); + if ($sth->err()) { + &Apache::lonnet::logthis("Unable to execute MySQL request:"); + &Apache::lonnet::logthis("\n".$statement."\n"); + &Apache::lonnet::logthis("error is:".$sth->errstr()); + return undef; + } + foreach my $row (@{$sth->fetchall_arrayref}) { + my ($id,$student,$section,$start,$end) = (@$row); + if (ref($roster{$student}) eq 'HASH') { + if (($roster{$student}{'section'} ne $section) || + ($roster{$student}{'start'} ne $start) || + ($roster{$student}{'end'} ne $end)) { + $sqldata{$id} = { + section => $roster{$student}{'section'}, + start => $roster{$student}{'start'}, + end => $roster{$student}{'end'}, + }; + } + } + } + $sth->finish(); + if (keys(%sqldata)) { + foreach my $id (sort { $a <=> $b } keys(%sqldata)) { + my $request = "UPDATE $student_table SET section='$sqldata{$id}{section}'". + ", start='$sqldata{$id}{start}'". + ", end='$sqldata{$id}{end}' WHERE student_id='$id'"; + $dbh->do($request); + } + } + return; +} + sub get_student_data_from_performance_cache { my ($sname,$sdom,$symb,$courseid)=@_; my $student = $sname.':'.$sdom if (defined($sname) && defined($sdom)); @@ -1585,22 +1755,13 @@ sub get_problem_statistics { # $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table. ' WHERE awarded>0'; - if (defined($time_requirements)) { - $request .= ' AND '.$time_requirements; - } my ($max,$min) = &execute_SQL_request($dbh,$request); # $request = 'SELECT SUM(awarded) FROM '.$stats_table; - if (defined($time_requirements)) { - $request .= ' AND '.$time_requirements; - } my ($Solved) = &execute_SQL_request($dbh,$request); # $request = 'SELECT SUM(awarded) FROM '.$stats_table. " WHERE solved='correct_by_override'"; - if (defined($time_requirements)) { - $request .= ' AND '.$time_requirements; - } my ($solved) = &execute_SQL_request($dbh,$request); # $Solved -= $solved; @@ -1683,7 +1844,7 @@ sub populate_weight_table { my @resources; foreach my $seq (@sequences) { push(@resources,$navmap->retrieveResources($seq, - sub {shift->is_problem();}, + sub {shift->is_gradable();}, 0,0,0)); } if (! scalar(@resources)) { @@ -1841,7 +2002,7 @@ sub rank_students_by_scores_on_resources $limits =~ s/( AND )$//; # Remove extra conjunction $request .= "WHERE $limits"; } - $request .= " $award_clause GROUP BY a.student_id ORDER BY score"; + $request .= " $award_clause GROUP BY a.student_id ORDER BY score, b.student"; #&Apache::lonnet::logthis('request = '.$/.$request); my $sth = $dbh->prepare($request) or die "Can't prepare $request"; $sth->execute(); @@ -2392,14 +2553,15 @@ sub CL_STATUS { return 7; } sub CL_TYPE { return 8; } sub CL_LOCKEDTYPE { return 9; } sub CL_CREDITS { return 10; } -sub CL_GROUP { return 11; } -sub CL_PERMANENTEMAIL { return 12; } -sub CL_ROLE { return 13; } -sub CL_EXTENT { return 14; } -sub CL_PHOTO { return 15; } -sub CL_THUMBNAIL { return 16; } -sub CL_AUTHORQUOTA { return 17; } -sub CL_AUTHORUSAGE { return 18; } +sub CL_INSTSEC { return 11; } +sub CL_GROUP { return 12; } +sub CL_PERMANENTEMAIL { return 13; } +sub CL_ROLE { return 14; } +sub CL_EXTENT { return 15; } +sub CL_PHOTO { return 16; } +sub CL_THUMBNAIL { return 17; } +sub CL_AUTHORQUOTA { return 18; } +sub CL_AUTHORUSAGE { return 19; } sub get_classlist { my ($cdom,$cnum) = @_; @@ -2419,9 +2581,9 @@ sub get_classlist { } my ($sname,$sdom) = split(/:/,$student); my @Values = split(/:/,$info); - my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits); + my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec); if (@Values > 2) { - ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits) = @Values; + ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec) = @Values; } else { # We have to get the data ourselves ($end,$start) = @Values; $section = &Apache::lonnet::getsection($sdom,$sname,$cid); @@ -2461,12 +2623,12 @@ sub get_classlist { } $classlist{$student} = [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type, - $lockedtype,$credits]; + $lockedtype,$credits,$instsec]; } if (wantarray()) { return (\%classlist,['domain','username','end','start','id', 'section','fullname','status','type', - 'lockedtype','credits']); + 'lockedtype','credits','instsec']); } else { return \%classlist; } @@ -3197,7 +3359,7 @@ $env{'course.'.$cid.'.domain'}, and $env Returns a reference to a hash which contains: keys '$sname:$sdom' values [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type, - $lockedtype,$credits] + $lockedtype,$credits,$instsec] The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used as indices into the returned list to future-proof clients against