--- loncom/interface/loncoursedata.pm 2002/08/28 21:50:27 1.22 +++ loncom/interface/loncoursedata.pm 2002/10/16 13:43:34 1.36 @@ -1,7 +1,7 @@ # The LearningOnline Network with CAPA # (Publication Handler # -# $Id: loncoursedata.pm,v 1.22 2002/08/28 21:50:27 stredwic Exp $ +# $Id: loncoursedata.pm,v 1.36 2002/10/16 13:43:34 matthew Exp $ # # Copyright Michigan State University Board of Trustees # @@ -89,7 +89,7 @@ Input: $courseID, $lastDownloadTime, $c $courseID: The id of the course $lastDownloadTime: This is the date stamp for when this information was -last gathered. If it is set to 'Not downloaded', it will gather the data +last gathered. If it is set to Not downloaded, it will gather the data again, though it currently does not remove the old data. $c: The connection class that can determine if the browser has aborted. It @@ -103,7 +103,7 @@ Output: \%classlist -A list of student name:domain (as keys) (known below as $name) -A hash pointer for each student containing lastname, generation, firstname, -middlename, and PID : Key is $name.'studentInformation' +middlename, and PID : Key is $name.studentInformation -A hash pointer to each students section data : Key is $name.section @@ -125,7 +125,7 @@ sub DownloadClasslist { $Apache::lonnet::perlvar{'lonUsersDir'}); # Always download the information if lastDownloadTime is set to - # 'Not downloaded', otherwise it is only downloaded if the file + # Not downloaded, otherwise it is only downloaded if the file # has been updated and has a more recent date stamp if($lastDownloadTime ne 'Not downloaded' && $lastDownloadTime >= $modifiedTime && $modifiedTime >= 0) { @@ -139,7 +139,7 @@ sub DownloadClasslist { %classlist=&Apache::lonnet::dump('classlist',$courseDomain, $courseNumber); foreach(keys (%classlist)) { if(/^(con_lost|error|no_such_host)/i) { - return \%classlist; + return; } } @@ -195,14 +195,14 @@ $namedata: student name:domain $courseID: The id of the course $lastDownloadTime: This is the date stamp for when this information was -last gathered. If it is set to 'Not downloaded', it will gather the data +last gathered. If it is set to Not downloaded, it will gather the data again, though it currently does not remove the old data. $WhatIWant: Regular expression used to get selected data with dump Output: \%courseData -\%courseData: A hash pointer to the raw data from the student's course +\%courseData: A hash pointer to the raw data from the students course database. =back @@ -254,7 +254,7 @@ sub DownloadCourseInformation { These functions process all the data for all the students. Also, they are the functions that access the cache database for writing the majority of the time. The downloading and caching were separated to reduce problems -with stopping downloading then can't tie hash to database later. +with stopping downloading then can not tie hash to database later. =cut @@ -309,17 +309,28 @@ sub ProcessTopResourceMap { return 'Can not open Coursemap.'; } + my $oldkeys; + if(defined($cache->{'ResourceKeys'})) { + $oldkeys = $cache->{'ResourceKeys'}; + foreach (split(':::', $cache->{'ResourceKeys'})) { + delete $cache->{$_}; + } + delete $cache->{'ResourceKeys'}; + } + # Initialize state machine. Set information pointing to top level map. my (@sequences, @currentResource, @finishResource); my ($currentSequence, $currentResourceID, $lastResourceID); - $currentResourceID=$hash{'ids_/res/'.$ENV{'request.course.uri'}}; + $currentResourceID=$hash{'ids_'. + &Apache::lonnet::clutter($ENV{'request.course.uri'})}; push(@currentResource, $currentResourceID); $lastResourceID=-1; $currentSequence=-1; my $topLevelSequenceNumber = $currentSequence; my %sequenceRecord; + my %allkeys; while(1) { if($c->aborted()) { last; @@ -344,6 +355,7 @@ sub ProcessTopResourceMap { } else { $cache->{'orderedSequences'}.=':'.$currentSequence; } + $allkeys{'orderedSequences'}++; $lastResourceID=$hash{'map_finish_'. $hash{'src_'.$currentResourceID}}; @@ -378,20 +390,24 @@ sub ProcessTopResourceMap { $currentResourceID})); $cache->{$currentResourceID.':problem'}=$Problem; + $allkeys{$currentResourceID.':problem'}++; if(!defined($cache->{$currentSequence.':problems'})) { $cache->{$currentSequence.':problems'}=$currentResourceID; } else { $cache->{$currentSequence.':problems'}.= ':'.$currentResourceID; } + $allkeys{$currentSequence.':problems'}++; my $meta=$hash{'src_'.$currentResourceID}; # $cache->{$currentResourceID.':title'}= # &Apache::lonnet::metdata($meta,'title'); $cache->{$currentResourceID.':title'}= $hash{'title_'.$currentResourceID}; + $allkeys{$currentResourceID.':title'}++; $cache->{$currentResourceID.':source'}= $hash{'src_'.$currentResourceID}; + $allkeys{$currentResourceID.':source'}++; # Get Parts for problem my %beenHere; @@ -408,6 +424,8 @@ sub ProcessTopResourceMap { $cache->{$currentSequence.':'.$currentResourceID. ':parts'}.=':'.$partId; } + $allkeys{$currentSequence.':'.$currentResourceID. + ':parts'}++; } if($beenHere{'r:'.$partId.':'.$responseId} == 0) { $beenHere{'r:'.$partId.':'.$responseId}++; @@ -421,6 +439,8 @@ sub ProcessTopResourceMap { ':'.$partId.':responseIDs'}.=':'. $responseId; } + $allkeys{$currentSequence.':'.$currentResourceID.':'. + $partId.':responseIDs'}++; } if(/^optionresponse/ && $beenHere{'o:'.$partId.':'.$currentResourceID} == 0) { @@ -434,6 +454,7 @@ sub ProcessTopResourceMap { $currentResourceID.':'. $partId.':'.$responseId; } + $allkeys{'OptionResponses'}++; } } } @@ -449,8 +470,10 @@ sub ProcessTopResourceMap { # Capture sequence information here $cache->{$currentSequence.':title'}= $hash{'title_'.$currentResourceID}; + $allkeys{$currentSequence.':title'}++; $cache->{$currentSequence.':source'}= $hash{'src_'.$currentResourceID}; + $allkeys{$currentSequence.':source'}++; my $totalProblems=0; foreach my $currentProblem (split(/\:/, @@ -474,6 +497,7 @@ sub ProcessTopResourceMap { $cache->{$currentSequence.':columnWidth'}= (scalar @titleLength); } + $allkeys{$currentSequence.':columnWidth'}++; } else { # Remove sequence from list, if it contains no problems to # display. @@ -510,6 +534,15 @@ sub ProcessTopResourceMap { $currentResourceID=pop(@currentResource); } + my @theKeys = keys(%allkeys); + my $newkeys = join(':::', @theKeys); + $cache->{'ResourceKeys'} = join(':::', $newkeys); + if($newkeys ne $oldkeys) { + $cache->{'ResourceUpdated'} = 'true'; + } else { + $cache->{'ResourceUpdated'} = 'false'; + } + unless (untie(%hash)) { &Apache::lonnet::logthis("WARNING: ". "Could not untie coursemap $fn (browse)". @@ -540,7 +573,7 @@ $cache: A hash pointer to store the data $classlist: The hash of data collected about a student from &DownloadClasslist(). The hash contains a list of students, a pointer -to a hash of student information for each student, and each student's section +to a hash of student information for each student, and each students section number. $courseID: The course ID @@ -553,7 +586,24 @@ browser Output: @names @names: An array of students whose information has been processed, and are to -be considered in an arbitrary order. +be considered in an arbitrary order. The entries in @names are of the form +username:domain. + +The values in $cache are as follows: + + *NOTE: for the following $name implies username:domain + $name.':error' only defined if an error occured. Value + contains the error message + $name.':lastDownloadTime' unconverted time of the last update of a + student\'s course data + $name.'updateTime' coverted time of the last update of a + student\'s course data + $name.':username' username of a student + $name.':domain' domain of a student + $name.':fullname' full name of a student + $name.':id' PID of a student + $name.':Status' active/expired status of a student + $name.':section' section of a student =back @@ -576,9 +626,8 @@ sub ProcessClasslist { if($c->aborted()) { return (); } - my $studentInformation = $classlist->{$name.':studentInformation'}, - my $sectionData = $classlist->{$name.':sections'}, - my $date = $classlist->{$name}, + my $studentInformation = $classlist->{$name.':studentInformation'}; + my $date = $classlist->{$name}; my ($studentName,$studentDomain) = split(/\:/,$name); $cache->{$name.':username'}=$studentName; @@ -613,6 +662,7 @@ sub ProcessClasslist { $courseID=~s/^(\w)/\/$1/; my $sec=''; + my $sectionData = $classlist->{$name.':sections'}; foreach my $key (keys (%$sectionData)) { my $value = $sectionData->{$key}; if ($key=~/^$courseID(?:\/)*(\w+)*\_st$/) { @@ -620,7 +670,7 @@ sub ProcessClasslist { if($key eq $courseID.'_st') { $tempsection=''; } - my ($dummy,$roleend,$rolestart)=split(/\_/,$value); + my (undef,$roleend,$rolestart)=split(/\_/,$value); if($roleend eq $end && $rolestart eq $start) { $sec = $tempsection; last; @@ -661,7 +711,7 @@ Takes the course data downloaded for a s &DownloadCourseInformation() and breaks it up into key value pairs to be stored in the cached data. The keys are comprised of the $username:$domain:$keyFromCourseDatabase. The student username:domain is -stored away signifying that the student's information has been downloaded and +stored away signifying that the students information has been downloaded and can be reused from cached data. =over 4 @@ -693,11 +743,26 @@ sub ProcessStudentData { return; } + # This little delete thing, should not be here. Move some other + # time though. + if(defined($cache->{$name.':keys'})) { + foreach (split(':::', $cache->{$name.':keys'})) { + delete $cache->{$name.':'.$_}; + } + delete $cache->{$name.':keys'}; + } + + my %courseKeys; # user name:domain was prepended earlier in DownloadCourseInformation foreach (keys %$courseData) { + my $currentKey = $_; + $currentKey =~ s/^$name//; + $courseKeys{$currentKey}++; $cache->{$_}=$courseData->{$_}; } + $cache->{$name.':keys'} = join(':::', keys(%courseKeys)); + return; } @@ -753,6 +818,16 @@ sub ExtractStudentData { return; } + # This little delete thing, should not be here. Move some other + # time though. + my %allkeys; + if(defined($output->{$name.':keys'})) { + foreach (split(':::', $output->{$name.':keys'})) { + delete $output->{$name.':'.$_}; + } + delete $output->{$name.':keys'}; + } + my ($username,$domain)=split(':',$name); my $Version; @@ -768,6 +843,8 @@ sub ExtractStudentData { # Output dashes for all the parts of this problem if there # is no version information about the current problem. + $output->{$name.':'.$problemID.':NoVersion'} = 'false'; + $allkeys{$name.':'.$problemID.':NoVersion'}++; if(!$LatestVersion) { foreach my $part (split(/\:/,$data->{$sequence.':'. $problemID. @@ -775,6 +852,9 @@ sub ExtractStudentData { $output->{$name.':'.$problemID.':'.$part.':tries'} = 0; $output->{$name.':'.$problemID.':'.$part.':awarded'} = 0; $output->{$name.':'.$problemID.':'.$part.':code'} = ' '; + $allkeys{$name.':'.$problemID.':'.$part.':tries'}++; + $allkeys{$name.':'.$problemID.':'.$part.':awarded'}++; + $allkeys{$name.':'.$problemID.':'.$part.':code'}++; $totalProblems++; } $output->{$name.':'.$problemID.':NoVersion'} = 'true'; @@ -860,6 +940,7 @@ sub ExtractStudentData { ':parts'})) { $output->{$name.':'.$problemID.':'.$part.':wrong'} = $partData{$part.':tries'}; + $allkeys{$name.':'.$problemID.':'.$part.':wrong'}++; if($partData{$part.':code'} eq '*') { $output->{$name.':'.$problemID.':'.$part.':wrong'}--; @@ -875,14 +956,22 @@ sub ExtractStudentData { $partData{$part.':code'}; $output->{$name.':'.$problemID.':'.$part.':awarded'} = $partData{$part.':awarded'}; + $allkeys{$name.':'.$problemID.':'.$part.':tries'}++; + $allkeys{$name.':'.$problemID.':'.$part.':code'}++; + $allkeys{$name.':'.$problemID.':'.$part.':awarded'}++; + $totalAwarded += $partData{$part.':awarded'}; $output->{$name.':'.$problemID.':'.$part.':timestamp'} = $partData{$part.':timestamp'}; + $allkeys{$name.':'.$problemID.':'.$part.':timestamp'}++; + foreach my $response (split(':', $data->{$sequence.':'. $problemID.':'. $part.':responseIDs'})) { $output->{$name.':'.$problemID.':'.$part.':'.$response. ':submission'}=join(':::',@submissions); + $allkeys{$name.':'.$problemID.':'.$part.':'.$response. + ':submission'}++; } if($partData{$part.':code'} ne 'x') { @@ -892,6 +981,7 @@ sub ExtractStudentData { } $output->{$name.':'.$sequence.':problemsCorrect'} = $problemsCorrect; + $allkeys{$name.':'.$sequence.':problemsCorrect'}++; $problemsSolved += $problemsCorrect; $problemsCorrect=0; } @@ -899,6 +989,11 @@ sub ExtractStudentData { $output->{$name.':problemsSolved'} = $problemsSolved; $output->{$name.':totalProblems'} = $totalProblems; $output->{$name.':totalAwarded'} = $totalAwarded; + $allkeys{$name.':problemsSolved'}++; + $allkeys{$name.':totalProblems'}++; + $allkeys{$name.':totalAwarded'}++; + + $output->{$name.':keys'} = join(':::', keys(%allkeys)); return; } @@ -989,37 +1084,22 @@ sub ProcessFullName { my ($lastname, $generation, $firstname, $middlename)=@_; my $Str = ''; + # Strip whitespace preceeding & following name components. + $lastname =~ s/(\s+$|^\s+)//g; + $generation =~ s/(\s+$|^\s+)//g; + $firstname =~ s/(\s+$|^\s+)//g; + $middlename =~ s/(\s+$|^\s+)//g; + if($lastname ne '') { - $Str .= $lastname.' '; - if($generation ne '') { - $Str .= $generation; - } else { - chop($Str); - } - $Str .= ', '; - if($firstname ne '') { - $Str .= $firstname.' '; - } - if($middlename ne '') { - $Str .= $middlename; - } else { - chop($Str); - if($firstname eq '') { - chop($Str); - } - } + $Str .= $lastname; + $Str .= ' '.$generation if ($generation ne ''); + $Str .= ','; + $Str .= ' '.$firstname if ($firstname ne ''); + $Str .= ' '.$middlename if ($middlename ne ''); } else { - if($firstname ne '') { - $Str .= $firstname.' '; - } - if($middlename ne '') { - $Str .= $middlename.' '; - } - if($generation ne '') { - $Str .= $generation; - } else { - chop($Str); - } + $Str .= $firstname if ($firstname ne ''); + $Str .= ' '.$middlename if ($middlename ne ''); + $Str .= ' '.$generation if ($generation ne ''); } return $Str; @@ -1043,7 +1123,7 @@ $ChartDB: The name of the cache database Output: -1, 0, 1 --1: Couldn't tie database +-1: Could not tie database 0: Use cached data 1: New cache database created, use that. @@ -1095,8 +1175,6 @@ sub DownloadStudentCourseData { my $title = 'LON-CAPA Statistics'; my $heading = 'Download and Process Course Data'; my $studentCount = scalar(@$students); - my %cache; - my $WhatIWant; $WhatIWant = '(^version:|'; @@ -1113,6 +1191,8 @@ sub DownloadStudentCourseData { my $displayString; my $count=0; foreach (@$students) { + my %cache; + if($c->aborted()) { return 'Aborted'; } if($status eq 'true') { @@ -1122,35 +1202,38 @@ sub DownloadStudentCourseData { } my $downloadTime='Not downloaded'; + my $needUpdate = 'false'; if($checkDate eq 'true' && tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) { $downloadTime = $cache{$_.':lastDownloadTime'}; + $needUpdate = $cache{'ResourceUpdated'}; untie(%cache); } if($c->aborted()) { return 'Aborted'; } - if($downloadTime eq 'Not downloaded') { - my $courseData = - &DownloadCourseInformation($_, $courseID, $downloadTime, - $WhatIWant); - if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) { - foreach my $key (keys(%$courseData)) { - if($key =~ /^(con_lost|error|no_such_host)/i) { - $courseData->{$_.':error'} = 'No course data for '.$_; - last; - } - } - if($extract eq 'true') { - &ExtractStudentData($courseData, \%cache, \%cache, $_); - } else { - &ProcessStudentData(\%cache, $courseData, $_); - } - untie(%cache); - } else { - next; - } - } + if($needUpdate eq 'true') { + $downloadTime = 'Not downloaded'; + } + my $courseData = + &DownloadCourseInformation($_, $courseID, $downloadTime, + $WhatIWant); + if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) { + foreach my $key (keys(%$courseData)) { + if($key =~ /^(con_lost|error|no_such_host)/i) { + $courseData->{$_.':error'} = 'No course data for '.$_; + last; + } + } + if($extract eq 'true') { + &ExtractStudentData($courseData, \%cache, \%cache, $_); + } else { + &ProcessStudentData(\%cache, $courseData, $_); + } + untie(%cache); + } else { + next; + } } if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); } @@ -1170,9 +1253,7 @@ sub DownloadStudentCourseDataSeparate { $WhatIWant .= '|timestamp)'; $WhatIWant .= ')'; - &CheckForResidualDownload($courseID, $cacheDB, $students, $c); - - my %cache; + &CheckForResidualDownload($cacheDB, 'true', 'true', $courseID, $r, $c); my $studentCount = scalar(@$students); if($status eq 'true') { @@ -1191,10 +1272,13 @@ sub DownloadStudentCourseDataSeparate { &Apache::lonhtmlcommon::Update_PrgWin($displayString, $r); } + my %cache; my $downloadTime='Not downloaded'; + my $needUpdate = 'false'; if($checkDate eq 'true' && tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) { $downloadTime = $cache{$_.':lastDownloadTime'}; + $needUpdate = $cache{'ResourceUpdated'}; untie(%cache); } @@ -1202,31 +1286,33 @@ sub DownloadStudentCourseDataSeparate { return 'Aborted'; } - if($downloadTime eq 'Not downloaded') { - my $error = 0; - my $courseData = - &DownloadCourseInformation($_, $courseID, $downloadTime, - $WhatIWant); - my %downloadData; - unless(tie(%downloadData,'GDBM_File',$residualFile, - &GDBM_WRCREAT(),0640)) { - return 'Failed to tie temporary download hash.'; - } - foreach my $key (keys(%$courseData)) { - $downloadData{$key} = $courseData->{$key}; - if($key =~ /^(con_lost|error|no_such_host)/i) { - $error = 1; - last; - } + if($needUpdate eq 'true') { + $downloadTime = 'Not downloaded'; + } + + my $error = 0; + my $courseData = + &DownloadCourseInformation($_, $courseID, $downloadTime, + $WhatIWant); + my %downloadData; + unless(tie(%downloadData,'GDBM_File',$residualFile, + &GDBM_WRCREAT(),0640)) { + return 'Failed to tie temporary download hash.'; + } + foreach my $key (keys(%$courseData)) { + $downloadData{$key} = $courseData->{$key}; + if($key =~ /^(con_lost|error|no_such_host)/i) { + $error = 1; + last; } - if($error) { - foreach my $deleteKey (keys(%$courseData)) { - delete $downloadData{$deleteKey}; - } - $downloadData{$_.':error'} = 'No course data for '.$_; + } + if($error) { + foreach my $deleteKey (keys(%$courseData)) { + delete $downloadData{$deleteKey}; } - untie(%downloadData); + $downloadData{$_.':error'} = 'No course data for '.$_; } + untie(%downloadData); } if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); } @@ -1301,7 +1387,93 @@ sub CheckForResidualDownload { return 'OK'; } +################################################ +################################################ + +=pod + +=item &get_classlist(); + +Retrieve the classist of a given class or of the current class. Student +information is returned from the classlist.db file and, if needed, +from the students environment. + +Optional arguments are $cid, $cdom, and $cnum (course id, course domain, +and course number, respectively). Any omitted arguments will be taken +from the current environment ($ENV{'request.course.id'}, +$ENV{'course.'.$cid.'.domain'}, and $ENV{'course.'.$cid.'.num'}). + +Returns a reference to a hash which contains: + keys '$sname:$sdom' + values [$end,$start,$id,$section,$fullname] + +=cut + +################################################ +################################################ + +sub get_classlist { + my ($cid,$cdom,$cnum) = @_; + $cid = $cid || $ENV{'request.course.id'}; + $cdom = $cdom || $ENV{'course.'.$cid.'.domain'}; + $cnum = $cnum || $ENV{'course.'.$cid.'.num'}; + my $now = time; + # + my %classlist=&Apache::lonnet::dump('classlist',$cdom,$cnum); + while (my ($student,$info) = each(%classlist)) { + return undef if ($student =~ /^(con_lost|error|no_such_host)/i); + my ($sname,$sdom) = split(/:/,$student); + my @Values = split(/:/,$info); + my ($end,$start,$id,$section,$fullname); + if (@Values > 2) { + ($end,$start,$id,$section,$fullname) = @Values; + } else { # We have to get the data ourselves + ($end,$start) = @Values; + $section = &Apache::lonnet::usection($sdom,$sname,$cid); + my %info=&Apache::lonnet::get('environment', + ['firstname','middlename', + 'lastname','generation','id'], + $sdom, $sname); + my ($tmp) = keys(%info); + if ($tmp =~/^(con_lost|error|no_such_host)/i) { + $fullname = 'not available'; + $id = 'not available'; + } else { + $fullname = &ProcessFullName(@info{qw/lastname generation + firstname middlename/}); + $id = $info{'id'}; + } + # Update the classlist with this students information + if ($fullname ne 'not available') { + my $enrolldata = join(':',$end,$start,$id,$section,$fullname); + my $reply=&Apache::lonnet::cput('classlist', + {$student => $enrolldata}, + $cdom,$cnum); + if ($reply !~ /^(ok|delayed)/) { + &Apache::lonnet::logthis('Unable to update classlist for '. + 'student '.$sname.':'.$sdom. + ' error:'.$reply); + } + } + } + my $status='Expired'; + if(((!$end) || $now < $end) && ((!$start) || ($now > $start))) { + $status='Active'; + } + $classlist{$student} = + [$sdom,$sname,$end,$start,$id,$section,$fullname,$status]; + } + if (wantarray()) { + return (\%classlist,['domain','username','end','start','id', + 'section','fullname','status']); + } else { + return \%classlist; + } +} + # ----- END HELPER FUNCTIONS -------------------------------------------- 1; __END__ + +