File:
[LON-CAPA] /
loncom /
interface /
loncoursedata.pm
Revision
1.60:
download - view:
text,
annotated -
select for diffs
Fri Mar 21 16:04:10 2003 UTC (21 years, 6 months ago) by
matthew
Branches:
MAIN
CVS tags:
HEAD
Local caching:
Added 'weight' to performance_table and EXT call to get the weight.
Added timestamp checking for cache devalidation.
The addition of 'weight' (and the EXT call) seem to have slowed the caching
down considerably (it now takes 50% longer to store the data). The timestamp
checking appears to have had a much more modest effect on data retrieval
( < 0.02 sec/student is my estimate).
get_classlist:
Added slightly more verbose error logging.
1: # The LearningOnline Network with CAPA
2: #
3: # $Id: loncoursedata.pm,v 1.60 2003/03/21 16:04:10 matthew Exp $
4: #
5: # Copyright Michigan State University Board of Trustees
6: #
7: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
8: #
9: # LON-CAPA is free software; you can redistribute it and/or modify
10: # it under the terms of the GNU General Public License as published by
11: # the Free Software Foundation; either version 2 of the License, or
12: # (at your option) any later version.
13: #
14: # LON-CAPA is distributed in the hope that it will be useful,
15: # but WITHOUT ANY WARRANTY; without even the implied warranty of
16: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17: # GNU General Public License for more details.
18: #
19: # You should have received a copy of the GNU General Public License
20: # along with LON-CAPA; if not, write to the Free Software
21: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22: #
23: # /home/httpd/html/adm/gpl.txt
24: #
25: # http://www.lon-capa.org/
26: #
27: ###
28:
29: =pod
30:
31: =head1 NAME
32:
33: loncoursedata
34:
35: =head1 SYNOPSIS
36:
37: Set of functions that download and process student and course information.
38:
39: =head1 PACKAGES USED
40:
41: Apache::Constants qw(:common :http)
42: Apache::lonnet()
43: Apache::lonhtmlcommon
44: HTML::TokeParser
45: GDBM_File
46:
47: =cut
48:
49: package Apache::loncoursedata;
50:
51: use strict;
52: use Apache::Constants qw(:common :http);
53: use Apache::lonnet();
54: use Apache::lonhtmlcommon;
55: use Time::HiRes;
56: use Apache::lonmysql;
57: use HTML::TokeParser;
58: use GDBM_File;
59:
60: =pod
61:
62: =head1 DOWNLOAD INFORMATION
63:
64: This section contains all the functions that get data from other servers
65: and/or itself.
66:
67: =cut
68:
69: # ----- DOWNLOAD INFORMATION -------------------------------------------
70:
71: =pod
72:
73: =item &DownloadClasslist()
74:
75: Collects lastname, generation, middlename, firstname, PID, and section for each
76: student from their environment database. The section data is also download, though
77: it is in a rough format, and is processed later. The list of students is built from
78: collecting a classlist for the course that is to be displayed. Once the classlist
79: has been downloaded, its date stamp is recorded. Unless the datestamp for the
80: class database is reset or is modified, this data will not be downloaded again.
81: Also, there was talk about putting the fullname and section
82: and perhaps other pieces of data into the classlist file. This would
83: reduce the number of different file accesses and reduce the amount of
84: processing on this side.
85:
86: =over 4
87:
88: Input: $courseID, $lastDownloadTime, $c
89:
90: $courseID: The id of the course
91:
92: $lastDownloadTime: This is the date stamp for when this information was
93: last gathered. If it is set to Not downloaded, it will gather the data
94: again, though it currently does not remove the old data.
95:
96: $c: The connection class that can determine if the browser has aborted. It
97: is used to short circuit this function so that it does not continue to
98: get information when there is no need.
99:
100: Output: \%classlist
101:
102: \%classlist: A pointer to a hash containing the following data:
103:
104: -A list of student name:domain (as keys) (known below as $name)
105:
106: -A hash pointer for each student containing lastname, generation, firstname,
107: middlename, and PID : Key is $name.studentInformation
108:
109: -A hash pointer to each students section data : Key is $name.section
110:
111: -If there was an error in dump, it will be returned in the hash. See
112: the error codes for dump in lonnet. Also, an error key will be
113: generated if an abort occurs.
114:
115: =back
116:
117: =cut
118:
119: sub DownloadClasslist {
120: my ($courseID, $lastDownloadTime, $c)=@_;
121: my ($courseDomain,$courseNumber)=split(/\_/,$courseID);
122: my %classlist;
123:
124: my $modifiedTime = &Apache::lonnet::GetFileTimestamp($courseDomain,
125: $courseNumber,
126: 'classlist.db',
127: $Apache::lonnet::perlvar{'lonUsersDir'});
128:
129: # Always download the information if lastDownloadTime is set to
130: # Not downloaded, otherwise it is only downloaded if the file
131: # has been updated and has a more recent date stamp
132: if($lastDownloadTime ne 'Not downloaded' &&
133: $lastDownloadTime >= $modifiedTime && $modifiedTime >= 0) {
134: # Data is not gathered so return UpToDate as true. This
135: # will be interpreted in ProcessClasslist
136: $classlist{'lastDownloadTime'}=time;
137: $classlist{'UpToDate'} = 'true';
138: return \%classlist;
139: }
140:
141: %classlist=&Apache::lonnet::dump('classlist',$courseDomain, $courseNumber);
142: foreach(keys (%classlist)) {
143: if(/^(con_lost|error|no_such_host)/i) {
144: return;
145: }
146: }
147:
148: foreach my $name (keys(%classlist)) {
149: if(defined($c) && ($c->aborted())) {
150: $classlist{'error'}='aborted';
151: return \%classlist;
152: }
153:
154: my ($studentName,$studentDomain) = split(/\:/,$name);
155: # Download student environment data, specifically the full name and id.
156: my %studentInformation=&Apache::lonnet::get('environment',
157: ['lastname','generation',
158: 'firstname','middlename',
159: 'id'],
160: $studentDomain,
161: $studentName);
162: $classlist{$name.':studentInformation'}=\%studentInformation;
163:
164: if($c->aborted()) {
165: $classlist{'error'}='aborted';
166: return \%classlist;
167: }
168:
169: #Section
170: my %section=&Apache::lonnet::dump('roles',$studentDomain,$studentName);
171: $classlist{$name.':sections'}=\%section;
172: }
173:
174: $classlist{'UpToDate'} = 'false';
175: $classlist{'lastDownloadTime'}=time;
176:
177: return \%classlist;
178: }
179:
180: =pod
181:
182: =item &DownloadCourseInformation()
183:
184: Dump of all the course information for a single student. The data can be
185: pruned by making use of dumps regular expression arguement. This function
186: also takes a regular expression which it passes straight through to dump.
187: The data is no escaped, because it is done elsewhere. It also
188: checks the timestamp of the students course database file and only downloads
189: if it has been modified since the last download.
190:
191: =over 4
192:
193: Input: $namedata, $courseID, $lastDownloadTime, $WhatIWant
194:
195: $namedata: student name:domain
196:
197: $courseID: The id of the course
198:
199: $lastDownloadTime: This is the date stamp for when this information was
200: last gathered. If it is set to Not downloaded, it will gather the data
201: again, though it currently does not remove the old data.
202:
203: $WhatIWant: Regular expression used to get selected data with dump
204:
205: Output: \%courseData
206:
207: \%courseData: A hash pointer to the raw data from the students course
208: database.
209:
210: =back
211:
212: =cut
213:
214: sub DownloadCourseInformation {
215: my ($namedata,$courseID,$lastDownloadTime,$WhatIWant)=@_;
216: my %courseData;
217: my ($name,$domain) = split(/\:/,$namedata);
218:
219: my $modifiedTime = &Apache::lonnet::GetFileTimestamp($domain, $name,
220: $courseID.'.db',
221: $Apache::lonnet::perlvar{'lonUsersDir'});
222:
223: if($lastDownloadTime ne 'Not downloaded' &&
224: $lastDownloadTime >= $modifiedTime && $modifiedTime >= 0) {
225: # Data is not gathered so return UpToDate as true. This
226: # will be interpreted in ProcessClasslist
227: $courseData{$namedata.':lastDownloadTime'}=time;
228: $courseData{$namedata.':UpToDate'} = 'true';
229: return \%courseData;
230: }
231:
232: # Download course data
233: if(!defined($WhatIWant)) {
234: # set the regular expression to everything by setting it to period
235: $WhatIWant = '.';
236: }
237: %courseData=&Apache::lonnet::dump($courseID, $domain, $name, $WhatIWant);
238: $courseData{'UpToDate'} = 'false';
239: $courseData{'lastDownloadTime'}=time;
240:
241: my %newData;
242: foreach (keys(%courseData)) {
243: # need to have the keys to be prepended with the name:domain of the
244: # student to reduce data collision later.
245: $newData{$namedata.':'.$_} = $courseData{$_};
246: }
247:
248: return \%newData;
249: }
250:
251: # ----- END DOWNLOAD INFORMATION ---------------------------------------
252:
253: =pod
254:
255: =head1 PROCESSING FUNCTIONS
256:
257: These functions process all the data for all the students. Also, they
258: are the functions that access the cache database for writing the majority of
259: the time. The downloading and caching were separated to reduce problems
260: with stopping downloading then can not tie hash to database later.
261:
262: =cut
263:
264: # ----- PROCESSING FUNCTIONS ---------------------------------------
265:
266: ####################################################
267: ####################################################
268:
269: =pod
270:
271: =item &get_sequence_assessment_data()
272:
273: AT THIS TIME THE USE OF THIS FUNCTION IS *NOT* RECOMMENDED
274:
275: Use lonnavmaps to build a data structure describing the order and
276: assessment contents of each sequence in the current course.
277:
278: The returned structure is a hash reference.
279:
280: { title => 'title',
281: symb => 'symb',
282: source => '/s/o/u/r/c/e',
283: type => (container|assessment),
284: num_assess => 2, # only for container
285: parts => [11,13,15], # only for assessment
286: response_ids => [12,14,16], # only for assessment
287: contents => [........] # only for container
288: }
289:
290: $hash->{'contents'} is a reference to an array of hashes of the same structure.
291:
292: Also returned are array references to the sequences and assessments contained
293: in the course.
294:
295:
296: =cut
297:
298: ####################################################
299: ####################################################
300: sub get_sequence_assessment_data {
301: my $fn=$ENV{'request.course.fn'};
302: ##
303: ## use navmaps
304: my $navmap = Apache::lonnavmaps::navmap->new($fn.".db",$fn."_parms.db",
305: 1,0);
306: if (!defined($navmap)) {
307: return 'Can not open Coursemap';
308: }
309: my $iterator = $navmap->getIterator(undef, undef, undef, 1);
310: ##
311: ## Prime the pump
312: ##
313: ## We are going to loop until we run out of sequences/pages to explore for
314: ## resources. This means we have to start out with something to look
315: ## at.
316: my $title = $ENV{'course.'.$ENV{'request.course.id'}.'.description'};
317: my $symb = 'top';
318: my $src = 'not applicable';
319: #
320: my @Sequences;
321: my @Assessments;
322: my @Nested_Sequences = (); # Stack of sequences, keeps track of depth
323: my $top = { title => $title,
324: src => $src,
325: symb => $symb,
326: type => 'container',
327: num_assess => 0,
328: num_assess_parts => 0,
329: contents => [], };
330: push (@Sequences,$top);
331: push (@Nested_Sequences, $top);
332: #
333: # We need to keep track of which sequences contain homework problems
334: #
335: my $previous;
336: my $curRes = $iterator->next(); # BEGIN_MAP
337: $curRes = $iterator->next(); # The first item in the top level map.
338: while (scalar(@Nested_Sequences)) {
339: $previous = $curRes;
340: $curRes = $iterator->next();
341: my $currentmap = $Nested_Sequences[-1]; # Last one on the stack
342: if ($curRes == $iterator->BEGIN_MAP()) {
343: # get the map itself, instead of BEGIN_MAP
344: $title = $previous->title();
345: $symb = $previous->symb();
346: $src = $previous->src();
347: my $newmap = { title => $title,
348: src => $src,
349: symb => $symb,
350: type => 'container',
351: num_assess => 0,
352: contents => [],
353: };
354: push (@{$currentmap->{'contents'}},$newmap); # this is permanent
355: push (@Sequences,$newmap);
356: push (@Nested_Sequences, $newmap); # this is a stack
357: next;
358: }
359: if ($curRes == $iterator->END_MAP()) {
360: pop(@Nested_Sequences);
361: next;
362: }
363: next if (! ref($curRes));
364: next if (! $curRes->is_problem());# && !$curRes->randomout);
365: # Okay, from here on out we only deal with assessments
366: $title = $curRes->title();
367: $symb = $curRes->symb();
368: $src = $curRes->src();
369: my $parts = $curRes->parts();
370: my $assessment = { title => $title,
371: src => $src,
372: symb => $symb,
373: type => 'assessment',
374: parts => $parts,
375: num_parts => scalar(@$parts),
376: };
377: push(@Assessments,$assessment);
378: push(@{$currentmap->{'contents'}},$assessment);
379: $currentmap->{'num_assess'}++;
380: $currentmap->{'num_assess_parts'}+= scalar(@$parts);
381: }
382: $navmap->untieHashes();
383: return ($top,\@Sequences,\@Assessments);
384: }
385:
386: #################################################
387: #################################################
388:
389: =pod
390:
391: =item &ProcessTopResourceMap()
392:
393: Trace through the "big hash" created in rat/lonuserstate.pm::loadmap.
394: Basically, this function organizes a subset of the data and stores it in
395: cached data. The data stored is the problems, sequences, sequence titles,
396: parts of problems, and their ordering. Column width information is also
397: partially handled here on a per sequence basis.
398:
399: =over 4
400:
401: Input: $cache, $c
402:
403: $cache: A pointer to a hash to store the information
404:
405: $c: The connection class used to determine if an abort has been sent to the
406: browser
407:
408: Output: A string that contains an error message or "OK" if everything went
409: smoothly.
410:
411: =back
412:
413: =cut
414:
415: sub ProcessTopResourceMap {
416: my ($cache,$c)=@_;
417: my %hash;
418: my $fn=$ENV{'request.course.fn'};
419: if(-e "$fn.db") {
420: my $tieTries=0;
421: while($tieTries < 3) {
422: if($c->aborted()) {
423: return;
424: }
425: if(tie(%hash,'GDBM_File',"$fn.db",&GDBM_READER(),0640)) {
426: last;
427: }
428: $tieTries++;
429: sleep 1;
430: }
431: if($tieTries >= 3) {
432: return 'Coursemap undefined.';
433: }
434: } else {
435: return 'Can not open Coursemap.';
436: }
437:
438: my $oldkeys;
439: delete $cache->{'OptionResponses'};
440: if(defined($cache->{'ResourceKeys'})) {
441: $oldkeys = $cache->{'ResourceKeys'};
442: foreach (split(':::', $cache->{'ResourceKeys'})) {
443: delete $cache->{$_};
444: }
445: delete $cache->{'ResourceKeys'};
446: }
447:
448: # Initialize state machine. Set information pointing to top level map.
449: my (@sequences, @currentResource, @finishResource);
450: my ($currentSequence, $currentResourceID, $lastResourceID);
451:
452: $currentResourceID=$hash{'ids_'.
453: &Apache::lonnet::clutter($ENV{'request.course.uri'})};
454: push(@currentResource, $currentResourceID);
455: $lastResourceID=-1;
456: $currentSequence=-1;
457: my $topLevelSequenceNumber = $currentSequence;
458:
459: my %sequenceRecord;
460: my %allkeys;
461: while(1) {
462: if($c->aborted()) {
463: last;
464: }
465: # HANDLE NEW SEQUENCE!
466: #if page || sequence
467: if(defined($hash{'map_pc_'.$hash{'src_'.$currentResourceID}}) &&
468: !defined($sequenceRecord{$currentResourceID})) {
469: $sequenceRecord{$currentResourceID}++;
470: push(@sequences, $currentSequence);
471: push(@currentResource, $currentResourceID);
472: push(@finishResource, $lastResourceID);
473:
474: $currentSequence=$hash{'map_pc_'.$hash{'src_'.$currentResourceID}};
475:
476: # Mark sequence as containing problems. If it doesn't, then
477: # it will be removed when processing for this sequence is
478: # complete. This allows the problems in a sequence
479: # to be outputed before problems in the subsequences
480: if(!defined($cache->{'orderedSequences'})) {
481: $cache->{'orderedSequences'}=$currentSequence;
482: } else {
483: $cache->{'orderedSequences'}.=':'.$currentSequence;
484: }
485: $allkeys{'orderedSequences'}++;
486:
487: $lastResourceID=$hash{'map_finish_'.
488: $hash{'src_'.$currentResourceID}};
489: $currentResourceID=$hash{'map_start_'.
490: $hash{'src_'.$currentResourceID}};
491:
492: if(!($currentResourceID) || !($lastResourceID)) {
493: $currentSequence=pop(@sequences);
494: $currentResourceID=pop(@currentResource);
495: $lastResourceID=pop(@finishResource);
496: if($currentSequence eq $topLevelSequenceNumber) {
497: last;
498: }
499: }
500: next;
501: }
502:
503: # Handle gradable resources: exams, problems, etc
504: $currentResourceID=~/(\d+)\.(\d+)/;
505: my $partA=$1;
506: my $partB=$2;
507: if($hash{'src_'.$currentResourceID}=~
508: /\.(problem|exam|quiz|assess|survey|form)$/ &&
509: $partA eq $currentSequence &&
510: !defined($sequenceRecord{$currentSequence.':'.
511: $currentResourceID})) {
512: $sequenceRecord{$currentSequence.':'.$currentResourceID}++;
513: my $Problem = &Apache::lonnet::symbclean(
514: &Apache::lonnet::declutter($hash{'map_id_'.$partA}).
515: '___'.$partB.'___'.
516: &Apache::lonnet::declutter($hash{'src_'.
517: $currentResourceID}));
518:
519: $cache->{$currentResourceID.':problem'}=$Problem;
520: $allkeys{$currentResourceID.':problem'}++;
521: if(!defined($cache->{$currentSequence.':problems'})) {
522: $cache->{$currentSequence.':problems'}=$currentResourceID;
523: } else {
524: $cache->{$currentSequence.':problems'}.=
525: ':'.$currentResourceID;
526: }
527: $allkeys{$currentSequence.':problems'}++;
528:
529: my $meta=$hash{'src_'.$currentResourceID};
530: # $cache->{$currentResourceID.':title'}=
531: # &Apache::lonnet::metdata($meta,'title');
532: $cache->{$currentResourceID.':title'}=
533: $hash{'title_'.$currentResourceID};
534: $allkeys{$currentResourceID.':title'}++;
535: $cache->{$currentResourceID.':source'}=
536: $hash{'src_'.$currentResourceID};
537: $allkeys{$currentResourceID.':source'}++;
538:
539: # Get Parts for problem
540: my %beenHere;
541: foreach (split(/\,/,&Apache::lonnet::metadata($meta,'packages'))) {
542: if(/^\w+response_\d+.*/) {
543: my (undef, $partId, $responseId) = split(/_/,$_);
544: if($beenHere{'p:'.$partId} == 0) {
545: $beenHere{'p:'.$partId}++;
546: if(!defined($cache->{$currentSequence.':'.
547: $currentResourceID.':parts'})) {
548: $cache->{$currentSequence.':'.$currentResourceID.
549: ':parts'}=$partId;
550: } else {
551: $cache->{$currentSequence.':'.$currentResourceID.
552: ':parts'}.=':'.$partId;
553: }
554: $allkeys{$currentSequence.':'.$currentResourceID.
555: ':parts'}++;
556: }
557: if($beenHere{'r:'.$partId.':'.$responseId} == 0) {
558: $beenHere{'r:'.$partId.':'.$responseId}++;
559: if(!defined($cache->{$currentSequence.':'.
560: $currentResourceID.':'.$partId.
561: ':responseIDs'})) {
562: $cache->{$currentSequence.':'.$currentResourceID.
563: ':'.$partId.':responseIDs'}=$responseId;
564: } else {
565: $cache->{$currentSequence.':'.$currentResourceID.
566: ':'.$partId.':responseIDs'}.=':'.
567: $responseId;
568: }
569: $allkeys{$currentSequence.':'.$currentResourceID.':'.
570: $partId.':responseIDs'}++;
571: }
572: if(/^optionresponse/ &&
573: $beenHere{'o:'.$partId.':'.$currentResourceID} == 0) {
574: $beenHere{'o:'.$partId.$currentResourceID}++;
575: if(defined($cache->{'OptionResponses'})) {
576: $cache->{'OptionResponses'}.= ':::'.
577: $currentSequence.':'.$currentResourceID.':'.
578: $partId.':'.$responseId;
579: } else {
580: $cache->{'OptionResponses'}= $currentSequence.':'.
581: $currentResourceID.':'.
582: $partId.':'.$responseId;
583: }
584: $allkeys{'OptionResponses'}++;
585: }
586: }
587: }
588: }
589:
590: # if resource == finish resource, then it is the end of a sequence/page
591: if($currentResourceID eq $lastResourceID) {
592: # pop off last resource of sequence
593: $currentResourceID=pop(@currentResource);
594: $lastResourceID=pop(@finishResource);
595:
596: if(defined($cache->{$currentSequence.':problems'})) {
597: # Capture sequence information here
598: $cache->{$currentSequence.':title'}=
599: $hash{'title_'.$currentResourceID};
600: $allkeys{$currentSequence.':title'}++;
601: $cache->{$currentSequence.':source'}=
602: $hash{'src_'.$currentResourceID};
603: $allkeys{$currentSequence.':source'}++;
604:
605: my $totalProblems=0;
606: foreach my $currentProblem (split(/\:/,
607: $cache->{$currentSequence.
608: ':problems'})) {
609: foreach (split(/\:/,$cache->{$currentSequence.':'.
610: $currentProblem.
611: ':parts'})) {
612: $totalProblems++;
613: }
614: }
615: my @titleLength=split(//,$cache->{$currentSequence.
616: ':title'});
617: # $extra is 5 for problems correct and 3 for space
618: # between problems correct and problem output
619: my $extra = 8;
620: if(($totalProblems + $extra) > (scalar @titleLength)) {
621: $cache->{$currentSequence.':columnWidth'}=
622: $totalProblems + $extra;
623: } else {
624: $cache->{$currentSequence.':columnWidth'}=
625: (scalar @titleLength);
626: }
627: $allkeys{$currentSequence.':columnWidth'}++;
628: } else {
629: # Remove sequence from list, if it contains no problems to
630: # display.
631: $cache->{'orderedSequences'}=~s/$currentSequence//;
632: $cache->{'orderedSequences'}=~s/::/:/g;
633: $cache->{'orderedSequences'}=~s/^:|:$//g;
634: }
635:
636: $currentSequence=pop(@sequences);
637: if($currentSequence eq $topLevelSequenceNumber) {
638: last;
639: }
640: }
641:
642: # MOVE!!!
643: # move to next resource
644: unless(defined($hash{'to_'.$currentResourceID})) {
645: # big problem, need to handle. Next is probably wrong
646: my $errorMessage = 'Big problem in ';
647: $errorMessage .= 'loncoursedata::ProcessTopLevelMap.';
648: $errorMessage .= " bighash to_$currentResourceID not defined!";
649: &Apache::lonnet::logthis($errorMessage);
650: if (!defined($currentResourceID)) {last;}
651: }
652: my @nextResources=();
653: foreach (split(/\,/,$hash{'to_'.$currentResourceID})) {
654: if(!defined($sequenceRecord{$currentSequence.':'.
655: $hash{'goesto_'.$_}})) {
656: push(@nextResources, $hash{'goesto_'.$_});
657: }
658: }
659: push(@currentResource, @nextResources);
660: # Set the next resource to be processed
661: $currentResourceID=pop(@currentResource);
662: }
663:
664: my @theKeys = keys(%allkeys);
665: my $newkeys = join(':::', @theKeys);
666: $cache->{'ResourceKeys'} = join(':::', $newkeys);
667: if($newkeys ne $oldkeys) {
668: $cache->{'ResourceUpdated'} = 'true';
669: } else {
670: $cache->{'ResourceUpdated'} = 'false';
671: }
672:
673: unless (untie(%hash)) {
674: &Apache::lonnet::logthis("<font color=blue>WARNING: ".
675: "Could not untie coursemap $fn (browse)".
676: ".</font>");
677: }
678:
679: return 'OK';
680: }
681:
682: =pod
683:
684: =item &ProcessClasslist()
685:
686: Taking the class list dumped from &DownloadClasslist(), all the
687: students and their non-class information is processed using the
688: &ProcessStudentInformation() function. A date stamp is also recorded for
689: when the data was processed.
690:
691: Takes data downloaded for a student and breaks it up into managable pieces and
692: stored in cache data. The username, domain, class related date, PID,
693: full name, and section are all processed here.
694:
695: =over 4
696:
697: Input: $cache, $classlist, $courseID, $ChartDB, $c
698:
699: $cache: A hash pointer to store the data
700:
701: $classlist: The hash of data collected about a student from
702: &DownloadClasslist(). The hash contains a list of students, a pointer
703: to a hash of student information for each student, and each students section
704: number.
705:
706: $courseID: The course ID
707:
708: $ChartDB: The name of the cache database file.
709:
710: $c: The connection class used to determine if an abort has been sent to the
711: browser
712:
713: Output: @names
714:
715: @names: An array of students whose information has been processed, and are to
716: be considered in an arbitrary order. The entries in @names are of the form
717: username:domain.
718:
719: The values in $cache are as follows:
720:
721: *NOTE: for the following $name implies username:domain
722: $name.':error' only defined if an error occured. Value
723: contains the error message
724: $name.':lastDownloadTime' unconverted time of the last update of a
725: student\'s course data
726: $name.'updateTime' coverted time of the last update of a
727: student\'s course data
728: $name.':username' username of a student
729: $name.':domain' domain of a student
730: $name.':fullname' full name of a student
731: $name.':id' PID of a student
732: $name.':Status' active/expired status of a student
733: $name.':section' section of a student
734:
735: =back
736:
737: =cut
738:
739: sub ProcessClasslist {
740: my ($cache,$classlist,$courseID,$c)=@_;
741: my @names=();
742:
743: $cache->{'ClasslistTimeStamp'}=$classlist->{'lastDownloadTime'};
744: if($classlist->{'UpToDate'} eq 'true') {
745: return split(/:::/,$cache->{'NamesOfStudents'});;
746: }
747:
748: foreach my $name (keys(%$classlist)) {
749: if($name =~ /\:section/ || $name =~ /\:studentInformation/ ||
750: $name eq '' || $name eq 'UpToDate' || $name eq 'lastDownloadTime') {
751: next;
752: }
753: if($c->aborted()) {
754: return ();
755: }
756: my $studentInformation = $classlist->{$name.':studentInformation'};
757: my $date = $classlist->{$name};
758: my ($studentName,$studentDomain) = split(/\:/,$name);
759:
760: $cache->{$name.':username'}=$studentName;
761: $cache->{$name.':domain'}=$studentDomain;
762: # Initialize timestamp for student
763: if(!defined($cache->{$name.':lastDownloadTime'})) {
764: $cache->{$name.':lastDownloadTime'}='Not downloaded';
765: $cache->{$name.':updateTime'}=' Not updated';
766: }
767:
768: my $error = 0;
769: foreach(keys(%$studentInformation)) {
770: if(/^(con_lost|error|no_such_host)/i) {
771: $cache->{$name.':error'}=
772: 'Could not download student environment data.';
773: $cache->{$name.':fullname'}='';
774: $cache->{$name.':id'}='';
775: $error = 1;
776: }
777: }
778: next if($error);
779: push(@names,$name);
780: $cache->{$name.':fullname'}=&ProcessFullName(
781: $studentInformation->{'lastname'},
782: $studentInformation->{'generation'},
783: $studentInformation->{'firstname'},
784: $studentInformation->{'middlename'});
785: $cache->{$name.':id'}=$studentInformation->{'id'};
786:
787: my ($end, $start)=split(':',$date);
788: $courseID=~s/\_/\//g;
789: $courseID=~s/^(\w)/\/$1/;
790:
791: my $sec='';
792: my $sectionData = $classlist->{$name.':sections'};
793: foreach my $key (keys (%$sectionData)) {
794: my $value = $sectionData->{$key};
795: if ($key=~/^$courseID(?:\/)*(\w+)*\_st$/) {
796: my $tempsection=$1;
797: if($key eq $courseID.'_st') {
798: $tempsection='';
799: }
800: my (undef,$roleend,$rolestart)=split(/\_/,$value);
801: if($roleend eq $end && $rolestart eq $start) {
802: $sec = $tempsection;
803: last;
804: }
805: }
806: }
807:
808: my $status='Expired';
809: if(((!$end) || time < $end) && ((!$start) || (time > $start))) {
810: $status='Active';
811: }
812: $cache->{$name.':Status'}=$status;
813: $cache->{$name.':section'}=$sec;
814:
815: if($sec eq '' || !defined($sec) || $sec eq ' ') {
816: $sec = 'none';
817: }
818: if(defined($cache->{'sectionList'})) {
819: if($cache->{'sectionList'} !~ /(^$sec:|^$sec$|:$sec$|:$sec:)/) {
820: $cache->{'sectionList'} .= ':'.$sec;
821: }
822: } else {
823: $cache->{'sectionList'} = $sec;
824: }
825: }
826:
827: $cache->{'ClasslistTimestamp'}=time;
828: $cache->{'NamesOfStudents'}=join(':::',@names);
829:
830: return @names;
831: }
832:
833: =pod
834:
835: =item &ProcessStudentData()
836:
837: Takes the course data downloaded for a student in
838: &DownloadCourseInformation() and breaks it up into key value pairs
839: to be stored in the cached data. The keys are comprised of the
840: $username:$domain:$keyFromCourseDatabase. The student username:domain is
841: stored away signifying that the students information has been downloaded and
842: can be reused from cached data.
843:
844: =over 4
845:
846: Input: $cache, $courseData, $name
847:
848: $cache: A hash pointer to store data
849:
850: $courseData: A hash pointer that points to the course data downloaded for a
851: student.
852:
853: $name: username:domain
854:
855: Output: None
856:
857: *NOTE: There is no output, but an error message is stored away in the cache
858: data. This is checked in &FormatStudentData(). The key username:domain:error
859: will only exist if an error occured. The error is an error from
860: &DownloadCourseInformation().
861:
862: =back
863:
864: =cut
865:
866: sub ProcessStudentData {
867: my ($cache,$courseData,$name)=@_;
868:
869: if(!&CheckDateStampError($courseData, $cache, $name)) {
870: return;
871: }
872:
873: # This little delete thing, should not be here. Move some other
874: # time though.
875: if(defined($cache->{$name.':keys'})) {
876: foreach (split(':::', $cache->{$name.':keys'})) {
877: delete $cache->{$name.':'.$_};
878: }
879: delete $cache->{$name.':keys'};
880: }
881:
882: my %courseKeys;
883: # user name:domain was prepended earlier in DownloadCourseInformation
884: foreach (keys %$courseData) {
885: my $currentKey = $_;
886: $currentKey =~ s/^$name//;
887: $courseKeys{$currentKey}++;
888: $cache->{$_}=$courseData->{$_};
889: }
890:
891: $cache->{$name.':keys'} = join(':::', keys(%courseKeys));
892:
893: return;
894: }
895:
896: =pod
897:
898: =item &ExtractStudentData()
899:
900: HISTORY: This function originally existed in every statistics module,
901: and performed different tasks, the had some overlap. Due to the need
902: for the data from the different modules, they were combined into
903: a single function.
904:
905: This function now extracts all the necessary course data for a student
906: from what was downloaded from their homeserver. There is some extra
907: time overhead compared to the ProcessStudentInformation function, but
908: it would have had to occurred at some point anyways. This is now
909: typically called while downloading the data it will process. It is
910: the brother function to ProcessStudentInformation.
911:
912: =over 4
913:
914: Input: $input, $output, $data, $name
915:
916: $input: A hash that contains the input data to be processed
917:
918: $output: A hash to contain the processed data
919:
920: $data: A hash containing the information on what is to be
921: processed and how (basically).
922:
923: $name: username:domain
924:
925: The input is slightly different here, but is quite simple.
926: It is currently used where the $input, $output, and $data
927: can and are often the same hashes, but they do not need
928: to be.
929:
930: Output: None
931:
932: *NOTE: There is no output, but an error message is stored away in the cache
933: data. This is checked in &FormatStudentData(). The key username:domain:error
934: will only exist if an error occured. The error is an error from
935: &DownloadCourseInformation().
936:
937: =back
938:
939: =cut
940:
941: sub ExtractStudentData {
942: my ($input, $output, $data, $name)=@_;
943:
944: if(!&CheckDateStampError($input, $data, $name)) {
945: return;
946: }
947:
948: # This little delete thing, should not be here. Move some other
949: # time though.
950: my %allkeys;
951: if(defined($output->{$name.':keys'})) {
952: foreach (split(':::', $output->{$name.':keys'})) {
953: delete $output->{$name.':'.$_};
954: }
955: delete $output->{$name.':keys'};
956: }
957:
958: my ($username,$domain)=split(':',$name);
959:
960: my $Version;
961: my $problemsCorrect = 0;
962: my $totalProblems = 0;
963: my $problemsSolved = 0;
964: my $numberOfParts = 0;
965: my $totalAwarded = 0;
966: foreach my $sequence (split(':', $data->{'orderedSequences'})) {
967: foreach my $problemID (split(':', $data->{$sequence.':problems'})) {
968: my $problem = $data->{$problemID.':problem'};
969: my $LatestVersion = $input->{$name.':version:'.$problem};
970:
971: # Output dashes for all the parts of this problem if there
972: # is no version information about the current problem.
973: $output->{$name.':'.$problemID.':NoVersion'} = 'false';
974: $allkeys{$name.':'.$problemID.':NoVersion'}++;
975: if(!$LatestVersion) {
976: foreach my $part (split(/\:/,$data->{$sequence.':'.
977: $problemID.
978: ':parts'})) {
979: $output->{$name.':'.$problemID.':'.$part.':tries'} = 0;
980: $output->{$name.':'.$problemID.':'.$part.':awarded'} = 0;
981: $output->{$name.':'.$problemID.':'.$part.':code'} = ' ';
982: $allkeys{$name.':'.$problemID.':'.$part.':tries'}++;
983: $allkeys{$name.':'.$problemID.':'.$part.':awarded'}++;
984: $allkeys{$name.':'.$problemID.':'.$part.':code'}++;
985: $totalProblems++;
986: }
987: $output->{$name.':'.$problemID.':NoVersion'} = 'true';
988: next;
989: }
990:
991: my %partData=undef;
992: # Initialize part data, display skips correctly
993: # Skip refers to when a student made no submissions on that
994: # part/problem.
995: foreach my $part (split(/\:/,$data->{$sequence.':'.
996: $problemID.
997: ':parts'})) {
998: $partData{$part.':tries'}=0;
999: $partData{$part.':code'}=' ';
1000: $partData{$part.':awarded'}=0;
1001: $partData{$part.':timestamp'}=0;
1002: foreach my $response (split(':', $data->{$sequence.':'.
1003: $problemID.':'.
1004: $part.':responseIDs'})) {
1005: $partData{$part.':'.$response.':submission'}='';
1006: }
1007: }
1008:
1009: # Looping through all the versions of each part, starting with the
1010: # oldest version. Basically, it gets the most recent
1011: # set of grade data for each part.
1012: my @submissions = ();
1013: for(my $Version=1; $Version<=$LatestVersion; $Version++) {
1014: foreach my $part (split(/\:/,$data->{$sequence.':'.
1015: $problemID.
1016: ':parts'})) {
1017:
1018: if(!defined($input->{"$name:$Version:$problem".
1019: ":resource.$part.solved"})) {
1020: # No grade for this submission, so skip
1021: next;
1022: }
1023:
1024: my $tries=0;
1025: my $code=' ';
1026: my $awarded=0;
1027:
1028: $tries = $input->{$name.':'.$Version.':'.$problem.
1029: ':resource.'.$part.'.tries'};
1030: $awarded = $input->{$name.':'.$Version.':'.$problem.
1031: ':resource.'.$part.'.awarded'};
1032:
1033: $partData{$part.':awarded'}=($awarded) ? $awarded : 0;
1034: $partData{$part.':tries'}=($tries) ? $tries : 0;
1035:
1036: $partData{$part.':timestamp'}=$input->{$name.':'.$Version.':'.
1037: $problem.
1038: ':timestamp'};
1039: if(!$input->{$name.':'.$Version.':'.$problem.':resource.'.$part.
1040: '.previous'}) {
1041: foreach my $response (split(':',
1042: $data->{$sequence.':'.
1043: $problemID.':'.
1044: $part.':responseIDs'})) {
1045: @submissions=($input->{$name.':'.$Version.':'.
1046: $problem.
1047: ':resource.'.$part.'.'.
1048: $response.'.submission'},
1049: @submissions);
1050: }
1051: }
1052:
1053: my $val = $input->{$name.':'.$Version.':'.$problem.
1054: ':resource.'.$part.'.solved'};
1055: if ($val eq 'correct_by_student') {$code = '*';}
1056: elsif ($val eq 'correct_by_override') {$code = '+';}
1057: elsif ($val eq 'incorrect_attempted') {$code = '.';}
1058: elsif ($val eq 'incorrect_by_override'){$code = '-';}
1059: elsif ($val eq 'excused') {$code = 'x';}
1060: elsif ($val eq 'ungraded_attempted') {$code = '#';}
1061: else {$code = ' ';}
1062: $partData{$part.':code'}=$code;
1063: }
1064: }
1065:
1066: foreach my $part (split(/\:/,$data->{$sequence.':'.$problemID.
1067: ':parts'})) {
1068: $output->{$name.':'.$problemID.':'.$part.':wrong'} =
1069: $partData{$part.':tries'};
1070: $allkeys{$name.':'.$problemID.':'.$part.':wrong'}++;
1071:
1072: if($partData{$part.':code'} eq '*') {
1073: $output->{$name.':'.$problemID.':'.$part.':wrong'}--;
1074: $problemsCorrect++;
1075: } elsif($partData{$part.':code'} eq '+') {
1076: $output->{$name.':'.$problemID.':'.$part.':wrong'}--;
1077: $problemsCorrect++;
1078: }
1079:
1080: $output->{$name.':'.$problemID.':'.$part.':tries'} =
1081: $partData{$part.':tries'};
1082: $output->{$name.':'.$problemID.':'.$part.':code'} =
1083: $partData{$part.':code'};
1084: $output->{$name.':'.$problemID.':'.$part.':awarded'} =
1085: $partData{$part.':awarded'};
1086: $allkeys{$name.':'.$problemID.':'.$part.':tries'}++;
1087: $allkeys{$name.':'.$problemID.':'.$part.':code'}++;
1088: $allkeys{$name.':'.$problemID.':'.$part.':awarded'}++;
1089:
1090: $totalAwarded += $partData{$part.':awarded'};
1091: $output->{$name.':'.$problemID.':'.$part.':timestamp'} =
1092: $partData{$part.':timestamp'};
1093: $allkeys{$name.':'.$problemID.':'.$part.':timestamp'}++;
1094:
1095: foreach my $response (split(':', $data->{$sequence.':'.
1096: $problemID.':'.
1097: $part.':responseIDs'})) {
1098: $output->{$name.':'.$problemID.':'.$part.':'.$response.
1099: ':submission'}=join(':::',@submissions);
1100: $allkeys{$name.':'.$problemID.':'.$part.':'.$response.
1101: ':submission'}++;
1102: }
1103:
1104: if($partData{$part.':code'} ne 'x') {
1105: $totalProblems++;
1106: }
1107: }
1108: }
1109:
1110: $output->{$name.':'.$sequence.':problemsCorrect'} = $problemsCorrect;
1111: $allkeys{$name.':'.$sequence.':problemsCorrect'}++;
1112: $problemsSolved += $problemsCorrect;
1113: $problemsCorrect=0;
1114: }
1115:
1116: $output->{$name.':problemsSolved'} = $problemsSolved;
1117: $output->{$name.':totalProblems'} = $totalProblems;
1118: $output->{$name.':totalAwarded'} = $totalAwarded;
1119: $allkeys{$name.':problemsSolved'}++;
1120: $allkeys{$name.':totalProblems'}++;
1121: $allkeys{$name.':totalAwarded'}++;
1122:
1123: $output->{$name.':keys'} = join(':::', keys(%allkeys));
1124:
1125: return;
1126: }
1127:
1128: sub LoadDiscussion {
1129: my ($courseID)=@_;
1130: my %Discuss=();
1131: my %contrib=&Apache::lonnet::dump(
1132: $courseID,
1133: $ENV{'course.'.$courseID.'.domain'},
1134: $ENV{'course.'.$courseID.'.num'});
1135:
1136: #my %contrib=&DownloadCourseInformation($name, $courseID, 0);
1137:
1138: foreach my $temp(keys %contrib) {
1139: if ($temp=~/^version/) {
1140: my $ver=$contrib{$temp};
1141: my ($dummy,$prb)=split(':',$temp);
1142: for (my $idx=1; $idx<=$ver; $idx++ ) {
1143: my $name=$contrib{"$idx:$prb:sendername"};
1144: $Discuss{"$name:$prb"}=$idx;
1145: }
1146: }
1147: }
1148:
1149: return \%Discuss;
1150: }
1151:
1152: # ----- END PROCESSING FUNCTIONS ---------------------------------------
1153:
1154: =pod
1155:
1156: =head1 HELPER FUNCTIONS
1157:
1158: These are just a couple of functions do various odd and end
1159: jobs. There was also a couple of bulk functions added. These are
1160: &DownloadStudentCourseData(), &DownloadStudentCourseDataSeparate(), and
1161: &CheckForResidualDownload(). These functions now act as the interface
1162: for downloading student course data. The statistical modules should
1163: no longer make the calls to dump and download and process etc. They
1164: make calls to these bulk functions to get their data.
1165:
1166: =cut
1167:
1168: # ----- HELPER FUNCTIONS -----------------------------------------------
1169:
1170: sub CheckDateStampError {
1171: my ($courseData, $cache, $name)=@_;
1172: if($courseData->{$name.':UpToDate'} eq 'true') {
1173: $cache->{$name.':lastDownloadTime'} =
1174: $courseData->{$name.':lastDownloadTime'};
1175: if($courseData->{$name.':lastDownloadTime'} eq 'Not downloaded') {
1176: $cache->{$name.':updateTime'} = ' Not updated';
1177: } else {
1178: $cache->{$name.':updateTime'}=
1179: localtime($courseData->{$name.':lastDownloadTime'});
1180: }
1181: return 0;
1182: }
1183:
1184: $cache->{$name.':lastDownloadTime'}=$courseData->{$name.':lastDownloadTime'};
1185: if($courseData->{$name.':lastDownloadTime'} eq 'Not downloaded') {
1186: $cache->{$name.':updateTime'} = ' Not updated';
1187: } else {
1188: $cache->{$name.':updateTime'}=
1189: localtime($courseData->{$name.':lastDownloadTime'});
1190: }
1191:
1192: if(defined($courseData->{$name.':error'})) {
1193: $cache->{$name.':error'}=$courseData->{$name.':error'};
1194: return 0;
1195: }
1196:
1197: return 1;
1198: }
1199:
1200: =pod
1201:
1202: =item &ProcessFullName()
1203:
1204: Takes lastname, generation, firstname, and middlename (or some partial
1205: set of this data) and returns the full name version as a string. Format
1206: is Lastname generation, firstname middlename or a subset of this.
1207:
1208: =cut
1209:
1210: sub ProcessFullName {
1211: my ($lastname, $generation, $firstname, $middlename)=@_;
1212: my $Str = '';
1213:
1214: # Strip whitespace preceeding & following name components.
1215: $lastname =~ s/(\s+$|^\s+)//g;
1216: $generation =~ s/(\s+$|^\s+)//g;
1217: $firstname =~ s/(\s+$|^\s+)//g;
1218: $middlename =~ s/(\s+$|^\s+)//g;
1219:
1220: if($lastname ne '') {
1221: $Str .= $lastname;
1222: $Str .= ' '.$generation if ($generation ne '');
1223: $Str .= ',';
1224: $Str .= ' '.$firstname if ($firstname ne '');
1225: $Str .= ' '.$middlename if ($middlename ne '');
1226: } else {
1227: $Str .= $firstname if ($firstname ne '');
1228: $Str .= ' '.$middlename if ($middlename ne '');
1229: $Str .= ' '.$generation if ($generation ne '');
1230: }
1231:
1232: return $Str;
1233: }
1234:
1235: =pod
1236:
1237: =item &TestCacheData()
1238:
1239: Determine if the cache database can be accessed with a tie. It waits up to
1240: ten seconds before returning failure. This function exists to help with
1241: the problems with stopping the data download. When an abort occurs and the
1242: user quickly presses a form button and httpd child is created. This
1243: child needs to wait for the other to finish (hopefully within ten seconds).
1244:
1245: =over 4
1246:
1247: Input: $ChartDB
1248:
1249: $ChartDB: The name of the cache database to be opened
1250:
1251: Output: -1, 0, 1
1252:
1253: -1: Could not tie database
1254: 0: Use cached data
1255: 1: New cache database created, use that.
1256:
1257: =back
1258:
1259: =cut
1260:
1261: sub TestCacheData {
1262: my ($ChartDB,$isRecalculate,$totalDelay)=@_;
1263: my $isCached=-1;
1264: my %testData;
1265: my $tieTries=0;
1266:
1267: if(!defined($totalDelay)) {
1268: $totalDelay = 10;
1269: }
1270:
1271: if ((-e "$ChartDB") && (!$isRecalculate)) {
1272: $isCached = 1;
1273: } else {
1274: $isCached = 0;
1275: }
1276:
1277: while($tieTries < $totalDelay) {
1278: my $result=0;
1279: if($isCached) {
1280: $result=tie(%testData,'GDBM_File',$ChartDB,&GDBM_READER(),0640);
1281: } else {
1282: $result=tie(%testData,'GDBM_File',$ChartDB,&GDBM_NEWDB(),0640);
1283: }
1284: if($result) {
1285: last;
1286: }
1287: $tieTries++;
1288: sleep 1;
1289: }
1290: if($tieTries >= $totalDelay) {
1291: return -1;
1292: }
1293:
1294: untie(%testData);
1295:
1296: return $isCached;
1297: }
1298:
1299: sub DownloadStudentCourseData {
1300: my ($students,$checkDate,$cacheDB,$extract,$status,$courseID,$r,$c)=@_;
1301:
1302: my $title = 'LON-CAPA Statistics';
1303: my $heading = 'Download and Process Course Data';
1304: my $studentCount = scalar(@$students);
1305:
1306: my $WhatIWant;
1307: $WhatIWant = '(^version:|';
1308: $WhatIWant .= '^\d+:.+?:(resource\.\d+\.';
1309: $WhatIWant .= '(solved|tries|previous|awarded|(\d+\.submission))\s*$';#'
1310: $WhatIWant .= '|timestamp)';
1311: $WhatIWant .= ')';
1312: # $WhatIWant = '.';
1313:
1314: my %prog_state;
1315: if($status eq 'true') {
1316: %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r, $title,
1317: $heading,($#$students)+1);
1318: }
1319:
1320: foreach (@$students) {
1321: my %cache;
1322:
1323: if($c->aborted()) { return 'Aborted'; }
1324:
1325: if($status eq 'true') {
1326: &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
1327: 'last student '.$_);
1328: }
1329:
1330: my $downloadTime='Not downloaded';
1331: my $needUpdate = 'false';
1332: if($checkDate eq 'true' &&
1333: tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
1334: $downloadTime = $cache{$_.':lastDownloadTime'};
1335: $needUpdate = $cache{'ResourceUpdated'};
1336: untie(%cache);
1337: }
1338:
1339: if($c->aborted()) { return 'Aborted'; }
1340:
1341: if($needUpdate eq 'true') {
1342: $downloadTime = 'Not downloaded';
1343: }
1344: my $courseData =
1345: &DownloadCourseInformation($_, $courseID, $downloadTime,
1346: $WhatIWant);
1347: if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
1348: foreach my $key (keys(%$courseData)) {
1349: if($key =~ /^(con_lost|error|no_such_host)/i) {
1350: $courseData->{$_.':error'} = 'No course data for '.$_;
1351: last;
1352: }
1353: }
1354: if($extract eq 'true') {
1355: &ExtractStudentData($courseData, \%cache, \%cache, $_);
1356: } else {
1357: &ProcessStudentData(\%cache, $courseData, $_);
1358: }
1359: untie(%cache);
1360: } else {
1361: next;
1362: }
1363: }
1364: if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state); }
1365:
1366: return 'OK';
1367: }
1368:
1369: sub DownloadStudentCourseDataSeparate {
1370: my ($students,$checkDate,$cacheDB,$extract,$status,$courseID,$r,$c)=@_;
1371: my $residualFile = $Apache::lonnet::tmpdir.$courseID.'DownloadFile.db';
1372: my $title = 'LON-CAPA Statistics';
1373: my $heading = 'Download Course Data';
1374:
1375: my $WhatIWant;
1376: $WhatIWant = '(^version:|';
1377: $WhatIWant .= '^\d+:.+?:(resource\.\d+\.';
1378: $WhatIWant .= '(solved|tries|previous|awarded|(\d+\.submission))\s*$';#'
1379: $WhatIWant .= '|timestamp)';
1380: $WhatIWant .= ')';
1381:
1382: &CheckForResidualDownload($cacheDB, 'true', 'true', $courseID, $r, $c);
1383:
1384: my $studentCount = scalar(@$students);
1385: my %prog_state;
1386: if($status eq 'true') {
1387: %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r, $title,
1388: $heading,($#$students)+1);
1389: }
1390: my $displayString='';
1391: foreach (@$students) {
1392: if($c->aborted()) {
1393: return 'Aborted';
1394: }
1395:
1396: if($status eq 'true') {
1397: &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
1398: 'last student '.$_);
1399: }
1400:
1401: my %cache;
1402: my $downloadTime='Not downloaded';
1403: my $needUpdate = 'false';
1404: if($checkDate eq 'true' &&
1405: tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
1406: $downloadTime = $cache{$_.':lastDownloadTime'};
1407: $needUpdate = $cache{'ResourceUpdated'};
1408: untie(%cache);
1409: }
1410:
1411: if($c->aborted()) {
1412: return 'Aborted';
1413: }
1414:
1415: if($needUpdate eq 'true') {
1416: $downloadTime = 'Not downloaded';
1417: }
1418:
1419: my $error = 0;
1420: my $courseData =
1421: &DownloadCourseInformation($_, $courseID, $downloadTime,
1422: $WhatIWant);
1423: my %downloadData;
1424: unless(tie(%downloadData,'GDBM_File',$residualFile,
1425: &GDBM_WRCREAT(),0640)) {
1426: return 'Failed to tie temporary download hash.';
1427: }
1428: foreach my $key (keys(%$courseData)) {
1429: $downloadData{$key} = $courseData->{$key};
1430: if($key =~ /^(con_lost|error|no_such_host)/i) {
1431: $error = 1;
1432: last;
1433: }
1434: }
1435: if($error) {
1436: foreach my $deleteKey (keys(%$courseData)) {
1437: delete $downloadData{$deleteKey};
1438: }
1439: $downloadData{$_.':error'} = 'No course data for '.$_;
1440: }
1441: untie(%downloadData);
1442: }
1443: if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r,
1444: \%prog_state); }
1445:
1446: return &CheckForResidualDownload($cacheDB, 'true', 'true',
1447: $courseID, $r, $c);
1448: }
1449:
1450: sub CheckForResidualDownload {
1451: my ($cacheDB,$extract,$status,$courseID,$r,$c)=@_;
1452:
1453: my $residualFile = $Apache::lonnet::tmpdir.$courseID.'DownloadFile.db';
1454: if(!-e $residualFile) {
1455: return 'OK';
1456: }
1457:
1458: my %downloadData;
1459: my %cache;
1460: unless(tie(%downloadData,'GDBM_File',$residualFile,&GDBM_READER(),0640)) {
1461: return 'Can not tie database for check for residual download: tempDB';
1462: }
1463: unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
1464: untie(%downloadData);
1465: return 'Can not tie database for check for residual download: cacheDB';
1466: }
1467:
1468: my @students=();
1469: my %checkStudent;
1470: my $key;
1471: while(($key, undef) = each %downloadData) {
1472: my @temp = split(':', $key);
1473: my $student = $temp[0].':'.$temp[1];
1474: if(!defined($checkStudent{$student})) {
1475: $checkStudent{$student}++;
1476: push(@students, $student);
1477: }
1478: }
1479:
1480: my $heading = 'Process Course Data';
1481: my $title = 'LON-CAPA Statistics';
1482: my $studentCount = scalar(@students);
1483: my %prog_state;
1484: if($status eq 'true') {
1485: %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r, $title,
1486: $heading,$#students+1);
1487: }
1488:
1489: my $count=1;
1490: foreach my $name (@students) {
1491: last if($c->aborted());
1492:
1493: if($status eq 'true') {
1494: &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
1495: 'last student '.$name);
1496: }
1497:
1498: if($extract eq 'true') {
1499: &ExtractStudentData(\%downloadData, \%cache, \%cache, $name);
1500: } else {
1501: &ProcessStudentData(\%cache, \%downloadData, $name);
1502: }
1503: $count++;
1504: }
1505:
1506: if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r,
1507: \%prog_state); }
1508:
1509: untie(%cache);
1510: untie(%downloadData);
1511:
1512: if(!$c->aborted()) {
1513: my @files = ($residualFile);
1514: unlink(@files);
1515: }
1516:
1517: return 'OK';
1518: }
1519:
1520:
1521: ################################################
1522: ################################################
1523:
1524: =pod
1525:
1526: =item &make_into_hash($values);
1527:
1528: Returns a reference to a hash as described by $values. $values is
1529: assumed to be the result of
1530: join(':',map {&Apache::lonnet::escape($_)} %orighash);
1531:
1532: This is a helper function for get_current_state.
1533:
1534: =cut
1535:
1536: ################################################
1537: ################################################
1538: sub make_into_hash {
1539: my $values = shift;
1540: my %tmp = map { &Apache::lonnet::unescape($_); }
1541: split(':',$values);
1542: return \%tmp;
1543: }
1544:
1545:
1546: ################################################
1547: ################################################
1548:
1549: =pod
1550:
1551: =head1 LOCAL DATA CACHING SUBROUTINES
1552:
1553: The local caching is done using MySQL. There is no fall-back implementation
1554: if MySQL is not running.
1555:
1556: The programmers interface is to call &get_current_state() or some other
1557: primary interface subroutine (described below). The internals of this
1558: storage system are documented here.
1559:
1560: There are six tables used to store student performance data (the results of
1561: a dumpcurrent). Each of these tables is created in MySQL with a name of
1562: $courseid_*****, where ***** is 'symb', 'part', or whatever is appropriate
1563: for the table. The tables and their purposes are described below.
1564:
1565: Some notes before we get started.
1566:
1567: Each table must have a PRIMARY KEY, which is a column or set of columns which
1568: will serve to uniquely identify a row of data. NULL is not allowed!
1569:
1570: INDEXes work best on integer data.
1571:
1572: JOIN is used to combine data from many tables into one output.
1573:
1574: lonmysql.pm is used for some of the interface, specifically the table creation
1575: calls. The inserts are done in bulk by directly calling the database handler.
1576: The SELECT ... JOIN statement used to retrieve the data does not have an
1577: interface in lonmysql.pm and I shudder at the thought of writing one.
1578:
1579: =head3 Table Descriptions
1580:
1581: =over 4
1582:
1583: =item $symb_table
1584:
1585: The symb_table has two columns. The first is a 'symb_id' and the second
1586: is the text name for the 'symb' (limited to 64k). The 'symb_id' is generated
1587: automatically by MySQL so inserts should be done on this table with an
1588: empty first element. This table has its PRIMARY KEY on the 'symb_id'.
1589:
1590: =item $part_table
1591:
1592: The part_table has two columns. The first is a 'part_id' and the second
1593: is the text name for the 'part' (limited to 100 characters). The 'part_id' is
1594: generated automatically by MySQL so inserts should be done on this table with
1595: an empty first element. This table has its PRIMARY KEY on the 'part' (100
1596: characters) and a KEY on 'part_id'.
1597:
1598: =item $student_table
1599:
1600: The student_table has two columns. The first is a 'student_id' and the second
1601: is the text description of the 'student' (typically username:domain) (less
1602: than 100 characters). The 'student_id' is automatically generated by MySQL.
1603: The use of the name 'student_id' is loaded, I know, but this ID is used ONLY
1604: internally to the MySQL database and is not the same as the students ID
1605: (stored in the students environment). This table has its PRIMARY KEY on the
1606: 'student' (100 characters).
1607:
1608: =item $updatetime_table
1609:
1610: The updatetime_table has two columns. The first is 'student' (100 characters,
1611: typically username:domain). The second is 'updatetime', which is an unsigned
1612: integer, NOT a MySQL date. This table has its PRIMARY KEY on 'student' (100
1613: characters).
1614:
1615: =item $performance_table
1616:
1617: The performance_table has 9 columns. The first three are 'symb_id',
1618: 'student_id', and 'part_id'. These comprise the PRIMARY KEY for this table
1619: and are directly related to the $symb_table, $student_table, and $part_table
1620: described above. MySQL does better indexing on numeric items than text,
1621: so we use these three "index tables". The remaining columns are
1622: 'solved', 'tries', 'awarded', 'award', 'awarddetail', and 'timestamp'.
1623: These are either the MySQL type TINYTEXT or various integers ('tries' and
1624: 'timestamp'). This table has KEYs of 'student_id' and 'symb_id'.
1625: For use of this table, see the functions described below.
1626:
1627: =item $parameters_table
1628:
1629: The parameters_table holds the data that does not fit neatly into the
1630: performance_table. The parameters table has four columns: 'symb_id',
1631: 'student_id', 'parameter', and 'value'. 'symb_id', 'student_id', and
1632: 'parameter' comprise the PRIMARY KEY for this table. 'parameter' is
1633: limited to 255 characters. 'value' is limited to 64k characters.
1634:
1635: =back
1636:
1637: =head3 Important Subroutines
1638:
1639: Here is a brief overview of the subroutines which are likely to be of
1640: interest:
1641:
1642: =over 4
1643:
1644: =item &get_current_state(): programmers interface.
1645:
1646: =item &init_dbs(): table creation
1647:
1648: =item &update_student_data(): data storage calls
1649:
1650: =item &get_student_data_from_performance_cache(): data retrieval
1651:
1652: =back
1653:
1654: =head3 Main Documentation
1655:
1656: =over 4
1657:
1658: =cut
1659:
1660: ################################################
1661: ################################################
1662:
1663: ################################################
1664: ################################################
1665: {
1666:
1667: my $current_course ='';
1668: my $symb_table;
1669: my $part_table;
1670: my $student_table;
1671: my $updatetime_table;
1672: my $performance_table;
1673: my $parameters_table;
1674:
1675: ################################################
1676: ################################################
1677:
1678: =pod
1679:
1680: =item &setup_table_names()
1681:
1682: input: course id
1683:
1684: output: none
1685:
1686: Sets the package variables for the MySQL table names:
1687:
1688: =over 4
1689:
1690: =item $symb_table
1691:
1692: =item $part_table
1693:
1694: =item $student_table
1695:
1696: =item $updatetime_table
1697:
1698: =item $performance_table
1699:
1700: =item $parameters_table
1701:
1702: =back
1703:
1704: =cut
1705:
1706: ################################################
1707: ################################################
1708: sub setup_table_names {
1709: my $courseid = shift;
1710: if (! defined($courseid)) {
1711: $courseid = $ENV{'request.course.id'};
1712: }
1713: #
1714: # Set up database names
1715: my $base_id = $courseid;
1716: $symb_table = $base_id.'_'.'symb';
1717: $part_table = $base_id.'_'.'part';
1718: $student_table = $base_id.'_'.'student';
1719: $updatetime_table = $base_id.'_'.'updatetime';
1720: $performance_table = $base_id.'_'.'performance';
1721: $parameters_table = $base_id.'_'.'parameters';
1722: return;
1723: }
1724:
1725: ################################################
1726: ################################################
1727:
1728: =pod
1729:
1730: =item &init_dbs()
1731:
1732: Input: course id
1733:
1734: Output: 0 on success, positive integer on error
1735:
1736: This routine issues the calls to lonmysql to create the tables used to
1737: store student data.
1738:
1739: =cut
1740:
1741: ################################################
1742: ################################################
1743: sub init_dbs {
1744: my $courseid = shift;
1745: &setup_table_names($courseid);
1746: #
1747: # Note - changes to this table must be reflected in the code that
1748: # stores the data (calls &Apache::lonmysql::store_row with this table
1749: # id
1750: my $symb_table_def = {
1751: id => $symb_table,
1752: permanent => 'no',
1753: columns => [{ name => 'symb_id',
1754: type => 'MEDIUMINT UNSIGNED',
1755: restrictions => 'NOT NULL',
1756: auto_inc => 'yes', },
1757: { name => 'symb',
1758: type => 'MEDIUMTEXT',
1759: restrictions => 'NOT NULL'},
1760: ],
1761: 'PRIMARY KEY' => ['symb_id'],
1762: };
1763: #
1764: my $part_table_def = {
1765: id => $part_table,
1766: permanent => 'no',
1767: columns => [{ name => 'part_id',
1768: type => 'MEDIUMINT UNSIGNED',
1769: restrictions => 'NOT NULL',
1770: auto_inc => 'yes', },
1771: { name => 'part',
1772: type => 'VARCHAR(100)',
1773: restrictions => 'NOT NULL'},
1774: ],
1775: 'PRIMARY KEY' => ['part (100)'],
1776: 'KEY' => [{ columns => ['part_id']},],
1777: };
1778: #
1779: my $student_table_def = {
1780: id => $student_table,
1781: permanent => 'no',
1782: columns => [{ name => 'student_id',
1783: type => 'MEDIUMINT UNSIGNED',
1784: restrictions => 'NOT NULL',
1785: auto_inc => 'yes', },
1786: { name => 'student',
1787: type => 'VARCHAR(100)',
1788: restrictions => 'NOT NULL'},
1789: ],
1790: 'PRIMARY KEY' => ['student (100)'],
1791: 'KEY' => [{ columns => ['student_id']},],
1792: };
1793: #
1794: my $updatetime_table_def = {
1795: id => $updatetime_table,
1796: permanent => 'no',
1797: columns => [{ name => 'student',
1798: type => 'VARCHAR(100)',
1799: restrictions => 'NOT NULL UNIQUE',},
1800: { name => 'updatetime',
1801: type => 'INT UNSIGNED',
1802: restrictions => 'NOT NULL' },
1803: ],
1804: 'PRIMARY KEY' => ['student (100)'],
1805: };
1806: #
1807: my $performance_table_def = {
1808: id => $performance_table,
1809: permanent => 'no',
1810: columns => [{ name => 'symb_id',
1811: type => 'MEDIUMINT UNSIGNED',
1812: restrictions => 'NOT NULL' },
1813: { name => 'student_id',
1814: type => 'MEDIUMINT UNSIGNED',
1815: restrictions => 'NOT NULL' },
1816: { name => 'part_id',
1817: type => 'MEDIUMINT UNSIGNED',
1818: restrictions => 'NOT NULL' },
1819: { name => 'solved',
1820: type => 'TINYTEXT' },
1821: { name => 'tries',
1822: type => 'SMALLINT UNSIGNED' },
1823: { name => 'awarded',
1824: type => 'TINYTEXT' },
1825: { name => 'award',
1826: type => 'TINYTEXT' },
1827: { name => 'awarddetail',
1828: type => 'TINYTEXT' },
1829: { name => 'timestamp',
1830: type => 'INT UNSIGNED'},
1831: { name => 'weight',
1832: type => 'INT UNSIGNED'},
1833: ],
1834: 'PRIMARY KEY' => ['symb_id','student_id','part_id'],
1835: 'KEY' => [{ columns=>['student_id'] },
1836: { columns=>['symb_id'] },],
1837: };
1838: #
1839: my $parameters_table_def = {
1840: id => $parameters_table,
1841: permanent => 'no',
1842: columns => [{ name => 'symb_id',
1843: type => 'MEDIUMINT UNSIGNED',
1844: restrictions => 'NOT NULL' },
1845: { name => 'student_id',
1846: type => 'MEDIUMINT UNSIGNED',
1847: restrictions => 'NOT NULL' },
1848: { name => 'parameter',
1849: type => 'TINYTEXT',
1850: restrictions => 'NOT NULL' },
1851: { name => 'value',
1852: type => 'MEDIUMTEXT' },
1853: ],
1854: 'PRIMARY KEY' => ['symb_id','student_id','parameter (255)'],
1855: };
1856: #
1857: # Create the tables
1858: my $tableid;
1859: $tableid = &Apache::lonmysql::create_table($symb_table_def);
1860: if (! defined($tableid)) {
1861: &Apache::lonnet::logthis("error creating symb_table: ".
1862: &Apache::lonmysql::get_error());
1863: return 1;
1864: }
1865: #
1866: $tableid = &Apache::lonmysql::create_table($part_table_def);
1867: if (! defined($tableid)) {
1868: &Apache::lonnet::logthis("error creating part_table: ".
1869: &Apache::lonmysql::get_error());
1870: return 2;
1871: }
1872: #
1873: $tableid = &Apache::lonmysql::create_table($student_table_def);
1874: if (! defined($tableid)) {
1875: &Apache::lonnet::logthis("error creating student_table: ".
1876: &Apache::lonmysql::get_error());
1877: return 3;
1878: }
1879: #
1880: $tableid = &Apache::lonmysql::create_table($updatetime_table_def);
1881: if (! defined($tableid)) {
1882: &Apache::lonnet::logthis("error creating updatetime_table: ".
1883: &Apache::lonmysql::get_error());
1884: return 4;
1885: }
1886: #
1887: $tableid = &Apache::lonmysql::create_table($performance_table_def);
1888: if (! defined($tableid)) {
1889: &Apache::lonnet::logthis("error creating preformance_table: ".
1890: &Apache::lonmysql::get_error());
1891: return 5;
1892: }
1893: #
1894: $tableid = &Apache::lonmysql::create_table($parameters_table_def);
1895: if (! defined($tableid)) {
1896: &Apache::lonnet::logthis("error creating parameters_table: ".
1897: &Apache::lonmysql::get_error());
1898: return 6;
1899: }
1900: return 0;
1901: }
1902:
1903: ################################################
1904: ################################################
1905:
1906: =pod
1907:
1908: =item &get_part_id()
1909:
1910: Get the MySQL id of a problem part string.
1911:
1912: Input: $part
1913:
1914: Output: undef on error, integer $part_id on success.
1915:
1916: =item &get_part()
1917:
1918: Get the string describing a part from the MySQL id of the problem part.
1919:
1920: Input: $part_id
1921:
1922: Output: undef on error, $part string on success.
1923:
1924: =cut
1925:
1926: ################################################
1927: ################################################
1928:
1929: my %ids_by_part;
1930: my %parts_by_id;
1931:
1932: sub get_part_id {
1933: my ($part) = @_;
1934: if (! exists($ids_by_part{$part})) {
1935: &Apache::lonmysql::store_row($part_table,[undef,$part]);
1936: undef(%ids_by_part);
1937: my @Result = &Apache::lonmysql::get_rows($part_table);
1938: foreach (@Result) {
1939: $ids_by_part{$_->[1]}=$_->[0];
1940: }
1941: }
1942: return $ids_by_part{$part} if (exists($ids_by_part{$part}));
1943: return undef; # error
1944: }
1945:
1946: sub get_part {
1947: my ($part_id) = @_;
1948: if (! exists($parts_by_id{$part_id}) ||
1949: ! defined($parts_by_id{$part_id}) ||
1950: $parts_by_id{$part_id} eq '') {
1951: my @Result = &Apache::lonmysql::get_rows($part_table);
1952: foreach (@Result) {
1953: $parts_by_id{$_->[0]}=$_->[1];
1954: }
1955: }
1956: return $parts_by_id{$part_id} if(exists($parts_by_id{$part_id}));
1957: return undef; # error
1958: }
1959:
1960: ################################################
1961: ################################################
1962:
1963: =pod
1964:
1965: =item &get_symb_id()
1966:
1967: Get the MySQL id of a symb.
1968:
1969: Input: $symb
1970:
1971: Output: undef on error, integer $symb_id on success.
1972:
1973: =item &get_symb()
1974:
1975: Get the symb associated with a MySQL symb_id.
1976:
1977: Input: $symb_id
1978:
1979: Output: undef on error, $symb on success.
1980:
1981: =cut
1982:
1983: ################################################
1984: ################################################
1985:
1986: my %ids_by_symb;
1987: my %symbs_by_id;
1988:
1989: sub get_symb_id {
1990: my ($symb) = @_;
1991: if (! exists($ids_by_symb{$symb})) {
1992: &Apache::lonmysql::store_row($symb_table,[undef,$symb]);
1993: undef(%ids_by_symb);
1994: my @Result = &Apache::lonmysql::get_rows($symb_table);
1995: foreach (@Result) {
1996: $ids_by_symb{$_->[1]}=$_->[0];
1997: }
1998: }
1999: return $ids_by_symb{$symb} if(exists( $ids_by_symb{$symb}));
2000: return undef; # error
2001: }
2002:
2003: sub get_symb {
2004: my ($symb_id) = @_;
2005: if (! exists($symbs_by_id{$symb_id}) ||
2006: ! defined($symbs_by_id{$symb_id}) ||
2007: $symbs_by_id{$symb_id} eq '') {
2008: my @Result = &Apache::lonmysql::get_rows($symb_table);
2009: foreach (@Result) {
2010: $symbs_by_id{$_->[0]}=$_->[1];
2011: }
2012: }
2013: return $symbs_by_id{$symb_id} if(exists( $symbs_by_id{$symb_id}));
2014: return undef; # error
2015: }
2016:
2017: ################################################
2018: ################################################
2019:
2020: =pod
2021:
2022: =item &get_student_id()
2023:
2024: Get the MySQL id of a student.
2025:
2026: Input: $sname, $dom
2027:
2028: Output: undef on error, integer $student_id on success.
2029:
2030: =item &get_student()
2031:
2032: Get student username:domain associated with the MySQL student_id.
2033:
2034: Input: $student_id
2035:
2036: Output: undef on error, string $student (username:domain) on success.
2037:
2038: =cut
2039:
2040: ################################################
2041: ################################################
2042:
2043: my %ids_by_student;
2044: my %students_by_id;
2045:
2046: sub get_student_id {
2047: my ($sname,$sdom) = @_;
2048: my $student = $sname.':'.$sdom;
2049: if (! exists($ids_by_student{$student})) {
2050: &Apache::lonmysql::store_row($student_table,[undef,$student]);
2051: undef(%ids_by_student);
2052: my @Result = &Apache::lonmysql::get_rows($student_table);
2053: foreach (@Result) {
2054: $ids_by_student{$_->[1]}=$_->[0];
2055: }
2056: }
2057: return $ids_by_student{$student} if(exists( $ids_by_student{$student}));
2058: return undef; # error
2059: }
2060:
2061: sub get_student {
2062: my ($student_id) = @_;
2063: if (! exists($students_by_id{$student_id}) ||
2064: ! defined($students_by_id{$student_id}) ||
2065: $students_by_id{$student_id} eq '') {
2066: my @Result = &Apache::lonmysql::get_rows($student_table);
2067: foreach (@Result) {
2068: $students_by_id{$_->[0]}=$_->[1];
2069: }
2070: }
2071: return $students_by_id{$student_id} if(exists($students_by_id{$student_id}));
2072: return undef; # error
2073: }
2074:
2075: ################################################
2076: ################################################
2077:
2078: =pod
2079:
2080: =item &update_student_data()
2081:
2082: Input: $sname, $sdom, $courseid
2083:
2084: Output: $returnstatus, \%student_data
2085:
2086: $returnstatus is a string describing any errors that occured. 'okay' is the
2087: default.
2088: \%student_data is the data returned by a call to lonnet::currentdump.
2089:
2090: This subroutine loads a students data using lonnet::currentdump and inserts
2091: it into the MySQL database. The inserts are done on two tables,
2092: $performance_table and $parameters_table. $parameters_table holds the data
2093: that is not included in $performance_table. See the description of
2094: $performance_table elsewhere in this file. The INSERT calls are made
2095: directly by this subroutine, not through lonmysql because we do a 'bulk'
2096: insert which takes advantage of MySQLs non-SQL compliant INSERT command to
2097: insert multiple rows at a time. If anything has gone wrong during this
2098: process, $returnstatus is updated with a description of the error and
2099: \%student_data is returned.
2100:
2101: Notice we do not insert the data and immediately query it. This means it
2102: is possible for there to be data returned this first time that is not
2103: available the second time. CYA.
2104:
2105: =cut
2106:
2107: ################################################
2108: ################################################
2109: sub update_student_data {
2110: my ($sname,$sdom,$courseid) = @_;
2111: #
2112: # Set up database names
2113: &setup_table_names($courseid);
2114: #
2115: my $student_id = &get_student_id($sname,$sdom);
2116: my $student = $sname.':'.$sdom;
2117: #
2118: my $returnstatus = 'okay';
2119: #
2120: # Download students data
2121: my $time_of_retrieval = time;
2122: my @tmp = &Apache::lonnet::currentdump($courseid,$sdom,$sname);
2123: if ((scalar(@tmp) > 0) && ($tmp[0] =~ /^error:/)) {
2124: &Apache::lonnet::logthis('error getting data for '.
2125: $sname.':'.$sdom.' in course '.$courseid.
2126: ':'.$tmp[0]);
2127: $returnstatus = 'error getting data';
2128: return $returnstatus;
2129: }
2130: if (scalar(@tmp) < 1) {
2131: return ('no data',undef);
2132: }
2133: my %student_data = @tmp;
2134: #
2135: # Remove all of the students data from the table
2136: my $dbh = &Apache::lonmysql::get_dbh();
2137: $dbh->do('DELETE FROM '.$performance_table.' WHERE student_id='.
2138: $student_id);
2139: $dbh->do('DELETE FROM '.$parameters_table.' WHERE student_id='.
2140: $student_id);
2141: #
2142: # Store away the data
2143: #
2144: my $starttime = Time::HiRes::time;
2145: my $elapsed = 0;
2146: my $rows_stored;
2147: my $store_parameters_command = 'INSERT INTO '.$parameters_table.
2148: ' VALUES '."\n";
2149: my $store_performance_command = 'INSERT INTO '.$performance_table.
2150: ' VALUES '."\n";
2151: return 'error' if (! defined($dbh));
2152: while (my ($current_symb,$param_hash) = each(%student_data)) {
2153: #
2154: # make sure the symb is set up properly
2155: my $symb_id = &get_symb_id($current_symb);
2156: #
2157: # Load data into the tables
2158: while (my ($parameter,$value) = each (%$param_hash)) {
2159: my $newstring;
2160: if ($parameter !~ /(timestamp|resource\.(.*)\.(solved|tries|awarded|award|awarddetail|previous))/) {
2161: $newstring = "('".join("','",
2162: $symb_id,$student_id,
2163: $parameter,$value)."'),\n";
2164: if ($newstring !~ /''/) {
2165: $store_parameters_command .= $newstring;
2166: $rows_stored++;
2167: }
2168: }
2169: next if ($parameter !~ /^resource\.(.*)\.solved$/);
2170: #
2171: my $part = $1;
2172: my $part_id = &get_part_id($part);
2173: next if (!defined($part_id));
2174: my $solved = $value;
2175: my $tries = $param_hash->{'resource.'.$part.'.tries'};
2176: my $awarded = $param_hash->{'resource.'.$part.'.awarded'};
2177: my $award = $param_hash->{'resource.'.$part.'.award'};
2178: my $awarddetail = $param_hash->{'resource.'.$part.'.awarddetail'};
2179: my $timestamp = $param_hash->{'timestamp'};
2180: # use EXT to get the weight
2181: my $weight = &Apache::lonnet::EXT('resource.'.$part.'.weight',
2182: $current_symb,$sdom,$sname);
2183: # Give the weight back to the user
2184: $param_hash->{'resource.'.$part.'.weight'}=$weight;
2185: #
2186: $solved = '' if (! defined($awarded));
2187: $tries = '' if (! defined($tries));
2188: $awarded = '' if (! defined($awarded));
2189: $award = '' if (! defined($award));
2190: $awarddetail = '' if (! defined($awarddetail));
2191: $newstring = "('".join("','",$symb_id,$student_id,$part_id,
2192: $solved,$tries,$awarded,$award,
2193: $awarddetail,$timestamp,$weight)."'),\n";
2194: $store_performance_command .= $newstring;
2195: $rows_stored++;
2196: }
2197: }
2198: chop $store_parameters_command;
2199: chop $store_parameters_command;
2200: chop $store_performance_command;
2201: chop $store_performance_command;
2202: my $start = Time::HiRes::time;
2203: $dbh->do($store_parameters_command);
2204: if ($dbh->err()) {
2205: &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr());
2206: &Apache::lonnet::logthis('command = '.$store_performance_command);
2207: $returnstatus = 'error: unable to insert parameters into database';
2208: return $returnstatus,\%student_data;
2209: }
2210: $dbh->do($store_performance_command);
2211: if ($dbh->err()) {
2212: &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr());
2213: &Apache::lonnet::logthis('command = '.$store_parameters_command);
2214: $returnstatus = 'error: unable to insert performance into database';
2215: return $returnstatus,\%student_data;
2216: }
2217: $elapsed += Time::HiRes::time - $start;
2218: #
2219: # Set the students update time
2220: &Apache::lonmysql::replace_row($updatetime_table,
2221: [$student,$time_of_retrieval]);
2222: return ($returnstatus,\%student_data);
2223: }
2224:
2225: ################################################
2226: ################################################
2227:
2228: =pod
2229:
2230: =item &ensure_current_data()
2231:
2232: Input: $sname, $sdom, $courseid
2233:
2234: Output: $status, $data
2235:
2236: This routine ensures the data for a given student is up to date. It calls
2237: &init_dbs() if the tables do not exist. The $updatetime_table is queried
2238: to determine the time of the last update. If the students data is out of
2239: date, &update_student_data() is called. The return values from the call
2240: to &update_student_data() are returned.
2241:
2242: =cut
2243:
2244: ################################################
2245: ################################################
2246: sub ensure_current_data {
2247: my ($sname,$sdom,$courseid) = @_;
2248: my $status = 'okay'; # return value
2249: #
2250: &setup_table_names($courseid);
2251: #
2252: # if the tables do not exist, make them
2253: my @CurrentTable = &Apache::lonmysql::tables_in_db();
2254: my ($found_symb,$found_student,$found_part,$found_update,
2255: $found_performance,$found_parameters);
2256: foreach (@CurrentTable) {
2257: $found_symb = 1 if ($_ eq $symb_table);
2258: $found_student = 1 if ($_ eq $student_table);
2259: $found_part = 1 if ($_ eq $part_table);
2260: $found_update = 1 if ($_ eq $updatetime_table);
2261: $found_performance = 1 if ($_ eq $performance_table);
2262: $found_parameters = 1 if ($_ eq $parameters_table);
2263: }
2264: if (!$found_symb || !$found_update ||
2265: !$found_student || !$found_part ||
2266: !$found_performance || !$found_parameters) {
2267: if (&init_dbs($courseid)) {
2268: return 'error';
2269: }
2270: }
2271: #
2272: # Get the update time for the user
2273: my $updatetime = 0;
2274: my $modifiedtime = &Apache::lonnet::GetFileTimestamp
2275: ($sdom,$sname,$courseid.'.db',
2276: $Apache::lonnet::perlvar{'lonUsersDir'});
2277: #
2278: my $student = $sname.':'.$sdom;
2279: my @Result = &Apache::lonmysql::get_rows($updatetime_table,
2280: "student ='$student'");
2281: my $data = undef;
2282: if (@Result) {
2283: $updatetime = $Result[0]->[1];
2284: }
2285: if ($modifiedtime > $updatetime) {
2286: ($status,$data) = &update_student_data($sname,$sdom,$courseid);
2287: }
2288: return ($status,$data);
2289: }
2290:
2291: ################################################
2292: ################################################
2293:
2294: =pod
2295:
2296: =item &get_student_data_from_performance_cache()
2297:
2298: Input: $sname, $sdom, $symb, $courseid
2299:
2300: Output: hash reference containing the data for the given student.
2301: If $symb is undef, all the students data is returned.
2302:
2303: This routine is the heart of the local caching system. See the description
2304: of $performance_table, $symb_table, $student_table, and $part_table. The
2305: main task is building the MySQL request. The tables appear in the request
2306: in the order in which they should be parsed by MySQL. When searching
2307: on a student the $student_table is used to locate the 'student_id'. All
2308: rows in $performance_table which have a matching 'student_id' are returned,
2309: with data from $part_table and $symb_table which match the entries in
2310: $performance_table, 'part_id' and 'symb_id'. When searching on a symb,
2311: the $symb_table is processed first, with matching rows grabbed from
2312: $performance_table and filled in from $part_table and $student_table in
2313: that order.
2314:
2315: Running 'EXPLAIN ' on the 'SELECT' statements generated can be quite
2316: interesting, especially if you play with the order the tables are listed.
2317:
2318: =cut
2319:
2320: ################################################
2321: ################################################
2322: sub get_student_data_from_performance_cache {
2323: my ($sname,$sdom,$symb,$courseid)=@_;
2324: my $student = $sname.':'.$sdom if (defined($sname) && defined($sdom));
2325: &setup_table_names();
2326: #
2327: # Return hash
2328: my $studentdata;
2329: #
2330: my $dbh = &Apache::lonmysql::get_dbh();
2331: my $request = "SELECT ".
2332: "d.symb,c.part,a.solved,a.tries,a.awarded,a.award,a.awarddetail,".
2333: "a.timestamp,a.weight ";
2334: if (defined($student)) {
2335: $request .= "FROM $student_table AS b ".
2336: "LEFT JOIN $performance_table AS a ON b.student_id=a.student_id ".
2337: "LEFT JOIN $part_table AS c ON c.part_id = a.part_id ".
2338: "LEFT JOIN $symb_table AS d ON d.symb_id = a.symb_id ".
2339: "WHERE student='$student'";
2340: if (defined($symb) && $symb ne '') {
2341: $request .= " AND d.symb='".$dbh->quote($symb)."'";
2342: }
2343: } elsif (defined($symb) && $symb ne '') {
2344: $request .= "FROM $symb_table as d ".
2345: "LEFT JOIN $performance_table AS a ON d.symb_id=a.symb_id ".
2346: "LEFT JOIN $part_table AS c ON c.part_id = a.part_id ".
2347: "LEFT JOIN $student_table AS b ON b.student_id = a.student_id ".
2348: "WHERE symb='".$dbh->quote($symb)."'";
2349: }
2350: my $starttime = Time::HiRes::time;
2351: my $rows_retrieved = 0;
2352: my $sth = $dbh->prepare($request);
2353: $sth->execute();
2354: if ($sth->err()) {
2355: &Apache::lonnet::logthis("Unable to execute MySQL request:");
2356: &Apache::lonnet::logthis("\n".$request."\n");
2357: &Apache::lonnet::logthis("error is:".$sth->errstr());
2358: return undef;
2359: }
2360: foreach my $row (@{$sth->fetchall_arrayref}) {
2361: $rows_retrieved++;
2362: my ($symb,$part,$solved,$tries,$awarded,$award,$awarddetail,$time,$weight) =
2363: (@$row);
2364: my $base = 'resource.'.$part;
2365: $studentdata->{$symb}->{$base.'.solved'} = $solved;
2366: $studentdata->{$symb}->{$base.'.tries'} = $tries;
2367: $studentdata->{$symb}->{$base.'.awarded'} = $awarded;
2368: $studentdata->{$symb}->{$base.'.award'} = $award;
2369: $studentdata->{$symb}->{$base.'.awarddetail'} = $awarddetail;
2370: $studentdata->{$symb}->{'timestamp'} = $time if (defined($time) && $time ne '');
2371: $studentdata->{$symb}->{'resource.'.$part.'.weight'}=$weight;
2372: }
2373: return $studentdata;
2374: }
2375:
2376: ################################################
2377: ################################################
2378:
2379: =pod
2380:
2381: =item &get_current_state()
2382:
2383: Input: $sname,$sdom,$symb,$courseid
2384:
2385: Output: Described below
2386:
2387: Retrieve the current status of a students performance. $sname and
2388: $sdom are the only required parameters. If $symb is undef the results
2389: of an &Apache::lonnet::currentdump() will be returned.
2390: If $courseid is undef it will be retrieved from the environment.
2391:
2392: The return structure is based on &Apache::lonnet::currentdump. If
2393: $symb is unspecified, all the students data is returned in a hash of
2394: the form:
2395: (
2396: symb1 => { param1 => value1, param2 => value2 ... },
2397: symb2 => { param1 => value1, param2 => value2 ... },
2398: )
2399:
2400: If $symb is specified, a hash of
2401: (
2402: param1 => value1,
2403: param2 => value2,
2404: )
2405: is returned.
2406:
2407: If no data is found for $symb, or if the student has no performance data,
2408: an empty list is returned.
2409:
2410: =cut
2411:
2412: ################################################
2413: ################################################
2414: sub get_current_state {
2415: my ($sname,$sdom,$symb,$courseid,$forcedownload)=@_;
2416: if ($current_course ne $courseid) {
2417: # Clear out variables
2418: undef(%ids_by_part);
2419: undef(%parts_by_id);
2420: undef(%ids_by_symb);
2421: undef(%symbs_by_id);
2422: undef(%ids_by_student);
2423: undef(%students_by_id);
2424: $current_course = $courseid;
2425: }
2426: return () if (! defined($sname) || ! defined($sdom));
2427: #
2428: $courseid = $ENV{'request.course.id'} if (! defined($courseid));
2429: #
2430: my ($status,$data) = &ensure_current_data($sname,$sdom,$courseid);
2431: #
2432: if (defined($data)) {
2433: return %$data;
2434: } elsif ($status eq 'no data') {
2435: return ();
2436: } else {
2437: if ($status ne 'okay' && $status ne '') {
2438: &Apache::lonnet::logthis('status = '.$status);
2439: return ();
2440: }
2441: my $returnhash = &get_student_data_from_performance_cache($sname,$sdom,
2442: $symb,$courseid);
2443: return %$returnhash if (defined($returnhash));
2444: }
2445: return ();
2446: }
2447:
2448: ################################################
2449: ################################################
2450:
2451: =pod
2452:
2453: =back
2454:
2455: =item End of Local Data Caching Subroutines
2456:
2457: =cut
2458:
2459: ################################################
2460: ################################################
2461:
2462:
2463: }
2464: ################################################
2465: ################################################
2466:
2467: =pod
2468:
2469: =head3 Classlist Subroutines
2470:
2471: =item &get_classlist();
2472:
2473: Retrieve the classist of a given class or of the current class. Student
2474: information is returned from the classlist.db file and, if needed,
2475: from the students environment.
2476:
2477: Optional arguments are $cid, $cdom, and $cnum (course id, course domain,
2478: and course number, respectively). Any omitted arguments will be taken
2479: from the current environment ($ENV{'request.course.id'},
2480: $ENV{'course.'.$cid.'.domain'}, and $ENV{'course.'.$cid.'.num'}).
2481:
2482: Returns a reference to a hash which contains:
2483: keys '$sname:$sdom'
2484: values [$sdom,$sname,$end,$start,$id,$section,$fullname,$status]
2485:
2486: The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
2487: as indices into the returned list to future-proof clients against
2488: changes in the list order.
2489:
2490: =cut
2491:
2492: ################################################
2493: ################################################
2494:
2495: sub CL_SDOM { return 0; }
2496: sub CL_SNAME { return 1; }
2497: sub CL_END { return 2; }
2498: sub CL_START { return 3; }
2499: sub CL_ID { return 4; }
2500: sub CL_SECTION { return 5; }
2501: sub CL_FULLNAME { return 6; }
2502: sub CL_STATUS { return 7; }
2503:
2504: sub get_classlist {
2505: my ($cid,$cdom,$cnum) = @_;
2506: $cid = $cid || $ENV{'request.course.id'};
2507: $cdom = $cdom || $ENV{'course.'.$cid.'.domain'};
2508: $cnum = $cnum || $ENV{'course.'.$cid.'.num'};
2509: my $now = time;
2510: #
2511: my %classlist=&Apache::lonnet::dump('classlist',$cdom,$cnum);
2512: while (my ($student,$info) = each(%classlist)) {
2513: if ($student =~ /^(con_lost|error|no_such_host)/i) {
2514: &Apache::lonnet::logthis('get_classlist error for '.$cid.':'.$student);
2515: return undef;
2516: }
2517: my ($sname,$sdom) = split(/:/,$student);
2518: my @Values = split(/:/,$info);
2519: my ($end,$start,$id,$section,$fullname);
2520: if (@Values > 2) {
2521: ($end,$start,$id,$section,$fullname) = @Values;
2522: } else { # We have to get the data ourselves
2523: ($end,$start) = @Values;
2524: $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
2525: my %info=&Apache::lonnet::get('environment',
2526: ['firstname','middlename',
2527: 'lastname','generation','id'],
2528: $sdom, $sname);
2529: my ($tmp) = keys(%info);
2530: if ($tmp =~/^(con_lost|error|no_such_host)/i) {
2531: $fullname = 'not available';
2532: $id = 'not available';
2533: &Apache::lonnet::logthis('unable to retrieve environment '.
2534: 'for '.$sname.':'.$sdom);
2535: } else {
2536: $fullname = &ProcessFullName(@info{qw/lastname generation
2537: firstname middlename/});
2538: $id = $info{'id'};
2539: }
2540: # Update the classlist with this students information
2541: if ($fullname ne 'not available') {
2542: my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
2543: my $reply=&Apache::lonnet::cput('classlist',
2544: {$student => $enrolldata},
2545: $cdom,$cnum);
2546: if ($reply !~ /^(ok|delayed)/) {
2547: &Apache::lonnet::logthis('Unable to update classlist for '.
2548: 'student '.$sname.':'.$sdom.
2549: ' error:'.$reply);
2550: }
2551: }
2552: }
2553: my $status='Expired';
2554: if(((!$end) || $now < $end) && ((!$start) || ($now > $start))) {
2555: $status='Active';
2556: }
2557: $classlist{$student} =
2558: [$sdom,$sname,$end,$start,$id,$section,$fullname,$status];
2559: }
2560: if (wantarray()) {
2561: return (\%classlist,['domain','username','end','start','id',
2562: 'section','fullname','status']);
2563: } else {
2564: return \%classlist;
2565: }
2566: }
2567:
2568: # ----- END HELPER FUNCTIONS --------------------------------------------
2569:
2570: 1;
2571: __END__
2572:
2573:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>