Diff for /loncom/interface/loncoursedata.pm between versions 1.1 and 1.196

version 1.1, 2002/07/09 15:43:49 version 1.196, 2015/03/10 02:45:56
Line 1 Line 1
 # The LearningOnline Network with CAPA  # The LearningOnline Network with CAPA
 # (Publication Handler  
 #  #
 # $Id$  # $Id$
 #  #
Line 26 Line 25
 # http://www.lon-capa.org/  # http://www.lon-capa.org/
 #  #
 ###  ###
   
 =pod  =pod
   
 =head1 NAME  =head1 NAME
   
 loncoursedata  Apache::loncoursedata
   
 =head1 SYNOPSIS  =head1 SYNOPSIS
   
 Set of functions that download and process student information.  Set of functions that download and process student and course information.
   
 =head1 PACKAGES USED  =head1 PACKAGES USED
   
  Apache::Constants qw(:common :http)    Apache::lonnet
  Apache::lonnet()    Apache::longroup
  HTML::TokeParser    Time::HiRes
  GDBM_File    Apache::lonmysql
     LONCAPA
     Digest::MD5
   
 =cut  =cut
   
 package Apache::loncoursedata;  package Apache::loncoursedata;
   
 use strict;  use strict;
 use Apache::Constants qw(:common :http);  use Apache::lonnet;
 use Apache::lonnet();  use Apache::longroup();
 use HTML::TokeParser;  use Time::HiRes();
 use GDBM_File;  use Apache::lonmysql();
   use LONCAPA;
   use Digest::MD5();
   
   =pod 
   
   =head2 make_into_hash
   
   Turn a colon separated string into a hash and return a reference
   to it.  Numbering from 0 even elements are keys and odd elements
   are values e.g. a:b:c:d creates a hash like
     a => b, c =>d
   
   =cut
   
   sub make_into_hash {
       my $values = shift;
       my %tmp = map { &unescape($_); } split(':',$values);
       return \%tmp;
   }
   
   
   { # Begin scope of table identifiers
   
   my $current_course ='';
   my $symb_table;
   my $part_table;
   my $student_table;
   my $groupnames_table;
   my $students_groups_table;
   my $performance_table;
   my $parameters_table;
   my $fulldump_response_table;
   my $fulldump_part_table;
   my $fulldump_timestamp_table;
   my $weight_table;
   
   my @Tables;
   
   
   
   sub init_dbs {
       my ($courseid,$drop) = @_;
       &setup_table_names($courseid);
       #
       # Drop any of the existing tables
       if ($drop) {
           foreach my $table (@Tables) {
               &Apache::lonmysql::drop_table($table);
           }
       }
       #
       # Note - changes to this table must be reflected in the code that 
       # stores the data (calls &Apache::lonmysql::store_row with this table
       # id
       my $symb_table_def = {
           id => $symb_table,
           permanent => 'no',
           columns => [{ name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL',
                         auto_inc     => 'yes', },
                       { name => 'symb',
                         type => 'MEDIUMTEXT',
                         restrictions => 'NOT NULL'},
                       ],
           'PRIMARY KEY' => ['symb_id'],
       };
       #
       my $part_table_def = {
           id => $part_table,
           permanent => 'no',
           columns => [{ name => 'part_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL',
                         auto_inc     => 'yes', },
                       { name => 'part',
                         type => 'VARCHAR(100) BINARY',
                         restrictions => 'NOT NULL'},
                       ],
           'PRIMARY KEY' => ['part (100)'],
           'KEY' => [{ columns => ['part_id']},],
       };
       #
       my $student_table_def = {
           id => $student_table,
           permanent => 'no',
           columns => [{ name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL',
                         auto_inc     => 'yes', },
                       { name => 'student',
                         type => 'VARCHAR(100) BINARY',
                         restrictions => 'NOT NULL UNIQUE'},
                       { name => 'section',
                         type => 'VARCHAR(100) BINARY',
                         restrictions => 'NOT NULL'},
                       { name => 'start',
                         type => 'INT',
                         restrictions => 'NOT NULL'},
                       { name => 'end',
                         type => 'INT',
                         restrictions => 'NOT NULL'},
                       { name => 'classification',
                         type => 'VARCHAR(100) BINARY', },
                       { name => 'updatetime',
                         type => 'INT UNSIGNED'},
                       { name => 'fullupdatetime',
                         type => 'INT UNSIGNED'},
                       ],
           'PRIMARY KEY' => ['student_id'],
           'KEY' => [{ columns => ['student (100)',
                                   'section (100)',
                                   'start',
    'end']},],
       };
       #
       my $groupnames_table_def = {
           id => $groupnames_table,
           permanent => 'no',
           columns => [{ name => 'group_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL',
                         auto_inc => 'yes', },
                       { name => 'groupname',
                         type => 'VARCHAR(100) BINARY',
                         restrictions => 'NOT NULL UNIQUE'},
                      ],
           'PRIMARY KEY' => ['group_id'],
           'KEY' => [{ columns => ['groupname (100)',]},],
       };
       #
       my $students_groups_table_def = {
           id => $students_groups_table,
           permanent => 'no',
           columns => [{ name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL', },
                       { name => 'group_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL', },
                      ],
           'PRIMARY KEY' => ['student_id','group_id'],
           'KEY' => [{ columns => ['student_id'] },
                     { columns => ['group_id'] },],
       };
       #
       my $performance_table_def = {
           id => $performance_table,
           permanent => 'no',
           columns => [{ name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'part_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'part',
                         type => 'VARCHAR(100) BINARY',
                         restrictions => 'NOT NULL'},                    
                       { name => 'solved',
                         type => 'TINYTEXT' },
                       { name => 'tries',
                         type => 'SMALLINT UNSIGNED' },
                       { name => 'awarded',
                         type => 'REAL' },
                       { name => 'award',
                         type => 'TINYTEXT' },
                       { name => 'awarddetail',
                         type => 'TINYTEXT' },
                       { name => 'timestamp',
                         type => 'INT UNSIGNED'},
                       ],
           'PRIMARY KEY' => ['symb_id','student_id','part_id'],
           'KEY' => [{ columns=>['student_id'] },
                     { columns=>['symb_id'] },],
       };
       #
       my $fulldump_part_table_def = {
           id => $fulldump_part_table,
           permanent => 'no',
           columns => [
                       { name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'part_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'transaction',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'tries',
                         type => 'SMALLINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'award',
                         type => 'TINYTEXT' },
                       { name => 'awarded',
                         type => 'REAL' },
                       { name => 'previous',
                         type => 'SMALLINT UNSIGNED' },
   #                    { name => 'regrader',
   #                      type => 'TINYTEXT' },
   #                    { name => 'afterduedate',
   #                      type => 'TINYTEXT' },
                       ],
           'PRIMARY KEY' => ['symb_id','part_id','student_id','transaction'],
           'KEY' => [
                     { columns=>['symb_id'] },
                     { columns=>['part_id'] },
                     { columns=>['student_id'] },
                     ],
       };
       #
       my $fulldump_response_table_def = {
           id => $fulldump_response_table,
           permanent => 'no',
           columns => [
                       { name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'part_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'response_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'transaction',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'awarddetail',
                         type => 'TINYTEXT' },
   #                    { name => 'message',
   #                      type => 'CHAR BINARY'},
                       { name => 'response_specific',
                         type => 'TINYTEXT' },
                       { name => 'response_specific_value',
                         type => 'TINYTEXT' },
                       { name => 'response_specific_2',
                         type => 'TINYTEXT' },
                       { name => 'response_specific_value_2',
                         type => 'TINYTEXT' },
                       { name => 'submission',
                         type => 'TEXT'},
                       ],
               'PRIMARY KEY' => ['symb_id','part_id','response_id','student_id',
                                 'transaction'],
               'KEY' => [
                         { columns=>['symb_id'] },
                         { columns=>['part_id','response_id'] },
                         { columns=>['student_id'] },
                         ],
       };
       my $fulldump_timestamp_table_def = {
           id => $fulldump_timestamp_table,
           permanent => 'no',
           columns => [
                       { name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'transaction',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL' },
                       { name => 'timestamp',
                         type => 'INT UNSIGNED'},
                       ],
           'PRIMARY KEY' => ['symb_id','student_id','transaction'],
           'KEY' => [
                     { columns=>['symb_id'] },
                     { columns=>['student_id'] },
                     { columns=>['transaction'] },
                     ],
       };
       #
       my $parameters_table_def = {
           id => $parameters_table,
           permanent => 'no',
           columns => [{ name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'student_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'parameter',
                         type => 'TINYTEXT',
                         restrictions => 'NOT NULL'  },
                       { name => 'value',
                         type => 'MEDIUMTEXT' },
                       ],
           'PRIMARY KEY' => ['symb_id','student_id','parameter (255)'],
       };
       #
       my $weight_table_def = {
           id => $weight_table,
           permanent => 'no',
           columns => [{ name => 'symb_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'part_id',
                         type => 'MEDIUMINT UNSIGNED',
                         restrictions => 'NOT NULL'  },
                       { name => 'weight',
                         type => 'REAL',
                         restrictions => 'NOT NULL'  },
                       ],
           'PRIMARY KEY' => ['symb_id','part_id'],
       };
       #
       # Create the tables
       my $tableid;
       $tableid = &Apache::lonmysql::create_table($symb_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating symb_table: ".
                                    &Apache::lonmysql::get_error());
           return 1;
       }
       #
       $tableid = &Apache::lonmysql::create_table($part_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating part_table: ".
                                    &Apache::lonmysql::get_error());
           return 2;
       }
       #
       $tableid = &Apache::lonmysql::create_table($student_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating student_table: ".
                                    &Apache::lonmysql::get_error());
           return 3;
       }
       #
       $tableid = &Apache::lonmysql::create_table($performance_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating preformance_table: ".
                                    &Apache::lonmysql::get_error());
           return 5;
       }
       #
       $tableid = &Apache::lonmysql::create_table($parameters_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating parameters_table: ".
                                    &Apache::lonmysql::get_error());
           return 6;
       }
       #
       $tableid = &Apache::lonmysql::create_table($fulldump_part_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating fulldump_part_table: ".
                                    &Apache::lonmysql::get_error());
           return 7;
       }
       #
       $tableid = &Apache::lonmysql::create_table($fulldump_response_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating fulldump_response_table: ".
                                    &Apache::lonmysql::get_error());
           return 8;
       }
       $tableid = &Apache::lonmysql::create_table($fulldump_timestamp_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating fulldump_timestamp_table: ".
                                    &Apache::lonmysql::get_error());
           return 9;
       }
       $tableid = &Apache::lonmysql::create_table($weight_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating weight_table: ".
                                    &Apache::lonmysql::get_error());
           return 10;
       }
       $tableid = &Apache::lonmysql::create_table($groupnames_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating groupnames_table: ".
                                    &Apache::lonmysql::get_error());
           return 11;
       }
       $tableid = &Apache::lonmysql::create_table($students_groups_table_def);
       if (! defined($tableid)) {
           &Apache::lonnet::logthis("error creating student_groups_table: ".
                                    &Apache::lonmysql::get_error());
           return 12;
       }
       return 0;
   }
   
 =pod  =pod
   
 =head1 DOWNLOAD INFORMATION  =head2 delete_caches
   
 This section contains all the files that get data from other servers   Drops all of the tables in the local mysql cache associated with the
 and/or itself.  There is one function that has a call to get remote  specified course id.
 information but isn't included here which is ProcessTopLevelMap.  The  
 usage was small enough to be ignored, but that portion may be moved  TODO:  The drops should be pushed into lonmysql to further isolate 
 here in the future.  mysql code from other modules.
   
 =cut  =cut
   sub delete_caches {
       my $courseid = shift;
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       #
       &setup_table_names($courseid);
       #
       my $dbh = &Apache::lonmysql::get_dbh();
       foreach my $table (@Tables) {
           my $command = 'DROP TABLE '.$table.';';
           $dbh->do($command);
           if ($dbh->err) {
               &Apache::lonnet::logthis($command.' resulted in error: '.$dbh->errstr);
           }
       }
       return;
   }
   
 # ----- DOWNLOAD INFORMATION -------------------------------------------  
   
 =pod  my $have_read_part_table = 0;
   my %ids_by_part;
   my %parts_by_id;
   
   sub get_part_id {
       my ($part) = @_;
       $part = 0 if (! defined($part));
       if (! $have_read_part_table) {
           my @Result = &Apache::lonmysql::get_rows($part_table);
           foreach (@Result) {
               $ids_by_part{$_->[1]}=$_->[0];
           }
           $have_read_part_table = 1;
       }
       if (! exists($ids_by_part{$part})) {
           &Apache::lonmysql::store_row($part_table,[undef,$part]);
           undef(%ids_by_part);
           my @Result = &Apache::lonmysql::get_rows($part_table);
           foreach (@Result) {
               $ids_by_part{$_->[1]}=$_->[0];
           }
       }
       return $ids_by_part{$part} if (exists($ids_by_part{$part}));
       return undef; # error
   }
   
 =item &DownloadNamePIDSection()  sub get_part {
       my ($part_id) = @_;
       if (! exists($parts_by_id{$part_id})  || 
           ! defined($parts_by_id{$part_id}) ||
           $parts_by_id{$part_id} eq '') {
           my @Result = &Apache::lonmysql::get_rows($part_table);
           foreach (@Result) {
               $parts_by_id{$_->[0]}=$_->[1];
           }
       }
       return $parts_by_id{$part_id} if(exists($parts_by_id{$part_id}));
       return undef; # error
   }
   
 Collects lastname, generation, middlename, firstname, PID, and section for each  
 student from their environment database.  The list of students is built from  
 collecting a classlist for the course that is to be displayed.  
   
 =over 4  my $have_read_symb_table = 0;
   my %ids_by_symb;
   my %symbs_by_id;
   
   sub get_symb_id {
       my ($symb) = @_;
       if (! $have_read_symb_table) {
           my @Result = &Apache::lonmysql::get_rows($symb_table);
           foreach (@Result) {
               $ids_by_symb{$_->[1]}=$_->[0];
           }
           $have_read_symb_table = 1;
       }
       if (! exists($ids_by_symb{$symb})) {
           &Apache::lonmysql::store_row($symb_table,[undef,$symb]);
           undef(%ids_by_symb);
           my @Result = &Apache::lonmysql::get_rows($symb_table);
           foreach (@Result) {
               $ids_by_symb{$_->[1]}=$_->[0];
           }
       }
       return $ids_by_symb{$symb} if(exists( $ids_by_symb{$symb}));
       return undef; # error
   }
   
   sub get_symb {
       my ($symb_id) = @_;
       if (! exists($symbs_by_id{$symb_id})  || 
           ! defined($symbs_by_id{$symb_id}) ||
           $symbs_by_id{$symb_id} eq '') {
           my @Result = &Apache::lonmysql::get_rows($symb_table);
           foreach (@Result) {
               $symbs_by_id{$_->[0]}=$_->[1];
           }
       }
       return $symbs_by_id{$symb_id} if(exists( $symbs_by_id{$symb_id}));
       return undef; # error
   }
   
   my $have_read_student_table = 0;
   my %ids_by_student;
   my %students_by_id;
   
   sub get_student_id {
       my ($sname,$sdom) = @_;
       my $student = $sname.':'.$sdom;
       if (! $have_read_student_table) {
           my @Result = &Apache::lonmysql::get_rows($student_table);
           foreach (@Result) {
               $ids_by_student{$_->[1]}=$_->[0];
           }
           $have_read_student_table = 1;
       }
       if (! exists($ids_by_student{$student})) {
           &populate_student_table();
           undef(%ids_by_student);
           undef(%students_by_id);
           my @Result = &Apache::lonmysql::get_rows($student_table);
           foreach (@Result) {
               $ids_by_student{$_->[1]}=$_->[0];
           }
       }
       return $ids_by_student{$student} if(exists( $ids_by_student{$student}));
       return undef; # error
   }
   
   sub get_student {
       my ($student_id) = @_;
       if (! exists($students_by_id{$student_id})  || 
           ! defined($students_by_id{$student_id}) ||
           $students_by_id{$student_id} eq '') {
           my @Result = &Apache::lonmysql::get_rows($student_table);
           foreach (@Result) {
               $students_by_id{$_->[0]}=$_->[1];
           }
       }
       return $students_by_id{$student_id} if(exists($students_by_id{$student_id}));
       return undef; # error
   }
   
   sub populate_student_table {
       my ($courseid) = @_;
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       #
       &setup_table_names($courseid);
       &init_dbs($courseid,0);
       my $dbh = &Apache::lonmysql::get_dbh();
       my $request = 'INSERT IGNORE INTO '.$student_table.
           "(student,section,start,end) VALUES ";
       my $cdom = $env{'course.'.$courseid.'.domain'};
       my $cnum = $env{'course.'.$courseid.'.num'};
       my $classlist = &get_classlist($cdom,$cnum);
       my $student_count=0;
       while (my ($student,$data) = each %$classlist) {
           my ($section,$start,$end) = ($data->[&CL_SECTION()],
        $data->[&CL_START()],
        $data->[&CL_END()]);
           if ($section eq '' || $section =~ /^\s*$/) {
               $section = 'none';
           }
    if (!defined($start)) { $start = 0; }
    if (!defined($end))   { $end   = 0; }
           $request .= "('".$student."','".$section."','".$start."','".$end."'),";
           $student_count++;
       }
       return if ($student_count == 0);
       chop($request);
       $dbh->do($request);
       if ($dbh->err()) {
           &Apache::lonnet::logthis("error ".$dbh->errstr().
                                    " occurred executing \n".
                                    $request);
       }
       return;
   }
   
 Input: $courseID, $c  my $have_read_groupnames_table = 0;
   my %ids_by_groupname;
   
 $courseID:  The id of the course  sub get_group_id {
       my ($groupname) = @_;
       if (! $have_read_groupnames_table) {
           my @Result = &Apache::lonmysql::get_rows($groupnames_table);
           foreach (@Result) {
               $ids_by_groupname{$_->[1]}=$_->[0];
           }
           $have_read_groupnames_table = 1;
       }
       if (! exists($ids_by_groupname{$groupname})) {
           &populate_groupnames_table();
           undef(%ids_by_groupname);
           my @Result = &Apache::lonmysql::get_rows($groupnames_table);
           foreach (@Result) {
               $ids_by_groupname{$_->[1]}=$_->[0];
           }
       }
       if (exists($ids_by_groupname{$groupname})) {
           return $ids_by_groupname{$groupname};
       }
       return undef; # error
   }
   
 $c: The connection class that can determine if the browser has aborted.  It  sub populate_groupnames_table {
 is used to short circuit this function so that it doesn't continue to       my ($courseid) = @_;
 get information when there is no need.      if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       &setup_table_names($courseid);
       &init_dbs($courseid,0);
       my $dbh = &Apache::lonmysql::get_dbh();
       my $cdom = $env{'course.'.$courseid.'.domain'};
       my $cnum = $env{'course.'.$courseid.'.num'};
       my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum);
       return if (!%curr_groups);
       my $request = 'INSERT IGNORE INTO '.$groupnames_table.
                     '(groupname) VALUES ';
       foreach my $groupname (sort(keys(%curr_groups)),'none') {
           $request .= "('".$groupname."'),";
       }
       chop($request);
       $dbh->do($request);
       if ($dbh->err()) {
           &Apache::lonnet::logthis("error ".$dbh->errstr().
                                    " occurred executing \n".
                                    $request);
       }
       return;
   }
   
 Output: \%classlist  my $have_read_studentsgroups_table = 0;
   my %groupids_by_studentid;
   
 \%classlist: A pointer to a hash containing the following data:  sub get_students_groupids {
       my ($student_id) = @_;
       if (! $have_read_studentsgroups_table) {
           my @Result = &Apache::lonmysql::get_rows($students_groups_table);
           foreach (@Result) {
               push(@{$groupids_by_studentid{$_->[0]}},$_->[1]);
           }
           $have_read_studentsgroups_table = 1;
       }
       if (! exists($groupids_by_studentid{$student_id})) {
           &populate_students_groups_table();
           undef(%groupids_by_studentid);
           my @Result = &Apache::lonmysql::get_rows($students_groups_table);
           foreach (@Result) {
               push(@{$groupids_by_studentid{$_->[0]}},$_->[1]);
           }
       }
       if (exists($groupids_by_studentid{$student_id})) {
           if (ref($groupids_by_studentid{$student_id}) eq 'ARRAY') {
               return @{$groupids_by_studentid{$student_id}};
           }
       }
       return undef; # error
   }
   
 -A list of student name:domain (as keys) (known below as $name)  
   
 -A hash pointer for each student containing lastname, generation, firstname,  sub populate_students_groups_table {
 middlename, and PID : Key is $name.'studentInformation'      my ($courseid) = @_;
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       #
       &setup_table_names($courseid);
       &init_dbs($courseid,0);
       my $dbh = &Apache::lonmysql::get_dbh();
       my $request = 'INSERT IGNORE INTO '.$students_groups_table.
           "(student_id,group_id) VALUES ";
       my $cdom = $env{'course.'.$courseid.'.domain'};
       my $cnum = $env{'course.'.$courseid.'.num'};
       my ($classlist,$keylist) = &get_classlist($cdom,$cnum);
       my ($classgroups,$studentgroups) = &get_group_memberships($classlist,
                                                                 $keylist,
                                                                 $cdom,$cnum);
       my $record_count = 0;
       foreach my $student (sort(keys(%{$classgroups}))) {
           my $student_id = &get_student_id(split(':',$student));
           my @studentsgroups = &get_students_groups($student,'Active',$classgroups);
           if (@studentsgroups < 1) {
               @studentsgroups = ('none');
           }
           foreach my $groupname (@studentsgroups) {
               my $group_id = &get_group_id($groupname);
               $request .= "('".$student_id."','".$group_id."'),";
               $record_count++;
           }
       }
       return if ($record_count == 0);
       chop($request);
       $dbh->do($request);
       if ($dbh->err()) {
           &Apache::lonnet::logthis("error ".$dbh->errstr().
                                    " occurred executing \n".
                                    $request);
       }
       return;
   }
   
 -A hash pointer to each students section data : Key is $name.section  sub clear_internal_caches {
       $have_read_part_table = 0;
       undef(%ids_by_part);
       undef(%parts_by_id);
       $have_read_symb_table = 0;
       undef(%ids_by_symb);
       undef(%symbs_by_id);
       $have_read_student_table = 0;
       undef(%ids_by_student);
       undef(%students_by_id);
       $have_read_groupnames_table = 0;
       undef(%ids_by_groupname);
   }
   
 =back  sub symb_is_for_task {
       my ($symb) = @_;
       return ($symb =~ /\.task$/);
   }
   
 =cut  
   
 sub DownloadStudentNamePIDSection {  sub update_full_student_data {
     my ($courseID, $c)=@_;      my ($sname,$sdom,$courseid) = @_;
     my ($courseDomain,$courseNumber)=split(/\_/,$courseID);      #
       # Set up database names
     my %classlist=&Apache::lonnet::dump('classlist',$courseDomain,      &setup_table_names($courseid);
                                         $courseNumber);      #
     my ($checkForError)=keys (%classlist);      my $student_id = &get_student_id($sname,$sdom);
     if($checkForError =~ /^(con_lost|error|no_such_host)/i) {      my $student = $sname.':'.$sdom;
         return \%classlist;      #
       my $returnstatus = 'okay';
       #
       # Download students data
       my $time_of_retrieval = time;
       my @tmp = &Apache::lonnet::dumpstore($courseid,$sdom,$sname);
       if (@tmp && $tmp[0] =~ /^error/) {
           $returnstatus = 'error retrieving full student data';
           return $returnstatus;
       } elsif (! @tmp) {
           $returnstatus = 'okay: no student data';
           return $returnstatus;
       }
       my %studentdata = @tmp;
       #
       # Get database handle and clean out the tables 
       my $dbh = &Apache::lonmysql::get_dbh();
       $dbh->do('DELETE FROM '.$fulldump_response_table.' WHERE student_id='.
                $student_id);
       $dbh->do('DELETE FROM '.$fulldump_part_table.' WHERE student_id='.
                $student_id);
       $dbh->do('DELETE FROM '.$fulldump_timestamp_table.' WHERE student_id='.
                $student_id);
       #
       # Parse and store the data into a form we can handle
       my $partdata;
       my $respdata;
       while (my ($key,$value) = each(%studentdata)) {
           next if ($key =~ /^(\d+):(resource$|subnum$|keys:)/);
           my ($transaction,$symb,$parameter) = split(':',$key);
    $symb = &unescape($symb);
    $parameter = &unescape($parameter);
           my $symb_id = &get_symb_id($symb);
           if ($parameter eq 'timestamp') {
               # We can deal with 'timestamp' right away
               my @timestamp_storage = ($symb_id,$student_id,
                                        $transaction,$value);
               my $store_command = 'INSERT IGNORE INTO '.$fulldump_timestamp_table.
                   " VALUES ('".join("','",@timestamp_storage)."');";
               $dbh->do($store_command);
               if ($dbh->err()) {
                   &Apache::lonnet::logthis('unable to execute '.$store_command);
                   &Apache::lonnet::logthis($dbh->errstr());
               }
               next;
           } elsif ($parameter eq 'version') {
               next;
    } elsif (&symb_is_for_task($symb)) {
       next if ($parameter !~ /^resource\.(.*)\.(award|
         awarded|
         solved|
         submission|
         portfiles|
         status|
         version|
         regrader)\s*$/x);
       my ($version_and_part_id, $field) = ($1,$2);
   
       next if ($version_and_part_id !~ /\./ 
        && $field ne 'regrader' && $field ne 'version');
   
       my ($version, $part, $instance) = 
    split(/\./,$version_and_part_id);
   
       #skip and instance dimension or criteria specific data
       next if (defined($instance) 
        && $instance ne $field
        && $instance ne 'bridgetask');
       
       if (!defined($part)) {
    $part = $version;
       }
       my $resp_id = &get_part_id('0');
       my $part_id = &get_part_id($part);
       
       if ($field eq 'version') {
    # for tasks each version is an attempt at it thus
    #     version -> tries
    $partdata->{$symb_id}{$part_id}{$transaction}{'tries'}=
       $value;
    # at new version time the record gets reset thus adding a
    # virtual response awarddetail of 'new_version'
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific'}='status';
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value'}='new_version';
   
       } elsif ($field eq 'award' || $field eq 'awarded' 
        || $field eq 'solved') {
    $partdata->{$symb_id}{$part_id}{$transaction}{$field}=
       $value;
       } elsif ($field eq 'portfiles') {
    # tasks only accepts portfolio submissions
    $value = $dbh->quote($value);
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'submission'}=$value;
       } elsif ($field eq 'status') {
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific'}=$field;
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value'}=$value;
       } elsif ($field eq 'regrader') {
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_2'}=$field;
    $respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value_2'}=$value;
       }
    } elsif ($parameter =~ /^resource\.(.*)\.(tries|
                                                     award|
                                                     awarded|
                                                     previous|
                                                     solved|
                                                     awarddetail|
                                                     submission|
                                                     submissiongrading|
                                                     molecule)\s*$/x){
               # we do not have enough information to store an 
               # entire row, so we save it up until later.
               my ($part_and_resp_id,$field) = ($1,$2);
               my ($part,$part_id,$resp,$resp_id);
               if ($part_and_resp_id =~ /\./) {
                   ($part,$resp) = split(/\./,$part_and_resp_id);
                   $part_id = &get_part_id($part);
                   $resp_id = &get_part_id($resp);
               } else {
                   $part_id = &get_part_id($part_and_resp_id);
               }
               # Deal with part specific data
               if ($field =~ /^(tries|award|awarded|previous)$/) {
                   $partdata->{$symb_id}->{$part_id}->{$transaction}->{$field}=$value;
               }
               # deal with response specific data
               if (defined($resp_id) &&
                   $field =~ /^(awarddetail|
                                submission|
                                submissiongrading|
                                molecule)$/x) {
                   if ($field eq 'submission') {
                       # We have to be careful with user supplied input.
                       # most of the time we are okay because it is escaped.
                       # However, there is one wrinkle: submissions which end in
                       # and odd number of '\' cause insert errors to occur.  
                       # Best trap this somehow...
                       $value = $dbh->quote($value);
                   }
                   if ($field eq 'submissiongrading' || 
                       $field eq 'molecule') {
                       $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{'response_specific'}=$field;
                       $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{'response_specific_value'}=$value;
                   } else {
                       $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{$field}=$value;
                   }
               }
           }
       }
       ##
       ## Store the part data
       my $store_command = 'INSERT IGNORE INTO '.$fulldump_part_table.
           ' VALUES '."\n";
       my $store_rows = 0;
       while (my ($symb_id,$hash1) = each (%$partdata)) {
           while (my ($part_id,$hash2) = each (%$hash1)) {
               while (my ($transaction,$data) = each (%$hash2)) {
                   $store_command .= "('".join("','",$symb_id,$part_id,
                                               $student_id,
                                               $transaction,
                                               $data->{'tries'},
                                               $data->{'award'},
                                               $data->{'awarded'},
                                               $data->{'previous'})."'),";
                   $store_rows++;
               }
           }
       }
       if ($store_rows) {
           chop($store_command);
           $dbh->do($store_command);
           if ($dbh->err) {
               $returnstatus = 'error saving part data';
               &Apache::lonnet::logthis('insert error '.$dbh->errstr());
               &Apache::lonnet::logthis("While attempting\n".$store_command);
           }
       }
       ##
       ## Store the response data
       $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table.
           ' VALUES '."\n";
       $store_rows = 0;
       while (my ($symb_id,$hash1) = each (%$respdata)) {
           while (my ($part_id,$hash2) = each (%$hash1)) {
               while (my ($resp_id,$hash3) = each (%$hash2)) {
                   while (my ($transaction,$data) = each (%$hash3)) {
                       my $submission = $data->{'submission'};
                       # We have to be careful with user supplied input.
                       # most of the time we are okay because it is escaped.
                       # However, there is one wrinkle: submissions which end in
                       # and odd number of '\' cause insert errors to occur.  
                       # Best trap this somehow...
                       $submission = $dbh->quote($submission);
                       $store_command .= "('".
                           join("','",$symb_id,$part_id,
                                $resp_id,$student_id,
                                $transaction,
                                $data->{'awarddetail'},
                                $data->{'response_specific'},
                                $data->{'response_specific_value'},
                                $data->{'response_specific_2'},
                                $data->{'response_specific_value_2'}).
                                "',".$submission."),";
                       $store_rows++;
                   }
               }
           }
       }
       if ($store_rows) {
           chop($store_command);
           $dbh->do($store_command);
           if ($dbh->err) {
               $returnstatus = 'error saving response data';
               &Apache::lonnet::logthis('insert error '.$dbh->errstr());
               &Apache::lonnet::logthis("While attempting\n".$store_command);
           }
       }
       ##
       ## Update the students "current" data in the performance 
       ## and parameters tables.
       my ($status,undef) = &store_student_data
           ($sname,$sdom,$courseid,
            &Apache::lonnet::convert_dump_to_currentdump(\%studentdata));
       if ($returnstatus eq 'okay' && $status ne 'okay') {
           $returnstatus = 'error saving current data:'.$status;
       } elsif ($status ne 'okay') {
           $returnstatus .= ' error saving current data:'.$status;
       }        
       ##
       ## Update the students time......
       if ($returnstatus eq 'okay') {
           &store_updatetime($student_id,$time_of_retrieval,$time_of_retrieval);
           if ($dbh->err) {
               if ($returnstatus eq 'okay') {
                   $returnstatus = 'error updating student time';
               } else {
                   $returnstatus = 'error updating student time';
               }
           }
       }
       return $returnstatus;
   }
   
   
   sub update_student_data {
       my ($sname,$sdom,$courseid) = @_;
       #
       # Set up database names
       &setup_table_names($courseid);
       #
       my $student_id = &get_student_id($sname,$sdom);
       my $student = $sname.':'.$sdom;
       #
       my $returnstatus = 'okay';
       #
       # Download students data
       my $time_of_retrieval = time;
       my %student_data = &Apache::lonnet::currentdump($courseid,$sdom,$sname);
       if (&Apache::lonnet::error(%student_data)) {
           &Apache::lonnet::logthis('error getting data for '.
                                    $sname.':'.$sdom.' in course '.$courseid.
                                    ':'.(%student_data)[0]);
           $returnstatus =(%student_data)[0] ;
           return ($returnstatus,undef);
       }
       if (scalar(keys(%student_data)) < 1) {
           return ('no data',undef);
       }
       my @Results = &store_student_data($sname,$sdom,$courseid,\%student_data);
       #
       # Set the students update time
       if ($Results[0] eq 'okay') {
           &store_updatetime($student_id,$time_of_retrieval);
       }
       #
       return @Results;
   }
   
   sub store_updatetime {
       my ($student_id,$updatetime,$fullupdatetime)=@_;
       my $values = '';
       if (defined($updatetime)) {
           $values = 'updatetime='.$updatetime.' ';
     }      }
       if (defined($fullupdatetime)) {
           if ($values ne '') {
               $values .= ',';
           }
           $values .= 'fullupdatetime='.$fullupdatetime.' ';
       }
       return if ($values eq '');
       my $dbh = &Apache::lonmysql::get_dbh();
       my $request = 'UPDATE '.$student_table.' SET '.$values.
           ' WHERE student_id='.$student_id.' LIMIT 1';
       $dbh->do($request);
   }
   
     foreach my $name (keys(%classlist)) {  sub store_student_data {
         if($c->aborted()) {      my ($sname,$sdom,$courseid,$student_data) = @_;
             $classlist{'error'}='aborted';      #
             return \%classlist;      my $student_id = &get_student_id($sname,$sdom);
       my $student = $sname.':'.$sdom;
       #
       my $returnstatus = 'okay';
       #
       # Remove all of the students data from the table
       my $dbh = &Apache::lonmysql::get_dbh();
       $dbh->do('DELETE FROM '.$performance_table.' WHERE student_id='.
                $student_id);
       $dbh->do('DELETE FROM '.$parameters_table.' WHERE student_id='.
                $student_id);
       #
       # Store away the data
       #
       my $starttime = Time::HiRes::time;
       my $elapsed = 0;
       my $rows_stored;
       my $store_parameters_command  = 'INSERT IGNORE INTO '.$parameters_table.
           ' VALUES '."\n";
       my $num_parameters = 0;
       my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table.
           ' VALUES '."\n";
       return ('error',undef) if (! defined($dbh));
       while (my ($current_symb,$param_hash) = each(%{$student_data})) {
           #
           # make sure the symb is set up properly
           my $symb_id = &get_symb_id($current_symb);
           #
           # Parameters
           while (my ($parameter,$value) = each(%$param_hash)) {
               if ($parameter !~ /(timestamp|resource\.(.*)\.(solved|tries|awarded|award|awarddetail|previous))/) {
                   my $sql_parameter = "('".join("','",
                                                 $symb_id,$student_id,
                                                 $parameter)."',".
                                                     $dbh->quote($value)."),\n";
                   $num_parameters ++;
                   if ($sql_parameter !~ /''/) {
                       $store_parameters_command .= $sql_parameter;
                       #$rows_stored++;
                   }
               }
           }
           # Performance
           my %stored;
           while (my ($parameter,$value) = each(%$param_hash)) {
               next if ($parameter !~ /^resource\.(.*)\.(solved|awarded)$/);
               my $part  = $1;
       my $which = $2;
       next if ($part =~ /\./);
               next if (exists($stored{$part}));
               $stored{$part}++;
               #
               my $part_id = &get_part_id($part);
               next if (!defined($part_id));
       
               my ($solved,$awarded);
       if ($which eq 'solved') {
    $solved  = $value;
    $awarded = $param_hash->{'resource.'.$part.'.awarded'};
       } else {
    $solved  = $param_hash->{'resource.'.$part.'.solved'};
    $awarded = $value;
       }
               my $award   = $param_hash->{'resource.'.$part.'.award'};
               my $awarddetail = $param_hash->{'resource.'.$part.'.awarddetail'};
               my $timestamp = $param_hash->{'timestamp'};
       my $tries   = $param_hash->{'resource.'.$part.'.tries'};
       if (&symb_is_for_task($current_symb)) {
    $tries   = $param_hash->{'resource.'.$part.'.version'};
       }
               #
               $solved      = '' if (! defined($solved));
               $tries       = '' if (! defined($tries));
               $awarded     = '' if (! defined($awarded));
               $award       = '' if (! defined($award));
               $awarddetail = '' if (! defined($awarddetail));
               my $sql_performance = 
                   "('".join("','",$symb_id,$student_id,$part_id,$part,
                                   $solved,$tries,$awarded,$award,
                                   $awarddetail,$timestamp)."'),\n";
               $store_performance_command .= $sql_performance;
               $rows_stored++;
         }          }
       }
       if (! $rows_stored) { return ($returnstatus, undef); }
       $store_parameters_command =~ s|,\n$||;
       $store_performance_command =~ s|,\n$||;
       my $start = Time::HiRes::time;
       $dbh->do($store_performance_command);
       if ($dbh->err()) {
           &Apache::lonnet::logthis('performance bigass insert error:'.
                                    $dbh->errstr());
           &Apache::lonnet::logthis('command = '.$/.$store_performance_command);
           $returnstatus = 'error: unable to insert performance into database';
           return ($returnstatus,$student_data);
       }
       $dbh->do($store_parameters_command) if ($num_parameters>0);
       if ($dbh->err()) {
           &Apache::lonnet::logthis('parameters bigass insert error:'.
                                    $dbh->errstr());
           &Apache::lonnet::logthis('command = '.$/.$store_parameters_command);
           &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
           &Apache::lonnet::logthis('student_id = '.$student_id);
           $returnstatus = 'error: unable to insert parameters into database';
           return ($returnstatus,$student_data);
       }
       $elapsed += Time::HiRes::time - $start;
       return ($returnstatus,$student_data);
   }
   
         my ($studentName,$studentDomain) = split(/\:/,$name);  
         # Download student environment data, specifically the full name and id.  
         my %studentInformation=&Apache::lonnet::get('environment',  
                                                     ['lastname','generation',  
                                                      'firstname','middlename',  
                                                      'id'],  
                                                     $studentDomain,  
                                                     $studentName);  
         $classlist{$name.':studentInformation'}=\%studentInformation;  
   
         if($c->aborted()) {  sub ensure_tables_are_set_up {
             $classlist{'error'}='aborted';      my ($courseid) = @_;
             return \%classlist;      $courseid = $env{'request.course.id'} if (! defined($courseid));
       # 
       # Clean out package variables
       &setup_table_names($courseid);
       #
       # if the tables do not exist, make them
       my @CurrentTable = &Apache::lonmysql::tables_in_db();
       my ($found_symb,$found_student,$found_groups,$found_groupnames,$found_part,
           $found_performance,$found_parameters,$found_fulldump_part,
           $found_fulldump_response,$found_fulldump_timestamp,
           $found_weight);
       foreach (@CurrentTable) {
           $found_symb        = 1 if ($_ eq $symb_table);
           $found_student     = 1 if ($_ eq $student_table);
           $found_groups      = 1 if ($_ eq $students_groups_table);
           $found_groupnames  = 1 if ($_ eq $groupnames_table);
           $found_part        = 1 if ($_ eq $part_table);
           $found_performance = 1 if ($_ eq $performance_table);
           $found_parameters  = 1 if ($_ eq $parameters_table);
           $found_fulldump_part      = 1 if ($_ eq $fulldump_part_table);
           $found_fulldump_response  = 1 if ($_ eq $fulldump_response_table);
           $found_fulldump_timestamp = 1 if ($_ eq $fulldump_timestamp_table);
           $found_weight      = 1 if ($_ eq $weight_table);
       }
       if (!$found_symb          || 
           !$found_student       || !$found_part              ||
           !$found_performance   || !$found_parameters        ||
           !$found_fulldump_part || !$found_fulldump_response ||
           !$found_fulldump_timestamp || !$found_weight ) {
           if (&init_dbs($courseid,1)) {
               return 'error';
         }          }
       }
   }
   
         #Section  sub ensure_current_data {
         my %section=&Apache::lonnet::dump('roles',$studentDomain,$studentName);      my ($sname,$sdom,$courseid) = @_;
         $classlist{$name.':section'}=\%section;      my $status = 'okay';   # return value
       #
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       &ensure_tables_are_set_up($courseid);
       #
       # Get the update time for the user
       my $updatetime = 0;
       my $getuserdir = 1;
       my $modifiedtime = &Apache::lonnet::GetFileTimestamp
           ($sdom,$sname,$courseid.'.db',$getuserdir);
       #
       if ($modifiedtime == -1) {
    return ('no data',undef);
     }      }
   
     return \%classlist;      my $student_id = &get_student_id($sname,$sdom);
       &get_students_groupids($student_id);
       my @Result = &Apache::lonmysql::get_rows($student_table,
                                                "student_id ='$student_id'");
       my $data = undef;
       if (@Result) {
           $updatetime = $Result[0]->[6];  # Ack!  This is dumb!
       }
       if ($modifiedtime > $updatetime) {
           ($status,$data) = &update_student_data($sname,$sdom,$courseid);
       }
       return ($status,$data);
 }  }
   
 =pod  
   
 =item &DownloadStudentCourseInformation()  sub ensure_current_full_data {
       my ($sname,$sdom,$courseid) = @_;
       my $status = 'okay';   # return value
       #
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       &ensure_tables_are_set_up($courseid);
       #
       # Get the update time for the user
       my $getuserdir = 1;
       my $modifiedtime = &Apache::lonnet::GetFileTimestamp
           ($sdom,$sname,$courseid.'.db',$getuserdir);
       #
       my $student_id = &get_student_id($sname,$sdom);
       &get_students_groupids($student_id);
       my @Result = &Apache::lonmysql::get_rows($student_table,
                                                "student_id ='$student_id'");
       my $updatetime;
       if (@Result && ref($Result[0]) eq 'ARRAY') {
           $updatetime = $Result[0]->[7];
       }
       if (! defined($updatetime) || $modifiedtime > $updatetime) {
           $status = &update_full_student_data($sname,$sdom,$courseid);
       }
       return $status;
   }
   
 Dump of all the course information for a single student.  There is no  
 pruning of data, it is all stored in a hash and returned.  
   
 =over 4  sub get_student_data_from_performance_cache {
       my ($sname,$sdom,$symb,$courseid)=@_;
       my $student = $sname.':'.$sdom if (defined($sname) && defined($sdom));
       &setup_table_names($courseid);
       #
       # Return hash
       my $studentdata;
       #
       my $dbh = &Apache::lonmysql::get_dbh();
       my $request = "SELECT ".
           "d.symb,a.part,a.solved,a.tries,a.awarded,a.award,a.awarddetail,".
               "a.timestamp ";
       if (defined($student)) {
           $request .= "FROM $student_table AS b ".
               "LEFT JOIN $performance_table AS a ON b.student_id=a.student_id ".
   #            "LEFT JOIN $part_table AS c ON c.part_id = a.part_id ".
               "LEFT JOIN $symb_table AS d ON d.symb_id = a.symb_id ".
                   "WHERE student='$student'";
           if (defined($symb) && $symb ne '') {
               $request .= " AND d.symb=".$dbh->quote($symb);
           }
       } elsif (defined($symb) && $symb ne '') {
           $request .= "FROM $symb_table as d ".
               "LEFT JOIN $performance_table AS a ON d.symb_id=a.symb_id ".
   #            "LEFT JOIN $part_table    AS c ON c.part_id = a.part_id ".
               "LEFT JOIN $student_table AS b ON b.student_id = a.student_id ".
                   "WHERE symb='".$dbh->quote($symb)."'";
       }
       my $starttime = Time::HiRes::time;
       my $rows_retrieved = 0;
       my $sth = $dbh->prepare($request);
       $sth->execute();
       if ($sth->err()) {
           &Apache::lonnet::logthis("Unable to execute MySQL request:");
           &Apache::lonnet::logthis("\n".$request."\n");
           &Apache::lonnet::logthis("error is:".$sth->errstr());
           return undef;
       }
       foreach my $row (@{$sth->fetchall_arrayref}) {
           $rows_retrieved++;
           my ($symb,$part,$solved,$tries,$awarded,$award,$awarddetail,$time) = 
               (@$row);
           my $base = 'resource.'.$part;
           $studentdata->{$symb}->{$base.'.solved'}  = $solved;
           $studentdata->{$symb}->{$base.'.tries'}   = $tries;
           $studentdata->{$symb}->{$base.'.awarded'} = $awarded;
           $studentdata->{$symb}->{$base.'.award'}   = $award;
           $studentdata->{$symb}->{$base.'.awarddetail'} = $awarddetail;
           $studentdata->{$symb}->{'timestamp'} = $time if (defined($time) && $time ne '');
       }
       ## Get misc parameters
       $request = 'SELECT c.symb,a.parameter,a.value '.
           "FROM $student_table AS b ".
           "LEFT JOIN $parameters_table AS a ON b.student_id=a.student_id ".
           "LEFT JOIN $symb_table AS c ON c.symb_id = a.symb_id ".
           "WHERE student='$student'";
       if (defined($symb) && $symb ne '') {
           $request .= " AND c.symb=".$dbh->quote($symb);
       }
       $sth = $dbh->prepare($request);
       $sth->execute();
       if ($sth->err()) {
           &Apache::lonnet::logthis("Unable to execute MySQL request:");
           &Apache::lonnet::logthis("\n".$request."\n");
           &Apache::lonnet::logthis("error is:".$sth->errstr());
           if (defined($symb) && $symb ne '') {
               $studentdata = $studentdata->{$symb};
           }
           return $studentdata;
       }
       #
       foreach my $row (@{$sth->fetchall_arrayref}) {
           $rows_retrieved++;
           my ($symb,$parameter,$value) = (@$row);
           $studentdata->{$symb}->{$parameter}  = $value;
       }
       #
       if (defined($symb) && $symb ne '') {
           $studentdata = $studentdata->{$symb};
       }
       return $studentdata;
   }
   
 Input: $name, $courseID  
   
 $name: student name:domain  sub get_current_state {
       my ($sname,$sdom,$symb,$courseid,$forcedownload)=@_;
       #
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       #
       return () if (! defined($sname) || ! defined($sdom));
       #
       my ($status,$data) = &ensure_current_data($sname,$sdom,$courseid);
   #    &Apache::lonnet::logthis
   #        ('sname = '.$sname.
   #         ' domain = '.$sdom.
   #         ' status = '.$status.
   #         ' data is '.(defined($data)?'defined':'undefined'));
   #    while (my ($symb,$hash) = each(%$data)) {
   #        &Apache::lonnet::logthis($symb."\n----------------------------------");
   #        while (my ($key,$value) = each (%$hash)) {
   #            &Apache::lonnet::logthis("   ".$key." = ".$value);
   #        }
   #    }
       #
       if (defined($data) && defined($symb) && ref($data->{$symb})) {
           return %{$data->{$symb}};
       } elsif (defined($data) && ! defined($symb) && ref($data)) {
           return %$data;
       } 
       if ($status eq 'no data') {
           return ();
       } else {
           if ($status ne 'okay' && $status ne '') {
               &Apache::lonnet::logthis('status = '.$status);
               return ('error: '.$status,undef);
           }
           my $returnhash = &get_student_data_from_performance_cache($sname,$sdom,
                                                         $symb,$courseid);
           return %$returnhash if (defined($returnhash));
       }
       return ();
   }
   
 $courseID:  The id of the course  
   
 Output: \%courseData  sub get_problem_statistics {
       my ($Sections,$Groups,$status,$symb,$part,$courseid,$starttime,$endtime) = @_;
       return if (! defined($symb) || ! defined($part));
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       #
       &setup_table_names($courseid);
       my $symb_id = &get_symb_id($symb);
       my $part_id = &get_part_id($part);
       my $stats_table = &temp_table_name($courseid,'problem_stats');
       #
       my $dbh = &Apache::lonmysql::get_dbh();
       return undef if (! defined($dbh));
       #
       # Clean out the table
       $dbh->do('DROP TABLE '.$stats_table);  # May return an error
       my $request = 
           'CREATE TEMPORARY TABLE '.$stats_table.' '.
           'SELECT a.student_id,a.solved,a.award,a.awarded,a.tries '.
           'FROM '.$performance_table.' AS a ';
       #
       # See if we need to include some requirements on the students
       if ((defined($Sections) && lc($Sections->[0]) ne 'all') || 
           (defined($status)   && lc($status)        ne 'any')) {
           $request .= 'NATURAL LEFT JOIN '.$student_table.' AS b ';
       }
       my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
       if (defined($groups_join)) {
           $request .= $groups_join;
       }
       $request .= ' WHERE a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
       #
       # Limit the students included to those specified
       my ($section_limits,$enrollment_limits)=
           &limit_by_section_and_status($Sections,$status,'b');
       #
       # Limit by starttime and endtime
       my $time_requirements = undef;
       if (defined($starttime)) {
           $time_requirements .= 'a.timestamp>='.$starttime;
           if (defined($endtime)) {
               $time_requirements .= ' AND a.timestamp<='.$endtime;
           }
       } elsif (defined($endtime)) {
           $time_requirements .= 'a.timestamp<='.$endtime;
       }
       if (defined($time_requirements)) {
           $request .= ' AND '.$time_requirements;
       }
       if (defined($section_limits)) {
           $request .= ' AND '.$section_limits;
       }
       if (defined($enrollment_limits)) {
           $request .= ' AND '.$enrollment_limits;
       }
       # Limit by group, as required
       if (defined($group_limits)) {
           $request .= ' AND '.$group_limits;
       }
       #
       # Finally, execute the request to create the temporary table
       $dbh->do($request);
       #
       # Collect the first suite of statistics
       $request = 'SELECT COUNT(*),SUM(tries),'.
           'AVG(tries),STD(tries) '.
           'FROM '.$stats_table;
       my ($num,$tries,$mean,$STD) = &execute_SQL_request
           ($dbh,$request);
       #
       $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table.
           ' WHERE awarded>0';
       if (defined($time_requirements)) {
           $request .= ' AND '.$time_requirements;
       }
       my ($max,$min) = &execute_SQL_request($dbh,$request);
       #
       $request = 'SELECT SUM(awarded) FROM '.$stats_table;
       if (defined($time_requirements)) {
           $request .= ' AND '.$time_requirements;
       }
       my ($Solved) = &execute_SQL_request($dbh,$request);
       #
       $request = 'SELECT SUM(awarded) FROM '.$stats_table.
           " WHERE solved='correct_by_override'";
       if (defined($time_requirements)) {
           $request .= ' AND '.$time_requirements;
       }
       my ($solved) = &execute_SQL_request($dbh,$request);
       #
       $Solved -= $solved;
       #
       $num    = 0 if (! defined($num));
       $tries  = 0 if (! defined($tries));
       $max    = 0 if (! defined($max));
       $min    = 0 if (! defined($min));
       $STD    = 0 if (! defined($STD));
       $Solved = 0 if (! defined($Solved) || $Solved < 0);
       $solved = 0 if (! defined($solved));
       #
       # Compute the more complicated statistics
       my $DegOfDiff = 'nan';
       $DegOfDiff = 1-($Solved)/$tries if ($tries>0);
       #
       my $SKEW = 'nan';
       my $wrongpercent = 0;
       my $numwrong = 'nan';
       if ($num > 0) {
           ($SKEW) = &execute_SQL_request($dbh,'SELECT SQRT(SUM('.
                                        'POWER(tries - '.$STD.',3)'.
                                        '))/'.$num.' FROM '.$stats_table);
           $numwrong = $num-$Solved;
           $wrongpercent=int(10*100*$numwrong/$num)/10;
       }
       #
       # Drop the temporary table
       $dbh->do('DROP TABLE '.$stats_table);  # May return an error
       #
       # Return result
       return { num_students => $num,
                tries        => $tries,
                max_tries    => $max,
                min_tries    => $min,
                mean_tries   => $mean,
                std_tries    => $STD,
                skew_tries   => $SKEW,
                num_solved   => $Solved,
                num_override => $solved,
                num_wrong    => $numwrong,
                per_wrong    => $wrongpercent,
                deg_of_diff  => $DegOfDiff };
   }
   
 \%courseData:  A hash pointer to the raw data from the student's course  ##
 database.  ## This is a helper for get_statistics
   sub execute_SQL_request {
       my ($dbh,$request)=@_;
   #    &Apache::lonnet::logthis($request);
       my $sth = $dbh->prepare($request);
       if (!$sth) {
    die($dbh->errstr . " SQL: $request");
       }
       $sth->execute();
       my $row = $sth->fetchrow_arrayref();
       if (ref($row) eq 'ARRAY' && scalar(@$row)>0) {
           return @$row;
       }
       return ();
   }
   
 =back  
   
 =cut  sub populate_weight_table {
       my ($courseid) = @_;
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       #
       &setup_table_names($courseid);
       my $navmap = Apache::lonnavmaps::navmap->new();
       if (!defined($navmap)) {
           &Apache::lonnet::logthis('loncoursedata::populate_weight_table:'.$/.
                                    '  unable to get navmaps resource'.$/.
                                    '  '.join(' ',(caller)));
           return;
       }
       my @sequences = $navmap->retrieveResources(undef,
                                                  sub { shift->is_map(); },1,0,1);
       my @resources;
       foreach my $seq (@sequences) {
           push(@resources,$navmap->retrieveResources($seq,
                                                      sub {shift->is_problem();},
                                                      0,0,0));
       }
       if (! scalar(@resources)) {
           &Apache::lonnet::logthis('loncoursedata::populate_weight_table:'.$/.
                                    ' no resources returned for '.$courseid);
           return;
       }
       #       Since we use lonnet::EXT to retrieve problem weights,
       #       to ensure current data we must clear the caches out.
       &Apache::lonnet::clear_EXT_cache_status();
       my $dbh = &Apache::lonmysql::get_dbh();
       my $request = 'INSERT IGNORE INTO '.$weight_table.
           "(symb_id,part_id,weight) VALUES ";
       my $weight;
       foreach my $res (@resources) {
           my $symb_id = &get_symb_id($res->symb);
           foreach my $part (@{$res->parts}) {
               my $part_id = &get_part_id($part);
               $weight = &Apache::lonnet::EXT('resource.'.$part.'.weight',
                                              $res->symb,
                                              undef,undef,undef);
               if (!defined($weight) || ($weight eq '')) { 
                   $weight=1;
               }
               $request .= "('".$symb_id."','".$part_id."','".$weight."'),";
           }
       }
       $request =~ s/(,)$//;
   #    &Apache::lonnet::logthis('request = '.$/.$request);
       $dbh->do($request);
       if ($dbh->err()) {
           &Apache::lonnet::logthis("error ".$dbh->errstr().
                                    " occurred executing \n".
                                    $request);
       }
       return;
   }
   
   sub limit_by_start_end_time {
       my ($starttime,$endtime,$table) = @_;
       my $time_requirements = undef;
       if (defined($starttime)) {
           $time_requirements .= $table.".timestamp>='".$starttime."'";
           if (defined($endtime)) {
               $time_requirements .= " AND ".$table.".timestamp<='".$endtime."'";
           }
       } elsif (defined($endtime)) {
           $time_requirements .= $table.".timestamp<='".$endtime."'";
       }
       return $time_requirements;
   }
   
 sub DownloadStudentCourseInformation {  
     my ($name,$courseID)=@_;  
     my ($studentName,$studentDomain) = split(/\:/,$name);  
   
     # Download student course data  sub limit_by_section_and_status {
     my %courseData=&Apache::lonnet::dump($courseID,$studentDomain,      my ($Sections,$enrollment,$tablename) = @_;
  $studentName);      my $student_requirements = undef;
     return \%courseData;      if ( (defined($Sections) && $Sections->[0] ne 'all')) {
           $student_requirements = '('.
               join(' OR ', map { $tablename.".section='".$_."'" } @$Sections
                    ).')';
       }
       my $enrollment_requirements=undef;
       if (defined($enrollment) && $enrollment ne 'Any') {
    my $now = time();
    if ( $enrollment eq 'Future' ) {
       $enrollment_requirements = 
    "( $tablename.start > $now AND ".
    "( $tablename.end = 0 OR $tablename.end > $now))";
    } elsif ( $enrollment eq 'Active' ) {
       $enrollment_requirements = 
    "(( $tablename.start = 0 OR $tablename.start < $now )  AND ".
    " ( $tablename.end   = 0 OR $tablename.end   > $now ))";
    } elsif ( $enrollment eq 'Expired' ) {
       $enrollment_requirements = 
    "(( $tablename.start < $now )  AND ".
    " ( $tablename.end   < $now ))";
    }
       }
       return ($student_requirements,$enrollment_requirements);
 }  }
   
 # ----- END DOWNLOAD INFORMATION ---------------------------------------  
   
 =pod  
   
 =head1 PROCESSING FUNCTIONS  sub limit_by_group {
       my ($Groups,$stutable,$grptable,$stugrptab) = @_;
       my $groups_join = undef;
       my $group_limits = undef;
       if ( (defined($Groups) && $Groups->[0] ne 'all')) {
           $groups_join =
             ' LEFT JOIN '.$students_groups_table.
                        ' AS '.$stugrptab.' ON '.
                        $stugrptab.'.student_id = '.$stutable.'.student_id'.
             ' LEFT JOIN '.$groupnames_table.
                        ' AS '.$grptable.' ON '.
                        $stugrptab.'.group_id = '.$grptable.'.group_id ';
           $group_limits =
             ' ('.
                join(' OR ', map {  "$grptable.groupname='".$_."'" } @$Groups
              ).')';
       }
       return ($groups_join,$group_limits);
   }
   
 These functions process all the data for all the students.  Also, they  
 are the only functions that access the cache database for writing.  Thus  
 they are the only functions that cache data.  The downloading and caching  
 were separated to reduce problems with stopping downloading then can't  
 tie hash to database later.  
   
 =cut  sub RNK_student { return 0; };
   sub RNK_score   { return 1; };
   
 # ----- PROCESSING FUNCTIONS ---------------------------------------  sub rank_students_by_scores_on_resources {
       my ($resources,$Sections,$Groups,$enrollment,$courseid,$starttime,$endtime,
           $has_award_for) = @_;
       return if (! defined($resources) || ! ref($resources) eq 'ARRAY');
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       #
       &setup_table_names($courseid);
       my $dbh = &Apache::lonmysql::get_dbh();
       my ($section_limits,$enrollment_limits)=
           &limit_by_section_and_status($Sections,$enrollment,'b');
       my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
       my $symb_limits = '('.join(' OR ',map {'a.symb_id='.&get_symb_id($_);
                                          } @$resources
                                  ).')';
       my ($award_col, $award_join, $award_clause) = ('', '', '');
       if ($has_award_for) {
           my $resource_id = &get_symb_id($has_award_for);
           $award_col = ", perf.awarded";
           $award_join = "LEFT JOIN $performance_table AS perf ON perf.symb_id"
               ." = $resource_id AND perf.student_id = b.student_id ";
           $award_clause = "AND perf.awarded IS NOT NULL";
       }
       my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
       my $request = "SELECT b.student,SUM(a.awarded*w.weight) AS score "
           ."$award_col FROM $performance_table AS a ".
           "NATURAL LEFT JOIN $weight_table AS w ".
           "LEFT JOIN $student_table AS b ON a.student_id=b.student_id ".
           "$award_join $groups_join "; 
       my $limits;
       if (defined($section_limits)) {
           $limits .= $section_limits.' AND ';
       }
       if (defined($enrollment_limits)) {
           $limits .= $enrollment_limits.' AND ';
       }
       if (defined($time_limits)) {
           $limits .= $time_limits.' AND ';
       }
       if ($symb_limits ne '()') {
           $limits .= $symb_limits.' AND ';
       }
       if (defined($group_limits)) {
           $limits .= $group_limits.' AND ';
       }
       if ($limits) {
           $limits =~ s/( AND )$//;   # Remove extra conjunction
           $request .= "WHERE $limits";
       } 
       $request .= " $award_clause GROUP BY a.student_id ORDER BY score";
       #&Apache::lonnet::logthis('request = '.$/.$request);
       my $sth = $dbh->prepare($request) or die "Can't prepare $request";
       $sth->execute();
       my $rows = $sth->fetchall_arrayref();
       return ($rows);
   }
   
 =pod  sub get_sum_of_scores {
       my ($symb,$part,$students,$courseid,$starttime,$endtime) = @_;
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       if (defined($students) && 
           ((@$students == 0) ||
            (@$students == 1 && (! defined($students->[0]) || 
                                 $students->[0] eq ''))
            )
           ){
           undef($students);
       }
       #
       &setup_table_names($courseid);
       my $dbh = &Apache::lonmysql::get_dbh();
       my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
       my $request = 'SELECT SUM(a.awarded*w.weight),SUM(w.weight) FROM '.
           $performance_table.' AS a '.
           'NATURAL LEFT JOIN '.$weight_table.' AS w ';
       $request .= 'WHERE a.symb_id='.&get_symb_id($symb).
           ' AND a.part_id='.&get_part_id($part);
       if (defined($time_limits)) {
           $request .= ' AND '.$time_limits;
       }
       if (defined($students)) {
           $request .= ' AND ('.
               join(' OR ',map {'a.student_id='.&get_student_id(split(':',$_));
                            } @$students).
                                ')';
       }
       my $sth = $dbh->prepare($request);
       $sth->execute();
       my $rows = $sth->fetchrow_arrayref();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 1 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed, fetchrow_arrayrefed'.
                                    $/.$request);
           return (undef,undef);
       }
       return ($rows->[0],$rows->[1]);
   }
   
 =item &ProcessTopResourceMap()  
   
 Trace through the "big hash" created in rat/lonuserstate.pm::loadmap.    sub score_stats {
 Basically, this function organizes a subset of the data and stores it in      my ($Sections,$Groups,$enrollment,$symbs,$starttime,$endtime,$courseid)=@_;
 cached data.  The data stored is the problems, sequences, sequence titles,      if (! defined($courseid)) {
 parts of problems, and their ordering.  Column width information is also           $courseid = $env{'request.course.id'};
 partially handled here on a per sequence basis.      }
       #
       &setup_table_names($courseid);
       my $dbh = &Apache::lonmysql::get_dbh();
       #
       my ($section_limits,$enrollment_limits)=
           &limit_by_section_and_status($Sections,$enrollment,'b');
       my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
       my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
       my @Symbids = map { &get_symb_id($_); } @{$symbs};
       #
       my $stats_table = &temp_table_name($courseid,'problem_stats');
       my $symb_restriction = join(' OR ',map {'a.symb_id='.$_;} @Symbids);
       my $request = 'DROP TABLE '.$stats_table;
       $dbh->do($request);
       $request = 
           'CREATE TEMPORARY TABLE '.$stats_table.' '.
           'SELECT a.student_id,'.
           'SUM(a.awarded*w.weight) AS score FROM '.
           $performance_table.' AS a '.
           'NATURAL LEFT JOIN '.$weight_table.' AS w '.
           'LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id '.
           $groups_join;
       my $limit = ' WHERE ('.$symb_restriction.')';
       if ($time_limits) {
           $limit .= ' AND '.$time_limits;
       }
       if ($section_limits) {
           $limit .= ' AND '.$section_limits;
       }
       if ($enrollment_limits) {
           $limit .= ' AND '.$enrollment_limits;
       }
       if ($group_limits) {
           $limit .= ' AND '.$group_limits;
       }
       $request .= $limit.' GROUP BY a.student_id';
   #    &Apache::lonnet::logthis('request = '.$/.$request);
       my $sth = $dbh->prepare($request);
       $sth->execute();
       $request = 
           'SELECT AVG(score),STD(score),MAX(score),MIN(score),COUNT(score) '.
           'FROM '.$stats_table;
       my ($ave,$std,$max,$min,$count) = &execute_SQL_request($dbh,$request);
   #    &Apache::lonnet::logthis('request = '.$/.$request);
       
       $request = 'SELECT SUM(weight) FROM '.$weight_table.
           ' AS a WHERE ('.$symb_restriction.')';
       my ($max_possible) = &execute_SQL_request($dbh,$request);
       # &Apache::lonnet::logthis('request = '.$/.$request);
       return($min,$max,$ave,$std,$count,$max_possible);
   }
   
 =over 4  
   
 Input: $cache, $c  
   
 $cache:  A pointer to a hash to store the information  sub count_stats {
       my ($Sections,$Groups,$enrollment,$symbs,$starttime,$endtime,$courseid)=@_;
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       #
       &setup_table_names($courseid);
       my $dbh = &Apache::lonmysql::get_dbh();
       #
       my ($section_limits,$enrollment_limits)=
           &limit_by_section_and_status($Sections,$enrollment,'b');
       my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
       my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
       my @Symbids = map { &get_symb_id($_); } @{$symbs};
       #
       my $stats_table = &temp_table_name($courseid,'problem_stats');
       my $symb_restriction = join(' OR ',map {'a.symb_id='.$_;} @Symbids);
       my $request = 'DROP TABLE '.$stats_table;
       $dbh->do($request);
       $request = 
           'CREATE TEMPORARY TABLE '.$stats_table.' '.
           'SELECT a.student_id,'.
           'SUM(a.awarded) AS count FROM '.
           $performance_table.' AS a '.
           'LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id '.
           $groups_join;
       my $limit =  ' WHERE ('.$symb_restriction.')';
       if ($time_limits) {
           $limit .= ' AND '.$time_limits;
       }
       if ($section_limits) {
           $limit .= ' AND '.$section_limits;
       }
       if ($enrollment_limits) {
           $limit .= ' AND '.$enrollment_limits;
       }
       if ($group_limits) {
           $limit .= ' AND '.$group_limits;
       }
       $request .= $limit.' GROUP BY a.student_id';
   #    &Apache::lonnet::logthis('request = '.$/.$request);
       my $sth = $dbh->prepare($request);
       $sth->execute();
       $request = 
           'SELECT AVG(count),STD(count),MAX(count),MIN(count),COUNT(count) '.
           'FROM '.$stats_table;
       my ($ave,$std,$max,$min,$count) = &execute_SQL_request($dbh,$request);
   #    &Apache::lonnet::logthis('request = '.$/.$request);
       return($min,$max,$ave,$std,$count);
   }
   
 $c:  The connection class used to determine if an abort has been sent to the   
 browser  
   
 Output: A string that contains an error message or "OK" if everything went   sub get_student_data {
 smoothly.      my ($students,$courseid) = @_;
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       &setup_table_names($courseid);
       my $dbh = &Apache::lonmysql::get_dbh();
       return undef if (! defined($dbh));
       my $request = 'SELECT '.
           'student_id, student '.
           'FROM '.$student_table;
       if (defined($students)) {
           $request .= ' WHERE ('.
               join(' OR ', map {'student_id='.
                                     &get_student_id($_->{'username'},
                                                     $_->{'domain'})
                                 } @$students
                    ).')';
       }
       $request.= ' ORDER BY student_id';
       my $sth = $dbh->prepare($request);
       $sth->execute();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 2 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed '.$/.$request);
           return undef;
       }
       my $dataset = $sth->fetchall_arrayref();
       if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
           return $dataset;
       }
   }
   
 =back  sub RD_student_id      { return 0; }
   sub RD_awarddetail     { return 1; }
   sub RD_response_eval   { return 2; }
   sub RD_response_eval_2 { return 3; }
   sub RD_submission      { return 4; }
   sub RD_timestamp       { return 5; }
   sub RD_tries           { return 6; }
   sub RD_sname           { return 7; }
   
   sub get_response_data {
       my ($Sections,$Groups,$enrollment,$symb,$response,$courseid) = @_;
       return undef if (! defined($symb) || 
                  ! defined($response));
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       #
       &setup_table_names($courseid);
       my $symb_id = &get_symb_id($symb);
       if (! defined($symb_id)) {
           &Apache::lonnet::logthis('Unable to find symb for '.$symb.' in '.$courseid);
           return undef;
       }
       my $response_id = &get_part_id($response);
       if (! defined($response_id)) {
           &Apache::lonnet::logthis('Unable to find id for '.$response.' in '.$courseid);
           return undef;
       }
       #
       my $dbh = &Apache::lonmysql::get_dbh();
       return undef if (! defined($dbh));
       #
       my ($student_requirements,$enrollment_requirements) = 
           &limit_by_section_and_status($Sections,$enrollment,'d');
       my ($groups_join,$group_limits) = &limit_by_group($Groups,'d','e','f');
       my $request = 'SELECT '.
           'a.student_id, a.awarddetail, a.response_specific_value, '.
           'a.response_specific_value_2, a.submission, b.timestamp, '.
    'c.tries, d.student '.
           'FROM '.$fulldump_response_table.' AS a '.
           'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
           'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
           'a.transaction = b.transaction '.
           'LEFT JOIN '.$fulldump_part_table.' AS c '.
           'ON a.symb_id=c.symb_id AND a.student_id=c.student_id AND '.        
           'a.part_id=c.part_id AND a.transaction = c.transaction '.
           'LEFT JOIN '.$student_table.' AS d '.
           'ON a.student_id=d.student_id '.
           $groups_join;
       my $limit = ' WHERE '.
           'a.symb_id='.$symb_id.' AND a.response_id='.$response_id;
       if (defined($student_requirements)) {
           $limit .= ' AND '.$student_requirements;
       }
       if (defined($enrollment_requirements)) {
           $limit .= ' AND '.$enrollment_requirements;
       }
       if (defined($group_limits)) {
           $limit .= ' AND '.$group_limits;
       }
       $request .= $limit.' ORDER BY b.timestamp';
   #    &Apache::lonnet::logthis("request =\n".$request);
       my $sth = $dbh->prepare($request);
       $sth->execute();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 3 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed '.$/.$request);
           return undef;
       }
       my $dataset = $sth->fetchall_arrayref();
       if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
           # Clear the \'s from around the submission
           for (my $i =0;$i<scalar(@$dataset);$i++) {
               $dataset->[$i]->[&RD_submission()] =~ s/(\'$|^\')//g;
           }
           return $dataset;
       }
   }
   
 =cut  
   
 sub ProcessTopResourceMap {  sub RDs_awarddetail     { return 3; }
     my ($cache,$c)=@_;  sub RDs_submission      { return 2; }
     my %hash;  sub RDs_timestamp       { return 1; }
     my $fn=$ENV{'request.course.fn'};  sub RDs_tries           { return 0; }
     if(-e "$fn.db") {  sub RDs_awarded         { return 4; }
  my $tieTries=0;  sub RDs_response_eval   { return 5; }
  while($tieTries < 3) {  sub RDs_response_eval_2 { return 6; }
             if($c->aborted()) {  sub RDs_part_award      { return 7; }
                 return;  
             }  sub get_response_data_by_student {
     if(tie(%hash,'GDBM_File',"$fn.db",&GDBM_READER,0640)) {      my ($student,$symb,$response,$courseid) = @_;
  last;      return undef if (! defined($symb) || 
     }                       ! defined($response));
     $tieTries++;      $courseid = $env{'request.course.id'} if (! defined($courseid));
     sleep 1;      #
  }      &setup_table_names($courseid);
  if($tieTries >= 3) {      my $symb_id = &get_symb_id($symb);
             return 'Coursemap undefined.';      my $response_id = &get_part_id($response);
       #
       my $student_id = &get_student_id($student->{'username'},
                                        $student->{'domain'});
       #
       my $dbh = &Apache::lonmysql::get_dbh();
       return undef if (! defined($dbh));
       my $request = 'SELECT '.
           'c.tries, b.timestamp, a.submission, a.awarddetail, c.awarded, '.
    'a.response_specific_value, a.response_specific_value_2, c.award '.
           'FROM '.$fulldump_response_table.' AS a '.
           'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
           'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
           'a.transaction = b.transaction '.
           'LEFT JOIN '.$fulldump_part_table.' AS c '.
           'ON a.symb_id=c.symb_id AND a.student_id=c.student_id AND '.        
           'a.part_id=c.part_id AND a.transaction = c.transaction '.
           'LEFT JOIN '.$student_table.' AS d '.
           'ON a.student_id=d.student_id '.
           'LEFT JOIN '.$performance_table.' AS e '.
           'ON a.symb_id=e.symb_id AND a.part_id=e.part_id AND '.
           'a.student_id=e.student_id AND c.tries=e.tries '.
           'WHERE '.
           'a.symb_id='.$symb_id.' AND a.response_id='.$response_id.
           ' AND a.student_id='.$student_id.' ORDER BY b.timestamp';
   #    &Apache::lonnet::logthis("request =\n".$request);
       my $sth = $dbh->prepare($request);
       $sth->execute();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 4 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed '.$/.$request);
           return undef;
       }
       my $dataset = $sth->fetchall_arrayref();
       if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
           # Clear the \'s from around the submission
           for (my $i =0;$i<scalar(@$dataset);$i++) {
               $dataset->[$i]->[&RDs_submission] =~ s/(\'$|^\')//g;
         }          }
     } else {          return $dataset;
         return 'Can not open Coursemap.';      }
       return undef; # error occurred
   }
   
   sub RT_student_id { return 0; }
   sub RT_awarded    { return 1; }
   sub RT_tries      { return 2; }
   sub RT_timestamp  { return 3; }
   
   sub get_response_time_data {
       my ($sections,$groups,$enrollment,$symb,$part,$courseid) = @_;
       return undef if (! defined($symb) || 
                        ! defined($part));
       $courseid = $env{'request.course.id'} if (! defined($courseid));
       #
       &setup_table_names($courseid);
       my $symb_id = &get_symb_id($symb);
       if (! defined($symb_id)) {
           &Apache::lonnet::logthis('Unable to find symb for '.$symb.' in '.$courseid);
           return undef;
       }
       my $part_id = &get_part_id($part);
       if (! defined($part_id)) {
           &Apache::lonnet::logthis('Unable to find id for '.$part.' in '.$courseid);
           return undef;
       }
       #
       my $dbh = &Apache::lonmysql::get_dbh();
       return undef if (! defined($dbh));
       my ($student_requirements,$enrollment_requirements) = 
           &limit_by_section_and_status($sections,$enrollment,'d');
       my ($groups_join,$group_limits) = &limit_by_group($groups,'d','e','f');
       my $request = 'SELECT '.
           'a.student_id, a.awarded, a.tries, b.timestamp '.
           'FROM '.$fulldump_part_table.' AS a '.
           'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
           'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
           'a.transaction = b.transaction '.
           'LEFT JOIN '.$student_table.' as d '.
           'ON a.student_id=d.student_id '.
           $groups_join;
       my $limit = ' WHERE '.
           'a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
       if (defined($student_requirements)) {
           $limit .= ' AND '.$student_requirements;
       }
       if (defined($enrollment_requirements)) {
           $limit .= ' AND '.$enrollment_requirements;
       }
       if (defined($group_limits)) {
           $limit .= ' AND '.$group_limits;  
       }
       $request .= $limit.' ORDER BY b.timestamp';
   #    &Apache::lonnet::logthis("request =\n".$request);
       my $sth = $dbh->prepare($request);
       $sth->execute();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 5 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed '.$/.$request);
           return undef;
       }
       my $dataset = $sth->fetchall_arrayref();
       if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
           return $dataset;
     }      }
   
     # Initialize state machine.  Set information pointing to top level map.  }
     my (@sequences, @currentResource, @finishResource);  
     my ($currentSequence, $currentResourceID, $lastResourceID);  sub get_student_scores {
       my ($sections,$groups,$Symbs,$enrollment,$courseid,$starttime,$endtime) = @_;
     $currentResourceID=$hash{'ids_/res/'.$ENV{'request.course.uri'}};      $courseid = $env{'request.course.id'} if (! defined($courseid));
     push(@currentResource, $currentResourceID);      &setup_table_names($courseid);
     $lastResourceID=-1;      my $dbh = &Apache::lonmysql::get_dbh();
     $currentSequence=-1;      return (undef) if (! defined($dbh));
     my $topLevelSequenceNumber = $currentSequence;      my $tmptable = &temp_table_name($courseid,'temp_'.time);
       my $request = 'DROP TABLE IF EXISTS '.$tmptable;
     while(1) {  #    &Apache::lonnet::logthis('request = '.$/.$request);
         if($c->aborted()) {      $dbh->do($request);
             last;      #
         }      my $symb_requirements;
  # HANDLE NEW SEQUENCE!      if (defined($Symbs)  && @$Symbs) {
  #if page || sequence          $symb_requirements = '('.
  if(defined($hash{'map_pc_'.$hash{'src_'.$currentResourceID}})) {              join(' OR ', map{ "(a.symb_id='".&get_symb_id($_->{'symb'}).
     push(@sequences, $currentSequence);                                "' AND a.part_id='".&get_part_id($_->{'part'}).
     push(@currentResource, $currentResourceID);                                "')"
     push(@finishResource, $lastResourceID);                                } @$Symbs).')';
       }
     $currentSequence=$hash{'map_pc_'.$hash{'src_'.$currentResourceID}};      #
       my ($student_requirements,$enrollment_requirements) = 
             # Mark sequence as containing problems.  If it doesn't, then          &limit_by_section_and_status($sections,$enrollment,'b');
             # it will be removed when processing for this sequence is      #
             # complete.  This allows the problems in a sequence      my ($groups_join,$group_limits) = &limit_by_group($groups,'b','d','e');
             # to be outputed before problems in the subsequences      my $time_requirements = &limit_by_start_end_time($starttime,$endtime,'a');
             if(!defined($cache->{'orderedSequences'})) {      ##
                 $cache->{'orderedSequences'}=$currentSequence;      $request = 'CREATE TEMPORARY TABLE IF NOT EXISTS '.$tmptable.
           ' SELECT a.student_id,SUM(a.awarded*c.weight) AS score FROM '.
           $performance_table.' AS a ';
       $request .= "LEFT JOIN ".$weight_table.' AS c ON a.symb_id=c.symb_id AND a.part_id=c.part_id ';
       if (defined($student_requirements) || defined($enrollment_requirements)) {
           $request .= ' LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id ';
       }
       if (defined($groups_join)) {
           $request .= $groups_join;
       }
       if (defined($symb_requirements)       || 
           defined($student_requirements)    ||
           defined($enrollment_requirements) ||
           defined($group_limits) ) {
           $request .= ' WHERE ';
       }
       if (defined($symb_requirements)) {
           $request .= $symb_requirements.' AND ';
       }
       if (defined($student_requirements)) {
           $request .= $student_requirements.' AND ';
       }
       if (defined($enrollment_requirements)) {
           $request .= $enrollment_requirements.' AND ';
       }
       if (defined($time_requirements)) {
           $request .= $time_requirements.' AND ';
       }
       $request =~ s/ AND $//; # Strip of the trailing ' AND '.
       $request .= ' GROUP BY a.student_id';
   #    &Apache::lonnet::logthis("request = \n".$request);
       my $sth = $dbh->prepare($request);
       $sth->execute();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 6 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed '.$/.$request);
           return undef;
       }
       $request = 'SELECT score,COUNT(*) FROM '.$tmptable.' GROUP BY score';
   #    &Apache::lonnet::logthis("request = \n".$request);
       $sth = $dbh->prepare($request);
       $sth->execute();
       if ($dbh->err) {
           &Apache::lonnet::logthis('error 7 = '.$dbh->errstr());
           &Apache::lonnet::logthis('prepared then executed '.$/.$request);
           return undef;
       }
       my $dataset = $sth->fetchall_arrayref();
       return $dataset;
   }
   
   
   
   sub setup_table_names {
       my ($courseid) = @_;
       if (! defined($courseid)) {
           $courseid = $env{'request.course.id'};
       }
       #
       if (! defined($current_course) || $current_course ne $courseid) {
           # Clear out variables
           $have_read_part_table = 0;
           undef(%ids_by_part);
           undef(%parts_by_id);
           $have_read_symb_table = 0;
           undef(%ids_by_symb);
           undef(%symbs_by_id);
           $have_read_student_table = 0;
           undef(%ids_by_student);
           undef(%students_by_id);
           $have_read_groupnames_table = 0;
           undef(%ids_by_groupname);
           #
           $current_course = $courseid;
       }
       #
       # Set up database names
       my $base_id = 'md5_'.&Digest::MD5::md5_hex($courseid);
       $symb_table               = $base_id.'_'.'symb';
       $part_table               = $base_id.'_'.'part';
       $student_table            = $base_id.'_'.'student';
       $groupnames_table         = $base_id.'_'.'groupnames';
       $students_groups_table    = $base_id.'_'.'studentgroups';
       $performance_table        = $base_id.'_'.'performance';
       $parameters_table         = $base_id.'_'.'parameters';
       $fulldump_part_table      = $base_id.'_'.'partdata';
       $fulldump_response_table  = $base_id.'_'.'responsedata';
       $fulldump_timestamp_table = $base_id.'_'.'timestampdata';
       $weight_table             = $base_id.'_'.'weight';
       #
       @Tables = (
                  $symb_table,
                  $part_table,
                  $student_table,
                  $groupnames_table,
                  $students_groups_table,
                  $performance_table,
                  $parameters_table,
                  $fulldump_part_table,
                  $fulldump_response_table,
                  $fulldump_timestamp_table,
                  $weight_table,
                  );
       return;
   }
   
   sub temp_table_name {
       my ($courseid,$affix) = @_;
       my $base_id = 'md5_'.&Digest::MD5::md5_hex($courseid);
       return $base_id.'_'.$affix;
   }
   
   
   } # End scope of table identifiers
   
   
   
   sub CL_SDOM     { return 0; }
   sub CL_SNAME    { return 1; }
   sub CL_END      { return 2; }
   sub CL_START    { return 3; }
   sub CL_ID       { return 4; }
   sub CL_SECTION  { return 5; }
   sub CL_FULLNAME { return 6; }
   sub CL_STATUS   { return 7; }
   sub CL_TYPE     { return 8; }
   sub CL_LOCKEDTYPE   { return 9; }
   sub CL_CREDITS  { return 10; }
   sub CL_GROUP    { return 11; }
   sub CL_PERMANENTEMAIL { return 12; }
   sub CL_ROLE     { return 13; }
   sub CL_EXTENT   { return 14; }
   sub CL_PHOTO   { return 15; }
   sub CL_THUMBNAIL { return 16; }
   sub CL_AUTHORQUOTA { return 17; }
   sub CL_AUTHORUSAGE { return 18; }
   
   sub get_classlist {
       my ($cdom,$cnum) = @_;
       my $cid = $cdom.'_'.$cnum;
       if (!defined($cdom) || !defined($cnum)) {
    $cid =  $env{'request.course.id'};
    $cdom = $env{'course.'.$cid.'.domain'};
    $cnum = $env{'course.'.$cid.'.num'};
       }
       my $now = time;
       #
       my %classlist=&Apache::lonnet::dump('classlist',$cdom,$cnum);
       while (my ($student,$info) = each(%classlist)) {
           if ($student =~ /^(con_lost|error|no_such_host)/i) {
               &Apache::lonnet::logthis('get_classlist error for '.$cid.':'.$student);
               return undef;
           }
           my ($sname,$sdom) = split(/:/,$student);
           my @Values = split(/:/,$info);
           my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits);
           if (@Values > 2) {
               ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits) = @Values;
           } else { # We have to get the data ourselves
               ($end,$start) = @Values;
               $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
               my %info=&Apache::lonnet::get('environment',
                                             ['firstname','middlename',
                                              'lastname','generation','id'],
                                             $sdom, $sname);
               my ($tmp) = keys(%info);
               if ($tmp =~/^(con_lost|error|no_such_host)/i) {
                   $fullname = 'not available';
                   $id = 'not available';
                   &Apache::lonnet::logthis('unable to retrieve environment '.
                                            'for '.$sname.':'.$sdom);
             } else {              } else {
                 $cache->{'orderedSequences'}.=':'.$currentSequence;                  $fullname = &Apache::lonnet::format_name(@info{qw/firstname middlename lastname generation/},'lastname');
                   $id = $info{'id'};
               }
               # Update the classlist with this students information
               if ($fullname ne 'not available') {
    my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
    my $reply=&Apache::lonnet::cput('classlist',
                                                   {$student => $enrolldata},
                                                   $cdom,$cnum);
                   if ($reply !~ /^(ok|delayed)/) {
                       &Apache::lonnet::logthis('Unable to update classlist for '.
                                                'student '.$sname.':'.$sdom.
                                                ' error:'.$reply);
                   }
             }              }
           }
           my $status='Expired';
           if(((!$end) || $now < $end) && ((!$start) || ($now > $start))) {
               $status='Active';
           }
           if(($now < $start) && ((!$end) || $now < $end )) {
               $status='Future';
           }
           $classlist{$student} = 
               [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,
                $lockedtype,$credits];
       }
       if (wantarray()) {
           return (\%classlist,['domain','username','end','start','id',
                                'section','fullname','status','type',
                                'lockedtype','credits']);
       } else {
           return \%classlist;
       }
   }
   
     $lastResourceID=$hash{'map_finish_'.  sub get_group_memberships {
   $hash{'src_'.$currentResourceID}};      my ($classlist,$keylist,$cdom,$cnum) = @_;
     $currentResourceID=$hash{'map_start_'.  
      $hash{'src_'.$currentResourceID}};  
   
     if(!($currentResourceID) || !($lastResourceID)) {  
  $currentSequence=pop(@sequences);  
  $currentResourceID=pop(@currentResource);  
  $lastResourceID=pop(@finishResource);  
  if($currentSequence eq $topLevelSequenceNumber) {  
     last;  
  }  
     }  
  }  
   
  # Handle gradable resources: exams, problems, etc      return ({},{}) if (!ref($classlist) || !ref($keylist));
  $currentResourceID=~/(\d+)\.(\d+)/;  
         my $partA=$1;  
         my $partB=$2;  
  if($hash{'src_'.$currentResourceID}=~  
    /\.(problem|exam|quiz|assess|survey|form)$/ &&  
    $partA eq $currentSequence) {  
     my $Problem = &Apache::lonnet::symbclean(  
   &Apache::lonnet::declutter($hash{'map_id_'.$partA}).  
   '___'.$partB.'___'.  
   &Apache::lonnet::declutter($hash{'src_'.  
  $currentResourceID}));  
   
     $cache->{$currentResourceID.':problem'}=$Problem;  
     if(!defined($cache->{$currentSequence.':problems'})) {  
  $cache->{$currentSequence.':problems'}=$currentResourceID;  
     } else {  
  $cache->{$currentSequence.':problems'}.=  
     ':'.$currentResourceID;  
     }  
   
             # Get Parts for problem      my $cid = $cdom.'_'.$cnum;
     my $meta=$hash{'src_'.$currentResourceID};      if (!defined($cdom) || !defined($cnum)) {
     foreach (split(/\,/,&Apache::lonnet::metadata($meta,'keys'))) {          $cid =  $env{'request.course.id'};
  if($_=~/^stores\_(\d+)\_tries$/) {          $cdom = $env{'course.'.$cid.'.domain'};
     my $Part=&Apache::lonnet::metadata($meta,$_.'.part');          $cnum = $env{'course.'.$cid.'.num'};
                     if(!defined($cache->{$currentSequence.':'.      }
                                           $currentResourceID.':parts'})) {      my (%classgroups,%studentgroups);
                         $cache->{$currentSequence.':'.$currentResourceID.      my $now = time;
                                    ':parts'}=$Part;      my $access_end = $env{'course.'.$cid.'.default_enrollment_end_date'};
       my %curr_groups =&Apache::longroup::coursegroups($cdom,$cnum);
       if (%curr_groups) {
           my $grpindex = &CL_GROUP();
           my %groupmemberhash = 
       &Apache::lonnet::get_group_membership($cdom,$cnum);
           foreach my $student (keys(%{$classlist})) {
               %{$classgroups{$student}} = ();
               my $hasgroup = 0;
               foreach my $status ('previous','future','active','aftercourse') {
                   %{$classgroups{$student}{$status}} = ();
               }
               foreach my $group (keys(%curr_groups)) {
                   if (defined($groupmemberhash{$group.':'.$student})) {
                       my ($end,$start) = split(/:/,$groupmemberhash{$group.':'.
                                                                       $student});
                       if ($start == -1) {
                           next;
                     } else {                      } else {
                         $cache->{$currentSequence.':'.$currentResourceID.                          $studentgroups{$group} ++;
                                    ':parts'}.=':'.$Part;                          $hasgroup = 1;
                     }                          if ($end && $end < $now) {
  }                              $classgroups{$student}{'previous'}{$group} =
     }                                           $groupmemberhash{$group.':'.$student};
  }                              if ($classlist->{$student}[&CL_STATUS()] eq 'Expired') {
                                   if ($access_end && $access_end < $now) {
  # if resource == finish resource, then it is the end of a sequence/page                                      if ($access_end - $end < 86400) {
  if($currentResourceID eq $lastResourceID) {                                          $classgroups{$student}{'aftercourse'}{$group} = $groupmemberhash{$group.':'.$student};
     # pop off last resource of sequence                                      }
     $currentResourceID=pop(@currentResource);                                  }
     $lastResourceID=pop(@finishResource);                              }
                           } elsif ($now > $start) {
     if(defined($cache->{$currentSequence.':problems'})) {                              if (!$end || $end > $now) {
  # Capture sequence information here                                  $classgroups{$student}{'active'}{$group} =
  $cache->{$currentSequence.':title'}=                                           $groupmemberhash{$group.':'.$student};
     $hash{'title_'.$currentResourceID};                              }
                           } else {
                 my $totalProblems=0;                              $classgroups{$student}{'future'}{$group} =
                 foreach my $currentProblem (split(/\:/,                                           $groupmemberhash{$group.':'.$student};
                                                $cache->{$currentSequence.                          }
                                                ':problems'})) {  
                     foreach (split(/\:/,$cache->{$currentSequence.':'.  
                                                    $currentProblem.  
                                                    ':parts'})) {  
                         $totalProblems++;  
                     }                      }
                 }                  }
  my @titleLength=split(//,$cache->{$currentSequence.  
                                                     ':title'});  
                 # $extra is 3 for problems correct and 3 for space  
                 # between problems correct and problem output  
                 my $extra = 6;  
  if(($totalProblems + $extra) > (scalar @titleLength)) {  
     $cache->{$currentSequence.':columnWidth'}=  
                         $totalProblems + $extra;  
  } else {  
     $cache->{$currentSequence.':columnWidth'}=  
                         (scalar @titleLength);  
  }  
     } else {  
                 # Remove sequence from list, if it contains no problems to  
                 # display.  
                 $cache->{'orderedSequences'}=~s/$currentSequence//;  
                 $cache->{'orderedSequences'}=~s/::/:/g;  
                 $cache->{'orderedSequences'}=~s/^:|:$//g;  
             }              }
               if (!$hasgroup) {
                   $studentgroups{'none'} ++;
               } else {
                   $classlist->{$student}->[$grpindex] = join(',',
                                 sort(keys(%{$classgroups{$student}{'active'}})));
               }
           }
       }
       return (\%classgroups,\%studentgroups);
   }
                                                                                      
   sub get_students_groups {
       my ($student,$enrollment_status,$classgroups) = @_;
       my @studentsgroups = ();
       if (ref($$classgroups{$student}{'active'}) eq 'HASH') {
           push(@studentsgroups,keys(%{$$classgroups{$student}{'active'}}));
       }
       if ($enrollment_status eq 'Any') {
           foreach my $status ('previous','future') {
               if (ref($$classgroups{$student}{$status}) eq 'HASH') {
                   push(@studentsgroups,keys(%{$$classgroups{$student}{$status}}));
               }
           }
       } else {
           if (ref($$classgroups{$student}{'aftercourse'}) eq 'HASH') {
               push(@studentsgroups,keys(%{$$classgroups{$student}{'aftercourse'}}));
           }
       }
       return @studentsgroups;
   }
   
     $currentSequence=pop(@sequences);  
     if($currentSequence eq $topLevelSequenceNumber) {  
  last;  
     }  
  }  
   
  # MOVE!!!  # ----- END HELPER FUNCTIONS --------------------------------------------
  # move to next resource  
  unless(defined($hash{'to_'.$currentResourceID})) {  
     # big problem, need to handle.  Next is probably wrong  
     last;  
  }  
  my @nextResources=();  
  foreach (split(/\,/,$hash{'to_'.$currentResourceID})) {  
     push(@nextResources, $hash{'goesto_'.$_});  
  }  
  push(@currentResource, @nextResources);  
  # Set the next resource to be processed  
  $currentResourceID=pop(@currentResource);  
     }  
   
     unless (untie(%hash)) {  1;
         &Apache::lonnet::logthis("<font color=blue>WARNING: ".  __END__
                                  "Could not untie coursemap $fn (browse)".  
                                  ".</font>");   
     }  
   
     return 'OK';  
 }  
   
 =pod  =pod
   
 =item &ProcessSection()  =head1 NAME
   
 Determine the section number for a student for the class.  A student can have   Apache::loncoursedata
 multiple sections for the same class.  The correct one is chosen.  
   
 =over 4  =head1 SYNOPSIS
   
 Input: $sectionData, $courseid, $ActiveFlag  Set of functions that download and process student and course information.
   
 $sectionData:  A pointer to a hash containing all section data for this   =head1 PACKAGES USED
 student for the class  
   
 $courseid:  The course ID.    Apache::lonnet
     Apache::longroup
     Time::HiRes
     Apache::lonmysql
     LONCAPA
     Digest::MD5
    
   =head1 DOWNLOAD INFORMATION
   
 $ActiveFlag:  The student's active status (Active/Expired)  This section contains all the functions that get data from other servers 
   and/or itself.
   
 Output: $oldsection, $cursection, or -1  
   
 $oldsection and $cursection and sections number that will be displayed in the   
 chart.  
   
 -1 is returned if an error occurs.  =head1 LOCAL DATA CACHING SUBROUTINES
   
 =back  The local caching is done using MySQL.  There is no fall-back implementation
   if MySQL is not running.
   
 =cut  The programmers interface is to call &get_current_state() or some other
   primary interface subroutine (described below).  The internals of this 
   storage system are documented here.
   
 sub ProcessSection {  There are six tables used to store student performance data (the results of
     my ($sectionData,$courseid,$ActiveFlag)=@_;  a dumpcurrent).  Each of these tables is created in MySQL with a name of
     $courseid=~s/\_/\//g;  $courseid_*****, where ***** is 'symb', 'part', or whatever is appropriate 
     $courseid=~s/^(\w)/\/$1/;  for the table.  The tables and their purposes are described below.
   
     my $cursection='-1';  
     my $oldsection='-1';  
     my $status='Expired';  
     my $section='';  
     foreach my $key (keys (%$sectionData)) {  
  my $value = $sectionData->{$key};  
         if ($key=~/^$courseid(?:\/)*(\w+)*\_st$/) {  
     $section=$1;  
     if($key eq $courseid.'_st') {  
  $section='';  
     }  
     my ($dummy,$end,$start)=split(/\_/,$value);  
     my $now=time;  
     my $notactive=0;  
     if ($start) {  
  if($now<$start) {  
     $notactive=1;  
  }  
     }  
     if($end) {  
  if ($now>$end) {  
     $notactive=1;  
  }  
     }  
     if($notactive == 0) {  
  $status='Active';  
  $cursection=$section;  
  last;  
     }  
     if($notactive == 1) {  
  $oldsection=$section;  
     }  
  }  
     }  
     if($status eq $ActiveFlag) {  
  if($cursection eq '-1') {  
     return $oldsection;  
  }  
  return $cursection;  
     }  
     if($ActiveFlag eq 'Any') {  
  if($cursection eq '-1') {  
     return $oldsection;  
  }  
  return $cursection;  
     }  
     return '-1';  
 }  
   
 =pod  Some notes before we get started.
   
   Each table must have a PRIMARY KEY, which is a column or set of columns which
   will serve to uniquely identify a row of data.  NULL is not allowed!
   
   INDEXes work best on integer data.
   
 =item &ProcessNamePIDSection()  JOIN is used to combine data from many tables into one output.
   
 Takes data downloaded for a student and breaks it up into managable pieces and   lonmysql.pm is used for some of the interface, specifically the table creation
 stored in cache data.  The username, domain, class related date, PID,   calls.  The inserts are done in bulk by directly calling the database handler.
 full name, and section are all processed here.  The SELECT ... JOIN statement used to retrieve the data does not have an
   interface in lonmysql.pm and I shudder at the thought of writing one.
   
   =head2 Table Descriptions
   
   =over 4
   
   =head2 Tables used to store meta information
   
   The following tables hold data required to keep track of the current status
   of a students data in the tables or to look up the students data in the tables.
   
 =over 4  =over 4
   
 Input: $cache, $studentInformation, $section, $date, $name, $courseID  =item C<$symb_table>
   
   The symb_table has two columns.  The first is a 'symb_id' and the second
   is the text name for the 'symb' (limited to 64k).  The 'symb_id' is generated
   automatically by MySQL so inserts should be done on this table with an
   empty first element.  This table has its PRIMARY KEY on the 'symb_id'.
   
   =item C<$part_table>
   
   The part_table has two columns.  The first is a 'part_id' and the second
   is the text name for the 'part' (limited to 100 characters).  The 'part_id' is
   generated automatically by MySQL so inserts should be done on this table with
   an empty first element.  This table has its PRIMARY KEY on the 'part' (100
   characters) and a KEY on 'part_id'.
   
 $cache:  A hash pointer to store the data  =item C<$student_table>
   
 $studentInformation:  Student information is what was requested in   The student_table has 7 columns.  The first is a 'student_id' assigned by 
 &DownloadPrerequistedData().  See that function for what data is requested.  MySQL.  The second is 'student' which is username:domain.  The third through
   fifth are 'section', 'status' (enrollment status), and 'classification' 
   (to be used in the future).  The sixth and seventh ('updatetime' and 
   'fullupdatetime') contain the time of last update and full update of student
   data.  This table has its PRIMARY KEY on the 'student_id' column and is indexed
   on 'student', 'section', and 'status'.
   
 $section: A hash pointer to class section related information.  =item C<$groupnames_table>
   
 $date:  A composite of the start and end date for this class for this  The groupnames_table has 2 columns.  The first is a 'group_id' assigned by 
 student.  Format:  end:start  MySQL.  The second is 'groupname' which is the name of the group in the course.
   
 $name:  the username:domain information  =item C<$students_groups_table>
   
 $courseID: The course ID  The students_groups_table has 2 columns.  The first is the 'student_id', and the 
   second is the 'group_id'. These two columns comprise the PRIMARY KEY for this 
   table, as an individual student may be affiliated with more than one group at
   any time. This table is indexed on both student_id and group_id.
   
 Output: None  =back 
   
 *NOTE:  There is no return value, but if an error occurs a key is added to   =head2 Tables used to store current status data
 the cache data with the value being the error message.  The key is   
 username:domain:error.  It will only exist if an error occurs.  The following tables store data only about the students current status on 
   a problem, meaning only the data related to the last attempt on a problem.
   
   =over 4
   
   =item C<$performance_table>
   
   The performance_table has 9 columns.  The first three are 'symb_id', 
   'student_id', and 'part_id'.  These comprise the PRIMARY KEY for this table
   and are directly related to the $symb_table, $student_table, and $part_table
   described above.  MySQL does better indexing on numeric items than text,
   so we use these three "index tables".  The remaining columns are
   'solved', 'tries', 'awarded', 'award', 'awarddetail', and 'timestamp'.
   These are either the MySQL type TINYTEXT or various integers ('tries' and 
   'timestamp').  This table has KEYs of 'student_id' and 'symb_id'.
   For use of this table, see the functions described below.
   
   =item C<$parameters_table>
   
   The parameters_table holds the data that does not fit neatly into the
   performance_table.  The parameters table has four columns: 'symb_id',
   'student_id', 'parameter', and 'value'.  'symb_id', 'student_id', and
   'parameter' comprise the PRIMARY KEY for this table.  'parameter' is 
   limited to 255 characters.  'value' is limited to 64k characters.
   
 =back  =back
   
 =cut  =head2 Tables used for storing historic data
   
 sub ProcessStudentNamePIDSection {  The following tables are used to store almost all of the transactions a student
     my ($cache,$studentInformation,$section,$date,$name,$courseID,$status)=@_;  has made on a homework problem.  See loncapa/docs/homework/datastorage for 
     my ($studentName,$studentDomain) = split(/\:/,$name);  specific information about each of the parameters stored.  
   
     $cache->{$name.':username'}=$studentName;  
     $cache->{$name.':domain'}=$studentDomain;  
     $cache->{$name.':date'}=$date;  
   
     my ($checkForError)=keys(%$studentInformation);  
     if($checkForError =~ /^(con_lost|error|no_such_host)/i) {  
  $cache->{$name.':error'}=  
     'Could not download student environment data.';  
  $cache->{$name.':fullname'}='';  
  $cache->{$name.':id'}='';  
     } else {  
  $cache->{$name.':fullname'}=&ProcessFullName(  
                                           $studentInformation->{'lastname'},  
           $studentInformation->{'generation'},  
           $studentInformation->{'firstname'},  
                                           $studentInformation->{'middlename'});  
  $cache->{$name.':id'}=$studentInformation->{'id'};  
     }  
   
     # Get student's section number  =over 4
     my $sec=&ProcessSection($section, $courseID, $status);  
     if($sec != -1) {  
  $cache->{$name.':section'}=$sec;  
     } else {  
  $cache->{$name.':section'}='';  
     }  
   
     return;  =item C<$fulldump_response_table>
 }  
   The response table holds data (documented in loncapa/docs/homework/datastorage)
   associated with a particular response id which is stored when a student 
   attempts a problem.  The following are the columns of the table, in order:
   'symb_id','part_id','response_id','student_id','transaction','tries',
   'awarddetail', 'response_specific', 'response_specific_value',
   'response_specific_2', 'response_specific_value_2', and 'submission
   (the text of the students submission).  The primary key is based on the
   first five columns listed above.
   
   =item C<$fulldump_part_table()>
   
   The part table holds data (documented in loncapa/docs/homework/datastorage)
   associated with a particular part id which is stored when a student attempts
   a problem.  The following are the columns of the table, in order:
   'symb_id','part_id','student_id','transaction','tries','award','awarded',
   and 'previous'.  The primary key is based on the first five columns listed 
   above.
   
   =item C<$fulldump_timestamp_table()>
   
   The timestamp table holds the timestamps of the transactions which are
   stored in $fulldump_response_table and $fulldump_part_table.  This data is
   about both the response and part data.  Columns: 'symb_id','student_id',
   'transaction', and 'timestamp'.  
   The primary key is based on the first 3 columns.
   
   =item C<$weight_table()>
   
   The weight table holds the weight for the problems used in the class.
   Whereas the weight of a problem can vary by section and student the data
   here is applied to the class as a whole.
   Columns: 'symb_id','part_id','response_id','weight'.
   
   =back
   
 =pod  
   
 =item &ProcessClassList()  =head1 IMPORTANT SUBROUTINES
   
 Taking the class list dumped from &DownloadPrerequisiteData(), all the   Here is a brief overview of the subroutines which are likely to be of 
 students and their non-class information is processed using the   interest:
 &ProcessStudentInformation() function.  A date stamp is also recorded for  
 when the data was processed.  
   
 =over 4  =over 4
   
 Input: $cache, $classlist, $courseID, $ChartDB, $c  =item C<&get_current_state()>
   
 $cache: A hash pointer to store the data  programmers interface.
   
 $classlist:  The hash of data collected about a student from   =item C<&init_dbs()>
 &DownloadPrerequisteData().  The hash contains a list of students, a pointer   
 to a hash of student information for each student, and each student's section   
 number.  
   
 $courseID:  The course ID  table creation
   
 $ChartDB:  The name of the cache database file.  =item C<&update_student_data()>
   
 $c:  The connection class used to determine if an abort has been sent to the   data storage calls
 browser  
   
 Output: @names  =item C<&get_student_data_from_performance_cache()>
   
 @names:  An array of students whose information has been processed, and are to   data retrieval
 be considered in an arbitrary order.  
   
 =back  =back
   
 =cut  =head1 OTHER SUBROUTINES
   
 sub ProcessClassList {  =over 4
     my ($cache,$classlist,$courseID,$status,$c)=@_;  
     my @names=();  
   
     foreach my $name (keys(%$classlist)) {  
         if($name =~ /\:section/ || $name =~ /\:studentInformation/ ||  
            $name eq '') {  
             next;  
         }  
         if($c->aborted()) {  
             last;  
         }  
         push(@names,$name);  
         &ProcessStudentNamePIDSection($cache,  
                                      $classlist->{$name.':studentInformation'},  
                                      $classlist->{$name.':section'},  
                                      $classlist->{$name},  
                                      $name,$courseID,$status);  
     }  
   
     # Time of download  =item C<&make_into_hash($values)>
     $cache->{'time'}=localtime();  
   
     return @names;  Returns a reference to a hash as described by $values.  $values is
 }  assumed to be the result of 
       join(':',map {&escape($_)} %orighash);
   
 =pod  This is a helper function for get_current_state.
   
   =item C<&init_dbs()>
   
   Input: course id
   
   Output: 0 on success, positive integer on error
   
   This routine issues the calls to lonmysql to create the tables used to
   store student data.
   
   item C<&delete_caches()>
   
   This routine drops all the tables associated with a course from the 
   MySQL database.
   
   Input: course id (optional, determined by environment if omitted) 
   
   Returns: nothing
   
 =item &ProcessStudentData()  =item C<&get_part_id()>
   
 Takes the course data downloaded for a student in   Get the MySQL id of a problem part string.
 &DownloadStudentCourseInformation() and breaks it up into key value pairs  
 to be stored in the cached data.  The keys are comprised of the   Input: $part
 $username:$domain:$keyFromCourseDatabase.  The student username:domain is  
 stored away signifying that the student's information has been downloaded and   Output: undef on error, integer $part_id on success.
 can be reused from cached data.  
   =item C<&get_part()>
   
   Get the string describing a part from the MySQL id of the problem part.
   
   Input: $part_id
   
   Output: undef on error, $part string on success.
   
   =item C<&get_symb_id()>
   
   Get the MySQL id of a symb.
   
   Input: $symb
   
   Output: undef on error, integer $symb_id on success.
   
   =item C<&get_symb()>
   
   Get the symb associated with a MySQL symb_id.
   
   Input: $symb_id
   
   Output: undef on error, $symb on success.
   
   =item C<&get_student_id()>
   
   Get the MySQL id of a student.
   
   Input: $sname, $dom
   
   Output: undef on error, integer $student_id on success.
   
   =item C<&get_student()>
   
   Get student username:domain associated with the MySQL student_id.
   
   Input: $student_id
   
   Output: undef on error, string $student (username:domain) on success.
   
   =item C<&clear_internal_caches()>
   
   Causes the internal caches used in get_student_id, get_student,
   get_symb_id, get_symb, get_part_id, and get_part to be undef'd.
   
   Needs to be called before the first operation with the MySQL database
   for a given Apache request.
   
   =item C<&update_full_student_data($sname,$sdom,$courseid)>
   
   Does a lonnet::dump on a student to populate the courses tables.
   
   Input: $sname, $sdom, $courseid
   
   Output: $returnstatus
   
   $returnstatus is a string describing any errors that occurred.  'okay' is the
   default.
   
   This subroutine loads a students data using lonnet::dump and inserts
   it into the MySQL database.  The inserts are done on three tables, 
   $fulldump_response_table, $fulldump_part_table, and $fulldump_timestamp_table.
   The INSERT calls are made directly by this subroutine, not through lonmysql 
   because we do a 'bulk'insert which takes advantage of MySQLs non-SQL 
   compliant INSERT command to insert multiple rows at a time.  
   If anything has gone wrong during this process, $returnstatus is updated with 
   a description of the error.
   
   Once the "fulldump" tables are updated, the tables used for chart and
   spreadsheet (which hold only the current state of the student on their
   homework, not historical data) are updated.  If all updates have occurred 
   successfully, $student_table is updated to reflect the time of the update.
   
   Notice we do not insert the data and immediately query it.  This means it
   is possible for there to be data returned this first time that is not 
   available the second time.  CYA.
   
   
   =item C<&update_student_data()>
   
   Input: $sname, $sdom, $courseid
   
   Output: $returnstatus, \%student_data
   
   $returnstatus is a string describing any errors that occurred.  'okay' is the
   default.
   \%student_data is the data returned by a call to lonnet::currentdump.
   
   This subroutine loads a students data using lonnet::currentdump and inserts
   it into the MySQL database.  The inserts are done on two tables, 
   $performance_table and $parameters_table.  $parameters_table holds the data 
   that is not included in $performance_table.  See the description of 
   $performance_table elsewhere in this file.  The INSERT calls are made
   directly by this subroutine, not through lonmysql because we do a 'bulk'
   insert which takes advantage of MySQLs non-SQL compliant INSERT command to 
   insert multiple rows at a time.  If anything has gone wrong during this
   process, $returnstatus is updated with a description of the error and
   \%student_data is returned.  
   
   Notice we do not insert the data and immediately query it.  This means it
   is possible for there to be data returned this first time that is not 
   available the second time.  CYA.
   
   =item &ensure_tables_are_set_up($courseid)
   
   Checks to be sure the MySQL tables for the given class are set up.
   If $courseid is omitted it will be obtained from the environment.
   
   Returns nothing on success and 'error' on failure
   
   
   =item C<&ensure_current_data()>
   
   Input: $sname, $sdom, $courseid
   
   Output: $status, $data
   
   This routine ensures the data for a given student is up to date.
   The $student_table is queried to determine the time of the last update.  
   If the students data is out of date, &update_student_data() is called.  
   The return values from the call to &update_student_data() are returned.
   
   =item C<&ensure_current_full_data($sname,$sdom,$courseid)>
   
   Input: $sname, $sdom, $courseid
   
   Output: $status
   
   This routine ensures the fulldata (the data from a lonnet::dump, not a
   lonnet::currentdump) for a given student is up to date.
   The $student_table is queried to determine the time of the last update.  
   If the students fulldata is out of date, &update_full_student_data() is
   called.  
   
   The return value from the call to &update_full_student_data() is returned.
   
   =item C<&get_student_data_from_performance_cache()>
   
   Input: $sname, $sdom, $symb, $courseid
   
   Output: hash reference containing the data for the given student.
   If $symb is undef, all the students data is returned.
   
   This routine is the heart of the local caching system.  See the description
   of $performance_table, $symb_table, $student_table, and $part_table.  The
   main task is building the MySQL request.  The tables appear in the request
   in the order in which they should be parsed by MySQL.  When searching
   on a student the $student_table is used to locate the 'student_id'.  All
   rows in $performance_table which have a matching 'student_id' are returned,
   with data from $part_table and $symb_table which match the entries in
   $performance_table, 'part_id' and 'symb_id'.  When searching on a symb,
   the $symb_table is processed first, with matching rows grabbed from 
   $performance_table and filled in from $part_table and $student_table in
   that order.  
   
   Running 'EXPLAIN ' on the 'SELECT' statements generated can be quite 
   interesting, especially if you play with the order the tables are listed.  
   
   
   =item C<&get_current_state()>
   
   Input: $sname,$sdom,$symb,$courseid
   
   Output: Described below
   
   Retrieve the current status of a students performance.  $sname and
   $sdom are the only required parameters.  If $symb is undef the results
   of an &Apache::lonnet::currentdump() will be returned.  
   If $courseid is undef it will be retrieved from the environment.
   
   The return structure is based on &Apache::lonnet::currentdump.  If
   $symb is unspecified, all the students data is returned in a hash of
   the form:
   ( 
     symb1 => { param1 => value1, param2 => value2 ... },
     symb2 => { param1 => value1, param2 => value2 ... },
   )
   
   If $symb is specified, a hash of 
   (
     param1 => value1, 
     param2 => value2,
   )
   is returned.
   
   If no data is found for $symb, or if the student has no performance data,
   an empty list is returned.
   
   =item C<&get_problem_statistics()>
   
   Gather data on a given problem.  The database is assumed to be 
   populated and all local caching variables are assumed to be set
   properly.  This means you need to call &ensure_current_data for
   the students you are concerned with prior to calling this routine.
   
   Inputs: $Sections, Groups, $status, $symb, $part, $courseid, $starttime,
           $endtime
   
 =over 4  =over 4
   
 Input: $cache, $courseData, $name  =item $Sections Array ref containing section names for students.  
   'all' is allowed to be the first (and only) item in the array.
   
   =item $Groups Array ref containing group names for students.
   'all' is allowed to be the first (and only) item in the array.
   
 $cache: A hash pointer to store data  =item $status String describing the status of students
   
 $courseData:  A hash pointer that points to the course data downloaded for a   =item $symb is the symb for the problem.
 student.  
   
 $name:  username:domain  =item $part is the part id you need statistics for
   
 Output: None  =item $courseid is the course id, of course!
   
 *NOTE:  There is no output, but an error message is stored away in the cache   =item $starttime and $endtime are unix times which to use to limit
 data.  This is checked in &FormatStudentData().  The key username:domain:error   the statistical data.
 will only exist if an error occured.  The error is an error from   
 &DownloadStudentCourseInformation().  
   
 =back  =back
   
 =cut  Outputs: See the code for up to date information.  A hash reference is
   returned.  The hash has the following keys defined:
   
 sub ProcessStudentData {  =over 4
     my ($cache,$courseData,$name)=@_;  
   
     my ($checkForError) = keys(%$courseData);  =item * num_students 
     if($checkForError =~ /^(con_lost|error|no_such_host)/i) {  
         $cache->{$name.':error'}='Could not download course data.';  
     } else {  
         foreach my $key (keys (%$courseData)) {  
             $cache->{$name.':'.$key}=$courseData->{$key};  
         }  
         if(defined($cache->{'NamesOfStudents'})) {  
             $cache->{'NamesOfStudents'}.=':::'.$name;  
         } else {  
             $cache->{'NamesOfStudents'}=$name;  
         }  
     }  
   
     return;  The number of students attempting the problem
 }        
   =item tries 
   
 # ----- END PROCESSING FUNCTIONS ---------------------------------------  The total number of tries for the students
         
   =item max_tries 
   
 =pod  The maximum number of tries taken
         
   =item mean_tries 
   
 =head1 HELPER FUNCTIONS  The average number of tries
         
   =item num_solved T
   
 These are just a couple of functions do various odd and end   he number of students able to solve the problem
 jobs.        
   =item num_override 
   
 =cut  The number of students whose answer is 'correct_by_override'
         
   =item deg_of_diff 
   
 # ----- HELPER FUNCTIONS -----------------------------------------------  The degree of difficulty of the problem
         
   =item std_tries 
   
 =pod  The standard deviation of the number of tries
         
   =item skew_tries 
   
 =item &ProcessFullName()  The skew of the number of tries
   
 Takes lastname, generation, firstname, and middlename (or some partial  =item per_wrong 
 set of this data) and returns the full name version as a string.  Format  
 is Lastname generation, firstname middlename or a subset of this.  
   
 =cut  The number of students attempting the problem who were not
   able to answer it correctly.
   
 sub ProcessFullName {  =back
     my ($lastname, $generation, $firstname, $middlename)=@_;  
     my $Str = '';  
   
     if($lastname ne '') {  
  $Str .= $lastname.' ';  
  if($generation ne '') {  
     $Str .= $generation;  
  } else {  
     chop($Str);  
  }  
  $Str .= ', ';  
  if($firstname ne '') {  
     $Str .= $firstname.' ';  
  }  
  if($middlename ne '') {  
     $Str .= $middlename;  
  } else {  
     chop($Str);  
     if($firstname eq '') {  
  chop($Str);  
     }  
  }  
     } else {  
  if($firstname ne '') {  
     $Str .= $firstname.' ';  
  }  
  if($middlename ne '') {  
     $Str .= $middlename.' ';  
  }  
  if($generation ne '') {  
     $Str .= $generation;  
  } else {  
     chop($Str);  
  }  
     }  
   
     return $Str;  =item C<&populate_weight_table()>
 }  
   
 =pod  =item C<&limit_by_start_end_times()>
   
 =item &TestCacheData()  Build SQL WHERE condition which limits the data collected by the start
   and end times provided
   
 Determine if the cache database can be accessed with a tie.  It waits up to  Inputs: $starttime, $endtime, $table
 ten seconds before returning failure.  This function exists to help with  
 the problems with stopping the data download.  When an abort occurs and the  
 user quickly presses a form button and httpd child is created.  This  
 child needs to wait for the other to finish (hopefully within ten seconds).  
   
 =over 4  Returns: $time_limits
   
   
   =item C<&limit_by_section_and_status()C<
   
   Build SQL WHERE condition which limits the data collected by section and
   student status.
   
   Inputs: $Sections (array ref)
       $enrollment (string: 'any', 'expired', 'active')
       $tablename The name of the table that holds the student data
   
   Returns: $student_requirements,$enrollment_requirements
   
 Input: $ChartDB  =item C<&limit_by_group()>
                                                                                  
   Build SQL LEFT JOIN statement to include students_groups and groupnames tables and SQL WHERE condition which limits the data collected by group.
                                                                                  
   Inputs: $Groups (array ref)
       $stutable   The name of the table which holds the student data.
       $grptable   The name of the table which maps group_id to groupname.
       $stugrptab  The name of the table which holds student group affiliations.   
   Returns: $groups_join,$group_limits
      $groups_join  JOIN part of SQL statement (to include group related tables) 
      $group_limits SQL WHERE condition limiting to requested groups
   
 $ChartDB: The name of the cache database to be opened  =item C<rank_students_by_scores_on_resources()>
   
 Output: -1, 0, 1  Inputs: 
       $resources: array ref of hash ref.  Each hash ref needs key 'symb'.
       $Sections: array ref of sections to include,
       $Groups: array ref of groups to include.
       $enrollment: string,
       $courseid (may be omitted)
       $starttime (may be omitted)
       $endtime (may be omitted)
       $has_award_for (may be omitted)
   
 -1: Couldn't tie database  Returns; An array of arrays.  The sub arrays contain a student name and
  0: Use cached data  their score on the resources. $starttime and $endtime constrain the
  1: New cache database created, use that.  list to awards obtained during the given time limits. $has_score_on
   constrains the list to those students who at least attempted the
   resource identified by the given symb, which is used to filter out
   such students for statistics that would be adversely affected by such
   students.
   
   =item C<&get_sum_of_scores>
   
   Inputs: $resource (hash ref, needs {'symb'} key),
   $part, (the part id),
   $students (array ref, contents of array are scalars holding 'sname:sdom'),
   $courseid
   
   Returns: the sum of the score on the problem part over the students and the
      maximum possible value for the sum (taken from the weight table).
    
   
   =item C<&score_stats()>
   
   Inputs: $Sections, $enrollment, $symbs, $starttime,
           $endtime, $courseid
   
   $Sections, $enrollment, $starttime, $endtime, and $courseid are the same as 
   elsewhere in this module.  
   $symbs is an array ref of symbs
   
   Returns: minimum, maximum, mean, s.d., number of students, and maximum
     possible of student scores on the given resources
   
   =item C<&count_stats()>
   
   Inputs: $Sections, $Groups, $enrollment, $symbs, $starttime,
           $endtime, $courseid
   
   $Sections, $Groups $enrollment, $starttime, $endtime, and $courseid are the 
   same as elsewhere in this module.  
   $symbs is an array ref of symbs
   
   Returns: minimum, maximum, mean, s.d., and number of students
     of the number of items correct on the given resources
   
   =item C<get_student_data()>
   
   =item C<&get_student_scores($Sections,$Groups,$Symbs,$enrollment,$courseid)>
   
   =item C<&setup_table_names()>
   
   input: course id
   
   output: none
   
 =back  =back
   
 =cut  =head3 End of Local Data Caching Subroutines
   
 sub TestCacheData {  =head3 Classlist Subroutines
     my ($ChartDB,$isRecalculate,$totalDelay)=@_;  
     my $isCached=-1;  
     my %testData;  
     my $tieTries=0;  
   
     if(!defined($totalDelay)) {  =over
         $totalDelay = 10;  
     }  
   
     if ((-e "$ChartDB") && (!$isRecalculate)) {  =item &get_classlist();
  $isCached = 1;  
     } else {  
  $isCached = 0;  
     }  
   
     while($tieTries < $totalDelay) {  Retrieve the classist of a given class or of the current class.  Student
         my $result=0;  information is returned from the classlist.db file and, if needed,
         if($isCached) {  from the students environment.
             $result=tie(%testData,'GDBM_File',$ChartDB,&GDBM_READER,0640);  
         } else {  
             $result=tie(%testData,'GDBM_File',$ChartDB,&GDBM_NEWDB,0640);  
         }  
         if($result) {  
             last;  
         }  
         $tieTries++;  
         sleep 1;  
     }  
     if($tieTries >= $totalDelay) {  
         return -1;  
     }  
   
     untie(%testData);  Optional arguments are $cdom, and $cnum (course domain,
   and course number, respectively).  If either is ommitted the course
   will be taken from the current environment ($env{'request.course.id'},
   $env{'course.'.$cid.'.domain'}, and $env{'course.'.$cid.'.num'}).
   
     return $isCached;  Returns a reference to a hash which contains:
 }   keys    '$sname:$sdom'
    values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,
             $lockedtype,$credits]
   
   The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
   as indices into the returned list to future-proof clients against
   changes in the list order.
   
   =back
   
   =cut
   
 # ----- END HELPER FUNCTIONS --------------------------------------------  
   
 1;  
 __END__  

Removed from v.1.1  
changed lines
  Added in v.1.196


FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>