loncom/interface/loncoursedata.pm - diff

Return to loncoursedata.pm CVS log

Up to [LON-CAPA] / loncom / interface

Diff for /loncom/interface/loncoursedata.pm between versions 1.89 and 1.192

-version 1.89, 2003/09/26 18:31:31
+version 1.192, 2011/10/03 13:11:39
  Line 25
  # http://www.lon-capa.org/
  #
  ###
  =pod
  =head1 NAME
- loncoursedata
+ Apache::loncoursedata
  =head1 SYNOPSIS
- Line 38  Set of functions that download and proce
+ Line 37  Set of functions that download and proce
  =head1 PACKAGES USED
-  Apache::Constants qw(:common :http)
+   Apache::lonnet
-  Apache::lonnet()
+   Apache::longroup
-  Apache::lonhtmlcommon
+   Time::HiRes
-  HTML::TokeParser
+   Apache::lonmysql
-  GDBM_File
+   LONCAPA
+   Digest::MD5
  =cut
  package Apache::loncoursedata;
  use strict;
- use Apache::Constants qw(:common :http);
+ use Apache::lonnet;
- use Apache::lonnet();
+ use Apache::longroup();
- use Apache::lonhtmlcommon;
+ use Time::HiRes();
- use Time::HiRes;
+ use Apache::lonmysql();
- use Apache::lonmysql;
+ use LONCAPA;
- use HTML::TokeParser;
+ use Digest::MD5();
- use GDBM_File;
+ =pod
- =pod
+ =head 2 make_into_hash
- =head1 DOWNLOAD INFORMATION
+ Turn a colon separated string into a hash and return a reference
- This section contains all the functions that get data from other servers
+ to it.  Numbering from 0 even elements are keys and odd elements
- and/or itself.
+ are values e.g. a:b:c:d creates a hash like
+   a => b, c =>d
- =cut
- ####################################################
- ####################################################
- =pod
- =item &get_sequence_assessment_data()
- AT THIS TIME THE USE OF THIS FUNCTION IS *NOT* RECOMMENDED
- Use lonnavmaps to build a data structure describing the order and
- assessment contents of each sequence in the current course.
- The returned structure is a hash reference.
- { title => 'title',
-   symb  => 'symb',
-   src   => '/s/o/u/r/c/e',
-   type  => (container|assessment),
-   num_assess   => 2,               # only for container
-   parts        => [11,13,15],      # only for assessment
-   response_ids => [12,14,16],      # only for assessment
-   contents     => [........]       # only for container
- }
- $hash->{'contents'} is a reference to an array of hashes of the same structure.
- Also returned are array references to the sequences and assessments contained
- in the course.
- =cut
- ####################################################
- ####################################################
- sub get_sequence_assessment_data {
-     my $fn=$ENV{'request.course.fn'};
-     ##
-     ## use navmaps
-     my $navmap = Apache::lonnavmaps::navmap->new();
-     if (!defined($navmap)) {
-         return 'Can not open Coursemap';
-     }
-     # We explicity grab the top level map because I am not sure we
-     # are pulling it from the iterator.
-     my $top_level_map = $navmap->getById('0.0');
-     #
-     my $iterator = $navmap->getIterator(undef, undef, undef, 1);
-     my $curRes = $iterator->next(); # Top level sequence
-     ##
-     ## Prime the pump
-     ##
-     ## We are going to loop until we run out of sequences/pages to explore for
-     ## resources.  This means we have to start out with something to look
-     ## at.
-     my $title = $ENV{'course.'.$ENV{'request.course.id'}.'.description'};
-     my $symb  = $top_level_map->symb();
-     my $src   = $top_level_map->src();
-     my $randompick = $top_level_map->randompick();
-     #
-     my @Sequences;
-     my @Assessments;
-     my @Nested_Sequences = ();   # Stack of sequences, keeps track of depth
-     my $top = { title    => $title,
-                 src      => $src,
-                 symb     => $symb,
-                 type     => 'container',
-                 num_assess => 0,
-                 num_assess_parts => 0,
-                 contents   => [],
-                 randompick => $randompick,
-             };
-     push (@Sequences,$top);
-     push (@Nested_Sequences, $top);
-     #
-     # We need to keep track of which sequences contain homework problems
-     #
-     my $previous_too;
-     my $previous;
-     while (scalar(@Nested_Sequences)) {
-         $previous_too = $previous;
-         $previous = $curRes;
-         $curRes = $iterator->next();
-         my $currentmap = $Nested_Sequences[-1]; # Last one on the stack
-         if ($curRes == $iterator->BEGIN_MAP()) {
-             if (! ref($previous)) {
-                 $previous = $previous_too;
-             }
-             if (! ref($previous)) {
-                 next;
-             }
-             # get the map itself, instead of BEGIN_MAP
-             $title = $previous->title();
-             $title =~ s/\&colon;/\&\#058;/g;
-             $symb  = $previous->symb();
-             $src   = $previous->src();
-             # pick up the filename if there is no title available
-             if (! defined($title) || $title eq '') {
-                 ($title) = ($src=~/\/([^\/]*)$/);
-             }
-             $randompick = $previous->randompick();
-             my $newmap = { title    => $title,
-                            src      => $src,
-                            symb     => $symb,
-                            type     => 'container',
-                            num_assess => 0,
-                            randompick => $randompick,
-                            contents   => [],
-                        };
-             push (@{$currentmap->{'contents'}},$newmap); # this is permanent
-             push (@Sequences,$newmap);
-             push (@Nested_Sequences, $newmap); # this is a stack
-             next;
-         }
-         if ($curRes == $iterator->END_MAP()) {
-             pop(@Nested_Sequences);
-             next;
-         }
-         next if (! ref($curRes));
-         next if (! $curRes->is_problem());# && !$curRes->randomout);
-         # Okay, from here on out we only deal with assessments
-         $title = $curRes->title();
-         $title =~ s/\&colon;/\&\#058;/g;
-         $symb  = $curRes->symb();
-         $src   = $curRes->src();
-         my $parts = $curRes->parts();
-         my %partdata;
-         foreach my $part (@$parts) {
-             my @Responses = $curRes->responseType($part);
-             my @Ids       = $curRes->responseIds($part);
-             $partdata{$part}->{'ResponseTypes'}= \@Responses;
-             $partdata{$part}->{'ResponseIds'}  = \@Ids;
-         }
-         my $assessment = { title => $title,
-                            src   => $src,
-                            symb  => $symb,
-                            type  => 'assessment',
-                            parts => $parts,
-                            num_parts => scalar(@$parts),
-                            partdata => \%partdata,
-                        };
-         push(@Assessments,$assessment);
-         push(@{$currentmap->{'contents'}},$assessment);
-         $currentmap->{'num_assess'}++;
-         $currentmap->{'num_assess_parts'}+= scalar(@$parts);
-     }
-     $navmap->untieHashes();
-     return ($top,\@Sequences,\@Assessments);
- }
- sub LoadDiscussion {
-     my ($courseID)=@_;
-     my %Discuss=();
-     my %contrib=&Apache::lonnet::dump(
-                 $courseID,
-                 $ENV{'course.'.$courseID.'.domain'},
-                 $ENV{'course.'.$courseID.'.num'});
-     #my %contrib=&DownloadCourseInformation($name, $courseID, 0);
-     foreach my $temp(keys %contrib) {
- 	if ($temp=~/^version/) {
- 	    my $ver=$contrib{$temp};
- 	    my ($dummy,$prb)=split(':',$temp);
- 	    for (my $idx=1; $idx<=$ver; $idx++ ) {
- 		my $name=$contrib{"$idx:$prb:sendername"};
- 		$Discuss{"$name:$prb"}=$idx;
- 	    }
- 	}
-     }
-     return \%Discuss;
- }
- ################################################
- ################################################
- =pod
- =item &GetUserName(username,userdomain)
- Returns a hash with the following entries:
-    'firstname', 'middlename', 'lastname', 'generation', and 'fullname'
-    'fullname' is the result of &Apache::loncoursedata::ProcessFullName.
- =cut
- ################################################
- ################################################
- sub GetUserName {
-     my ($username,$userdomain) = @_;
-     $username = $ENV{'user.name'} if (! defined($username));
-     $userdomain = $ENV{'user.domain'} if (! defined($username));
-     my %userenv = &Apache::lonnet::get('environment',
-                            ['firstname','middlename','lastname','generation'],
-                                        $userdomain,$username);
-     $userenv{'fullname'} = &ProcessFullName($userenv{'lastname'},
-                                             $userenv{'generation'},
-                                             $userenv{'firstname'},
-                                             $userenv{'middlename'});
-     return %userenv;
- }
- ################################################
- ################################################
- =pod
- =item &ProcessFullName()
- Takes lastname, generation, firstname, and middlename (or some partial
- set of this data) and returns the full name version as a string.  Format
- is Lastname generation, firstname middlename or a subset of this.
- =cut
- ################################################
- ################################################
- sub ProcessFullName {
-     my ($lastname, $generation, $firstname, $middlename)=@_;
-     my $Str = '';
-     # Strip whitespace preceeding & following name components.
-     $lastname   =~ s/(\s+$|^\s+)//g;
-     $generation =~ s/(\s+$|^\s+)//g;
-     $firstname  =~ s/(\s+$|^\s+)//g;
-     $middlename =~ s/(\s+$|^\s+)//g;
-     if($lastname ne '') {
- 	$Str .= $lastname;
- 	$Str .= ' '.$generation if ($generation ne '');
- 	$Str .= ',';
-         $Str .= ' '.$firstname  if ($firstname ne '');
-         $Str .= ' '.$middlename if ($middlename ne '');
-     } else {
-         $Str .= $firstname      if ($firstname ne '');
-         $Str .= ' '.$middlename if ($middlename ne '');
-         $Str .= ' '.$generation if ($generation ne '');
-     }
-     return $Str;
- }
- ################################################
- ################################################
- =pod
- =item &make_into_hash($values);
- Returns a reference to a hash as described by $values.  $values is
- assumed to be the result of
-     join(':',map {&Apache::lonnet::escape($_)} %orighash);
- This is a helper function for get_current_state.
  =cut
- ################################################
- ################################################
  sub make_into_hash {
      my $values = shift;
-     my %tmp = map { &Apache::lonnet::unescape($_); }
+     my %tmp = map { &unescape($_); } split(':',$values);
-                                            split(':',$values);
      return \%tmp;
  }
- ################################################
- ################################################
- =pod
- =head1 LOCAL DATA CACHING SUBROUTINES
- The local caching is done using MySQL.  There is no fall-back implementation
- if MySQL is not running.
- The programmers interface is to call &get_current_state() or some other
- primary interface subroutine (described below).  The internals of this
- storage system are documented here.
- There are six tables used to store student performance data (the results of
- a dumpcurrent).  Each of these tables is created in MySQL with a name of
- $courseid_*****, where ***** is 'symb', 'part', or whatever is appropriate
- for the table.  The tables and their purposes are described below.
- Some notes before we get started.
- Each table must have a PRIMARY KEY, which is a column or set of columns which
- will serve to uniquely identify a row of data.  NULL is not allowed!
- INDEXes work best on integer data.
- JOIN is used to combine data from many tables into one output.
- lonmysql.pm is used for some of the interface, specifically the table creation
- calls.  The inserts are done in bulk by directly calling the database handler.
- The SELECT ... JOIN statement used to retrieve the data does not have an
- interface in lonmysql.pm and I shudder at the thought of writing one.
- =head3 Table Descriptions
- =over 4
- =item Tables used to store meta information
- The following tables hold data required to keep track of the current status
- of a students data in the tables or to look up the students data in the tables.
- =over 4
- =item $symb_table
- The symb_table has two columns.  The first is a 'symb_id' and the second
- is the text name for the 'symb' (limited to 64k).  The 'symb_id' is generated
- automatically by MySQL so inserts should be done on this table with an
- empty first element.  This table has its PRIMARY KEY on the 'symb_id'.
- =item $part_table
- The part_table has two columns.  The first is a 'part_id' and the second
- is the text name for the 'part' (limited to 100 characters).  The 'part_id' is
- generated automatically by MySQL so inserts should be done on this table with
- an empty first element.  This table has its PRIMARY KEY on the 'part' (100
- characters) and a KEY on 'part_id'.
- =item $student_table
- The student_table has two columns.  The first is a 'student_id' and the second
- is the text description of the 'student' (typically username:domain) (less
- than 100 characters).  The 'student_id' is automatically generated by MySQL.
- The use of the name 'student_id' is loaded, I know, but this ID is used ONLY
- internally to the MySQL database and is not the same as the students ID
- (stored in the students environment).  This table has its PRIMARY KEY on the
- 'student' (100 characters).
- =item $studentdata_table
- The studentdata_table has four columns:  'student_id' (the unique id of
- the student), 'updatetime' (the time the students data was last updated),
- 'fullupdatetime' (the time the students full data was last updated),
- 'section', and 'classification'( the students current classification).
- This table has its PRIMARY KEY on 'student_id'.
- =back
- =item Tables used to store current status data
- The following tables store data only about the students current status on
- a problem, meaning only the data related to the last attempt on a problem.
- =over 4
- =item $performance_table
- The performance_table has 9 columns.  The first three are 'symb_id',
- 'student_id', and 'part_id'.  These comprise the PRIMARY KEY for this table
- and are directly related to the $symb_table, $student_table, and $part_table
- described above.  MySQL does better indexing on numeric items than text,
- so we use these three "index tables".  The remaining columns are
- 'solved', 'tries', 'awarded', 'award', 'awarddetail', and 'timestamp'.
- These are either the MySQL type TINYTEXT or various integers ('tries' and
- 'timestamp').  This table has KEYs of 'student_id' and 'symb_id'.
- For use of this table, see the functions described below.
- =item $parameters_table
- The parameters_table holds the data that does not fit neatly into the
- performance_table.  The parameters table has four columns: 'symb_id',
- 'student_id', 'parameter', and 'value'.  'symb_id', 'student_id', and
- 'parameter' comprise the PRIMARY KEY for this table.  'parameter' is
- limited to 255 characters.  'value' is limited to 64k characters.
- =back
- =item Tables used for storing historic data
- The following tables are used to store almost all of the transactions a student
- has made on a homework problem.  See loncapa/docs/homework/datastorage for
- specific information about each of the parameters stored.
- =over 4
- =item $fulldump_response_table
- The response table holds data (documented in loncapa/docs/homework/datastorage)
- associated with a particular response id which is stored when a student
- attempts a problem.  The following are the columns of the table, in order:
- 'symb_id','part_id','response_id','student_id','transaction','tries',
- 'awarddetail', 'awarded','response_specific' (data particular to the response
- type), 'response_specific_value', and 'submission (the text of the students
- submission).  The primary key is based on the first five columns listed above.
- =item $fulldump_part_table
- The part table holds data (documented in loncapa/docs/homework/datastorage)
- associated with a particular part id which is stored when a student attempts
- a problem.  The following are the columns of the table, in order:
- 'symb_id','part_id','student_id','transaction','tries','award','awarded',
- and 'previous'.  The primary key is based on the first five columns listed
- above.
- =item $fulldump_timestamp_table
- The timestamp table holds the timestamps of the transactions which are
- stored in $fulldump_response_table and $fulldump_part_table.  This data is
- about both the response and part data.  Columns: 'symb_id','student_id',
- 'transaction', and 'timestamp'.
- The primary key is based on the first 3 columns.
- =back
- =back
- =head3 Important Subroutines
- Here is a brief overview of the subroutines which are likely to be of
- interest:
- =over 4
- =item &get_current_state(): programmers interface.
- =item &init_dbs(): table creation
- =item &update_student_data(): data storage calls
- =item &get_student_data_from_performance_cache(): data retrieval
- =back
- =head3 Main Documentation
- =over 4
- =cut
- ################################################
- ################################################
- ################################################
- ################################################
  { # Begin scope of table identifiers
  my $current_course ='';
  my $symb_table;
  my $part_table;
  my $student_table;
- my $studentdata_table;
+ my $groupnames_table;
+ my $students_groups_table;
  my $performance_table;
  my $parameters_table;
  my $fulldump_response_table;
  my $fulldump_part_table;
  my $fulldump_timestamp_table;
+ my $weight_table;
  my @Tables;
- ################################################
- ################################################
- =pod
- =item &init_dbs()
- Input: course id
- Output: 0 on success, positive integer on error
- This routine issues the calls to lonmysql to create the tables used to
- store student data.
- =cut
- ################################################
- ################################################
  sub init_dbs {
-     my $courseid = shift;
+     my ($courseid,$drop) = @_;
      &setup_table_names($courseid);
      #
      # Drop any of the existing tables
-     foreach my $table (@Tables) {
+     if ($drop) {
-         &Apache::lonmysql::drop_table($table);
+         foreach my $table (@Tables) {
+             &Apache::lonmysql::drop_table($table);
+         }
      }
      #
      # Note - changes to this table must be reflected in the code that
- Line 575  sub init_dbs {
+ Line 129  sub init_dbs {
                        restrictions => 'NOT NULL',
                        auto_inc     => 'yes', },
                      { name => 'part',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
                        restrictions => 'NOT NULL'},
                      ],
          'PRIMARY KEY' => ['part (100)'],
- Line 590  sub init_dbs {
+ Line 144  sub init_dbs {
                        restrictions => 'NOT NULL',
                        auto_inc     => 'yes', },
                      { name => 'student',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
+                       restrictions => 'NOT NULL UNIQUE'},
+                     { name => 'section',
+                       type => 'VARCHAR(100) BINARY',
+                       restrictions => 'NOT NULL'},
+                     { name => 'start',
+                       type => 'INT',
+                       restrictions => 'NOT NULL'},
+                     { name => 'end',
+                       type => 'INT',
                        restrictions => 'NOT NULL'},
                      { name => 'classification',
-                       type => 'varchar(100)', },
+                       type => 'VARCHAR(100) BINARY', },
-                     ],
-         'PRIMARY KEY' => ['student (100)'],
-         'KEY' => [{ columns => ['student_id']},],
-     };
-     #
-     my $studentdata_table_def = {
-         id => $studentdata_table,
-         permanent => 'no',
-         columns => [{ name => 'student_id',
-                       type => 'MEDIUMINT UNSIGNED',
-                       restrictions => 'NOT NULL UNIQUE',},
                      { name => 'updatetime',
                        type => 'INT UNSIGNED'},
                      { name => 'fullupdatetime',
                        type => 'INT UNSIGNED'},
-                     { name => 'section',
-                       type => 'VARCHAR(100)'},
-                     { name => 'classification',
-                       type => 'VARCHAR(100)', },
                      ],
          'PRIMARY KEY' => ['student_id'],
+         'KEY' => [{ columns => ['student (100)',
+                                 'section (100)',
+                                 'start',
+ 				'end']},],
+     };
+     #
+     my $groupnames_table_def = {
+         id => $groupnames_table,
+         permanent => 'no',
+         columns => [{ name => 'group_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL',
+                       auto_inc => 'yes', },
+                     { name => 'groupname',
+                       type => 'VARCHAR(100) BINARY',
+                       restrictions => 'NOT NULL UNIQUE'},
+                    ],
+         'PRIMARY KEY' => ['group_id'],
+         'KEY' => [{ columns => ['groupname (100)',]},],
+     };
+     #
+     my $students_groups_table_def = {
+         id => $students_groups_table,
+         permanent => 'no',
+         columns => [{ name => 'student_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL', },
+                     { name => 'group_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL', },
+                    ],
+         'PRIMARY KEY' => ['student_id','group_id'],
+         'KEY' => [{ columns => ['student_id'] },
+                   { columns => ['group_id'] },],
      };
      #
      my $performance_table_def = {
- Line 630  sub init_dbs {
+ Line 212  sub init_dbs {
                        type => 'MEDIUMINT UNSIGNED',
                        restrictions => 'NOT NULL' },
                      { name => 'part',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
                        restrictions => 'NOT NULL'},
                      { name => 'solved',
                        type => 'TINYTEXT' },
                      { name => 'tries',
                        type => 'SMALLINT UNSIGNED' },
                      { name => 'awarded',
-                       type => 'TINYTEXT' },
+                       type => 'REAL' },
                      { name => 'award',
                        type => 'TINYTEXT' },
                      { name => 'awarddetail',
- Line 672  sub init_dbs {
+ Line 254  sub init_dbs {
                      { name => 'award',
                        type => 'TINYTEXT' },
                      { name => 'awarded',
-                       type => 'TINYTEXT' },
+                       type => 'REAL' },
                      { name => 'previous',
                        type => 'SMALLINT UNSIGNED' },
  #                    { name => 'regrader',
- Line 707  sub init_dbs {
+ Line 289  sub init_dbs {
                      { name => 'transaction',
                        type => 'MEDIUMINT UNSIGNED',
                        restrictions => 'NOT NULL' },
-                     { name => 'tries',
-                       type => 'SMALLINT UNSIGNED',
-                       restrictions => 'NOT NULL' },
                      { name => 'awarddetail',
                        type => 'TINYTEXT' },
-                     { name => 'awarded',
-                       type => 'TINYTEXT' },
  #                    { name => 'message',
- #                      type => 'CHAR' },
+ #                      type => 'CHAR BINARY'},
                      { name => 'response_specific',
                        type => 'TINYTEXT' },
                      { name => 'response_specific_value',
                        type => 'TINYTEXT' },
+                     { name => 'response_specific_2',
+                       type => 'TINYTEXT' },
+                     { name => 'response_specific_value_2',
+                       type => 'TINYTEXT' },
                      { name => 'submission',
                        type => 'TEXT'},
                      ],
- Line 754  sub init_dbs {
+ Line 335  sub init_dbs {
                    { columns=>['transaction'] },
                    ],
      };
      #
      my $parameters_table_def = {
          id => $parameters_table,
- Line 774  sub init_dbs {
+ Line 354  sub init_dbs {
          'PRIMARY KEY' => ['symb_id','student_id','parameter (255)'],
      };
      #
+     my $weight_table_def = {
+         id => $weight_table,
+         permanent => 'no',
+         columns => [{ name => 'symb_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL'  },
+                     { name => 'part_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL'  },
+                     { name => 'weight',
+                       type => 'REAL',
+                       restrictions => 'NOT NULL'  },
+                     ],
+         'PRIMARY KEY' => ['symb_id','part_id'],
+     };
+     #
      # Create the tables
      my $tableid;
      $tableid = &Apache::lonmysql::create_table($symb_table_def);
- Line 797  sub init_dbs {
+ Line 393  sub init_dbs {
          return 3;
      }
      #
-     $tableid = &Apache::lonmysql::create_table($studentdata_table_def);
-     if (! defined($tableid)) {
-         &Apache::lonnet::logthis("error creating studentdata_table: ".
-                                  &Apache::lonmysql::get_error());
-         return 4;
-     }
-     #
      $tableid = &Apache::lonmysql::create_table($performance_table_def);
      if (! defined($tableid)) {
          &Apache::lonnet::logthis("error creating preformance_table: ".
- Line 837  sub init_dbs {
+ Line 426  sub init_dbs {
                                   &Apache::lonmysql::get_error());
          return 9;
      }
+     $tableid = &Apache::lonmysql::create_table($weight_table_def);
+     if (! defined($tableid)) {
+         &Apache::lonnet::logthis("error creating weight_table: ".
+                                  &Apache::lonmysql::get_error());
+         return 10;
+     }
+     $tableid = &Apache::lonmysql::create_table($groupnames_table_def);
+     if (! defined($tableid)) {
+         &Apache::lonnet::logthis("error creating groupnames_table: ".
+                                  &Apache::lonmysql::get_error());
+         return 11;
+     }
+     $tableid = &Apache::lonmysql::create_table($students_groups_table_def);
+     if (! defined($tableid)) {
+         &Apache::lonnet::logthis("error creating student_groups_table: ".
+                                  &Apache::lonmysql::get_error());
+         return 12;
+     }
      return 0;
  }
- ################################################
- ################################################
  =pod
- =item &delete_caches()
+ =head2 delete_caches
- =cut
+ Drops all of the tables in the local mysql cache associated with the
+ specified course id.
- ################################################
+ TODO:  The drops should be pushed into lonmysql to further isolate
- ################################################
+ mysql code from other modules.
+ =cut
  sub delete_caches {
      my $courseid = shift;
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      &setup_table_names($courseid);
      #
- Line 868  sub delete_caches {
+ Line 475  sub delete_caches {
      return;
  }
- ################################################
- ################################################
- =pod
- =item &get_part_id()
- Get the MySQL id of a problem part string.
- Input: $part
- Output: undef on error, integer $part_id on success.
- =item &get_part()
- Get the string describing a part from the MySQL id of the problem part.
- Input: $part_id
- Output: undef on error, $part string on success.
- =cut
- ################################################
- ################################################
  my $have_read_part_table = 0;
  my %ids_by_part;
- Line 934  sub get_part {
+ Line 516  sub get_part {
      return undef; # error
  }
- ################################################
- ################################################
- =pod
- =item &get_symb_id()
- Get the MySQL id of a symb.
- Input: $symb
- Output: undef on error, integer $symb_id on success.
- =item &get_symb()
- Get the symb associated with a MySQL symb_id.
- Input: $symb_id
- Output: undef on error, $symb on success.
- =cut
- ################################################
- ################################################
  my $have_read_symb_table = 0;
  my %ids_by_symb;
- Line 999  sub get_symb {
+ Line 556  sub get_symb {
      return undef; # error
  }
- ################################################
- ################################################
- =pod
- =item &get_student_id()
- Get the MySQL id of a student.
- Input: $sname, $dom
- Output: undef on error, integer $student_id on success.
- =item &get_student()
- Get student username:domain associated with the MySQL student_id.
- Input: $student_id
- Output: undef on error, string $student (username:domain) on success.
- =cut
- ################################################
- ################################################
  my $have_read_student_table = 0;
  my %ids_by_student;
  my %students_by_id;
- Line 1040  sub get_student_id {
+ Line 571  sub get_student_id {
          $have_read_student_table = 1;
      }
      if (! exists($ids_by_student{$student})) {
-         &Apache::lonmysql::store_row($student_table,[undef,$student,undef]);
+         &populate_student_table();
          undef(%ids_by_student);
+         undef(%students_by_id);
          my @Result = &Apache::lonmysql::get_rows($student_table);
          foreach (@Result) {
              $ids_by_student{$_->[1]}=$_->[0];
- Line 1065  sub get_student {
+ Line 597  sub get_student {
      return undef; # error
  }
- ################################################
+ sub populate_student_table {
- ################################################
+     my ($courseid) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     &init_dbs($courseid,0);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $request = 'INSERT IGNORE INTO '.$student_table.
+         "(student,section,start,end) VALUES ";
+     my $cdom = $env{'course.'.$courseid.'.domain'};
+     my $cnum = $env{'course.'.$courseid.'.num'};
+     my $classlist = &get_classlist($cdom,$cnum);
+     my $student_count=0;
+     while (my ($student,$data) = each %$classlist) {
+         my ($section,$start,$end) = ($data->[&CL_SECTION()],
+ 				     $data->[&CL_START()],
+ 				     $data->[&CL_END()]);
+         if ($section eq '' || $section =~ /^\s*$/) {
+             $section = 'none';
+         }
+ 	if (!defined($start)) { $start = 0; }
+ 	if (!defined($end))   { $end   = 0; }
+         $request .= "('".$student."','".$section."','".$start."','".$end."'),";
+         $student_count++;
+     }
+     return if ($student_count == 0);
+     chop($request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
- =pod
+ my $have_read_groupnames_table = 0;
+ my %ids_by_groupname;
- =item &update_full_student_data($sname,$sdom,$courseid)
+ sub get_group_id {
+     my ($groupname) = @_;
+     if (! $have_read_groupnames_table) {
+         my @Result = &Apache::lonmysql::get_rows($groupnames_table);
+         foreach (@Result) {
+             $ids_by_groupname{$_->[1]}=$_->[0];
+         }
+         $have_read_groupnames_table = 1;
+     }
+     if (! exists($ids_by_groupname{$groupname})) {
+         &populate_groupnames_table();
+         undef(%ids_by_groupname);
+         my @Result = &Apache::lonmysql::get_rows($groupnames_table);
+         foreach (@Result) {
+             $ids_by_groupname{$_->[1]}=$_->[0];
+         }
+     }
+     if (exists($ids_by_groupname{$groupname})) {
+         return $ids_by_groupname{$groupname};
+     }
+     return undef; # error
+ }
- Does a lonnet::dump on a student to populate the courses tables.
+ sub populate_groupnames_table {
+     my ($courseid) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     &setup_table_names($courseid);
+     &init_dbs($courseid,0);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $cdom = $env{'course.'.$courseid.'.domain'};
+     my $cnum = $env{'course.'.$courseid.'.num'};
+     my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum);
+     return if (!%curr_groups);
+     my $request = 'INSERT IGNORE INTO '.$groupnames_table.
+                   '(groupname) VALUES ';
+     foreach my $groupname (sort(keys(%curr_groups)),'none') {
+         $request .= "('".$groupname."'),";
+     }
+     chop($request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
- Input: $sname, $sdom, $courseid
+ my $have_read_studentsgroups_table = 0;
+ my %groupids_by_studentid;
- Output: $returnstatus
+ sub get_students_groupids {
+     my ($student_id) = @_;
+     if (! $have_read_studentsgroups_table) {
+         my @Result = &Apache::lonmysql::get_rows($students_groups_table);
+         foreach (@Result) {
+             push(@{$groupids_by_studentid{$_->[0]}},$_->[1]);
+         }
+         $have_read_studentsgroups_table = 1;
+     }
+     if (! exists($groupids_by_studentid{$student_id})) {
+         &populate_students_groups_table();
+         undef(%groupids_by_studentid);
+         my @Result = &Apache::lonmysql::get_rows($students_groups_table);
+         foreach (@Result) {
+             push(@{$groupids_by_studentid{$_->[0]}},$_->[1]);
+         }
+     }
+     if (exists($groupids_by_studentid{$student_id})) {
+         if (ref($groupids_by_studentid{$student_id}) eq 'ARRAY') {
+             return @{$groupids_by_studentid{$student_id}};
+         }
+     }
+     return undef; # error
+ }
- $returnstatus is a string describing any errors that occured.  'okay' is the
- default.
- This subroutine loads a students data using lonnet::dump and inserts
+ sub populate_students_groups_table {
- it into the MySQL database.  The inserts are done on three tables,
+     my ($courseid) = @_;
- $fulldump_response_table, $fulldump_part_table, and $fulldump_timestamp_table.
+     if (! defined($courseid)) {
- The INSERT calls are made directly by this subroutine, not through lonmysql
+         $courseid = $env{'request.course.id'};
- because we do a 'bulk'insert which takes advantage of MySQLs non-SQL
+     }
- compliant INSERT command to insert multiple rows at a time.
+     #
- If anything has gone wrong during this process, $returnstatus is updated with
+     &setup_table_names($courseid);
- a description of the error.
+     &init_dbs($courseid,0);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $request = 'INSERT IGNORE INTO '.$students_groups_table.
+         "(student_id,group_id) VALUES ";
+     my $cdom = $env{'course.'.$courseid.'.domain'};
+     my $cnum = $env{'course.'.$courseid.'.num'};
+     my ($classlist,$keylist) = &get_classlist($cdom,$cnum);
+     my ($classgroups,$studentgroups) = &get_group_memberships($classlist,
+                                                               $keylist,
+                                                               $cdom,$cnum);
+     my $record_count = 0;
+     foreach my $student (sort(keys(%{$classgroups}))) {
+         my $student_id = &get_student_id(split(':',$student));
+         my @studentsgroups = &get_students_groups($student,'Active',$classgroups);
+         if (@studentsgroups < 1) {
+             @studentsgroups = ('none');
+         }
+         foreach my $groupname (@studentsgroups) {
+             my $group_id = &get_group_id($groupname);
+             $request .= "('".$student_id."','".$group_id."'),";
+             $record_count++;
+         }
+     }
+     return if ($record_count == 0);
+     chop($request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
- Once the "fulldump" tables are updated, the tables used for chart and
+ sub clear_internal_caches {
- spreadsheet (which hold only the current state of the student on their
+     $have_read_part_table = 0;
- homework, not historical data) are updated.  If all updates have occured
+     undef(%ids_by_part);
- successfully, the studentdata table is updated to reflect the time of the
+     undef(%parts_by_id);
- update.
+     $have_read_symb_table = 0;
+     undef(%ids_by_symb);
+     undef(%symbs_by_id);
+     $have_read_student_table = 0;
+     undef(%ids_by_student);
+     undef(%students_by_id);
+     $have_read_groupnames_table = 0;
+     undef(%ids_by_groupname);
+ }
- Notice we do not insert the data and immediately query it.  This means it
+ sub symb_is_for_task {
- is possible for there to be data returned this first time that is not
+     my ($symb) = @_;
- available the second time.  CYA.
+     return ($symb =~ /\.task$/);
+ }
- =cut
- ################################################
- ################################################
  sub update_full_student_data {
      my ($sname,$sdom,$courseid) = @_;
      #
- Line 1117  sub update_full_student_data {
+ Line 791  sub update_full_student_data {
      #
      # Download students data
      my $time_of_retrieval = time;
-     my @tmp = &Apache::lonnet::dump($courseid,$sdom,$sname);
+     my @tmp = &Apache::lonnet::dumpstore($courseid,$sdom,$sname);
      if (@tmp && $tmp[0] =~ /^error/) {
          $returnstatus = 'error retrieving full student data';
          return $returnstatus;
- Line 1142  sub update_full_student_data {
+ Line 816  sub update_full_student_data {
      while (my ($key,$value) = each(%studentdata)) {
          next if ($key =~ /^(\d+):(resource$|subnum$|keys:)/);
          my ($transaction,$symb,$parameter) = split(':',$key);
+ 	$symb = &unescape($symb);
+ 	$parameter = &unescape($parameter);
          my $symb_id = &get_symb_id($symb);
          if ($parameter eq 'timestamp') {
              # We can deal with 'timestamp' right away
              my @timestamp_storage = ($symb_id,$student_id,
                                       $transaction,$value);
-             my $store_command = 'INSERT INTO '.$fulldump_timestamp_table.
+             my $store_command = 'INSERT IGNORE INTO '.$fulldump_timestamp_table.
                  " VALUES ('".join("','",@timestamp_storage)."');";
              $dbh->do($store_command);
              if ($dbh->err()) {
- Line 1157  sub update_full_student_data {
+ Line 833  sub update_full_student_data {
              next;
          } elsif ($parameter eq 'version') {
              next;
-         } elsif ($parameter =~ /^resource\.(.*)\.(tries|award|awarded|previous|solved|awarddetail|submission)\s*$/){
+ 	} elsif (&symb_is_for_task($symb)) {
+ 	    next if ($parameter !~ /^resource\.(.*)\.(award|
+ 						      awarded|
+ 						      solved|
+ 						      submission|
+ 						      portfiles|
+ 						      status|
+ 						      version|
+ 						      regrader)\s*$/x);
+ 	    my ($version_and_part_id, $field) = ($1,$2);
+ 	    next if ($version_and_part_id !~ /\./
+ 		     && $field ne 'regrader' && $field ne 'version');
+ 	    my ($version, $part, $instance) =
+ 		split(/\./,$version_and_part_id);
+ 	    #skip and instance dimension or criteria specific data
+ 	    next if (defined($instance)
+ 		     && $instance ne $field
+ 		     && $instance ne 'bridgetask');
+ 	    if (!defined($part)) {
+ 		$part = $version;
+ 	    }
+ 	    my $resp_id = &get_part_id('0');
+ 	    my $part_id = &get_part_id($part);
+ 	    if ($field eq 'version') {
+ 		# for tasks each version is an attempt at it thus
+ 		#     version -> tries
+ 		$partdata->{$symb_id}{$part_id}{$transaction}{'tries'}=
+ 		    $value;
+ 		# at new version time the record gets reset thus adding a
+ 		# virtual response awarddetail of 'new_version'
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific'}='status';
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value'}='new_version';
+ 	    } elsif ($field eq 'award' || $field eq 'awarded'
+ 		     || $field eq 'solved') {
+ 		$partdata->{$symb_id}{$part_id}{$transaction}{$field}=
+ 		    $value;
+ 	    } elsif ($field eq 'portfiles') {
+ 		# tasks only accepts portfolio submissions
+ 		$value = $dbh->quote($value);
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'submission'}=$value;
+ 	    } elsif ($field eq 'status') {
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific'}=$field;
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value'}=$value;
+ 	    } elsif ($field eq 'regrader') {
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_2'}=$field;
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value_2'}=$value;
+ 	    }
+ 	} elsif ($parameter =~ /^resource\.(.*)\.(tries|
+                                                   award|
+                                                   awarded|
+                                                   previous|
+                                                   solved|
+                                                   awarddetail|
+                                                   submission|
+                                                   submissiongrading|
+                                                   molecule)\s*$/x){
              # we do not have enough information to store an
              # entire row, so we save it up until later.
              my ($part_and_resp_id,$field) = ($1,$2);
- Line 1169  sub update_full_student_data {
+ Line 906  sub update_full_student_data {
              } else {
                  $part_id = &get_part_id($part_and_resp_id);
              }
+             # Deal with part specific data
              if ($field =~ /^(tries|award|awarded|previous)$/) {
                  $partdata->{$symb_id}->{$part_id}->{$transaction}->{$field}=$value;
              }
+             # deal with response specific data
              if (defined($resp_id) &&
-                 $field =~ /^(tries|awarddetail|awarded|submission)$/) {
+                 $field =~ /^(awarddetail|
+                              submission|
+                              submissiongrading|
+                              molecule)$/x) {
                  if ($field eq 'submission') {
                      # We have to be careful with user supplied input.
                      # most of the time we are okay because it is escaped.
                      # However, there is one wrinkle: submissions which end in
                      # and odd number of '\' cause insert errors to occur.
                      # Best trap this somehow...
-                     my ($offensive_string) = ($value =~ /(\\+)$/);
+                     $value = $dbh->quote($value);
-                     if (length($offensive_string) % 2) {
+                 }
-                         $value =~ s/\\$/\\\\/;
+                 if ($field eq 'submissiongrading' ||
-                     }
+                     $field eq 'molecule') {
+                     $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{'response_specific'}=$field;
+                     $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{'response_specific_value'}=$value;
+                 } else {
+                     $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{$field}=$value;
                  }
-                 $respdata->{$symb_id}->{$part_id}->{$resp_id}->{$transaction}->{$field}=$value;
              }
          }
      }
      ##
      ## Store the part data
-     my $store_command = 'INSERT INTO '.$fulldump_part_table.
+     my $store_command = 'INSERT IGNORE INTO '.$fulldump_part_table.
          ' VALUES '."\n";
      my $store_rows = 0;
      while (my ($symb_id,$hash1) = each (%$partdata)) {
- Line 1212  sub update_full_student_data {
+ Line 957  sub update_full_student_data {
          chop($store_command);
          $dbh->do($store_command);
          if ($dbh->err) {
-             $returnstatus = 'error storing part data';
+             $returnstatus = 'error saving part data';
              &Apache::lonnet::logthis('insert error '.$dbh->errstr());
              &Apache::lonnet::logthis("While attempting\n".$store_command);
          }
      }
      ##
      ## Store the response data
-     $store_command = 'INSERT INTO '.$fulldump_response_table.
+     $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table.
          ' VALUES '."\n";
      $store_rows = 0;
      while (my ($symb_id,$hash1) = each (%$respdata)) {
          while (my ($part_id,$hash2) = each (%$hash1)) {
              while (my ($resp_id,$hash3) = each (%$hash2)) {
                  while (my ($transaction,$data) = each (%$hash3)) {
-                     $store_command .= "('".join("','",$symb_id,$part_id,
+                     my $submission = $data->{'submission'};
-                                                 $resp_id,$student_id,
+                     # We have to be careful with user supplied input.
-                                                 $transaction,
+                     # most of the time we are okay because it is escaped.
-                                                 $data->{'tries'},
+                     # However, there is one wrinkle: submissions which end in
-                                                 $data->{'awarddetail'},
+                     # and odd number of '\' cause insert errors to occur.
-                                                 $data->{'awarded'},
+                     # Best trap this somehow...
-                                                 '','',
+                     $submission = $dbh->quote($submission);
-                                                 $data->{'submission'})."'),";
+                     $store_command .= "('".
+                         join("','",$symb_id,$part_id,
+                              $resp_id,$student_id,
+                              $transaction,
+                              $data->{'awarddetail'},
+                              $data->{'response_specific'},
+                              $data->{'response_specific_value'},
+                              $data->{'response_specific_2'},
+                              $data->{'response_specific_value_2'}).
+                              "',".$submission."),";
                      $store_rows++;
                  }
              }
- Line 1243  sub update_full_student_data {
+ Line 997  sub update_full_student_data {
          chop($store_command);
          $dbh->do($store_command);
          if ($dbh->err) {
-             $returnstatus = 'error storing response data';
+             $returnstatus = 'error saving response data';
              &Apache::lonnet::logthis('insert error '.$dbh->errstr());
              &Apache::lonnet::logthis("While attempting\n".$store_command);
          }
- Line 1255  sub update_full_student_data {
+ Line 1009  sub update_full_student_data {
          ($sname,$sdom,$courseid,
           &Apache::lonnet::convert_dump_to_currentdump(\%studentdata));
      if ($returnstatus eq 'okay' && $status ne 'okay') {
-         $returnstatus = 'error storing current data:'.$status;
+         $returnstatus = 'error saving current data:'.$status;
      } elsif ($status ne 'okay') {
-         $returnstatus .= ' error storing current data:'.$status;
+         $returnstatus .= ' error saving current data:'.$status;
      }
      ##
      ## Update the students time......
      if ($returnstatus eq 'okay') {
-         &Apache::lonmysql::replace_row
+         &store_updatetime($student_id,$time_of_retrieval,$time_of_retrieval);
-             ($studentdata_table,
+         if ($dbh->err) {
-              [$student_id,$time_of_retrieval,$time_of_retrieval,undef,undef]);
+             if ($returnstatus eq 'okay') {
+                 $returnstatus = 'error updating student time';
+             } else {
+                 $returnstatus = 'error updating student time';
+             }
+         }
      }
      return $returnstatus;
  }
- ################################################
- ################################################
- =pod
- =item &update_student_data()
- Input: $sname, $sdom, $courseid
- Output: $returnstatus, \%student_data
- $returnstatus is a string describing any errors that occured.  'okay' is the
- default.
- \%student_data is the data returned by a call to lonnet::currentdump.
- This subroutine loads a students data using lonnet::currentdump and inserts
- it into the MySQL database.  The inserts are done on two tables,
- $performance_table and $parameters_table.  $parameters_table holds the data
- that is not included in $performance_table.  See the description of
- $performance_table elsewhere in this file.  The INSERT calls are made
- directly by this subroutine, not through lonmysql because we do a 'bulk'
- insert which takes advantage of MySQLs non-SQL compliant INSERT command to
- insert multiple rows at a time.  If anything has gone wrong during this
- process, $returnstatus is updated with a description of the error and
- \%student_data is returned.
- Notice we do not insert the data and immediately query it.  This means it
- is possible for there to be data returned this first time that is not
- available the second time.  CYA.
- =cut
- ################################################
- ################################################
  sub update_student_data {
      my ($sname,$sdom,$courseid) = @_;
      #
- Line 1316  sub update_student_data {
+ Line 1042  sub update_student_data {
      #
      # Download students data
      my $time_of_retrieval = time;
-     my @tmp = &Apache::lonnet::currentdump($courseid,$sdom,$sname);
+     my %student_data = &Apache::lonnet::currentdump($courseid,$sdom,$sname);
-     if ((scalar(@tmp) > 0) && ($tmp[0] =~ /^error:/)) {
+     if (&Apache::lonnet::error(%student_data)) {
          &Apache::lonnet::logthis('error getting data for '.
                                   $sname.':'.$sdom.' in course '.$courseid.
-                                  ':'.$tmp[0]);
+                                  ':'.(%student_data)[0]);
-         $returnstatus = 'error getting data';
+         $returnstatus =(%student_data)[0] ;
          return ($returnstatus,undef);
      }
-     if (scalar(@tmp) < 1) {
+     if (scalar(keys(%student_data)) < 1) {
          return ('no data',undef);
      }
-     my %student_data = @tmp;
      my @Results = &store_student_data($sname,$sdom,$courseid,\%student_data);
      #
      # Set the students update time
-     &Apache::lonmysql::replace_row($studentdata_table,
+     if ($Results[0] eq 'okay') {
-                          [$student_id,$time_of_retrieval,undef,undef,undef]);
+         &store_updatetime($student_id,$time_of_retrieval);
+     }
      #
      return @Results;
  }
+ sub store_updatetime {
+     my ($student_id,$updatetime,$fullupdatetime)=@_;
+     my $values = '';
+     if (defined($updatetime)) {
+         $values = 'updatetime='.$updatetime.' ';
+     }
+     if (defined($fullupdatetime)) {
+         if ($values ne '') {
+             $values .= ',';
+         }
+         $values .= 'fullupdatetime='.$fullupdatetime.' ';
+     }
+     return if ($values eq '');
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $request = 'UPDATE '.$student_table.' SET '.$values.
+         ' WHERE student_id='.$student_id.' LIMIT 1';
+     $dbh->do($request);
+ }
  sub store_student_data {
      my ($sname,$sdom,$courseid,$student_data) = @_;
      #
- Line 1357  sub store_student_data {
+ Line 1102  sub store_student_data {
      my $starttime = Time::HiRes::time;
      my $elapsed = 0;
      my $rows_stored;
-     my $store_parameters_command  = 'INSERT INTO '.$parameters_table.
+     my $store_parameters_command  = 'INSERT IGNORE INTO '.$parameters_table.
          ' VALUES '."\n";
      my $num_parameters = 0;
-     my $store_performance_command = 'INSERT INTO '.$performance_table.
+     my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table.
          ' VALUES '."\n";
      return ('error',undef) if (! defined($dbh));
      while (my ($current_symb,$param_hash) = each(%{$student_data})) {
- Line 1368  sub store_student_data {
+ Line 1113  sub store_student_data {
          # make sure the symb is set up properly
          my $symb_id = &get_symb_id($current_symb);
          #
-         # Load data into the tables
+         # Parameters
          while (my ($parameter,$value) = each(%$param_hash)) {
-             my $newstring;
              if ($parameter !~ /(timestamp|resource\.(.*)\.(solved|tries|awarded|award|awarddetail|previous))/) {
-                 $newstring = "('".join("','",
+                 my $sql_parameter = "('".join("','",
-                                        $symb_id,$student_id,
+                                               $symb_id,$student_id,
-                                        $parameter)."',".
+                                               $parameter)."',".
-                                            $dbh->quote($value)."),\n";
+                                                   $dbh->quote($value)."),\n";
                  $num_parameters ++;
-                 if ($newstring !~ /''/) {
+                 if ($sql_parameter !~ /''/) {
-                     $store_parameters_command .= $newstring;
+                     $store_parameters_command .= $sql_parameter;
-                     $rows_stored++;
+                     #$rows_stored++;
                  }
              }
-             next if ($parameter !~ /^resource\.(.*)\.solved$/);
+         }
+         # Performance
+         my %stored;
+         while (my ($parameter,$value) = each(%$param_hash)) {
+             next if ($parameter !~ /^resource\.(.*)\.(solved|awarded)$/);
+             my $part  = $1;
+ 	    my $which = $2;
+ 	    next if ($part =~ /\./);
+             next if (exists($stored{$part}));
+             $stored{$part}++;
              #
-             my $part = $1;
              my $part_id = &get_part_id($part);
              next if (!defined($part_id));
-             my $solved  = $value;
-             my $tries   = $param_hash->{'resource.'.$part.'.tries'};
+             my ($solved,$awarded);
-             my $awarded = $param_hash->{'resource.'.$part.'.awarded'};
+ 	    if ($which eq 'solved') {
+ 		$solved  = $value;
+ 		$awarded = $param_hash->{'resource.'.$part.'.awarded'};
+ 	    } else {
+ 		$solved  = $param_hash->{'resource.'.$part.'.solved'};
+ 		$awarded = $value;
+ 	    }
              my $award   = $param_hash->{'resource.'.$part.'.award'};
              my $awarddetail = $param_hash->{'resource.'.$part.'.awarddetail'};
              my $timestamp = $param_hash->{'timestamp'};
+ 	    my $tries   = $param_hash->{'resource.'.$part.'.tries'};
+ 	    if (&symb_is_for_task($current_symb)) {
+ 		$tries   = $param_hash->{'resource.'.$part.'.version'};
+ 	    }
              #
              $solved      = '' if (! defined($solved));
              $tries       = '' if (! defined($tries));
              $awarded     = '' if (! defined($awarded));
              $award       = '' if (! defined($award));
              $awarddetail = '' if (! defined($awarddetail));
-             $newstring = "('".join("','",$symb_id,$student_id,$part_id,$part,
+             my $sql_performance =
-                                    $solved,$tries,$awarded,$award,
+                 "('".join("','",$symb_id,$student_id,$part_id,$part,
-                                    $awarddetail,$timestamp)."'),\n";
+                                 $solved,$tries,$awarded,$award,
-             $store_performance_command .= $newstring;
+                                 $awarddetail,$timestamp)."'),\n";
+             $store_performance_command .= $sql_performance;
              $rows_stored++;
          }
      }
-     chop $store_parameters_command;
+     if (! $rows_stored) { return ($returnstatus, undef); }
-     chop $store_parameters_command;
+     $store_parameters_command =~ s|,\n$||;
-     chop $store_performance_command;
+     $store_performance_command =~ s|,\n$||;
-     chop $store_performance_command;
      my $start = Time::HiRes::time;
+     $dbh->do($store_performance_command);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis('performance bigass insert error:'.
+                                  $dbh->errstr());
+         &Apache::lonnet::logthis('command = '.$/.$store_performance_command);
+         $returnstatus = 'error: unable to insert performance into database';
+         return ($returnstatus,$student_data);
+     }
      $dbh->do($store_parameters_command) if ($num_parameters>0);
      if ($dbh->err()) {
-         &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr());
+         &Apache::lonnet::logthis('parameters bigass insert error:'.
-         &Apache::lonnet::logthis('command = '.$store_parameters_command);
+                                  $dbh->errstr());
+         &Apache::lonnet::logthis('command = '.$/.$store_parameters_command);
          &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
          &Apache::lonnet::logthis('student_id = '.$student_id);
          $returnstatus = 'error: unable to insert parameters into database';
          return ($returnstatus,$student_data);
      }
-     $dbh->do($store_performance_command);
-     if ($dbh->err()) {
-         &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr());
-         &Apache::lonnet::logthis('command = '.$store_performance_command);
-         $returnstatus = 'error: unable to insert performance into database';
-         return ($returnstatus,$student_data);
-     }
      $elapsed += Time::HiRes::time - $start;
      return ($returnstatus,$student_data);
  }
- ######################################
- ######################################
- =pod
- =item &ensure_tables_are_set_up($courseid)
- Checks to be sure the MySQL tables for the given class are set up.
- If $courseid is omitted it will be obtained from the environment.
- Returns nothing on success and 'error' on failure
- =cut
- ######################################
- ######################################
  sub ensure_tables_are_set_up {
      my ($courseid) = @_;
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      # Clean out package variables
      &setup_table_names($courseid);
      #
      # if the tables do not exist, make them
      my @CurrentTable = &Apache::lonmysql::tables_in_db();
-     my ($found_symb,$found_student,$found_part,$found_studentdata,
+     my ($found_symb,$found_student,$found_groups,$found_groupnames,$found_part,
          $found_performance,$found_parameters,$found_fulldump_part,
-         $found_fulldump_response,$found_fulldump_timestamp);
+         $found_fulldump_response,$found_fulldump_timestamp,
+         $found_weight);
      foreach (@CurrentTable) {
          $found_symb        = 1 if ($_ eq $symb_table);
          $found_student     = 1 if ($_ eq $student_table);
+         $found_groups      = 1 if ($_ eq $students_groups_table);
+         $found_groupnames  = 1 if ($_ eq $groupnames_table);
          $found_part        = 1 if ($_ eq $part_table);
-         $found_studentdata = 1 if ($_ eq $studentdata_table);
          $found_performance = 1 if ($_ eq $performance_table);
          $found_parameters  = 1 if ($_ eq $parameters_table);
          $found_fulldump_part      = 1 if ($_ eq $fulldump_part_table);
          $found_fulldump_response  = 1 if ($_ eq $fulldump_response_table);
          $found_fulldump_timestamp = 1 if ($_ eq $fulldump_timestamp_table);
+         $found_weight      = 1 if ($_ eq $weight_table);
      }
-     if (!$found_symb        || !$found_studentdata ||
+     if (!$found_symb          ||
-         !$found_student     || !$found_part   ||
+         !$found_student       || !$found_part              ||
-         !$found_performance || !$found_parameters ||
+         !$found_performance   || !$found_parameters        ||
          !$found_fulldump_part || !$found_fulldump_response ||
-         !$found_fulldump_timestamp ) {
+         !$found_fulldump_timestamp || !$found_weight ) {
-         if (&init_dbs($courseid)) {
+         if (&init_dbs($courseid,1)) {
              return 'error';
          }
      }
  }
- ################################################
- ################################################
- =pod
- =item &ensure_current_data()
- Input: $sname, $sdom, $courseid
- Output: $status, $data
- This routine ensures the data for a given student is up to date.
- The $studentdata_table is queried to determine the time of the last update.
- If the students data is out of date, &update_student_data() is called.
- The return values from the call to &update_student_data() are returned.
- =cut
- ################################################
- ################################################
  sub ensure_current_data {
      my ($sname,$sdom,$courseid) = @_;
      my $status = 'okay';   # return value
      #
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      &ensure_tables_are_set_up($courseid);
      #
      # Get the update time for the user
      my $updatetime = 0;
+     my $getuserdir = 1;
      my $modifiedtime = &Apache::lonnet::GetFileTimestamp
-         ($sdom,$sname,$courseid.'.db',
+         ($sdom,$sname,$courseid.'.db',$getuserdir);
-          $Apache::lonnet::perlvar{'lonUsersDir'});
      #
+     if ($modifiedtime == -1) {
+ 	return ('no data',undef);
+     }
      my $student_id = &get_student_id($sname,$sdom);
-     my @Result = &Apache::lonmysql::get_rows($studentdata_table,
+     my @Result = &Apache::lonmysql::get_rows($student_table,
                                               "student_id ='$student_id'");
      my $data = undef;
      if (@Result) {
-         $updatetime = $Result[0]->[1];
+         $updatetime = $Result[0]->[6];  # Ack!  This is dumb!
      }
      if ($modifiedtime > $updatetime) {
          ($status,$data) = &update_student_data($sname,$sdom,$courseid);
- Line 1527  sub ensure_current_data {
+ Line 1263  sub ensure_current_data {
      return ($status,$data);
  }
- ################################################
- ################################################
- =pod
- =item &ensure_current_full_data($sname,$sdom,$courseid)
- Input: $sname, $sdom, $courseid
- Output: $status
- This routine ensures the fulldata (the data from a lonnet::dump, not a
- lonnet::currentdump) for a given student is up to date.
- The $studentdata_table is queried to determine the time of the last update.
- If the students fulldata is out of date, &update_full_student_data() is
- called.
- The return value from the call to &update_full_student_data() is returned.
- =cut
- ################################################
- ################################################
  sub ensure_current_full_data {
      my ($sname,$sdom,$courseid) = @_;
      my $status = 'okay';   # return value
      #
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      &ensure_tables_are_set_up($courseid);
      #
      # Get the update time for the user
+     my $getuserdir = 1;
      my $modifiedtime = &Apache::lonnet::GetFileTimestamp
-         ($sdom,$sname,$courseid.'.db',
+         ($sdom,$sname,$courseid.'.db',$getuserdir);
-          $Apache::lonnet::perlvar{'lonUsersDir'});
      #
      my $student_id = &get_student_id($sname,$sdom);
-     my @Result = &Apache::lonmysql::get_rows($studentdata_table,
+     my @Result = &Apache::lonmysql::get_rows($student_table,
                                               "student_id ='$student_id'");
      my $updatetime;
      if (@Result && ref($Result[0]) eq 'ARRAY') {
-         $updatetime = $Result[0]->[2];
+         $updatetime = $Result[0]->[7];
      }
      if (! defined($updatetime) || $modifiedtime > $updatetime) {
          $status = &update_full_student_data($sname,$sdom,$courseid);
- Line 1575  sub ensure_current_full_data {
+ Line 1289  sub ensure_current_full_data {
      return $status;
  }
- ################################################
- ################################################
- =pod
- =item &get_student_data_from_performance_cache()
- Input: $sname, $sdom, $symb, $courseid
- Output: hash reference containing the data for the given student.
- If $symb is undef, all the students data is returned.
- This routine is the heart of the local caching system.  See the description
- of $performance_table, $symb_table, $student_table, and $part_table.  The
- main task is building the MySQL request.  The tables appear in the request
- in the order in which they should be parsed by MySQL.  When searching
- on a student the $student_table is used to locate the 'student_id'.  All
- rows in $performance_table which have a matching 'student_id' are returned,
- with data from $part_table and $symb_table which match the entries in
- $performance_table, 'part_id' and 'symb_id'.  When searching on a symb,
- the $symb_table is processed first, with matching rows grabbed from
- $performance_table and filled in from $part_table and $student_table in
- that order.
- Running 'EXPLAIN ' on the 'SELECT' statements generated can be quite
- interesting, especially if you play with the order the tables are listed.
- =cut
- ################################################
- ################################################
  sub get_student_data_from_performance_cache {
      my ($sname,$sdom,$symb,$courseid)=@_;
      my $student = $sname.':'.$sdom if (defined($sname) && defined($sdom));
- Line 1656  sub get_student_data_from_performance_ca
+ Line 1340  sub get_student_data_from_performance_ca
          $studentdata->{$symb}->{$base.'.awarddetail'} = $awarddetail;
          $studentdata->{$symb}->{'timestamp'} = $time if (defined($time) && $time ne '');
      }
+     ## Get misc parameters
+     $request = 'SELECT c.symb,a.parameter,a.value '.
+         "FROM $student_table AS b ".
+         "LEFT JOIN $parameters_table AS a ON b.student_id=a.student_id ".
+         "LEFT JOIN $symb_table AS c ON c.symb_id = a.symb_id ".
+         "WHERE student='$student'";
+     if (defined($symb) && $symb ne '') {
+         $request .= " AND c.symb=".$dbh->quote($symb);
+     }
+     $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($sth->err()) {
+         &Apache::lonnet::logthis("Unable to execute MySQL request:");
+         &Apache::lonnet::logthis("\n".$request."\n");
+         &Apache::lonnet::logthis("error is:".$sth->errstr());
+         if (defined($symb) && $symb ne '') {
+             $studentdata = $studentdata->{$symb};
+         }
+         return $studentdata;
+     }
+     #
+     foreach my $row (@{$sth->fetchall_arrayref}) {
+         $rows_retrieved++;
+         my ($symb,$parameter,$value) = (@$row);
+         $studentdata->{$symb}->{$parameter}  = $value;
+     }
+     #
      if (defined($symb) && $symb ne '') {
          $studentdata = $studentdata->{$symb};
      }
      return $studentdata;
  }
- ################################################
- ################################################
- =pod
- =item &get_current_state()
- Input: $sname,$sdom,$symb,$courseid
- Output: Described below
- Retrieve the current status of a students performance.  $sname and
- $sdom are the only required parameters.  If $symb is undef the results
- of an &Apache::lonnet::currentdump() will be returned.
- If $courseid is undef it will be retrieved from the environment.
- The return structure is based on &Apache::lonnet::currentdump.  If
- $symb is unspecified, all the students data is returned in a hash of
- the form:
- (
-   symb1 => { param1 => value1, param2 => value2 ... },
-   symb2 => { param1 => value1, param2 => value2 ... },
- )
- If $symb is specified, a hash of
- (
-   param1 => value1,
-   param2 => value2,
- )
- is returned.
- If no data is found for $symb, or if the student has no performance data,
- an empty list is returned.
- =cut
- ################################################
- ################################################
  sub get_current_state {
      my ($sname,$sdom,$symb,$courseid,$forcedownload)=@_;
      #
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      return () if (! defined($sname) || ! defined($sdom));
      #
- Line 1730  sub get_current_state {
+ Line 1404  sub get_current_state {
      } else {
          if ($status ne 'okay' && $status ne '') {
              &Apache::lonnet::logthis('status = '.$status);
-             return ();
+             return ('error: '.$status,undef);
          }
          my $returnhash = &get_student_data_from_performance_cache($sname,$sdom,
                                                        $symb,$courseid);
- Line 1739  sub get_current_state {
+ Line 1413  sub get_current_state {
      return ();
  }
- ################################################
- ################################################
- =pod
- =item &get_problem_statistics()
- Gather data on a given problem.  The database is assumed to be
- populated and all local caching variables are assumed to be set
- properly.  This means you need to call &ensure_current_data for
- the students you are concerned with prior to calling this routine.
- Inputs: $students, $symb, $part, $courseid
- =over 4
- =item $students is an array of hash references.
- Each hash must contain at least the 'username' and 'domain' of a student.
- =item $symb is the symb for the problem.
- =item $part is the part id you need statistics for
- =item $courseid is the course id, of course!
- =back
- Outputs: See the code for up to date information.  A hash reference is
- returned.  The hash has the following keys defined:
- =over 4
- =item num_students The number of students attempting the problem
- =item tries The total number of tries for the students
- =item max_tries The maximum number of tries taken
- =item mean_tries The average number of tries
- =item num_solved The number of students able to solve the problem
- =item num_override The number of students whose answer is 'correct_by_override'
- =item deg_of_diff The degree of difficulty of the problem
- =item std_tries The standard deviation of the number of tries
- =item skew_tries The skew of the number of tries
- =item per_wrong The number of students attempting the problem who were not
- able to answer it correctly.
- =back
- =cut
- ################################################
- ################################################
  sub get_problem_statistics {
-     my ($students,$symb,$part,$courseid) = @_;
+     my ($Sections,$Groups,$status,$symb,$part,$courseid,$starttime,$endtime) = @_;
      return if (! defined($symb) || ! defined($part));
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
+     &setup_table_names($courseid);
      my $symb_id = &get_symb_id($symb);
      my $part_id = &get_part_id($part);
-     my $stats_table = $courseid.'_problem_stats';
+     my $stats_table = &temp_table_name($courseid,'problem_stats');
      #
      my $dbh = &Apache::lonmysql::get_dbh();
      return undef if (! defined($dbh));
      #
-     # A) Number of Students attempting problem
+     # Clean out the table
-     # B) Total number of tries of students attempting problem
-     # C) Mod (largest number of tries for solving the problem)
-     # D) Mean (average number of tries for solving the problem)
-     # E) Number of students to solve the problem
-     # F) Number of students to solve the problem by override
-     # G) Number of students unable to solve the problem
-     # H) Degree of difficulty : 1-(E+F)/B
-     # I) Standard deviation of number of tries
-     # J) Skew of tries: sqrt(sum(Xi-D)^3)/A
-     #
      $dbh->do('DROP TABLE '.$stats_table);  # May return an error
      my $request =
-         'CREATE TEMPORARY TABLE '.$stats_table.
+         'CREATE TEMPORARY TABLE '.$stats_table.' '.
-             ' SELECT student_id,solved,award,tries FROM '.$performance_table.
+         'SELECT a.student_id,a.solved,a.award,a.awarded,a.tries '.
-                 ' WHERE symb_id='.$symb_id.' AND part_id='.$part_id;
+         'FROM '.$performance_table.' AS a ';
-     if (defined($students)) {
+     #
-         $request .= ' AND ('.
+     # See if we need to include some requirements on the students
-             join(' OR ', map {'student_id='.
+     if ((defined($Sections) && lc($Sections->[0]) ne 'all') ||
-                                   &get_student_id($_->{'username'},
+         (defined($status)   && lc($status)        ne 'any')) {
-                                                   $_->{'domain'})
+         $request .= 'NATURAL LEFT JOIN '.$student_table.' AS b ';
-                                   } @$students
-                  ).')';
      }
- #    &Apache::lonnet::logthis($request);
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     if (defined($groups_join)) {
+         $request .= $groups_join;
+     }
+     $request .= ' WHERE a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
+     #
+     # Limit the students included to those specified
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$status,'b');
+     #
+     # Limit by starttime and endtime
+     my $time_requirements = undef;
+     if (defined($starttime)) {
+         $time_requirements .= 'a.timestamp>='.$starttime;
+         if (defined($endtime)) {
+             $time_requirements .= ' AND a.timestamp<='.$endtime;
+         }
+     } elsif (defined($endtime)) {
+         $time_requirements .= 'a.timestamp<='.$endtime;
+     }
+     if (defined($time_requirements)) {
+         $request .= ' AND '.$time_requirements;
+     }
+     if (defined($section_limits)) {
+         $request .= ' AND '.$section_limits;
+     }
+     if (defined($enrollment_limits)) {
+         $request .= ' AND '.$enrollment_limits;
+     }
+     # Limit by group, as required
+     if (defined($group_limits)) {
+         $request .= ' AND '.$group_limits;
+     }
+     #
+     # Finally, execute the request to create the temporary table
      $dbh->do($request);
-     my ($num,$tries,$mod,$mean,$STD) = &execute_SQL_request
+     #
-         ($dbh,
+     # Collect the first suite of statistics
-          'SELECT COUNT(*),SUM(tries),MAX(tries),AVG(tries),STD(tries) FROM '.
+     $request = 'SELECT COUNT(*),SUM(tries),'.
-          $stats_table);
+         'AVG(tries),STD(tries) '.
-     my ($Solved) = &execute_SQL_request($dbh,'SELECT COUNT(tries) FROM '.
+         'FROM '.$stats_table;
-                                         $stats_table.
+     my ($num,$tries,$mean,$STD) = &execute_SQL_request
-                                         " WHERE solved='correct_by_student'");
+         ($dbh,$request);
-     my ($solved) = &execute_SQL_request($dbh,'SELECT COUNT(tries) FROM '.
+     #
-                                         $stats_table.
+     $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table.
-                                         " WHERE solved='correct_by_override'");
+         ' WHERE awarded>0';
+     if (defined($time_requirements)) {
+         $request .= ' AND '.$time_requirements;
+     }
+     my ($max,$min) = &execute_SQL_request($dbh,$request);
+     #
+     $request = 'SELECT SUM(awarded) FROM '.$stats_table;
+     if (defined($time_requirements)) {
+         $request .= ' AND '.$time_requirements;
+     }
+     my ($Solved) = &execute_SQL_request($dbh,$request);
+     #
+     $request = 'SELECT SUM(awarded) FROM '.$stats_table.
+         " WHERE solved='correct_by_override'";
+     if (defined($time_requirements)) {
+         $request .= ' AND '.$time_requirements;
+     }
+     my ($solved) = &execute_SQL_request($dbh,$request);
+     #
+     $Solved -= $solved;
+     #
      $num    = 0 if (! defined($num));
      $tries  = 0 if (! defined($tries));
-     $mod    = 0 if (! defined($mod));
+     $max    = 0 if (! defined($max));
+     $min    = 0 if (! defined($min));
      $STD    = 0 if (! defined($STD));
-     $Solved = 0 if (! defined($Solved));
+     $Solved = 0 if (! defined($Solved) || $Solved < 0);
      $solved = 0 if (! defined($solved));
      #
+     # Compute the more complicated statistics
      my $DegOfDiff = 'nan';
      $DegOfDiff = 1-($Solved)/$tries if ($tries>0);
+     #
      my $SKEW = 'nan';
      my $wrongpercent = 0;
+     my $numwrong = 'nan';
      if ($num > 0) {
          ($SKEW) = &execute_SQL_request($dbh,'SELECT SQRT(SUM('.
                                       'POWER(tries - '.$STD.',3)'.
                                       '))/'.$num.' FROM '.$stats_table);
-         $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
+         $numwrong = $num-$Solved;
+         $wrongpercent=int(10*100*$numwrong/$num)/10;
      }
      #
+     # Drop the temporary table
      $dbh->do('DROP TABLE '.$stats_table);  # May return an error
      #
-     # Store in metadata
-     #
-     if ($num) {
- 	my %storestats=();
-         my $urlres=(&Apache::lonnet::decode_symb($symb))[2];
- 	$storestats{$courseid.'___'.$urlres.'___timestamp'}=time;
- 	$storestats{$courseid.'___'.$urlres.'___stdno'}=$num;
- 	$storestats{$courseid.'___'.$urlres.'___avetries'}=$mean;
- 	$storestats{$courseid.'___'.$urlres.'___difficulty'}=$DegOfDiff;
- 	$urlres=~/^(\w+)\/(\w+)/;
- 	&Apache::lonnet::put('nohist_resevaldata',\%storestats,$1,$2);
-     }
-     #
      # Return result
-     #
      return { num_students => $num,
               tries        => $tries,
-              max_tries    => $mod,
+              max_tries    => $max,
+              min_tries    => $min,
               mean_tries   => $mean,
               std_tries    => $STD,
               skew_tries   => $SKEW,
               num_solved   => $Solved,
               num_override => $solved,
+              num_wrong    => $numwrong,
               per_wrong    => $wrongpercent,
               deg_of_diff  => $DegOfDiff };
  }
+ ##
+ ## This is a helper for get_statistics
  sub execute_SQL_request {
      my ($dbh,$request)=@_;
  #    &Apache::lonnet::logthis($request);
      my $sth = $dbh->prepare($request);
+     if (!$sth) {
+ 	die($dbh->errstr . " SQL: $request");
+     }
      $sth->execute();
      my $row = $sth->fetchrow_arrayref();
      if (ref($row) eq 'ARRAY' && scalar(@$row)>0) {
- Line 1910  sub execute_SQL_request {
+ Line 1564  sub execute_SQL_request {
  }
- ################################################
+ sub populate_weight_table {
- ################################################
+     my ($courseid) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $navmap = Apache::lonnavmaps::navmap->new();
+     if (!defined($navmap)) {
+         &Apache::lonnet::logthis('loncoursedata::populate_weight_table:'.$/.
+                                  '  unable to get navmaps resource'.$/.
+                                  '  '.join(' ',(caller)));
+         return;
+     }
+     my @sequences = $navmap->retrieveResources(undef,
+                                                sub { shift->is_map(); },1,0,1);
+     my @resources;
+     foreach my $seq (@sequences) {
+         push(@resources,$navmap->retrieveResources($seq,
+                                                    sub {shift->is_problem();},
+,0,0));
+     }
+     if (! scalar(@resources)) {
+         &Apache::lonnet::logthis('loncoursedata::populate_weight_table:'.$/.
+                                  ' no resources returned for '.$courseid);
+         return;
+     }
+     #       Since we use lonnet::EXT to retrieve problem weights,
+     #       to ensure current data we must clear the caches out.
+     &Apache::lonnet::clear_EXT_cache_status();
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $request = 'INSERT IGNORE INTO '.$weight_table.
+         "(symb_id,part_id,weight) VALUES ";
+     my $weight;
+     foreach my $res (@resources) {
+         my $symb_id = &get_symb_id($res->symb);
+         foreach my $part (@{$res->parts}) {
+             my $part_id = &get_part_id($part);
+             $weight = &Apache::lonnet::EXT('resource.'.$part.'.weight',
+                                            $res->symb,
+                                            undef,undef,undef);
+             if (!defined($weight) || ($weight eq '')) {
+                 $weight=1;
+             }
+             $request .= "('".$symb_id."','".$part_id."','".$weight."'),";
+         }
+     }
+     $request =~ s/(,)$//;
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
- =pod
+ sub limit_by_start_end_time {
+     my ($starttime,$endtime,$table) = @_;
+     my $time_requirements = undef;
+     if (defined($starttime)) {
+         $time_requirements .= $table.".timestamp>='".$starttime."'";
+         if (defined($endtime)) {
+             $time_requirements .= " AND ".$table.".timestamp<='".$endtime."'";
+         }
+     } elsif (defined($endtime)) {
+         $time_requirements .= $table.".timestamp<='".$endtime."'";
+     }
+     return $time_requirements;
+ }
- =item &setup_table_names()
- input: course id
+ sub limit_by_section_and_status {
+     my ($Sections,$enrollment,$tablename) = @_;
+     my $student_requirements = undef;
+     if ( (defined($Sections) && $Sections->[0] ne 'all')) {
+         $student_requirements = '('.
+             join(' OR ', map { $tablename.".section='".$_."'" } @$Sections
+                  ).')';
+     }
+     my $enrollment_requirements=undef;
+     if (defined($enrollment) && $enrollment ne 'Any') {
+ 	my $now = time();
+ 	if ( $enrollment eq 'Future' ) {
+ 	    $enrollment_requirements =
+ 		"( $tablename.start > $now AND ".
+ 		"( $tablename.end = 0 OR $tablename.end > $now))";
+ 	} elsif ( $enrollment eq 'Active' ) {
+ 	    $enrollment_requirements =
+ 		"(( $tablename.start = 0 OR $tablename.start < $now )  AND ".
+ 		" ( $tablename.end   = 0 OR $tablename.end   > $now ))";
+ 	} elsif ( $enrollment eq 'Expired' ) {
+ 	    $enrollment_requirements =
+ 		"(( $tablename.start < $now )  AND ".
+ 		" ( $tablename.end   < $now ))";
+ 	}
+     }
+     return ($student_requirements,$enrollment_requirements);
+ }
- output: none
- Cleans up the package variables for local caching.
- =cut
+ sub limit_by_group {
+     my ($Groups,$stutable,$grptable,$stugrptab) = @_;
+     my $groups_join = undef;
+     my $group_limits = undef;
+     if ( (defined($Groups) && $Groups->[0] ne 'all')) {
+         $groups_join =
+           ' LEFT JOIN '.$students_groups_table.
+                      ' AS '.$stugrptab.' ON '.
+                      $stugrptab.'.student_id = '.$stutable.'.student_id'.
+           ' LEFT JOIN '.$groupnames_table.
+                      ' AS '.$grptable.' ON '.
+                      $stugrptab.'.group_id = '.$grptable.'.group_id ';
+         $group_limits =
+           ' ('.
+              join(' OR ', map {  "$grptable.groupname='".$_."'" } @$Groups
+            ).')';
+     }
+     return ($groups_join,$group_limits);
+ }
+ sub RNK_student { return 0; };
+ sub RNK_score   { return 1; };
+ sub rank_students_by_scores_on_resources {
+     my ($resources,$Sections,$Groups,$enrollment,$courseid,$starttime,$endtime,
+         $has_award_for) = @_;
+     return if (! defined($resources) || ! ref($resources) eq 'ARRAY');
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$enrollment,'b');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     my $symb_limits = '('.join(' OR ',map {'a.symb_id='.&get_symb_id($_);
+                                        } @$resources
+                                ).')';
+     my ($award_col, $award_join, $award_clause) = ('', '', '');
+     if ($has_award_for) {
+         my $resource_id = &get_symb_id($has_award_for);
+         $award_col = ", perf.awarded";
+         $award_join = "LEFT JOIN $performance_table AS perf ON perf.symb_id"
+             ." = $resource_id AND perf.student_id = b.student_id ";
+         $award_clause = "AND perf.awarded IS NOT NULL";
+     }
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my $request = "SELECT b.student,SUM(a.awarded*w.weight) AS score "
+         ."$award_col FROM $performance_table AS a ".
+         "NATURAL LEFT JOIN $weight_table AS w ".
+         "LEFT JOIN $student_table AS b ON a.student_id=b.student_id ".
+         "$award_join $groups_join ";
+     my $limits;
+     if (defined($section_limits)) {
+         $limits .= $section_limits.' AND ';
+     }
+     if (defined($enrollment_limits)) {
+         $limits .= $enrollment_limits.' AND ';
+     }
+     if (defined($time_limits)) {
+         $limits .= $time_limits.' AND ';
+     }
+     if ($symb_limits ne '()') {
+         $limits .= $symb_limits.' AND ';
+     }
+     if (defined($group_limits)) {
+         $limits .= $group_limits.' AND ';
+     }
+     if ($limits) {
+         $limits =~ s/( AND )$//;   # Remove extra conjunction
+         $request .= "WHERE $limits";
+     }
+     $request .= " $award_clause GROUP BY a.student_id ORDER BY score";
+     #&Apache::lonnet::logthis('request = '.$/.$request);
+     my $sth = $dbh->prepare($request) or die "Can't prepare $request";
+     $sth->execute();
+     my $rows = $sth->fetchall_arrayref();
+     return ($rows);
+ }
+ sub get_sum_of_scores {
+     my ($symb,$part,$students,$courseid,$starttime,$endtime) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     if (defined($students) &&
+         ((@$students == 0) ||
+          (@$students == 1 && (! defined($students->[0]) ||
+                               $students->[0] eq ''))
+          )
+         ){
+         undef($students);
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my $request = 'SELECT SUM(a.awarded*w.weight),SUM(w.weight) FROM '.
+         $performance_table.' AS a '.
+         'NATURAL LEFT JOIN '.$weight_table.' AS w ';
+     $request .= 'WHERE a.symb_id='.&get_symb_id($symb).
+         ' AND a.part_id='.&get_part_id($part);
+     if (defined($time_limits)) {
+         $request .= ' AND '.$time_limits;
+     }
+     if (defined($students)) {
+         $request .= ' AND ('.
+             join(' OR ',map {'a.student_id='.&get_student_id(split(':',$_));
+                          } @$students).
+                              ')';
+     }
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     my $rows = $sth->fetchrow_arrayref();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 1 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed, fetchrow_arrayrefed'.
+                                  $/.$request);
+         return (undef,undef);
+     }
+     return ($rows->[0],$rows->[1]);
+ }
+ sub score_stats {
+     my ($Sections,$Groups,$enrollment,$symbs,$starttime,$endtime,$courseid)=@_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     #
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$enrollment,'b');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my @Symbids = map { &get_symb_id($_); } @{$symbs};
+     #
+     my $stats_table = &temp_table_name($courseid,'problem_stats');
+     my $symb_restriction = join(' OR ',map {'a.symb_id='.$_;} @Symbids);
+     my $request = 'DROP TABLE '.$stats_table;
+     $dbh->do($request);
+     $request =
+         'CREATE TEMPORARY TABLE '.$stats_table.' '.
+         'SELECT a.student_id,'.
+         'SUM(a.awarded*w.weight) AS score FROM '.
+         $performance_table.' AS a '.
+         'NATURAL LEFT JOIN '.$weight_table.' AS w '.
+         'LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id '.
+         $groups_join;
+     my $limit = ' WHERE ('.$symb_restriction.')';
+     if ($time_limits) {
+         $limit .= ' AND '.$time_limits;
+     }
+     if ($section_limits) {
+         $limit .= ' AND '.$section_limits;
+     }
+     if ($enrollment_limits) {
+         $limit .= ' AND '.$enrollment_limits;
+     }
+     if ($group_limits) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' GROUP BY a.student_id';
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     $request =
+         'SELECT AVG(score),STD(score),MAX(score),MIN(score),COUNT(score) '.
+         'FROM '.$stats_table;
+     my ($ave,$std,$max,$min,$count) = &execute_SQL_request($dbh,$request);
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     $request = 'SELECT SUM(weight) FROM '.$weight_table.
+         ' AS a WHERE ('.$symb_restriction.')';
+     my ($max_possible) = &execute_SQL_request($dbh,$request);
+     # &Apache::lonnet::logthis('request = '.$/.$request);
+     return($min,$max,$ave,$std,$count,$max_possible);
+ }
+ sub count_stats {
+     my ($Sections,$Groups,$enrollment,$symbs,$starttime,$endtime,$courseid)=@_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     #
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$enrollment,'b');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my @Symbids = map { &get_symb_id($_); } @{$symbs};
+     #
+     my $stats_table = &temp_table_name($courseid,'problem_stats');
+     my $symb_restriction = join(' OR ',map {'a.symb_id='.$_;} @Symbids);
+     my $request = 'DROP TABLE '.$stats_table;
+     $dbh->do($request);
+     $request =
+         'CREATE TEMPORARY TABLE '.$stats_table.' '.
+         'SELECT a.student_id,'.
+         'SUM(a.awarded) AS count FROM '.
+         $performance_table.' AS a '.
+         'LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id '.
+         $groups_join;
+     my $limit =  ' WHERE ('.$symb_restriction.')';
+     if ($time_limits) {
+         $limit .= ' AND '.$time_limits;
+     }
+     if ($section_limits) {
+         $limit .= ' AND '.$section_limits;
+     }
+     if ($enrollment_limits) {
+         $limit .= ' AND '.$enrollment_limits;
+     }
+     if ($group_limits) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' GROUP BY a.student_id';
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     $request =
+         'SELECT AVG(count),STD(count),MAX(count),MIN(count),COUNT(count) '.
+         'FROM '.$stats_table;
+     my ($ave,$std,$max,$min,$count) = &execute_SQL_request($dbh,$request);
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     return($min,$max,$ave,$std,$count);
+ }
+ sub get_student_data {
+     my ($students,$courseid) = @_;
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     return undef if (! defined($dbh));
+     my $request = 'SELECT '.
+         'student_id, student '.
+         'FROM '.$student_table;
+     if (defined($students)) {
+         $request .= ' WHERE ('.
+             join(' OR ', map {'student_id='.
+                                   &get_student_id($_->{'username'},
+                                                   $_->{'domain'})
+                               } @$students
+                  ).')';
+     }
+     $request.= ' ORDER BY student_id';
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 2 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
+         return undef;
+     }
+     my $dataset = $sth->fetchall_arrayref();
+     if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+         return $dataset;
+     }
+ }
+ sub RD_student_id      { return 0; }
+ sub RD_awarddetail     { return 1; }
+ sub RD_response_eval   { return 2; }
+ sub RD_response_eval_2 { return 3; }
+ sub RD_submission      { return 4; }
+ sub RD_timestamp       { return 5; }
+ sub RD_tries           { return 6; }
+ sub RD_sname           { return 7; }
+ sub get_response_data {
+     my ($Sections,$Groups,$enrollment,$symb,$response,$courseid) = @_;
+     return undef if (! defined($symb) ||
+                ! defined($response));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
+     #
+     &setup_table_names($courseid);
+     my $symb_id = &get_symb_id($symb);
+     if (! defined($symb_id)) {
+         &Apache::lonnet::logthis('Unable to find symb for '.$symb.' in '.$courseid);
+         return undef;
+     }
+     my $response_id = &get_part_id($response);
+     if (! defined($response_id)) {
+         &Apache::lonnet::logthis('Unable to find id for '.$response.' in '.$courseid);
+         return undef;
+     }
+     #
+     my $dbh = &Apache::lonmysql::get_dbh();
+     return undef if (! defined($dbh));
+     #
+     my ($student_requirements,$enrollment_requirements) =
+         &limit_by_section_and_status($Sections,$enrollment,'d');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'d','e','f');
+     my $request = 'SELECT '.
+         'a.student_id, a.awarddetail, a.response_specific_value, '.
+         'a.response_specific_value_2, a.submission, b.timestamp, '.
+ 	'c.tries, d.student '.
+         'FROM '.$fulldump_response_table.' AS a '.
+         'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
+         'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
+         'a.transaction = b.transaction '.
+         'LEFT JOIN '.$fulldump_part_table.' AS c '.
+         'ON a.symb_id=c.symb_id AND a.student_id=c.student_id AND '.
+         'a.part_id=c.part_id AND a.transaction = c.transaction '.
+         'LEFT JOIN '.$student_table.' AS d '.
+         'ON a.student_id=d.student_id '.
+         $groups_join;
+     my $limit = ' WHERE '.
+         'a.symb_id='.$symb_id.' AND a.response_id='.$response_id;
+     if (defined($student_requirements)) {
+         $limit .= ' AND '.$student_requirements;
+     }
+     if (defined($enrollment_requirements)) {
+         $limit .= ' AND '.$enrollment_requirements;
+     }
+     if (defined($group_limits)) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' ORDER BY b.timestamp';
+ #    &Apache::lonnet::logthis("request =\n".$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 3 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
+         return undef;
+     }
+     my $dataset = $sth->fetchall_arrayref();
+     if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+         # Clear the \'s from around the submission
+         for (my $i =0;$i<scalar(@$dataset);$i++) {
+             $dataset->[$i]->[&RD_submission()] =~ s/(\'$|^\')//g;
+         }
+         return $dataset;
+     }
+ }
+ sub RDs_awarddetail     { return 3; }
+ sub RDs_submission      { return 2; }
+ sub RDs_timestamp       { return 1; }
+ sub RDs_tries           { return 0; }
+ sub RDs_awarded         { return 4; }
+ sub RDs_response_eval   { return 5; }
+ sub RDs_response_eval_2 { return 6; }
+ sub RDs_part_award      { return 7; }
+ sub get_response_data_by_student {
+     my ($student,$symb,$response,$courseid) = @_;
+     return undef if (! defined($symb) ||
+                      ! defined($response));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
+     #
+     &setup_table_names($courseid);
+     my $symb_id = &get_symb_id($symb);
+     my $response_id = &get_part_id($response);
+     #
+     my $student_id = &get_student_id($student->{'username'},
+                                      $student->{'domain'});
+     #
+     my $dbh = &Apache::lonmysql::get_dbh();
+     return undef if (! defined($dbh));
+     my $request = 'SELECT '.
+         'c.tries, b.timestamp, a.submission, a.awarddetail, c.awarded, '.
+ 	'a.response_specific_value, a.response_specific_value_2, c.award '.
+         'FROM '.$fulldump_response_table.' AS a '.
+         'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
+         'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
+         'a.transaction = b.transaction '.
+         'LEFT JOIN '.$fulldump_part_table.' AS c '.
+         'ON a.symb_id=c.symb_id AND a.student_id=c.student_id AND '.
+         'a.part_id=c.part_id AND a.transaction = c.transaction '.
+         'LEFT JOIN '.$student_table.' AS d '.
+         'ON a.student_id=d.student_id '.
+         'LEFT JOIN '.$performance_table.' AS e '.
+         'ON a.symb_id=e.symb_id AND a.part_id=e.part_id AND '.
+         'a.student_id=e.student_id AND c.tries=e.tries '.
+         'WHERE '.
+         'a.symb_id='.$symb_id.' AND a.response_id='.$response_id.
+         ' AND a.student_id='.$student_id.' ORDER BY b.timestamp';
+ #    &Apache::lonnet::logthis("request =\n".$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 4 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
+         return undef;
+     }
+     my $dataset = $sth->fetchall_arrayref();
+     if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+         # Clear the \'s from around the submission
+         for (my $i =0;$i<scalar(@$dataset);$i++) {
+             $dataset->[$i]->[&RDs_submission] =~ s/(\'$|^\')//g;
+         }
+         return $dataset;
+     }
+     return undef; # error occurred
+ }
+ sub RT_student_id { return 0; }
+ sub RT_awarded    { return 1; }
+ sub RT_tries      { return 2; }
+ sub RT_timestamp  { return 3; }
+ sub get_response_time_data {
+     my ($sections,$groups,$enrollment,$symb,$part,$courseid) = @_;
+     return undef if (! defined($symb) ||
+                      ! defined($part));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
+     #
+     &setup_table_names($courseid);
+     my $symb_id = &get_symb_id($symb);
+     if (! defined($symb_id)) {
+         &Apache::lonnet::logthis('Unable to find symb for '.$symb.' in '.$courseid);
+         return undef;
+     }
+     my $part_id = &get_part_id($part);
+     if (! defined($part_id)) {
+         &Apache::lonnet::logthis('Unable to find id for '.$part.' in '.$courseid);
+         return undef;
+     }
+     #
+     my $dbh = &Apache::lonmysql::get_dbh();
+     return undef if (! defined($dbh));
+     my ($student_requirements,$enrollment_requirements) =
+         &limit_by_section_and_status($sections,$enrollment,'d');
+     my ($groups_join,$group_limits) = &limit_by_group($groups,'d','e','f');
+     my $request = 'SELECT '.
+         'a.student_id, a.awarded, a.tries, b.timestamp '.
+         'FROM '.$fulldump_part_table.' AS a '.
+         'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
+         'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
+         'a.transaction = b.transaction '.
+         'LEFT JOIN '.$student_table.' as d '.
+         'ON a.student_id=d.student_id '.
+         $groups_join;
+     my $limit = ' WHERE '.
+         'a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
+     if (defined($student_requirements)) {
+         $limit .= ' AND '.$student_requirements;
+     }
+     if (defined($enrollment_requirements)) {
+         $limit .= ' AND '.$enrollment_requirements;
+     }
+     if (defined($group_limits)) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' ORDER BY b.timestamp';
+ #    &Apache::lonnet::logthis("request =\n".$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 5 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
+         return undef;
+     }
+     my $dataset = $sth->fetchall_arrayref();
+     if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+         return $dataset;
+     }
+ }
+ sub get_student_scores {
+     my ($sections,$groups,$Symbs,$enrollment,$courseid,$starttime,$endtime) = @_;
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     return (undef) if (! defined($dbh));
+     my $tmptable = &temp_table_name($courseid,'temp_'.time);
+     my $request = 'DROP TABLE IF EXISTS '.$tmptable;
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     $dbh->do($request);
+     #
+     my $symb_requirements;
+     if (defined($Symbs)  && @$Symbs) {
+         $symb_requirements = '('.
+             join(' OR ', map{ "(a.symb_id='".&get_symb_id($_->{'symb'}).
+                               "' AND a.part_id='".&get_part_id($_->{'part'}).
+                               "')"
+                               } @$Symbs).')';
+     }
+     #
+     my ($student_requirements,$enrollment_requirements) =
+         &limit_by_section_and_status($sections,$enrollment,'b');
+     #
+     my ($groups_join,$group_limits) = &limit_by_group($groups,'b','d','e');
+     my $time_requirements = &limit_by_start_end_time($starttime,$endtime,'a');
+     ##
+     $request = 'CREATE TEMPORARY TABLE IF NOT EXISTS '.$tmptable.
+         ' SELECT a.student_id,SUM(a.awarded*c.weight) AS score FROM '.
+         $performance_table.' AS a ';
+     $request .= "LEFT JOIN ".$weight_table.' AS c ON a.symb_id=c.symb_id AND a.part_id=c.part_id ';
+     if (defined($student_requirements) || defined($enrollment_requirements)) {
+         $request .= ' LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id ';
+     }
+     if (defined($groups_join)) {
+         $request .= $groups_join;
+     }
+     if (defined($symb_requirements)       ||
+         defined($student_requirements)    ||
+         defined($enrollment_requirements) ||
+         defined($group_limits) ) {
+         $request .= ' WHERE ';
+     }
+     if (defined($symb_requirements)) {
+         $request .= $symb_requirements.' AND ';
+     }
+     if (defined($student_requirements)) {
+         $request .= $student_requirements.' AND ';
+     }
+     if (defined($enrollment_requirements)) {
+         $request .= $enrollment_requirements.' AND ';
+     }
+     if (defined($time_requirements)) {
+         $request .= $time_requirements.' AND ';
+     }
+     $request =~ s/ AND $//; # Strip of the trailing ' AND '.
+     $request .= ' GROUP BY a.student_id';
+ #    &Apache::lonnet::logthis("request = \n".$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 6 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
+         return undef;
+     }
+     $request = 'SELECT score,COUNT(*) FROM '.$tmptable.' GROUP BY score';
+ #    &Apache::lonnet::logthis("request = \n".$request);
+     $sth = $dbh->prepare($request);
+     $sth->execute();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 7 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
+         return undef;
+     }
+     my $dataset = $sth->fetchall_arrayref();
+     return $dataset;
+ }
- ################################################
- ################################################
  sub setup_table_names {
      my ($courseid) = @_;
      if (! defined($courseid)) {
-         $courseid = $ENV{'request.course.id'};
+         $courseid = $env{'request.course.id'};
      }
      #
      if (! defined($current_course) || $current_course ne $courseid) {
- Line 1944  sub setup_table_names {
+ Line 2234  sub setup_table_names {
          $have_read_student_table = 0;
          undef(%ids_by_student);
          undef(%students_by_id);
+         $have_read_groupnames_table = 0;
+         undef(%ids_by_groupname);
          #
          $current_course = $courseid;
      }
      #
      # Set up database names
-     my $base_id = $courseid;
+     my $base_id = 'md5_'.&Digest::MD5::md5_hex($courseid);
-     $symb_table        = $base_id.'_'.'symb';
+     $symb_table               = $base_id.'_'.'symb';
-     $part_table        = $base_id.'_'.'part';
+     $part_table               = $base_id.'_'.'part';
-     $student_table     = $base_id.'_'.'student';
+     $student_table            = $base_id.'_'.'student';
-     $studentdata_table = $base_id.'_'.'studentdata';
+     $groupnames_table         = $base_id.'_'.'groupnames';
-     $performance_table = $base_id.'_'.'performance';
+     $students_groups_table    = $base_id.'_'.'studentgroups';
-     $parameters_table  = $base_id.'_'.'parameters';
+     $performance_table        = $base_id.'_'.'performance';
+     $parameters_table         = $base_id.'_'.'parameters';
      $fulldump_part_table      = $base_id.'_'.'partdata';
      $fulldump_response_table  = $base_id.'_'.'responsedata';
      $fulldump_timestamp_table = $base_id.'_'.'timestampdata';
+     $weight_table             = $base_id.'_'.'weight';
      #
      @Tables = (
                 $symb_table,
                 $part_table,
                 $student_table,
-                $studentdata_table,
+                $groupnames_table,
+                $students_groups_table,
                 $performance_table,
                 $parameters_table,
                 $fulldump_part_table,
                 $fulldump_response_table,
                 $fulldump_timestamp_table,
+                $weight_table,
                 );
      return;
  }
- ################################################
+ sub temp_table_name {
- ################################################
+     my ($courseid,$affix) = @_;
+     my $base_id = 'md5_'.&Digest::MD5::md5_hex($courseid);
- =pod
+     return $base_id.'_'.$affix;
+ }
- =back
- =item End of Local Data Caching Subroutines
- =cut
- ################################################
- ################################################
  } # End scope of table identifiers
- ################################################
- ################################################
- =pod
- =head3 Classlist Subroutines
- =item &get_classlist();
- Retrieve the classist of a given class or of the current class.  Student
- information is returned from the classlist.db file and, if needed,
- from the students environment.
- Optional arguments are $cid, $cdom, and $cnum (course id, course domain,
- and course number, respectively).  Any omitted arguments will be taken
- from the current environment ($ENV{'request.course.id'},
- $ENV{'course.'.$cid.'.domain'}, and $ENV{'course.'.$cid.'.num'}).
- Returns a reference to a hash which contains:
-  keys    '$sname:$sdom'
-  values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status]
- The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
- as indices into the returned list to future-proof clients against
- changes in the list order.
- =cut
- ################################################
- ################################################
  sub CL_SDOM     { return 0; }
  sub CL_SNAME    { return 1; }
- Line 2029  sub CL_ID       { return 4; }
+ Line 2289  sub CL_ID       { return 4; }
  sub CL_SECTION  { return 5; }
  sub CL_FULLNAME { return 6; }
  sub CL_STATUS   { return 7; }
+ sub CL_TYPE     { return 8; }
+ sub CL_LOCKEDTYPE   { return 9; }
+ sub CL_GROUP    { return 10; }
+ sub CL_PERMANENTEMAIL { return 11; }
+ sub CL_ROLE     { return 12; }
+ sub CL_EXTENT   { return 13; }
+ sub CL_PHOTO   { return 14; }
+ sub CL_THUMBNAIL { return 15; }
  sub get_classlist {
-     my ($cid,$cdom,$cnum) = @_;
+     my ($cdom,$cnum) = @_;
-     $cid = $cid || $ENV{'request.course.id'};
+     my $cid = $cdom.'_'.$cnum;
-     $cdom = $cdom || $ENV{'course.'.$cid.'.domain'};
+     if (!defined($cdom) || !defined($cnum)) {
-     $cnum = $cnum || $ENV{'course.'.$cid.'.num'};
+ 	$cid =  $env{'request.course.id'};
+ 	$cdom = $env{'course.'.$cid.'.domain'};
+ 	$cnum = $env{'course.'.$cid.'.num'};
+     }
      my $now = time;
      #
      my %classlist=&Apache::lonnet::dump('classlist',$cdom,$cnum);
- Line 2045  sub get_classlist {
+ Line 2316  sub get_classlist {
          }
          my ($sname,$sdom) = split(/:/,$student);
          my @Values = split(/:/,$info);
-         my ($end,$start,$id,$section,$fullname);
+         my ($end,$start,$id,$section,$fullname,$type,$lockedtype);
          if (@Values > 2) {
-             ($end,$start,$id,$section,$fullname) = @Values;
+             ($end,$start,$id,$section,$fullname,$type,$lockedtype) = @Values;
          } else { # We have to get the data ourselves
              ($end,$start) = @Values;
              $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
- Line 2062  sub get_classlist {
+ Line 2333  sub get_classlist {
                  &Apache::lonnet::logthis('unable to retrieve environment '.
                                           'for '.$sname.':'.$sdom);
              } else {
-                 $fullname = &ProcessFullName(@info{qw/lastname generation
+                 $fullname = &Apache::lonnet::format_name(@info{qw/firstname middlename lastname generation/},'lastname');
-                                                        firstname middlename/});
                  $id = $info{'id'};
              }
              # Update the classlist with this students information
              if ($fullname ne 'not available') {
-                 my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
+ 		my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
-                 my $reply=&Apache::lonnet::cput('classlist',
+ 		my $reply=&Apache::lonnet::cput('classlist',
                                                  {$student => $enrolldata},
                                                  $cdom,$cnum);
                  if ($reply !~ /^(ok|delayed)/) {
- Line 2083  sub get_classlist {
+ Line 2353  sub get_classlist {
          if(((!$end) || $now < $end) && ((!$start) || ($now > $start))) {
              $status='Active';
          }
+         if(($now < $start) && ((!$end) || $now < $end )) {
+             $status='Future';
+         }
          $classlist{$student} =
-             [$sdom,$sname,$end,$start,$id,$section,$fullname,$status];
+             [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,$lockedtype];
      }
      if (wantarray()) {
          return (\%classlist,['domain','username','end','start','id',
-                              'section','fullname','status']);
+                              'section','fullname','status','type','lockedtype']);
      } else {
          return \%classlist;
      }
  }
+ sub get_group_memberships {
+     my ($classlist,$keylist,$cdom,$cnum) = @_;
+     return ({},{}) if (!ref($classlist) || !ref($keylist));
+     my $cid = $cdom.'_'.$cnum;
+     if (!defined($cdom) || !defined($cnum)) {
+         $cid =  $env{'request.course.id'};
+         $cdom = $env{'course.'.$cid.'.domain'};
+         $cnum = $env{'course.'.$cid.'.num'};
+     }
+     my (%classgroups,%studentgroups);
+     my $now = time;
+     my $access_end = $env{'course.'.$cid.'.default_enrollment_end_date'};
+     my %curr_groups =&Apache::longroup::coursegroups($cdom,$cnum);
+     if (%curr_groups) {
+         my $grpindex = &CL_GROUP();
+         my %groupmemberhash =
+ 	    &Apache::lonnet::get_group_membership($cdom,$cnum);
+         foreach my $student (keys(%{$classlist})) {
+             %{$classgroups{$student}} = ();
+             my $hasgroup = 0;
+             foreach my $status ('previous','future','active','aftercourse') {
+                 %{$classgroups{$student}{$status}} = ();
+             }
+             foreach my $group (keys(%curr_groups)) {
+                 if (defined($groupmemberhash{$group.':'.$student})) {
+                     my ($end,$start) = split(/:/,$groupmemberhash{$group.':'.
+                                                                     $student});
+                     if ($start == -1) {
+                         next;
+                     } else {
+                         $studentgroups{$group} ++;
+                         $hasgroup = 1;
+                         if ($end && $end < $now) {
+                             $classgroups{$student}{'previous'}{$group} =
+                                          $groupmemberhash{$group.':'.$student};
+                             if ($classlist->{$student}[&CL_STATUS()] eq 'Expired') {
+                                 if ($access_end && $access_end < $now) {
+                                     if ($access_end - $end < 86400) {
+                                         $classgroups{$student}{'aftercourse'}{$group} = $groupmemberhash{$group.':'.$student};
+                                     }
+                                 }
+                             }
+                         } elsif ($now > $start) {
+                             if (!$end || $end > $now) {
+                                 $classgroups{$student}{'active'}{$group} =
+                                          $groupmemberhash{$group.':'.$student};
+                             }
+                         } else {
+                             $classgroups{$student}{'future'}{$group} =
+                                          $groupmemberhash{$group.':'.$student};
+                         }
+                     }
+                 }
+             }
+             if (!$hasgroup) {
+                 $studentgroups{'none'} ++;
+             } else {
+                 $classlist->{$student}->[$grpindex] = join(',',
+                               sort(keys(%{$classgroups{$student}{'active'}})));
+             }
+         }
+     }
+     return (\%classgroups,\%studentgroups);
+ }
+ sub get_students_groups {
+     my ($student,$enrollment_status,$classgroups) = @_;
+     my @studentsgroups = ();
+     if (ref($$classgroups{$student}{'active'}) eq 'HASH') {
+         push(@studentsgroups,keys(%{$$classgroups{$student}{'active'}}));
+     }
+     if ($enrollment_status eq 'Any') {
+         foreach my $status ('previous','future') {
+             if (ref($$classgroups{$student}{$status}) eq 'HASH') {
+                 push(@studentsgroups,keys(%{$$classgroups{$student}{$status}}));
+             }
+         }
+     } else {
+         if (ref($$classgroups{$student}{'aftercourse'}) eq 'HASH') {
+             push(@studentsgroups,keys(%{$$classgroups{$student}{'aftercourse'}}));
+         }
+     }
+     return @studentsgroups;
+ }
  # ----- END HELPER FUNCTIONS --------------------------------------------
 ;
  __END__
+ =pod
+ =head1 NAME
+ Apache::loncoursedata
+ =head1 SYNOPSIS
+ Set of functions that download and process student and course information.
+ =head1 PACKAGES USED
+   Apache::lonnet
+   Apache::longroup
+   Time::HiRes
+   Apache::lonmysql
+   LONCAPA
+   Digest::MD5
+ =head1 DOWNLOAD INFORMATION
+ This section contains all the functions that get data from other servers
+ and/or itself.
+ =head1 LOCAL DATA CACHING SUBROUTINES
+ The local caching is done using MySQL.  There is no fall-back implementation
+ if MySQL is not running.
+ The programmers interface is to call &get_current_state() or some other
+ primary interface subroutine (described below).  The internals of this
+ storage system are documented here.
+ There are six tables used to store student performance data (the results of
+ a dumpcurrent).  Each of these tables is created in MySQL with a name of
+ $courseid_*****, where ***** is 'symb', 'part', or whatever is appropriate
+ for the table.  The tables and their purposes are described below.
+ Some notes before we get started.
+ Each table must have a PRIMARY KEY, which is a column or set of columns which
+ will serve to uniquely identify a row of data.  NULL is not allowed!
+ INDEXes work best on integer data.
+ JOIN is used to combine data from many tables into one output.
+ lonmysql.pm is used for some of the interface, specifically the table creation
+ calls.  The inserts are done in bulk by directly calling the database handler.
+ The SELECT ... JOIN statement used to retrieve the data does not have an
+ interface in lonmysql.pm and I shudder at the thought of writing one.
+ =head2 Table Descriptions
+ =over 4
+ =head2 Tables used to store meta information
+ The following tables hold data required to keep track of the current status
+ of a students data in the tables or to look up the students data in the tables.
+ =over 4
+ =item C<$symb_table>
+ The symb_table has two columns.  The first is a 'symb_id' and the second
+ is the text name for the 'symb' (limited to 64k).  The 'symb_id' is generated
+ automatically by MySQL so inserts should be done on this table with an
+ empty first element.  This table has its PRIMARY KEY on the 'symb_id'.
+ =item C<$part_table>
+ The part_table has two columns.  The first is a 'part_id' and the second
+ is the text name for the 'part' (limited to 100 characters).  The 'part_id' is
+ generated automatically by MySQL so inserts should be done on this table with
+ an empty first element.  This table has its PRIMARY KEY on the 'part' (100
+ characters) and a KEY on 'part_id'.
+ =item C<$student_table>
+ The student_table has 7 columns.  The first is a 'student_id' assigned by
+ MySQL.  The second is 'student' which is username:domain.  The third through
+ fifth are 'section', 'status' (enrollment status), and 'classification'
+ (to be used in the future).  The sixth and seventh ('updatetime' and
+ 'fullupdatetime') contain the time of last update and full update of student
+ data.  This table has its PRIMARY KEY on the 'student_id' column and is indexed
+ on 'student', 'section', and 'status'.
+ =item C<$groupnames_table>
+ The groupnames_table has 2 columns.  The first is a 'group_id' assigned by
+ MySQL.  The second is 'groupname' which is the name of the group in the course.
+ =item C<$students_groups_table>
+ The students_groups_table has 2 columns.  The first is the 'student_id', and the
+ second is the 'group_id'. These two columns comprise the PRIMARY KEY for this
+ table, as an individual student may be affiliated with more than one group at
+ any time. This table is indexed on both student_id and group_id.
+ =back
+ =head2 Tables used to store current status data
+ The following tables store data only about the students current status on
+ a problem, meaning only the data related to the last attempt on a problem.
+ =over 4
+ =item C<$performance_table>
+ The performance_table has 9 columns.  The first three are 'symb_id',
+ 'student_id', and 'part_id'.  These comprise the PRIMARY KEY for this table
+ and are directly related to the $symb_table, $student_table, and $part_table
+ described above.  MySQL does better indexing on numeric items than text,
+ so we use these three "index tables".  The remaining columns are
+ 'solved', 'tries', 'awarded', 'award', 'awarddetail', and 'timestamp'.
+ These are either the MySQL type TINYTEXT or various integers ('tries' and
+ 'timestamp').  This table has KEYs of 'student_id' and 'symb_id'.
+ For use of this table, see the functions described below.
+ =item C<$parameters_table>
+ The parameters_table holds the data that does not fit neatly into the
+ performance_table.  The parameters table has four columns: 'symb_id',
+ 'student_id', 'parameter', and 'value'.  'symb_id', 'student_id', and
+ 'parameter' comprise the PRIMARY KEY for this table.  'parameter' is
+ limited to 255 characters.  'value' is limited to 64k characters.
+ =back
+ =head2 Tables used for storing historic data
+ The following tables are used to store almost all of the transactions a student
+ has made on a homework problem.  See loncapa/docs/homework/datastorage for
+ specific information about each of the parameters stored.
+ =over 4
+ =item C<$fulldump_response_table>
+ The response table holds data (documented in loncapa/docs/homework/datastorage)
+ associated with a particular response id which is stored when a student
+ attempts a problem.  The following are the columns of the table, in order:
+ 'symb_id','part_id','response_id','student_id','transaction','tries',
+ 'awarddetail', 'response_specific', 'response_specific_value',
+ 'response_specific_2', 'response_specific_value_2', and 'submission
+ (the text of the students submission).  The primary key is based on the
+ first five columns listed above.
+ =item C<$fulldump_part_table()>
+ The part table holds data (documented in loncapa/docs/homework/datastorage)
+ associated with a particular part id which is stored when a student attempts
+ a problem.  The following are the columns of the table, in order:
+ 'symb_id','part_id','student_id','transaction','tries','award','awarded',
+ and 'previous'.  The primary key is based on the first five columns listed
+ above.
+ =item C<$fulldump_timestamp_table()>
+ The timestamp table holds the timestamps of the transactions which are
+ stored in $fulldump_response_table and $fulldump_part_table.  This data is
+ about both the response and part data.  Columns: 'symb_id','student_id',
+ 'transaction', and 'timestamp'.
+ The primary key is based on the first 3 columns.
+ =item C<$weight_table()>
+ The weight table holds the weight for the problems used in the class.
+ Whereas the weight of a problem can vary by section and student the data
+ here is applied to the class as a whole.
+ Columns: 'symb_id','part_id','response_id','weight'.
+ =back
+ =head1 IMPORTANT SUBROUTINES
+ Here is a brief overview of the subroutines which are likely to be of
+ interest:
+ =over 4
+ =item C<&get_current_state()>
+ programmers interface.
+ =item C<&init_dbs()>
+ table creation
+ =item C<&update_student_data()>
+ data storage calls
+ =item C<&get_student_data_from_performance_cache()>
+ data retrieval
+ =back
+ =head1 OTHER SUBROUTINES
+ =over 4
+ =item C<&make_into_hash($values)>
+ Returns a reference to a hash as described by $values.  $values is
+ assumed to be the result of
+     join(':',map {&escape($_)} %orighash);
+ This is a helper function for get_current_state.
+ =item C<&init_dbs()>
+ Input: course id
+ Output: 0 on success, positive integer on error
+ This routine issues the calls to lonmysql to create the tables used to
+ store student data.
+ item C<&delete_caches()>
+ This routine drops all the tables associated with a course from the
+ MySQL database.
+ Input: course id (optional, determined by environment if omitted)
+ Returns: nothing
+ =item C<&get_part_id()>
+ Get the MySQL id of a problem part string.
+ Input: $part
+ Output: undef on error, integer $part_id on success.
+ =item C<&get_part()>
+ Get the string describing a part from the MySQL id of the problem part.
+ Input: $part_id
+ Output: undef on error, $part string on success.
+ =item C<&get_symb_id()>
+ Get the MySQL id of a symb.
+ Input: $symb
+ Output: undef on error, integer $symb_id on success.
+ =item C<&get_symb()>
+ Get the symb associated with a MySQL symb_id.
+ Input: $symb_id
+ Output: undef on error, $symb on success.
+ =item C<&get_student_id()>
+ Get the MySQL id of a student.
+ Input: $sname, $dom
+ Output: undef on error, integer $student_id on success.
+ =item C<&get_student()>
+ Get student username:domain associated with the MySQL student_id.
+ Input: $student_id
+ Output: undef on error, string $student (username:domain) on success.
+ =item C<&clear_internal_caches()>
+ Causes the internal caches used in get_student_id, get_student,
+ get_symb_id, get_symb, get_part_id, and get_part to be undef'd.
+ Needs to be called before the first operation with the MySQL database
+ for a given Apache request.
+ =item C<&update_full_student_data($sname,$sdom,$courseid)>
+ Does a lonnet::dump on a student to populate the courses tables.
+ Input: $sname, $sdom, $courseid
+ Output: $returnstatus
+ $returnstatus is a string describing any errors that occurred.  'okay' is the
+ default.
+ This subroutine loads a students data using lonnet::dump and inserts
+ it into the MySQL database.  The inserts are done on three tables,
+ $fulldump_response_table, $fulldump_part_table, and $fulldump_timestamp_table.
+ The INSERT calls are made directly by this subroutine, not through lonmysql
+ because we do a 'bulk'insert which takes advantage of MySQLs non-SQL
+ compliant INSERT command to insert multiple rows at a time.
+ If anything has gone wrong during this process, $returnstatus is updated with
+ a description of the error.
+ Once the "fulldump" tables are updated, the tables used for chart and
+ spreadsheet (which hold only the current state of the student on their
+ homework, not historical data) are updated.  If all updates have occurred
+ successfully, $student_table is updated to reflect the time of the update.
+ Notice we do not insert the data and immediately query it.  This means it
+ is possible for there to be data returned this first time that is not
+ available the second time.  CYA.
+ =item C<&update_student_data()>
+ Input: $sname, $sdom, $courseid
+ Output: $returnstatus, \%student_data
+ $returnstatus is a string describing any errors that occurred.  'okay' is the
+ default.
+ \%student_data is the data returned by a call to lonnet::currentdump.
+ This subroutine loads a students data using lonnet::currentdump and inserts
+ it into the MySQL database.  The inserts are done on two tables,
+ $performance_table and $parameters_table.  $parameters_table holds the data
+ that is not included in $performance_table.  See the description of
+ $performance_table elsewhere in this file.  The INSERT calls are made
+ directly by this subroutine, not through lonmysql because we do a 'bulk'
+ insert which takes advantage of MySQLs non-SQL compliant INSERT command to
+ insert multiple rows at a time.  If anything has gone wrong during this
+ process, $returnstatus is updated with a description of the error and
+ \%student_data is returned.
+ Notice we do not insert the data and immediately query it.  This means it
+ is possible for there to be data returned this first time that is not
+ available the second time.  CYA.
+ =item &ensure_tables_are_set_up($courseid)
+ Checks to be sure the MySQL tables for the given class are set up.
+ If $courseid is omitted it will be obtained from the environment.
+ Returns nothing on success and 'error' on failure
+ =item C<&ensure_current_data()>
+ Input: $sname, $sdom, $courseid
+ Output: $status, $data
+ This routine ensures the data for a given student is up to date.
+ The $student_table is queried to determine the time of the last update.
+ If the students data is out of date, &update_student_data() is called.
+ The return values from the call to &update_student_data() are returned.
+ =item C<&ensure_current_full_data($sname,$sdom,$courseid)>
+ Input: $sname, $sdom, $courseid
+ Output: $status
+ This routine ensures the fulldata (the data from a lonnet::dump, not a
+ lonnet::currentdump) for a given student is up to date.
+ The $student_table is queried to determine the time of the last update.
+ If the students fulldata is out of date, &update_full_student_data() is
+ called.
+ The return value from the call to &update_full_student_data() is returned.
+ =item C<&get_student_data_from_performance_cache()>
+ Input: $sname, $sdom, $symb, $courseid
+ Output: hash reference containing the data for the given student.
+ If $symb is undef, all the students data is returned.
+ This routine is the heart of the local caching system.  See the description
+ of $performance_table, $symb_table, $student_table, and $part_table.  The
+ main task is building the MySQL request.  The tables appear in the request
+ in the order in which they should be parsed by MySQL.  When searching
+ on a student the $student_table is used to locate the 'student_id'.  All
+ rows in $performance_table which have a matching 'student_id' are returned,
+ with data from $part_table and $symb_table which match the entries in
+ $performance_table, 'part_id' and 'symb_id'.  When searching on a symb,
+ the $symb_table is processed first, with matching rows grabbed from
+ $performance_table and filled in from $part_table and $student_table in
+ that order.
+ Running 'EXPLAIN ' on the 'SELECT' statements generated can be quite
+ interesting, especially if you play with the order the tables are listed.
+ =item C<&get_current_state()>
+ Input: $sname,$sdom,$symb,$courseid
+ Output: Described below
+ Retrieve the current status of a students performance.  $sname and
+ $sdom are the only required parameters.  If $symb is undef the results
+ of an &Apache::lonnet::currentdump() will be returned.
+ If $courseid is undef it will be retrieved from the environment.
+ The return structure is based on &Apache::lonnet::currentdump.  If
+ $symb is unspecified, all the students data is returned in a hash of
+ the form:
+ (
+   symb1 => { param1 => value1, param2 => value2 ... },
+   symb2 => { param1 => value1, param2 => value2 ... },
+ )
+ If $symb is specified, a hash of
+ (
+   param1 => value1,
+   param2 => value2,
+ )
+ is returned.
+ If no data is found for $symb, or if the student has no performance data,
+ an empty list is returned.
+ =item C<&get_problem_statistics()>
+ Gather data on a given problem.  The database is assumed to be
+ populated and all local caching variables are assumed to be set
+ properly.  This means you need to call &ensure_current_data for
+ the students you are concerned with prior to calling this routine.
+ Inputs: $Sections, Groups, $status, $symb, $part, $courseid, $starttime,
+         $endtime
+ =over 4
+ =item $Sections Array ref containing section names for students.
+ 'all' is allowed to be the first (and only) item in the array.
+ =item $Groups Array ref containing group names for students.
+ 'all' is allowed to be the first (and only) item in the array.
+ =item $status String describing the status of students
+ =item $symb is the symb for the problem.
+ =item $part is the part id you need statistics for
+ =item $courseid is the course id, of course!
+ =item $starttime and $endtime are unix times which to use to limit
+ the statistical data.
+ =back
+ Outputs: See the code for up to date information.  A hash reference is
+ returned.  The hash has the following keys defined:
+ =over 4
+ =item * num_students
+ The number of students attempting the problem
+ =item tries
+ The total number of tries for the students
+ =item max_tries
+ The maximum number of tries taken
+ =item mean_tries
+ The average number of tries
+ =item num_solved T
+ he number of students able to solve the problem
+ =item num_override
+ The number of students whose answer is 'correct_by_override'
+ =item deg_of_diff
+ The degree of difficulty of the problem
+ =item std_tries
+ The standard deviation of the number of tries
+ =item skew_tries
+ The skew of the number of tries
+ =item per_wrong
+ The number of students attempting the problem who were not
+ able to answer it correctly.
+ =back
+ =item C<&populate_weight_table()>
+ =item C<&limit_by_start_end_times()>
+ Build SQL WHERE condition which limits the data collected by the start
+ and end times provided
+ Inputs: $starttime, $endtime, $table
+ Returns: $time_limits
+ =item C<&limit_by_section_and_status()C<
+ Build SQL WHERE condition which limits the data collected by section and
+ student status.
+ Inputs: $Sections (array ref)
+     $enrollment (string: 'any', 'expired', 'active')
+     $tablename The name of the table that holds the student data
+ Returns: $student_requirements,$enrollment_requirements
+ =item C<&limit_by_group()>
+ Build SQL LEFT JOIN statement to include students_groups and groupnames tables and SQL WHERE condition which limits the data collected by group.
+ Inputs: $Groups (array ref)
+     $stutable   The name of the table which holds the student data.
+     $grptable   The name of the table which maps group_id to groupname.
+     $stugrptab  The name of the table which holds student group affiliations.
+ Returns: $groups_join,$group_limits
+    $groups_join  JOIN part of SQL statement (to include group related tables)
+    $group_limits SQL WHERE condition limiting to requested groups
+ =item C<rank_students_by_scores_on_resources()>
+ Inputs:
+     $resources: array ref of hash ref.  Each hash ref needs key 'symb'.
+     $Sections: array ref of sections to include,
+     $Groups: array ref of groups to include.
+     $enrollment: string,
+     $courseid (may be omitted)
+     $starttime (may be omitted)
+     $endtime (may be omitted)
+     $has_award_for (may be omitted)
+ Returns; An array of arrays.  The sub arrays contain a student name and
+ their score on the resources. $starttime and $endtime constrain the
+ list to awards obtained during the given time limits. $has_score_on
+ constrains the list to those students who at least attempted the
+ resource identified by the given symb, which is used to filter out
+ such students for statistics that would be adversely affected by such
+ students.
+ =item C<&get_sum_of_scores>
+ Inputs: $resource (hash ref, needs {'symb'} key),
+ $part, (the part id),
+ $students (array ref, contents of array are scalars holding 'sname:sdom'),
+ $courseid
+ Returns: the sum of the score on the problem part over the students and the
+    maximum possible value for the sum (taken from the weight table).
+ =item C<&score_stats()>
+ Inputs: $Sections, $enrollment, $symbs, $starttime,
+         $endtime, $courseid
+ $Sections, $enrollment, $starttime, $endtime, and $courseid are the same as
+ elsewhere in this module.
+ $symbs is an array ref of symbs
+ Returns: minimum, maximum, mean, s.d., number of students, and maximum
+   possible of student scores on the given resources
+ =item C<&count_stats()>
+ Inputs: $Sections, $Groups, $enrollment, $symbs, $starttime,
+         $endtime, $courseid
+ $Sections, $Groups $enrollment, $starttime, $endtime, and $courseid are the
+ same as elsewhere in this module.
+ $symbs is an array ref of symbs
+ Returns: minimum, maximum, mean, s.d., and number of students
+   of the number of items correct on the given resources
+ =item C<get_student_data()>
+ =item C<&get_student_scores($Sections,$Groups,$Symbs,$enrollment,$courseid)>
+ =item C<&setup_table_names()>
+ input: course id
+ output: none
+ =back
+ =head3 End of Local Data Caching Subroutines
+ =head3 Classlist Subroutines
+ =over
+ =item &get_classlist();
+ Retrieve the classist of a given class or of the current class.  Student
+ information is returned from the classlist.db file and, if needed,
+ from the students environment.
+ Optional arguments are $cdom, and $cnum (course domain,
+ and course number, respectively).  If either is ommitted the course
+ will be taken from the current environment ($env{'request.course.id'},
+ $env{'course.'.$cid.'.domain'}, and $env{'course.'.$cid.'.num'}).
+ Returns a reference to a hash which contains:
+  keys    '$sname:$sdom'
+  values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,$lockedtype]
+ The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
+ as indices into the returned list to future-proof clients against
+ changes in the list order.
+ =back
+ =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.89
changed lines
	Added in v.1.192