--- loncom/metadata_database/LONCAPA/lonmetadata.pm	2004/01/12 21:48:38	1.3
+++ loncom/metadata_database/LONCAPA/lonmetadata.pm	2004/04/16 21:43:56	1.8
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonmetadata.pm,v 1.3 2004/01/12 21:48:38 matthew Exp $
+# $Id: lonmetadata.pm,v 1.8 2004/04/16 21:43:56 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -242,8 +242,8 @@ sub create_metadata_storage {
 
 =item store_metadata()
 
-Inputs: database handle ($dbh) and a hash or hash reference containing the 
-metadata for a single resource.
+Inputs: database handle ($dbh), a table name, and a hash or hash reference 
+containing the metadata for a single resource.
 
 Returns: 1 on success, 0 on failure to store.
 
@@ -263,10 +263,14 @@ Returns: 1 on success, 0 on failure to s
     ##  $dbh, so we can't check our validity.
     ##
     my $sth = undef;
+    my $sth_table = undef;
 
 sub create_statement_handler {
     my $dbh = shift();
-    my $request = 'INSERT INTO metadata VALUES(';
+    my $tablename = shift();
+    $tablename = 'metadata' if (! defined($tablename));
+    $sth_table = $tablename;
+    my $request = 'INSERT INTO '.$tablename.' VALUES(';
     foreach (@Metadata_Table_Description) {
         $request .= '?,';
     }
@@ -276,13 +280,16 @@ sub create_statement_handler {
     return;
 }
 
-sub clear_sth { $sth=undef; }
+sub clear_sth { $sth=undef; $sth_table=undef;}
 
 sub store_metadata {
     my $dbh = shift();
+    my $tablename = shift();
     my $errors = '';
-    if (! defined($sth)) {
-        &create_statement_handler($dbh);
+    if (! defined($sth) || 
+        ( defined($tablename) && ($sth_table ne $tablename)) || 
+        (! defined($tablename) && $sth_table ne 'metadata')) {
+        &create_statement_handler($dbh,$tablename);
     }
     my $successcount = 0;
     while (my $mdata = shift()) {
@@ -290,7 +297,11 @@ sub store_metadata {
         my @MData;
         foreach my $field (@Metadata_Table_Description) {
             if (exists($mdata->{$field->{'name'}})) {
-                push(@MData,$mdata->{$field->{'name'}});
+                if ($mdata->{$field->{'name'}} eq 'nan') {
+                    push(@MData,'NULL');
+                } else {
+                    push(@MData,$mdata->{$field->{'name'}});
+                }
             } else {
                 push(@MData,undef);
             }
@@ -361,7 +372,7 @@ sub lookup_metadata {
 
 =item delete_metadata()
 
-
+Not implemented yet
 
 =cut
 
@@ -372,6 +383,343 @@ sub delete_metadata {}
 ######################################################################
 ######################################################################
 
+=pod
+
+=item metdata_col_to_hash
+
+Input: Array of metadata columns
+
+Return: Hash with the metadata columns as keys and the array elements
+passed in as values
+
+=cut
+
+######################################################################
+######################################################################
+sub metadata_col_to_hash {
+    my @cols=@_;
+    my %hash=();
+    for (my $i=0; $i<=$#Metadata_Table_Description;$i++) {
+        $hash{$Metadata_Table_Description[$i]->{'name'}}=$cols[$i];
+    }
+    return %hash;
+}
+
+######################################################################
+######################################################################
+
+=pod
+
+=item nohist_resevaldata.db data structure
+
+The nohist_resevaldata.db file has the following possible keys:
+
+ Statistics Data (values are integers, perl times, or real numbers)
+ ------------------------------------------
+ $course___$resource___avetries
+ $course___$resource___count
+ $course___$resource___difficulty
+ $course___$resource___stdno
+ $course___$resource___timestamp
+
+ Evaluation Data (values are on a 1 to 5 scale)
+ ------------------------------------------
+ $username@$dom___$resource___clear
+ $username@$dom___$resource___comments
+ $username@$dom___$resource___depth
+ $username@$dom___$resource___technical
+ $username@$dom___$resource___helpful
+
+ Course Context Data
+ ------------------------------------------
+ $course___$resource___course       course id
+ $course___$resource___comefrom     resource preceeding this resource
+ $course___$resource___goto         resource following this resource
+ $course___$resource___usage        resource containing this resource
+
+ New statistical data storage
+ ------------------------------------------
+ $course&$sec&$numstud___$resource___stats
+    $sec is a string describing the sections: all, 1 2, 1 2 3,...
+    Value is a '&' deliminated list of key=value pairs.
+    Possible keys are (currently) disc,course,sections,difficulty, 
+    stdno, timestamp
+
+=cut
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &process_reseval_data 
+
+Process a nohist_resevaldata hash into a more complex data structure.
+
+Input: Hash reference containing reseval data
+
+Returns: Hash with the following structure:
+
+$hash{$url}->{'statistics'}->{$courseid}->{'avetries'}   = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'count'}      = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'difficulty'} = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'stdno'}      = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'timestamp'}  = $value
+
+$hash{$url}->{'evaluation'}->{$username}->{'clear'}     = $value
+$hash{$url}->{'evaluation'}->{$username}->{'comments'}  = $value
+$hash{$url}->{'evaluation'}->{$username}->{'depth'}     = $value
+$hash{$url}->{'evaluation'}->{$username}->{'technical'} = $value
+$hash{$url}->{'evaluation'}->{$username}->{'helpful'}   = $value
+
+$hash{$url}->{'course'}    = \@Courses
+$hash{$url}->{'comefrom'}  = \@Resources
+$hash{$url}->{'goto'}      = \@Resources
+$hash{$url}->{'usage'}     = \@Resources
+
+$hash{$url}->{'stats'}->{$courseid\_$section}->{$key} = $value
+
+=cut
+
+######################################################################
+######################################################################
+sub process_reseval_data {
+    my ($evaldata) = @_;
+    my %DynamicData;
+    #
+    # Process every stored element
+    while (my ($storedkey,$value) = each(%{$evaldata})) {
+        my ($source,$file,$type) = split('___',$storedkey);
+        $source = &unescape($source);
+        $file = &unescape($file);
+        $value = &unescape($value);
+         "    got ".$file."\n        ".$type." ".$source."\n";
+        if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
+            #
+            # Statistics: $source is course id
+            $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
+        } elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){
+            #
+            # Evaluation $source is username, check if they evaluated it
+            # more than once.  If so, pad the entry with a space.
+            while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
+                $source .= ' ';
+            }
+            $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
+        } elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
+            #
+            # Context $source is course id or resource
+            push(@{$DynamicData{$file}->{$type}},&unescape($source));
+        } elsif ($type eq 'stats') {
+            #
+            # Statistics storage...
+            # $source is $cid\_$sec\_$stdno
+            # $value is stat1=value&stat2=value&stat3=value,....
+            #
+            my ($cid,$sec,$stdno)=split('&',$source);
+            my $crssec = $cid.'&'.$sec;
+            my @Data = split('&',$value);
+            my %Statistics;
+            while (my ($key,$value) = split('=',pop(@Data))) {
+                $Statistics{$key} = $value;
+            }
+            $sec =~ s:("$|^")::g;
+            $Statistics{'sections'} = $sec;
+            #
+            # Only store the data if the number of students is greater
+            # than the data already stored
+            if (! exists($DynamicData{$file}->{'stats'}->{$crssec}) ||
+                $DynamicData{$file}->{'stats'}->{$crssec}->{'stdno'}<$stdno){
+                $DynamicData{$file}->{'stats'}->{$crssec}=\%Statistics;
+            }
+        }
+    }
+    return %DynamicData;
+}
+
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &process_dynamic_metadata
+
+Inputs: $url: the url of the item to process
+$DynamicData: hash reference for the results of &process_reseval_data
+
+Returns: Hash containing the following keys:
+    avetries, avetries_list, difficulty, difficulty_list, stdno, stdno_list,
+    course, course_list, goto, goto_list, comefrom, comefrom_list,
+    usage, clear, technical, correct, helpful, depth, comments
+
+    Each of the return keys is associated with either a number or a string
+    The *_list items are comma-seperated strings.  'comments' is a string
+    containing generically marked-up comments.
+
+=cut
+
+######################################################################
+######################################################################
+sub process_dynamic_metadata {
+    my ($url,$DynamicData) = @_;
+    my %data;
+    my $resdata = $DynamicData->{$url};
+    #
+    # Get the statistical data - Use a weighted average
+    foreach my $type (qw/avetries difficulty disc/) {
+        my $studentcount;
+        my $sum;
+        my @Values;
+        my @Students;
+        #
+        # Old data
+        foreach my $coursedata (values(%{$resdata->{'statistics'}}),
+                                values(%{$resdata->{'stats'}})) {
+            if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
+                $studentcount += $coursedata->{'stdno'};
+                $sum += ($coursedata->{$type}*$coursedata->{'stdno'});
+                push(@Values,$coursedata->{$type});
+                push(@Students,$coursedata->{'stdno'});
+            }
+        }
+        if (exists($resdata->{'stats'})) {
+            foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
+                my $coursedata = $resdata->{'stats'}->{$identifier};
+                $studentcount += $coursedata->{'stdno'};
+                $sum += $coursedata->{$type}*$coursedata->{'stdno'};
+                push(@Values,$coursedata->{$type});                
+                push(@Students,$coursedata->{'stdno'});
+            }
+        }
+        #
+        # New data
+        if (defined($studentcount) && $studentcount>0) {
+            $data{$type} = $sum/$studentcount;
+            $data{$type.'_list'} = join(',',@Values);
+        }
+    }
+    #
+    # Find out the number of students who have completed the resource...
+    my $stdno;
+    foreach my $coursedata (values(%{$resdata->{'statistics'}}),
+                            values(%{$resdata->{'stats'}})) {
+        if (ref($coursedata) eq 'HASH' && exists($coursedata->{'stdno'})) {
+            $stdno += $coursedata->{'stdno'};
+        }
+    }
+    if (exists($resdata->{'stats'})) {
+        #
+        # For the number of students, take the maximum found for the class
+        my $current_course;
+        my $coursemax=0;
+        foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
+            my $coursedata = $resdata->{'stats'}->{$identifier};
+            if (! defined($current_course)) {
+                $current_course = $coursedata->{'course'};
+            }
+            if ($current_course ne $coursedata->{'course'}) {
+                $stdno += $coursemax;
+                $coursemax = 0;
+                $current_course = $coursedata->{'course'};                
+            }
+            if ($coursemax < $coursedata->{'stdno'}) {
+                $coursemax = $coursedata->{'stdno'};
+            }
+        }
+        $stdno += $coursemax; # pick up the final course in the list
+    }
+    $data{'stdno'}=$stdno;
+    #
+    # Get the context data
+    foreach my $type (qw/course goto comefrom/) {
+        if (defined($resdata->{$type}) && 
+            ref($resdata->{$type}) eq 'ARRAY') {
+            $data{$type} = scalar(@{$resdata->{$type}});
+            $data{$type.'_list'} = join(',',@{$resdata->{$type}});
+        }
+    }
+    if (defined($resdata->{'usage'}) && 
+        ref($resdata->{'usage'}) eq 'ARRAY') {
+        $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
+        $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
+    }
+    #
+    # Get the evaluation data
+    foreach my $type (qw/clear technical correct helpful depth/) {
+        my $count;
+        my $sum;
+        foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
+            $sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
+            $count++;
+        }
+        if ($count > 0) {
+            $data{$type}=$sum/$count;
+        }
+    }
+    #
+    # put together comments
+    my $comments = '<div class="LCevalcomments">';
+    foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
+        $comments .= 
+            '<p>'.
+            '<b>'.$evaluator.'</b>:'.
+            $resdata->{'evaluation'}->{'comments'}->{$evaluator}.
+            '</p>';
+    }
+    $comments .= '</div>';
+    $data{'comments'} = $comments;
+    #
+    if (exists($resdata->{'stats'})) {
+        $data{'stats'} = $resdata->{'stats'};
+    }
+    #
+    return %data;
+}
+
+sub dynamic_metadata_storage {
+    my ($data) = @_;
+    my %Store;
+    my $courseid = $data->{'course'};
+    my $sections = $data->{'sections'};
+    my $numstu = $data->{'num_students'};
+    my $urlres = $data->{'urlres'};
+    my $key = $courseid.'&'.$sections.'&'.$numstu.'___'.$urlres.'___stats';
+    $Store{$key} =
+        'course='.$courseid.'&'.
+        'sections='.$sections.'&'.
+        'timestamp='.time.'&'.
+        'stdno='.$data->{'num_students'}.'&'.
+        'avetries='.$data->{'mean_tries'}.'&'.
+        'difficulty='.$data->{'deg_of_diff'};
+    if (exists($data->{'deg_of_disc'})) {
+        $Store{$key} .= '&'.'disc='.$data->{'deg_of_disc'};
+    }
+    return %Store;
+}
+
+######################################################################
+######################################################################
+##
+## The usual suspects, repeated here to reduce dependency hell
+##
+######################################################################
+######################################################################
+sub unescape {
+    my $str=shift;
+    $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+    return $str;
+}
+
+sub escape {
+    my $str=shift;
+    $str =~ s/(\W)/"%".unpack('H2',$1)/eg;
+    return $str;
+}
+
+
+
+
 1;
 
 __END__;