# The LearningOnline Network with CAPA
#
# $Id: lonmetadata.pm,v 1.12 2005/03/11 03:25:18 matthew Exp $
#
# Copyright Michigan State University Board of Trustees
#
# This file is part of the LearningOnline Network with CAPA (LON-CAPA).
#
# LON-CAPA is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LON-CAPA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with LON-CAPA; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# /home/httpd/html/adm/gpl.txt
#
# http://www.lon-capa.org/
#
######################################################################

package LONCAPA::lonmetadata;

use strict;
use DBI;

######################################################################
######################################################################

=pod 

=head1 Name

lonmetadata

=head1 Synopsis

lonmetadata holds a description of the metadata table and provides
wrappers for the storage and retrieval of metadata to/from the database.

=head1 Description

=head1 Methods

=over 4

=cut

######################################################################
######################################################################

=pod

=item Old table creation command

CREATE TABLE IF NOT EXISTS metadata 
(title TEXT, 
author TEXT, 
subject TEXT, 
url TEXT, 
keywords TEXT, 
version TEXT, 
notes TEXT, 
abstract TEXT, 
mime TEXT, 
language TEXT, 
creationdate DATETIME, 
lastrevisiondate DATETIME, 
owner TEXT, 
copyright TEXT, 
domain TEXT

FULLTEXT idx_title (title), 
FULLTEXT idx_author (author), 
FULLTEXT idx_subject (subject), 
FULLTEXT idx_url (url), 
FULLTEXT idx_keywords (keywords), 
FULLTEXT idx_version (version), 
FULLTEXT idx_notes (notes), 
FULLTEXT idx_abstract (abstract), 
FULLTEXT idx_mime (mime), 
FULLTEXT idx_language (language),
FULLTEXT idx_owner (owner), 
FULLTEXT idx_copyright (copyright)) 

TYPE=MYISAM;

=cut

######################################################################
######################################################################
my @Metadata_Table_Description = 
    (
     { name => 'title',     type=>'TEXT'},
     { name => 'author',    type=>'TEXT'},
     { name => 'subject',   type=>'TEXT'},
     { name => 'url',       type=>'TEXT', restrictions => 'NOT NULL' },
     { name => 'keywords',  type=>'TEXT'},
     { name => 'version',   type=>'TEXT'},
     { name => 'notes',     type=>'TEXT'},
     { name => 'abstract',  type=>'TEXT'},
     { name => 'mime',      type=>'TEXT'},
     { name => 'language',  type=>'TEXT'},
     { name => 'creationdate',     type=>'DATETIME'},
     { name => 'lastrevisiondate', type=>'DATETIME'},
     { name => 'owner',     type=>'TEXT'},
     { name => 'copyright', type=>'TEXT'}, 
     { name => 'domain',    type=>'TEXT'},
      #--------------------------------------------------
     { name => 'dependencies',   type=>'TEXT'},
     { name => 'modifyinguser',  type=>'TEXT'},
     { name => 'authorspace',    type=>'TEXT'},
     { name => 'lowestgradelevel',  type=>'INT'},
     { name => 'highestgradelevel', type=>'INT'},
     { name => 'standards',      type=>'TEXT'},
     { name => 'count',          type=>'INT'},
     { name => 'course',         type=>'INT'},
     { name => 'course_list',    type=>'TEXT'},
     { name => 'goto',           type=>'INT'},
     { name => 'goto_list',      type=>'TEXT'},
     { name => 'comefrom',       type=>'INT'},
     { name => 'comefrom_list',  type=>'TEXT'},
     { name => 'sequsage',       type=>'INT'},
     { name => 'sequsage_list',  type=>'TEXT'},
     { name => 'stdno',          type=>'INT'},
     { name => 'stdno_list',     type=>'TEXT'},
     { name => 'avetries',       type=>'FLOAT'},
     { name => 'avetries_list',  type=>'TEXT'},
     { name => 'difficulty',     type=>'FLOAT'},
     { name => 'difficulty_list',type=>'TEXT'},
     { name => 'disc',           type=>'FLOAT'},
     { name => 'disc_list',      type=>'TEXT'},
     { name => 'clear',          type=>'FLOAT'},
     { name => 'technical',      type=>'FLOAT'},
     { name => 'correct',        type=>'FLOAT'},
     { name => 'helpful',        type=>'FLOAT'},
     { name => 'depth',          type=>'FLOAT'},
     { name => 'hostname',       type=> 'TEXT'},
     #--------------------------------------------------
     );

my @Fulltext_indicies = qw/
    title
    author
    subject
    url
    keywords
    version
    notes
    abstract
    mime
    language
    owner
    copyright/;

######################################################################
######################################################################

=pod

=item &describe_metadata_storage

Input: None

Returns: An array of hash references describing the columns and indicies
of the metadata table(s).

=cut

######################################################################
######################################################################
sub describe_metadata_storage { 
    return (\@Metadata_Table_Description,\@Fulltext_indicies);
}

######################################################################
######################################################################

=pod

=item create_metadata_storage()

Inputs: table name (optional): the name of the table.  Default is 'metadata'.

Returns: A perl string which, when executed by MySQL, will cause the
metadata storage to be initialized.

=cut

######################################################################
######################################################################
sub create_metadata_storage { 
    my ($tablename) = @_;
    $tablename = 'metadata' if (! defined($tablename));
    my $request = "CREATE TABLE IF NOT EXISTS ".$tablename." ";
    #
    # Process the columns  (this code is stolen from lonmysql.pm)
    my @Columns;
    my $col_des; # mysql column description
    foreach my $coldata (@Metadata_Table_Description) {
        my $column = $coldata->{'name'};
        $col_des = '';
        if (lc($coldata->{'type'}) =~ /(enum|set)/) { # 'enum' or 'set'
            $col_des.=$column." ".$coldata->{'type'}."('".
                join("', '",@{$coldata->{'values'}})."')";
        } else {
            $col_des.=$column." ".$coldata->{'type'};
            if (exists($coldata->{'size'})) {
                $col_des.="(".$coldata->{'size'}.")";
            }
        }
        # Modifiers
        if (exists($coldata->{'restrictions'})){
            $col_des.=" ".$coldata->{'restrictions'};
        }
        if (exists($coldata->{'default'})) {
            $col_des.=" DEFAULT '".$coldata->{'default'}."'";
        }
        $col_des.=' AUTO_INCREMENT' if (exists($coldata->{'auto_inc'}) &&
                                        ($coldata->{'auto_inc'} eq 'yes'));
        $col_des.=' PRIMARY KEY'    if (exists($coldata->{'primary_key'}) &&
                                        ($coldata->{'primary_key'} eq 'yes'));
    } continue {
        # skip blank items.
        push (@Columns,$col_des) if ($col_des ne '');
    }
    foreach my $colname (@Fulltext_indicies) {
        my $text = 'FULLTEXT idx_'.$colname.' ('.$colname.')';
        push (@Columns,$text);
    }
    $request .= "(".join(", ",@Columns).") TYPE=MyISAM";
    return $request;
}

######################################################################
######################################################################

=pod

=item store_metadata()

Inputs: database handle ($dbh), a table name, and a hash or hash reference 
containing the metadata for a single resource.

Returns: 1 on success, 0 on failure to store.

=cut

######################################################################
######################################################################
{
    ##
    ##  WARNING: The following cleverness may cause trouble in cases where
    ##  the dbi connection is dropped and recreated - a stale statement
    ##  handler may linger around and cause trouble.
    ##
    ##  In most scripts, this will work fine.  If the dbi is going to be
    ##  dropped and (possibly) later recreated, call &clear_sth.  Yes it
    ##  is annoying but $sth appearantly does not have a link back to the 
    ##  $dbh, so we can't check our validity.
    ##
    my $sth = undef;
    my $sth_table = undef;

sub create_statement_handler {
    my $dbh = shift();
    my $tablename = shift();
    $tablename = 'metadata' if (! defined($tablename));
    $sth_table = $tablename;
    my $request = 'INSERT INTO '.$tablename.' VALUES(';
    foreach (@Metadata_Table_Description) {
        $request .= '?,';
    }
    chop $request;
    $request.= ')';
    $sth = $dbh->prepare($request);
    return;
}

sub clear_sth { $sth=undef; $sth_table=undef;}

sub store_metadata {
    my ($dbh,$tablename,@Metadata)=@_;
    my $errors = '';
    if (! defined($sth) || 
        ( defined($tablename) && ($sth_table ne $tablename)) || 
        (! defined($tablename) && $sth_table ne 'metadata')) {
        &create_statement_handler($dbh,$tablename);
    }
    my $successcount = 0;
    foreach my $mdata (@Metadata) {
        next if (ref($mdata) ne "HASH");
        my @MData;
        foreach my $field (@Metadata_Table_Description) {
            my $fname = $field->{'name'};
            if (exists($mdata->{$fname}) && 
                defined($mdata->{$fname}) &&
                $mdata->{$fname} ne '') {
                if ($mdata->{$fname} eq 'nan' ||
                    $mdata->{$fname} eq '') {
                    push(@MData,'NULL');
                } else {
                    push(@MData,$mdata->{$fname});
                }
            } else {
                push(@MData,undef);
            }
        }
        $sth->execute(@MData);
        if (! $sth->err) {
            $successcount++;
        } else {
            $errors = join(',',$errors,$sth->errstr);
        }
        $errors =~ s/^,//;
    }
    if (wantarray()) {
        return ($successcount,$errors);
    } else {
        return $successcount;
    }
}

}

######################################################################
######################################################################

=pod

=item lookup_metadata()

Inputs: database handle ($dbh) and a hash or hash reference containing 
metadata which will be used for a search.

Returns: scalar with error string on failure, array reference on success.
The array reference is the same one returned by $sth->fetchall_arrayref().

=cut

######################################################################
######################################################################
sub lookup_metadata {
    my ($dbh,$condition,$fetchparameter,$tablename) = @_;
    $tablename = 'metadata' if (! defined($tablename));
    my $error;
    my $returnvalue=[];
    my $request = 'SELECT * FROM '.$tablename;
    if (defined($condition)) {
        $request .= ' WHERE '.$condition;
    }
    my $sth = $dbh->prepare($request);
    if ($sth->err) {
        $error = $sth->errstr;
    }
    if (! $error) {
        $sth->execute();
        if ($sth->err) {
            $error = $sth->errstr;
        } else {
            $returnvalue = $sth->fetchall_arrayref($fetchparameter);
            if ($sth->err) {
                $error = $sth->errstr;
            }
        }
    }
    return ($error,$returnvalue);
}

######################################################################
######################################################################

=pod

=item delete_metadata()

Removes a single metadata record, based on its url.

Inputs: $dbh, the database handler.
$tablename, the name of the metadata table to remove from. default: 'metadata'
$url, the url of the resource to remove from the metadata database.

Returns: undef on success, dbh errorstr on failure.

=cut

######################################################################
######################################################################
sub delete_metadata {
    my ($dbh,$tablename,$url) = @_;
    $tablename = 'metadata' if (! defined($tablename));
    my $error;
    my $delete_command = 'DELETE FROM '.$tablename.' WHERE url='.
        $dbh->quote($url);
    $dbh->do($delete_command);
    if ($dbh->err) {
        $error = $dbh->errstr();
    }
    return $error;
}

######################################################################
######################################################################

=pod

=item update_metadata

Updates metadata record in mysql database.  It does not matter if the record
currently exists.  Fields not present in the new metadata will be taken
from the current record, if it exists.  To delete an entry for a key, set 
it to "" or undef.

Inputs: 
$dbh, database handle
$newmetadata, hash reference containing the new metadata
$tablename, metadata table name.  Defaults to 'metadata'.

Returns:
$error on failure.  undef on success.

=cut

######################################################################
######################################################################
sub update_metadata {
    my ($dbh,$tablename,$newmetadata)=@_;
    my $error;
    $tablename = 'metadata' if (! defined($tablename));
    if (! exists($newmetadata->{'url'})) {
        $error = 'Unable to update: no url specified';
    }
    return $error if (defined($error));
    # 
    # Retrieve current values
    my $row;
    ($error,$row) = &lookup_metadata($dbh,
                                   ' url='.$dbh->quote($newmetadata->{'url'}),
                                     undef,$tablename);
    return $error if ($error);
    my %metadata = &LONCAPA::lonmetadata::metadata_col_to_hash(@{$row->[0]});
    #
    # Update metadata values
    while (my ($key,$value) = each(%$newmetadata)) {
        $metadata{$key} = $value;
    }
    #
    # Delete old data (deleting a nonexistant record does not produce an error.
    $error = &delete_metadata($dbh,$tablename,$newmetadata->{'url'});
    return $error if (defined($error));
    #
    # Store updated metadata
    my $success;
    ($success,$error) = &store_metadata($dbh,$tablename,\%metadata);
    return $error;
}

######################################################################
######################################################################

=pod

=item metdata_col_to_hash

Input: Array of metadata columns

Return: Hash with the metadata columns as keys and the array elements
passed in as values

=cut

######################################################################
######################################################################
sub metadata_col_to_hash {
    my @cols=@_;
    my %hash=();
    for (my $i=0; $i<=$#Metadata_Table_Description;$i++) {
        $hash{$Metadata_Table_Description[$i]->{'name'}}=$cols[$i];
    }
    return %hash;
}

######################################################################
######################################################################

=pod

=item nohist_resevaldata.db data structure

The nohist_resevaldata.db file has the following possible keys:

 Statistics Data (values are integers, perl times, or real numbers)
 ------------------------------------------
 $course___$resource___avetries
 $course___$resource___count
 $course___$resource___difficulty
 $course___$resource___stdno
 $course___$resource___timestamp

 Evaluation Data (values are on a 1 to 5 scale)
 ------------------------------------------
 $username@$dom___$resource___clear
 $username@$dom___$resource___comments
 $username@$dom___$resource___depth
 $username@$dom___$resource___technical
 $username@$dom___$resource___helpful
 $username@$dom___$resource___correct

 Course Context Data
 ------------------------------------------
 $course___$resource___course       course id
 $course___$resource___comefrom     resource preceeding this resource
 $course___$resource___goto         resource following this resource
 $course___$resource___usage        resource containing this resource

 New statistical data storage
 ------------------------------------------
 $course&$sec&$numstud___$resource___stats
    $sec is a string describing the sections: all, 1 2, 1 2 3,...
    Value is a '&' deliminated list of key=value pairs.
    Possible keys are (currently) disc,course,sections,difficulty, 
    stdno, timestamp

=cut

######################################################################
######################################################################

=pod

=item &process_reseval_data 

Process a nohist_resevaldata hash into a more complex data structure.

Input: Hash reference containing reseval data

Returns: Hash with the following structure:

$hash{$url}->{'statistics'}->{$courseid}->{'avetries'}   = $value
$hash{$url}->{'statistics'}->{$courseid}->{'count'}      = $value
$hash{$url}->{'statistics'}->{$courseid}->{'difficulty'} = $value
$hash{$url}->{'statistics'}->{$courseid}->{'stdno'}      = $value
$hash{$url}->{'statistics'}->{$courseid}->{'timestamp'}  = $value

$hash{$url}->{'evaluation'}->{$username}->{'clear'}     = $value
$hash{$url}->{'evaluation'}->{$username}->{'comments'}  = $value
$hash{$url}->{'evaluation'}->{$username}->{'depth'}     = $value
$hash{$url}->{'evaluation'}->{$username}->{'technical'} = $value
$hash{$url}->{'evaluation'}->{$username}->{'helpful'}   = $value

$hash{$url}->{'course'}    = \@Courses
$hash{$url}->{'comefrom'}  = \@Resources
$hash{$url}->{'goto'}      = \@Resources
$hash{$url}->{'usage'}     = \@Resources

$hash{$url}->{'stats'}->{$courseid\_$section}->{$key} = $value

=cut

######################################################################
######################################################################
sub process_reseval_data {
    my ($evaldata) = @_;
    my %DynamicData;
    #
    # Process every stored element
    while (my ($storedkey,$value) = each(%{$evaldata})) {
        my ($source,$file,$type) = split('___',$storedkey);
        $source = &unescape($source);
        $file = &unescape($file);
        $value = &unescape($value);
         "    got ".$file."\n        ".$type." ".$source."\n";
        if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
            #
            # Statistics: $source is course id
            $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
        } elsif ($type =~ /^(clear|comments|depth|technical|helpful|correct)$/){
            #
            # Evaluation $source is username, check if they evaluated it
            # more than once.  If so, pad the entry with a space.
            while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
                $source .= ' ';
            }
            $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
        } elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
            #
            # Context $source is course id or resource
            push(@{$DynamicData{$file}->{$type}},&unescape($source));
        } elsif ($type eq 'stats') {
            #
            # Statistics storage...
            # $source is $cid\_$sec\_$stdno
            # $value is stat1=value&stat2=value&stat3=value,....
            #
            my ($cid,$sec,$stdno)=split('&',$source);
            my $crssec = $cid.'&'.$sec;
            my @Data = split('&',$value);
            my %Statistics;
            while (my ($key,$value) = split('=',pop(@Data))) {
                $Statistics{$key} = $value;
            }
            $sec =~ s:("$|^")::g;
            $Statistics{'sections'} = $sec;
            #
            # Only store the data if the number of students is greater
            # than the data already stored
            if (! exists($DynamicData{$file}->{'stats'}->{$crssec}) ||
                $DynamicData{$file}->{'stats'}->{$crssec}->{'stdno'}<$stdno){
                $DynamicData{$file}->{'stats'}->{$crssec}=\%Statistics;
            }
        }
    }
    return %DynamicData;
}


######################################################################
######################################################################

=pod

=item &process_dynamic_metadata

Inputs: $url: the url of the item to process
$DynamicData: hash reference for the results of &process_reseval_data

Returns: Hash containing the following keys:
    avetries, avetries_list, difficulty, difficulty_list, stdno, stdno_list,
    course, course_list, goto, goto_list, comefrom, comefrom_list,
    usage, clear, technical, correct, helpful, depth, comments

    Each of the return keys is associated with either a number or a string
    The *_list items are comma-seperated strings.  'comments' is a string
    containing generically marked-up comments.

=cut

######################################################################
######################################################################
sub process_dynamic_metadata {
    my ($url,$DynamicData) = @_;
    my %data;
    my $resdata = $DynamicData->{$url};
    #
    # Get the statistical data - Use a weighted average
    foreach my $type (qw/avetries difficulty disc/) {
        my $studentcount;
        my $sum;
        my @Values;
        my @Students;
        #
        # Old data
        foreach my $coursedata (values(%{$resdata->{'statistics'}}),
                                values(%{$resdata->{'stats'}})) {
            if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
                $studentcount += $coursedata->{'stdno'};
                $sum += ($coursedata->{$type}*$coursedata->{'stdno'});
                push(@Values,$coursedata->{$type});
                push(@Students,$coursedata->{'stdno'});
            }
        }
        if (exists($resdata->{'stats'})) {
            foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
                my $coursedata = $resdata->{'stats'}->{$identifier};
                $studentcount += $coursedata->{'stdno'};
                $sum += $coursedata->{$type}*$coursedata->{'stdno'};
                push(@Values,$coursedata->{$type});                
                push(@Students,$coursedata->{'stdno'});
            }
        }
        #
        # New data
        if (defined($studentcount) && $studentcount>0) {
            $data{$type} = $sum/$studentcount;
            $data{$type.'_list'} = join(',',@Values);
        }
    }
    #
    # Find out the number of students who have completed the resource...
    my $stdno;
    foreach my $coursedata (values(%{$resdata->{'statistics'}}),
                            values(%{$resdata->{'stats'}})) {
        if (ref($coursedata) eq 'HASH' && exists($coursedata->{'stdno'})) {
            $stdno += $coursedata->{'stdno'};
        }
    }
    if (exists($resdata->{'stats'})) {
        #
        # For the number of students, take the maximum found for the class
        my $current_course;
        my $coursemax=0;
        foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
            my $coursedata = $resdata->{'stats'}->{$identifier};
            if (! defined($current_course)) {
                $current_course = $coursedata->{'course'};
            }
            if ($current_course ne $coursedata->{'course'}) {
                $stdno += $coursemax;
                $coursemax = 0;
                $current_course = $coursedata->{'course'};                
            }
            if ($coursemax < $coursedata->{'stdno'}) {
                $coursemax = $coursedata->{'stdno'};
            }
        }
        $stdno += $coursemax; # pick up the final course in the list
    }
    $data{'stdno'}=$stdno;
    #
    # Get the context data
    foreach my $type (qw/course goto comefrom/) {
        if (defined($resdata->{$type}) && 
            ref($resdata->{$type}) eq 'ARRAY') {
            $data{$type} = scalar(@{$resdata->{$type}});
            $data{$type.'_list'} = join(',',@{$resdata->{$type}});
        }
    }
    if (defined($resdata->{'usage'}) && 
        ref($resdata->{'usage'}) eq 'ARRAY') {
        $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
        $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
    }
    #
    # Get the evaluation data
    foreach my $type (qw/clear technical correct helpful depth/) {
        my $count;
        my $sum;
        foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
            $sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
            $count++;
        }
        if ($count > 0) {
            $data{$type}=$sum/$count;
        }
    }
    #
    # put together comments
    my $comments = '<div class="LCevalcomments">';
    foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
        $comments .= 
            '<p>'.
            '<b>'.$evaluator.'</b>:'.
            $resdata->{'evaluation'}->{'comments'}->{$evaluator}.
            '</p>';
    }
    $comments .= '</div>';
    $data{'comments'} = $comments;
    #
    if (exists($resdata->{'stats'})) {
        $data{'stats'} = $resdata->{'stats'};
    }
    if (exists($DynamicData->{'domain'})) {
        $data{'domain'} = $DynamicData->{'domain'};
    }
    #
    return %data;
}

sub dynamic_metadata_storage {
    my ($data) = @_;
    my %Store;
    my $courseid = $data->{'course'};
    my $sections = $data->{'sections'};
    my $numstu = $data->{'num_students'};
    my $urlres = $data->{'urlres'};
    my $key = $courseid.'&'.$sections.'&'.$numstu.'___'.$urlres.'___stats';
    $Store{$key} =
        'course='.$courseid.'&'.
        'sections='.$sections.'&'.
        'timestamp='.time.'&'.
        'stdno='.$data->{'num_students'}.'&'.
        'avetries='.$data->{'mean_tries'}.'&'.
        'difficulty='.$data->{'deg_of_diff'};
    if (exists($data->{'deg_of_disc'})) {
        $Store{$key} .= '&'.'disc='.$data->{'deg_of_disc'};
    }
    return %Store;
}

######################################################################
######################################################################
##
## The usual suspects, repeated here to reduce dependency hell
##
######################################################################
######################################################################
sub unescape {
    my $str=shift;
    $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
    return $str;
}

sub escape {
    my $str=shift;
    $str =~ s/(\W)/"%".unpack('H2',$1)/eg;
    return $str;
}

1;

__END__;

=pod

=back

=cut