--- loncom/metadata_database/LONCAPA/lonmetadata.pm 2006/12/29 19:15:28 1.15
+++ loncom/metadata_database/LONCAPA/lonmetadata.pm 2011/05/31 14:45:53 1.33
@@ -1,6 +1,6 @@
# The LearningOnline Network with CAPA
#
-# $Id: lonmetadata.pm,v 1.15 2006/12/29 19:15:28 raeburn Exp $
+# $Id: lonmetadata.pm,v 1.33 2011/05/31 14:45:53 raeburn Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -30,8 +30,9 @@ package LONCAPA::lonmetadata;
use strict;
use DBI;
+use HTML::TokeParser;
use vars qw($Metadata_Table_Description $Portfolio_metadata_table_description
-$Portfolio_access_table_description $Fulltext_indicies $Portfolio_metadata_indices $Portfolio_access_indices $Portfolio_addedfields_table_description $Portfolio_addedfields_indices);
+$Portfolio_access_table_description $Fulltext_indicies $Portfolio_metadata_indices $Portfolio_access_indices $Portfolio_addedfields_table_description $Portfolio_addedfields_indices $Allusers_table_description $Allusers_indices);
######################################################################
######################################################################
@@ -92,7 +93,7 @@ FULLTEXT idx_language (language),
FULLTEXT idx_owner (owner),
FULLTEXT idx_copyright (copyright))
-TYPE=MYISAM;
+ENGINE=MYISAM;
=cut
@@ -183,8 +184,6 @@ $Portfolio_metadata_table_description =
{ name => 'domain', type=>'TEXT'},
{ name => 'groupname', type=>'TEXT'},
{ name => 'courserestricted', type=>'TEXT'},
- { name => 'addedfieldnames', type=>'TEXT'},
- { name => 'addedfieldvalues', type=>'TEXT'},
#--------------------------------------------------
{ name => 'dependencies', type=>'TEXT'},
{ name => 'modifyinguser', type=>'TEXT'},
@@ -248,6 +247,26 @@ $Portfolio_addedfields_indices = [qw/
######################################################################
######################################################################
+$Allusers_table_description =
+ [
+ { name => 'username', type=>'TEXT', restrictions => 'NOT NULL' },
+ { name => 'domain', type=>'TEXT', restrictions => 'NOT NULL' },
+ { name => 'lastname', type=>'TEXT',},
+ { name => 'firstname', type=>'TEXT'},
+ { name => 'middlename', type=>'TEXT'},
+ { name => 'generation', type=>'TEXT'},
+ { name => 'permanentemail', type=>'TEXT'},
+ { name => 'id', type=>'TEXT'},
+ ];
+
+$Allusers_indices = [qw/
+ username
+ domain
+ lastname
+ firstname/];
+
+######################################################################
+######################################################################
=pod
@@ -269,12 +288,14 @@ sub describe_metadata_storage {
portfolio_metadata => $Portfolio_metadata_table_description,
portfolio_access => $Portfolio_access_table_description,
portfolio_addedfields => $Portfolio_addedfields_table_description,
+ allusers => $Allusers_table_description,
);
my %index_description = (
metadata => $Fulltext_indicies,
portfolio_metadata => $Portfolio_metadata_indices,
portfolio_access => $Portfolio_access_indices,
portfolio_addedfields => $Portfolio_addedfields_indices,
+ allusers => $Allusers_indices,
);
if ($tabletype eq 'portfolio_search') {
my @portfolio_search_table = @{$table_description{portfolio_metadata}};
@@ -335,6 +356,9 @@ sub create_metadata_storage {
$col_des.="(".$coldata->{'size'}.")";
}
}
+ if (($tablename =~ /allusers/) && ($column eq 'username')) {
+ $col_des .= ' CHARACTER SET latin1 COLLATE latin1_general_cs';
+ }
# Modifiers
if (exists($coldata->{'restrictions'})){
$col_des.=" ".$coldata->{'restrictions'};
@@ -360,7 +384,7 @@ sub create_metadata_storage {
$text .= 'idx_'.$colname.' ('.$colname.')';
push (@Columns,$text);
}
- $request .= "(".join(", ",@Columns).") TYPE=MyISAM";
+ $request .= "(".join(", ",@Columns).") ENGINE=MyISAM";
return $request;
}
@@ -502,7 +526,7 @@ sub lookup_metadata {
$error = $sth->errstr;
}
}
- }
+ }
return ($error,$returnvalue);
}
@@ -517,7 +541,8 @@ Removes a single metadata record, based
Inputs: $dbh, the database handler.
$tablename, the name of the metadata table to remove from. default: 'metadata'
-$url, the url of the resource to remove from the metadata database.
+$delitem, the resource to remove from the metadata database, in the form:
+ url = quoted url
Returns: undef on success, dbh errorstr on failure.
@@ -526,14 +551,17 @@ Returns: undef on success, dbh errorstr
######################################################################
######################################################################
sub delete_metadata {
- my ($dbh,$tablename,$url) = @_;
+ my ($dbh,$tablename,$delitem) = @_;
$tablename = 'metadata' if (! defined($tablename));
- my $error;
- my $delete_command = 'DELETE FROM '.$tablename.' WHERE url='.
- $dbh->quote($url);
- $dbh->do($delete_command);
- if ($dbh->err) {
- $error = $dbh->errstr();
+ my ($error,$delete_command);
+ if ($delitem eq '') {
+ $error = 'deletion aborted - no resource specified';
+ } else {
+ $delete_command = 'DELETE FROM '.$tablename.' WHERE '.$delitem;
+ $dbh->do($delete_command);
+ if ($dbh->err) {
+ $error = $dbh->errstr();
+ }
}
return $error;
}
@@ -554,7 +582,10 @@ Inputs:
$dbh, database handle
$newmetadata, hash reference containing the new metadata
$tablename, metadata table name. Defaults to 'metadata'.
-$tabletype, type of table (metadata, portfolio_metadata, portfolio_access)
+$tabletype, type of table (metadata, portfolio_metadata, portfolio_access,
+ allusers)
+$conditions, optional hash of conditions to use in SQL queries;
+ default used if none provided.
Returns:
$error on failure. undef on success.
@@ -564,20 +595,32 @@ $error on failure. undef on success.
######################################################################
######################################################################
sub update_metadata {
- my ($dbh,$tablename,$tabletype,$newmetadata)=@_;
- my $error;
+ my ($dbh,$tablename,$tabletype,$newmetadata,$conditions)=@_;
+ my ($error,$condition);
$tablename = 'metadata' if (! defined($tablename));
$tabletype = 'metadata' if (! defined($tabletype));
- if (! exists($newmetadata->{'url'})) {
- $error = 'Unable to update: no url specified';
+ if (ref($conditions) eq 'HASH') {
+ my @items;
+ foreach my $key (keys(%{$conditions})) {
+ if (! exists($newmetadata->{$key})) {
+ $error .= "Unable to update: no $key specified";
+ } else {
+ push(@items,"$key = ".$dbh->quote($newmetadata->{$key}));
+ }
+ }
+ $condition = join(' AND ',@items);
+ } else {
+ if (! exists($newmetadata->{'url'})) {
+ $error = 'Unable to update: no url specified';
+ } else {
+ $condition = 'url = '.$dbh->quote($newmetadata->{'url'});
+ }
}
return $error if (defined($error));
#
# Retrieve current values
my $row;
- ($error,$row) = &lookup_metadata($dbh,
- ' url='.$dbh->quote($newmetadata->{'url'}),
- undef,$tablename);
+ ($error,$row) = &lookup_metadata($dbh,$condition,undef,$tablename);
return $error if ($error);
my %metadata = &LONCAPA::lonmetadata::metadata_col_to_hash($tabletype,@{$row->[0]});
#
@@ -587,7 +630,7 @@ sub update_metadata {
}
#
# Delete old data (deleting a nonexistant record does not produce an error.
- $error = &delete_metadata($dbh,$tablename,$newmetadata->{'url'});
+ $error = &delete_metadata($dbh,$tablename,$condition);
return $error if (defined($error));
#
# Store updated metadata
@@ -796,23 +839,17 @@ sub process_dynamic_metadata {
# Get the statistical data - Use a weighted average
foreach my $type (qw/avetries difficulty disc/) {
my $studentcount;
+ my %course_counted;
my $sum;
my @Values;
my @Students;
#
- # Old data
- foreach my $coursedata (values(%{$resdata->{'statistics'}}),
- values(%{$resdata->{'stats'}})) {
- if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
- $studentcount += $coursedata->{'stdno'};
- $sum += ($coursedata->{$type}*$coursedata->{'stdno'});
- push(@Values,$coursedata->{$type});
- push(@Students,$coursedata->{'stdno'});
- }
- }
+ # New data
if (exists($resdata->{'stats'})) {
foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
my $coursedata = $resdata->{'stats'}->{$identifier};
+ next if (lc($coursedata->{$type}) eq 'nan');
+ $course_counted{$coursedata->{'course'}}++;
$studentcount += $coursedata->{'stdno'};
$sum += $coursedata->{$type}*$coursedata->{'stdno'};
push(@Values,$coursedata->{$type});
@@ -820,7 +857,18 @@ sub process_dynamic_metadata {
}
}
#
- # New data
+ # Old data
+ foreach my $course (keys(%{$resdata->{'statistics'}})) {
+ next if (exists($course_counted{$course}));
+ my $coursedata = $resdata->{'statistics'}{$course};
+ if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
+ next if (lc($coursedata->{$type}) eq 'nan');
+ $studentcount += $coursedata->{'stdno'};
+ $sum += ($coursedata->{$type}*$coursedata->{'stdno'});
+ push(@Values,$coursedata->{$type});
+ push(@Students,$coursedata->{'stdno'});
+ }
+ }
if (defined($studentcount) && $studentcount>0) {
$data{$type} = $sum/$studentcount;
$data{$type.'_list'} = join(',',@Values);
@@ -829,12 +877,7 @@ sub process_dynamic_metadata {
#
# Find out the number of students who have completed the resource...
my $stdno;
- foreach my $coursedata (values(%{$resdata->{'statistics'}}),
- values(%{$resdata->{'stats'}})) {
- if (ref($coursedata) eq 'HASH' && exists($coursedata->{'stdno'})) {
- $stdno += $coursedata->{'stdno'};
- }
- }
+ my %course_counted;
if (exists($resdata->{'stats'})) {
#
# For the number of students, take the maximum found for the class
@@ -847,6 +890,7 @@ sub process_dynamic_metadata {
}
if ($current_course ne $coursedata->{'course'}) {
$stdno += $coursemax;
+ $course_counted{$coursedata->{'course'}}++;
$coursemax = 0;
$current_course = $coursedata->{'course'};
}
@@ -856,6 +900,14 @@ sub process_dynamic_metadata {
}
$stdno += $coursemax; # pick up the final course in the list
}
+ # check for old data that has not been run since the format was changed
+ foreach my $course (keys(%{$resdata->{'statistics'}})) {
+ next if (exists($course_counted{$course}));
+ my $coursedata = $resdata->{'statistics'}{$course};
+ if (ref($coursedata) eq 'HASH' && exists($coursedata->{'stdno'})) {
+ $stdno += $coursedata->{'stdno'};
+ }
+ }
$data{'stdno'}=$stdno;
#
# Get the context data
@@ -886,16 +938,20 @@ sub process_dynamic_metadata {
}
#
# put together comments
- my $comments = '';
- $data{'comments'} = $comments;
+ if ($comments) {
+ $comments = '';
+ $data{'comments'} = $comments;
+ }
#
if (exists($resdata->{'stats'})) {
$data{'stats'} = $resdata->{'stats'};
@@ -928,17 +984,290 @@ sub dynamic_metadata_storage {
return %Store;
}
+###############################################################
+###############################################################
+### ###
+### &portfolio_metadata($filepath,$dom,$uname,$group) ###
+### Retrieve metadata for the given file ###
+### Returns array - ###
+### contains reference to metadatahash and ###
+### optional reference to addedfields hash ###
+### ###
+###############################################################
+###############################################################
+
+sub portfolio_metadata {
+ my ($fullpath,$dom,$uname,$group)=@_;
+ my ($mime) = ( $fullpath=~/\.(\w+)$/ );
+ my %metacache=();
+ if ($fullpath !~ /\.meta$/) {
+ $fullpath .= '.meta';
+ }
+ my (@standard_fields,%addedfields);
+ my $colsref = $Portfolio_metadata_table_description;
+ if (ref($colsref) eq 'ARRAY') {
+ my @columns = @{$colsref};
+ foreach my $coldata (@columns) {
+ push(@standard_fields,$coldata->{'name'});
+ }
+ }
+ my $metastring=&getfile($fullpath);
+ if (! defined($metastring)) {
+ $metacache{'keys'}= 'owner,domain,mime';
+ $metacache{'owner'} = $uname.':'.$dom;
+ $metacache{'domain'} = $dom;
+ $metacache{'mime'} = $mime;
+ if ($group ne '') {
+ $metacache{'keys'} .= ',courserestricted';
+ $metacache{'courserestricted'} = 'course.'.$dom.'_'.$uname;
+ }
+ } else {
+ my $parser=HTML::TokeParser->new(\$metastring);
+ my $token;
+ while ($token=$parser->get_token) {
+ if ($token->[0] eq 'S') {
+ my $entry=$token->[1];
+ if ($metacache{'keys'}) {
+ $metacache{'keys'}.=','.$entry;
+ } else {
+ $metacache{'keys'}=$entry;
+ }
+ my $value = $parser->get_text('/'.$entry);
+ if (!grep(/^\Q$entry\E$/,@standard_fields)) {
+ my $clean_value = lc($value);
+ $clean_value =~ s/\s/_/g;
+ if ($clean_value ne $entry) {
+ if (defined($addedfields{$entry})) {
+ $addedfields{$entry} .=','.$value;
+ } else {
+ $addedfields{$entry} = $value;
+ }
+ }
+ } else {
+ $metacache{$entry} = $value;
+ }
+ }
+ } # End of ($token->[0] eq 'S')
+
+ if (!exists($metacache{'domain'})) {
+ $metacache{'domain'} = $dom;
+ }
+ }
+ return (\%metacache,$metacache{'courserestricted'},\%addedfields);
+}
+
+sub process_portfolio_access_data {
+ my ($dbh,$simulate,$newnames,$url,$fullpath,$access_hash,$caller) = @_;
+ my %loghash;
+ if ($caller eq 'update') {
+ # Delete old data (no error if deleting non-existent record).
+ my $error;
+ if ($url eq '') {
+ $error = 'No url specified';
+ } else {
+ my $delitem = 'url = '.$dbh->quote($url);
+ $error=&delete_metadata($dbh,$newnames->{'access'},$delitem);
+ }
+ if (defined($error)) {
+ $loghash{'access'}{'err'} = "MySQL Error Delete: ".$error;
+ return %loghash;
+ }
+ }
+ # Check the file exists
+ if (-e $fullpath) {
+ foreach my $key (keys(%{$access_hash})) {
+ my $acc_data;
+ $acc_data->{url} = $url;
+ $acc_data->{keynum} = $key;
+ my ($num,$scope,$end,$start) =
+ ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/);
+ next if (($scope ne 'public') && ($scope ne 'guest'));
+ $acc_data->{scope} = $scope;
+ my $sqltime_error;
+ if ($end != 0) {
+ $acc_data->{end} = &sqltime($end,\$sqltime_error);
+ }
+ $acc_data->{start} = &sqltime($start,\$sqltime_error);
+ if ($sqltime_error) {
+ $loghash{$key}{'err'} = $sqltime_error;
+ }
+ if (! $simulate) {
+ my ($count,$err) =
+ &store_metadata($dbh,$newnames->{'access'},
+ 'portfolio_access',$acc_data);
+ if ($err) {
+ $loghash{$key}{'err'} = "MySQL Error Insert: ".$err;
+ }
+ if ($count < 1) {
+ $loghash{$key}{'count'} =
+ "Unable to insert record into MySQL database for $url";
+ }
+ }
+ }
+ }
+ return %loghash;
+}
+
+sub process_portfolio_metadata {
+ my ($dbh,$simulate,$newnames,$url,$fullpath,$is_course,$dom,$uname,$group,$caller) = @_;
+ my %loghash;
+ if ($caller eq 'update') {
+ # Delete old data (no error if deleting non-existent record).
+ my ($error,$delitem);
+ if ($url eq '') {
+ $error = 'No url specified';
+ } else {
+ $delitem = 'url = '.$dbh->quote($url);
+ $error=&delete_metadata($dbh,$newnames->{'portfolio'},$delitem);
+ }
+ if (defined($error)) {
+ $loghash{'metadata'}{'err'} = "MySQL Error delete metadata: ".
+ $error;
+ return %loghash;
+ }
+ $error=&delete_metadata($dbh,$newnames->{'addedfields'},$delitem);
+ if (defined($error)) {
+ $loghash{'addedfields'}{'err'}="MySQL Error delete addedfields: ".$error;
+ }
+ }
+ # Check the file exists.
+ if (-e $fullpath) {
+ my ($ref,$crs,$addedfields) = &portfolio_metadata($fullpath,$dom,$uname,
+ $group);
+ my $sqltime_error;
+ &getfiledates($ref,$fullpath,\$sqltime_error);
+ if ($is_course) {
+ $ref->{'groupname'} = $group;
+ }
+ my %Data;
+ if (ref($ref) eq 'HASH') {
+ %Data = %{$ref};
+ }
+ %Data = (
+ %Data,
+ 'url'=>$url,
+ 'version'=>'current',
+ );
+ my %loghash;
+ if (! $simulate) {
+ if ($sqltime_error) {
+ $loghash{'metadata'."\0"}{'err'} = $sqltime_error;
+ }
+ my ($count,$err) =
+ &store_metadata($dbh,$newnames->{'portfolio'},'portfolio_metadata',
+ \%Data);
+ if ($err) {
+ $loghash{'metadata'."\0"}{'err'} = "MySQL Error Insert: ".$err;
+ }
+ if ($count < 1) {
+ $loghash{'metadata'."\0"}{'count'} = "Unable to insert record into MySQL portfolio_metadata database table for $url";
+ }
+ if (ref($addedfields) eq 'HASH') {
+ if (keys(%{$addedfields}) > 0) {
+ foreach my $key (keys(%{$addedfields})) {
+ my $added_data = {
+ 'url' => $url,
+ 'field' => $key,
+ 'value' => $addedfields->{$key},
+ 'courserestricted' => $crs,
+ };
+ my ($count,$err) =
+ &store_metadata($dbh,$newnames->{'addedfields'},
+ 'portfolio_addedfields',$added_data);
+ if ($err) {
+ $loghash{$key}{'err'} =
+ "MySQL Error Insert: ".$err;
+ }
+ if ($count < 1) {
+ $loghash{$key}{'count'} = "Unable to insert record into MySQL portfolio_addedfields database table for url = $url and field = $key";
+ }
+ }
+ }
+ }
+ }
+ }
+ return %loghash;
+}
+
+sub process_allusers_data {
+ my ($dbh,$simulate,$newnames,$uname,$udom,$userdata,$caller) = @_;
+ my %loghash;
+ if ($caller eq 'update') {
+ # Delete old data (no error if deleting non-existent record).
+ my ($error,$delitem);
+ if ($udom eq '' || $uname eq '' ) {
+ $error = 'No domain and/or username specified';
+ } else {
+ $delitem = 'domain = '.$dbh->quote($udom).' AND username '.
+ 'COLLATE latin1_general_cs = '.$dbh->quote($uname);
+ $error=&delete_metadata($dbh,$newnames->{'allusers'},$delitem);
+ }
+ if (defined($error)) {
+ $loghash{'err'} = 'MySQL Error in allusers delete: '.$error;
+ return %loghash;
+ }
+ }
+ if (!$simulate) {
+ if ($udom ne '' && $uname ne '') {
+ my ($count,$err) = &store_metadata($dbh,$newnames->{'allusers'},
+ 'allusers',$userdata);
+ if ($err) {
+ $loghash{'err'} = 'MySQL Error in allusers insert: '.$err;
+ }
+ if ($count < 1) {
+ $loghash{'count'} =
+ 'Unable to insert record into MySQL allusers database for '.
+ $uname.' in '.$udom;
+ }
+ } else {
+ $loghash{'err'} =
+ 'MySQL Error allusrs insert: missing username and/or domain';
+ }
+ }
+ return %loghash;
+}
+
######################################################################
######################################################################
-## Utility originally in searchcat.pl. Moved to be more widely available.
+sub getfile {
+ my $file = shift();
+ if (! -e $file ) {
+ return undef;
+ }
+ open(my $fh,"<$file");
+ my $contents = '';
+ while (<$fh>) {
+ $contents .= $_;
+ }
+ return $contents;
+}
+
+##
+## &getfiledates($ref,$target,$sqltime_error)
+## Converts creationdate and modifieddates to SQL format
+## Applies stat() to file to retrieve dates if missing
+sub getfiledates {
+ my ($ref,$target,$sqltime_error) = @_;
+ if (! defined($ref->{'creationdate'}) ||
+ $ref->{'creationdate'} =~ /^\s*$/) {
+ $ref->{'creationdate'} = (stat($target))[9];
+ }
+ if (! defined($ref->{'lastrevisiondate'}) ||
+ $ref->{'lastrevisiondate'} =~ /^\s*$/) {
+ $ref->{'lastrevisiondate'} = (stat($target))[9];
+ }
+ $ref->{'creationdate'} = &sqltime($ref->{'creationdate'},$sqltime_error);
+ $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'},$sqltime_error);
+}
+
##
-## &sqltime($timestamp)
+## &sqltime($timestamp,$sqltime_error)
##
## Convert perl $timestamp to MySQL time. MySQL expects YYYY-MM-DD HH:MM:SS
##
sub sqltime {
- my ($time) = @_;
+ my ($time,$sqltime_error) = @_;
my $mysqltime;
if ($time =~
/(\d+)-(\d+)-(\d+) # YYYY-MM-DD
@@ -961,7 +1290,9 @@ sub sqltime {
} elsif (! defined($time) || $time == 0) {
$mysqltime = 0;
} else {
- &log(0," sqltime:Unable to decode time ".$time);
+ if (ref($sqltime_error) eq 'SCALAR') {
+ $$sqltime_error = "sqltime:Unable to decode time ".$time;
+ }
$mysqltime = 0;
}
return $mysqltime;
'. - ''.$evaluator.':'. + ''.$evaluator.': '. $resdata->{'evaluation'}->{'comments'}->{$evaluator}. '
'; } - $comments .= '