--- loncom/metadata_database/LONCAPA/lonmetadata.pm	2007/05/12 23:03:30	1.20
+++ loncom/metadata_database/LONCAPA/lonmetadata.pm	2012/03/14 20:53:56	1.35
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonmetadata.pm,v 1.20 2007/05/12 23:03:30 albertel Exp $
+# $Id: lonmetadata.pm,v 1.35 2012/03/14 20:53:56 www Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -32,7 +32,7 @@ use strict;
 use DBI;
 use HTML::TokeParser;
 use vars qw($Metadata_Table_Description $Portfolio_metadata_table_description 
-$Portfolio_access_table_description $Fulltext_indicies $Portfolio_metadata_indices $Portfolio_access_indices $Portfolio_addedfields_table_description $Portfolio_addedfields_indices);
+$Portfolio_access_table_description $Fulltext_indicies $Portfolio_metadata_indices $Portfolio_access_indices $Portfolio_addedfields_table_description $Portfolio_addedfields_indices $Allusers_table_description $Allusers_indices);
 
 ######################################################################
 ######################################################################
@@ -93,7 +93,7 @@ FULLTEXT idx_language (language),
 FULLTEXT idx_owner (owner), 
 FULLTEXT idx_copyright (copyright)) 
 
-TYPE=MYISAM;
+ENGINE=MYISAM;
 
 =cut
 
@@ -247,6 +247,26 @@ $Portfolio_addedfields_indices = [qw/
 ######################################################################
 ######################################################################
 
+$Allusers_table_description =
+    [
+     { name => 'username',   type=>'TEXT', restrictions => 'NOT NULL' },
+     { name => 'domain', type=>'TEXT', restrictions => 'NOT NULL' },
+     { name => 'lastname', type=>'TEXT',},
+     { name => 'firstname', type=>'TEXT'},
+     { name => 'middlename', type=>'TEXT'},
+     { name => 'generation', type=>'TEXT'},
+     { name => 'permanentemail', type=>'TEXT'},
+     { name => 'id', type=>'TEXT'},
+   ];
+
+$Allusers_indices = [qw/
+    username
+    domain
+    lastname
+    firstname/];
+
+######################################################################
+######################################################################
 
 =pod
 
@@ -268,12 +288,14 @@ sub describe_metadata_storage {
         portfolio_metadata    => $Portfolio_metadata_table_description,
         portfolio_access      => $Portfolio_access_table_description,
         portfolio_addedfields => $Portfolio_addedfields_table_description, 
+        allusers              => $Allusers_table_description,
     );
     my %index_description = (
         metadata              => $Fulltext_indicies,
         portfolio_metadata    => $Portfolio_metadata_indices,
         portfolio_access      => $Portfolio_access_indices,
         portfolio_addedfields => $Portfolio_addedfields_indices,
+        allusers              => $Allusers_indices,
     );
     if ($tabletype eq 'portfolio_search') {
         my @portfolio_search_table = @{$table_description{portfolio_metadata}};
@@ -334,6 +356,9 @@ sub create_metadata_storage {
                 $col_des.="(".$coldata->{'size'}.")";
             }
         }
+        if (($tablename =~ /allusers/) && ($column eq 'username')) {  
+            $col_des .= ' CHARACTER SET latin1 COLLATE latin1_general_cs';
+        }
         # Modifiers
         if (exists($coldata->{'restrictions'})){
             $col_des.=" ".$coldata->{'restrictions'};
@@ -359,7 +384,7 @@ sub create_metadata_storage {
         $text .= 'idx_'.$colname.' ('.$colname.')';
         push (@Columns,$text);
     }
-    $request .= "(".join(", ",@Columns).") TYPE=MyISAM";
+    $request .= "(".join(", ",@Columns).") ENGINE=MyISAM";
     return $request;
 }
 
@@ -438,7 +463,8 @@ sub store_metadata {
                     $mdata->{$fname} eq '') {
                     push(@MData,'NULL');
                 } else {
-                    push(@MData,$mdata->{$fname});
+                    push(@MData, $field->{type} eq 'DATETIME' ? 
+                        sqltime($mdata->{$fname}) : $mdata->{$fname});
                 }
             } else {
                 push(@MData,undef);
@@ -516,7 +542,8 @@ Removes a single metadata record, based
 
 Inputs: $dbh, the database handler.
 $tablename, the name of the metadata table to remove from. default: 'metadata'
-$url, the url of the resource to remove from the metadata database.
+$delitem, the resource to remove from the metadata database, in the form: 
+          url = quoted url 
 
 Returns: undef on success, dbh errorstr on failure.
 
@@ -525,14 +552,17 @@ Returns: undef on success, dbh errorstr
 ######################################################################
 ######################################################################
 sub delete_metadata {
-    my ($dbh,$tablename,$url) = @_;
+    my ($dbh,$tablename,$delitem) = @_;
     $tablename = 'metadata' if (! defined($tablename));
-    my $error;
-    my $delete_command = 'DELETE FROM '.$tablename.' WHERE url='.
-        $dbh->quote($url);
-    $dbh->do($delete_command);
-    if ($dbh->err) {
-        $error = $dbh->errstr();
+    my ($error,$delete_command);
+    if ($delitem eq '') {
+        $error = 'deletion aborted - no resource specified';    
+    } else {
+        $delete_command = 'DELETE FROM '.$tablename.' WHERE '.$delitem;
+        $dbh->do($delete_command);
+        if ($dbh->err) {
+            $error = $dbh->errstr();
+        }
     }
     return $error;
 }
@@ -553,7 +583,10 @@ Inputs:
 $dbh, database handle
 $newmetadata, hash reference containing the new metadata
 $tablename, metadata table name.  Defaults to 'metadata'.
-$tabletype, type of table (metadata, portfolio_metadata, portfolio_access)  
+$tabletype, type of table (metadata, portfolio_metadata, portfolio_access, 
+                           allusers)
+$conditions, optional hash of conditions to use in SQL queries; 
+             default used if none provided.
 
 Returns:
 $error on failure.  undef on success.
@@ -563,20 +596,32 @@ $error on failure.  undef on success.
 ######################################################################
 ######################################################################
 sub update_metadata {
-    my ($dbh,$tablename,$tabletype,$newmetadata)=@_;
-    my $error;
+    my ($dbh,$tablename,$tabletype,$newmetadata,$conditions)=@_;
+    my ($error,$condition);
     $tablename = 'metadata' if (! defined($tablename));
     $tabletype = 'metadata' if (! defined($tabletype));
-    if (! exists($newmetadata->{'url'})) {
-        $error = 'Unable to update: no url specified';
+    if (ref($conditions) eq 'HASH') {
+        my @items;
+        foreach my $key (keys(%{$conditions})) {
+            if (! exists($newmetadata->{$key})) {
+                $error .= "Unable to update: no $key specified";
+            } else {
+                push(@items,"$key = ".$dbh->quote($newmetadata->{$key}));
+            }
+        }
+        $condition = join(' AND ',@items); 
+    } else {
+        if (! exists($newmetadata->{'url'})) {
+            $error = 'Unable to update: no url specified';
+        } else {
+            $condition = 'url = '.$dbh->quote($newmetadata->{'url'});
+        }
     }
     return $error if (defined($error));
     # 
     # Retrieve current values
     my $row;
-    ($error,$row) = &lookup_metadata($dbh,
-                                   ' url='.$dbh->quote($newmetadata->{'url'}),
-                                     undef,$tablename);
+    ($error,$row) = &lookup_metadata($dbh,$condition,undef,$tablename);
     return $error if ($error);
     my %metadata = &LONCAPA::lonmetadata::metadata_col_to_hash($tabletype,@{$row->[0]});
     #
@@ -586,7 +631,7 @@ sub update_metadata {
     }
     #
     # Delete old data (deleting a nonexistant record does not produce an error.
-    $error = &delete_metadata($dbh,$tablename,$newmetadata->{'url'});
+    $error = &delete_metadata($dbh,$tablename,$condition);
     return $error if (defined($error));
     #
     # Store updated metadata
@@ -795,23 +840,17 @@ sub process_dynamic_metadata {
     # Get the statistical data - Use a weighted average
     foreach my $type (qw/avetries difficulty disc/) {
         my $studentcount;
+	my %course_counted;
         my $sum;
         my @Values;
         my @Students;
         #
-        # Old data
-        foreach my $coursedata (values(%{$resdata->{'statistics'}}),
-                                values(%{$resdata->{'stats'}})) {
-            if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
-                $studentcount += $coursedata->{'stdno'};
-                $sum += ($coursedata->{$type}*$coursedata->{'stdno'});
-                push(@Values,$coursedata->{$type});
-                push(@Students,$coursedata->{'stdno'});
-            }
-        }
+        # New data
         if (exists($resdata->{'stats'})) {
             foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
                 my $coursedata = $resdata->{'stats'}->{$identifier};
+		next if (lc($coursedata->{$type}) eq 'nan');
+		$course_counted{$coursedata->{'course'}}++;
                 $studentcount += $coursedata->{'stdno'};
                 $sum += $coursedata->{$type}*$coursedata->{'stdno'};
                 push(@Values,$coursedata->{$type});                
@@ -819,7 +858,18 @@ sub process_dynamic_metadata {
             }
         }
         #
-        # New data
+        # Old data
+	foreach my $course (keys(%{$resdata->{'statistics'}})) {
+	    next if (exists($course_counted{$course}));
+	    my $coursedata = $resdata->{'statistics'}{$course};
+            if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
+		next if (lc($coursedata->{$type}) eq 'nan');
+                $studentcount += $coursedata->{'stdno'};
+                $sum += ($coursedata->{$type}*$coursedata->{'stdno'});
+                push(@Values,$coursedata->{$type});
+                push(@Students,$coursedata->{'stdno'});
+            }
+        }
         if (defined($studentcount) && $studentcount>0) {
             $data{$type} = $sum/$studentcount;
             $data{$type.'_list'} = join(',',@Values);
@@ -869,6 +919,10 @@ sub process_dynamic_metadata {
             $data{$type.'_list'} = join(',',@{$resdata->{$type}});
         }
     }
+#
+# NOTE: usage is named sequsage elsewhere in LON-CAPA
+# The translation happens here
+#
     if (defined($resdata->{'usage'}) && 
         ref($resdata->{'usage'}) eq 'ARRAY') {
         $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
@@ -889,16 +943,20 @@ sub process_dynamic_metadata {
     }
     #
     # put together comments
-    my $comments = '<div class="LCevalcomments">';
+    my $comments = '';
     foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
         $comments .= 
             '<p>'.
-            '<b>'.$evaluator.'</b>:'.
+            '<b>'.$evaluator.'</b>: '.
             $resdata->{'evaluation'}->{'comments'}->{$evaluator}.
             '</p>';
     }
-    $comments .= '</div>';
-    $data{'comments'} = $comments;
+    if ($comments) {
+        $comments = '<div class="LCevalcomments">'
+                   .$comments
+                   .'</div>';
+        $data{'comments'} = $comments;
+    }
     #
     if (exists($resdata->{'stats'})) {
         $data{'stats'} = $resdata->{'stats'};
@@ -995,6 +1053,10 @@ sub portfolio_metadata {
                 }
             }
         } # End of ($token->[0] eq 'S')
+
+	if (!exists($metacache{'domain'})) {
+	    $metacache{'domain'} = $dom;
+	}
     }
     return (\%metacache,$metacache{'courserestricted'},\%addedfields);
 }
@@ -1004,7 +1066,13 @@ sub process_portfolio_access_data {
     my %loghash;
     if ($caller eq 'update') {
         # Delete old data (no error if deleting non-existent record).
-        my $error=&delete_metadata($dbh,$newnames->{'access'},$url);
+        my $error;
+        if ($url eq '') {
+            $error = 'No url specified'; 
+        } else {
+            my $delitem = 'url = '.$dbh->quote($url);
+            $error=&delete_metadata($dbh,$newnames->{'access'},$delitem);
+        }
         if (defined($error)) {
             $loghash{'access'}{'err'} = "MySQL Error Delete: ".$error;
             return %loghash;
@@ -1020,10 +1088,14 @@ sub process_portfolio_access_data {
                             ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/);
             next if (($scope ne 'public') && ($scope ne 'guest'));
             $acc_data->{scope} = $scope;
+            my $sqltime_error;
             if ($end != 0) {
-                $acc_data->{end} = &sqltime($end);
+                $acc_data->{end} = &sqltime($end,\$sqltime_error);
+            }
+            $acc_data->{start} = &sqltime($start,\$sqltime_error);
+            if ($sqltime_error) {
+                $loghash{$key}{'err'} = $sqltime_error;
             }
-            $acc_data->{start} = &sqltime($start);
             if (! $simulate) {
                 my ($count,$err) =
                      &store_metadata($dbh,$newnames->{'access'},
@@ -1046,13 +1118,19 @@ sub process_portfolio_metadata {
     my %loghash;
     if ($caller eq 'update') {
         # Delete old data (no error if deleting non-existent record).
-        my $error=&delete_metadata($dbh,$newnames->{'portfolio'},$url);
+        my ($error,$delitem);
+        if ($url eq '') {
+            $error = 'No url specified';
+        } else {
+            $delitem = 'url = '.$dbh->quote($url);
+            $error=&delete_metadata($dbh,$newnames->{'portfolio'},$delitem);
+        }
         if (defined($error)) {
             $loghash{'metadata'}{'err'} = "MySQL Error delete metadata: ".
                                                $error;
             return %loghash;
         }
-        $error=&delete_metadata($dbh,$newnames->{'addedfields'},$url);
+        $error=&delete_metadata($dbh,$newnames->{'addedfields'},$delitem);
         if (defined($error)) {
             $loghash{'addedfields'}{'err'}="MySQL Error delete addedfields: ".$error;
         }
@@ -1061,7 +1139,8 @@ sub process_portfolio_metadata {
     if (-e $fullpath) {
         my ($ref,$crs,$addedfields) = &portfolio_metadata($fullpath,$dom,$uname,
                                                           $group);
-        &getfiledates($ref,$fullpath);
+        my $sqltime_error;
+        &getfiledates($ref,$fullpath,\$sqltime_error);
         if ($is_course) {
             $ref->{'groupname'} = $group;
         }
@@ -1076,6 +1155,9 @@ sub process_portfolio_metadata {
         );
         my %loghash;
         if (! $simulate) {
+            if ($sqltime_error) {
+                $loghash{'metadata'."\0"}{'err'} = $sqltime_error;
+            }
             my ($count,$err) =
             &store_metadata($dbh,$newnames->{'portfolio'},'portfolio_metadata',
                             \%Data);
@@ -1112,6 +1194,44 @@ sub process_portfolio_metadata {
     return %loghash;
 }
 
+sub process_allusers_data {
+    my ($dbh,$simulate,$newnames,$uname,$udom,$userdata,$caller) = @_;
+    my %loghash;
+    if ($caller eq 'update') {
+        # Delete old data (no error if deleting non-existent record).
+        my ($error,$delitem);
+        if ($udom eq '' || $uname eq '' ) {
+            $error = 'No domain and/or username specified';
+        } else {
+            $delitem = 'domain = '.$dbh->quote($udom).' AND username '.
+                       'COLLATE latin1_general_cs = '.$dbh->quote($uname);
+            $error=&delete_metadata($dbh,$newnames->{'allusers'},$delitem);
+        }
+        if (defined($error)) {
+            $loghash{'err'} = 'MySQL Error in allusers delete: '.$error;
+            return %loghash;
+        }
+    }
+    if (!$simulate) {
+        if ($udom ne '' && $uname ne '') {
+            my ($count,$err) = &store_metadata($dbh,$newnames->{'allusers'},
+                                               'allusers',$userdata);
+            if ($err) {
+                $loghash{'err'} = 'MySQL Error in allusers insert: '.$err;
+            }
+            if ($count < 1) {
+                $loghash{'count'} = 
+                    'Unable to insert record into MySQL allusers database for '.
+                    $uname.' in '.$udom;
+            }
+        } else {
+            $loghash{'err'} = 
+                'MySQL Error allusrs insert: missing username and/or domain';
+        }
+    }
+    return %loghash;
+}
+
 ######################################################################
 ######################################################################
 
@@ -1129,11 +1249,11 @@ sub getfile {
 }
 
 ##
-## &getfiledates()
+## &getfiledates($ref,$target,$sqltime_error)
 ## Converts creationdate and modifieddates to SQL format
 ## Applies stat() to file to retrieve dates if missing
 sub getfiledates {
-    my ($ref,$target) = @_;
+    my ($ref,$target,$sqltime_error) = @_;
     if (! defined($ref->{'creationdate'}) ||
         $ref->{'creationdate'} =~ /^\s*$/) {
         $ref->{'creationdate'} = (stat($target))[9];
@@ -1142,17 +1262,17 @@ sub getfiledates {
         $ref->{'lastrevisiondate'} =~ /^\s*$/) {
         $ref->{'lastrevisiondate'} = (stat($target))[9];
     }
-    $ref->{'creationdate'}     = &sqltime($ref->{'creationdate'});
-    $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'});
+    $ref->{'creationdate'}     = &sqltime($ref->{'creationdate'},$sqltime_error);
+    $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'},$sqltime_error);
 }
  
 ##
-## &sqltime($timestamp)
+## &sqltime($timestamp,$sqltime_error)
 ##
 ## Convert perl $timestamp to MySQL time.  MySQL expects YYYY-MM-DD HH:MM:SS
 ##
 sub sqltime {
-    my ($time) = @_;
+    my ($time,$sqltime_error) = @_;
     my $mysqltime;
     if ($time =~
         /(\d+)-(\d+)-(\d+) # YYYY-MM-DD
@@ -1175,7 +1295,9 @@ sub sqltime {
     } elsif (! defined($time) || $time == 0) {
         $mysqltime = 0;
     } else {
-        &log(0,"    sqltime:Unable to decode time ".$time);
+        if (ref($sqltime_error) eq 'SCALAR') {
+            $$sqltime_error = "sqltime:Unable to decode time ".$time;
+        }
         $mysqltime = 0;
     }
     return $mysqltime;