--- loncom/metadata_database/searchcat.pl	2007/01/02 07:26:11	1.73
+++ loncom/metadata_database/searchcat.pl	2013/07/25 19:11:10	1.79
@@ -2,7 +2,7 @@
 # The LearningOnline Network
 # searchcat.pl "Search Catalog" batch script
 #
-# $Id: searchcat.pl,v 1.73 2007/01/02 07:26:11 raeburn Exp $
+# $Id: searchcat.pl,v 1.79 2013/07/25 19:11:10 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -68,7 +68,7 @@ use strict;
 use DBI;
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::lonmetadata;
-
+use LONCAPA;
 use Getopt::Long;
 use IO::File;
 use HTML::TokeParser;
@@ -123,6 +123,7 @@ my %oldnames = (
                  'portfolio'   => 'portfolio_metadata',
                  'access'      => 'portfolio_access',
                  'addedfields' => 'portfolio_addedfields',
+                 'allusers'    => 'allusers',
                );
 
 my %newnames;
@@ -134,6 +135,8 @@ foreach my $key (keys(%oldnames)) {
 #
 # Only run if machine is a library server
 exit if ($Apache::lonnet::perlvar{'lonRole'} ne 'library');
+my $hostid = $Apache::lonnet::perlvar{'lonHostID'};
+
 #
 #  Make sure this process is running from user=www
 my $wwwid=getpwnam('www');
@@ -172,7 +175,7 @@ foreach my $key (keys(%newnames)) {
 }
 
 #
-# Create the new metadata and portfolio tables
+# Create the new metadata, portfolio and allusers tables
 foreach my $key (keys(%newnames)) {
     if ($newnames{$key} ne '') { 
         my $request =
@@ -221,23 +224,33 @@ foreach my $dom (@domains) {
               no_chdir   => 1,
              }, join('/',($Apache::lonnet::perlvar{'lonDocRoot'},'res',$dom,$user)) );
     }
-    # Search for public portfolio files
-    my %portusers;
+    # Search for all users and public portfolio files
+    my (%allusers,%portusers,%courses);
     if ($oneuser) {
         %portusers = (
                         $oneuser => '',
                        );
+        %allusers = (
+                        $oneuser => '',
+                       );
+        %courses = &courseiddump($dom,'.',1,'.','.',$oneuser,undef,
+                                 undef,'.');
     } else {
+        # get courseIDs for domain on current machine
+        %courses=&Apache::lonnet::courseiddump($dom,'.',1,'.','.','.',1,[$hostid],'.');
         my $dir = $Apache::lonnet::perlvar{lonUsersDir}.'/'.$dom;
-        &descend_tree($dir,0,\%portusers);
+        &descend_tree($dom,$dir,0,\%portusers,\%allusers);
     }
     foreach my $uname (keys(%portusers)) {
         my $urlstart = '/uploaded/'.$dom.'/'.$uname;
         my $pathstart = &propath($dom,$uname).'/userfiles';
-        my $is_course = &Apache::lonnet::is_course($dom,$uname);
+        my $is_course = '';
+        if (exists($courses{$dom.'_'.$uname})) {
+            $is_course = 1;
+        }
         my $curr_perm = &Apache::lonnet::get_portfile_permissions($dom,$uname);
         my %access = &Apache::lonnet::get_access_controls($curr_perm);
-        foreach my $file (keys(%access)) { 
+        foreach my $file (keys(%access)) {
             my ($group,$url,$fullpath);
             if ($is_course) {
                 ($group, my ($path)) = ($file =~ /^(\w+)(\/.+)$/);
@@ -248,10 +261,102 @@ foreach my $dom (@domains) {
                 $url = $urlstart.'/portfolio'.$file;
             }
             if (ref($access{$file}) eq 'HASH') {
-                &process_portfolio_access_data($url,$access{$file});
+                my %portaccesslog = 
+                    &LONCAPA::lonmetadata::process_portfolio_access_data($dbh,
+                           $simulate,\%newnames,$url,$fullpath,$access{$file});
+                &portfolio_logging(%portaccesslog);
+            }
+            my %portmetalog = &LONCAPA::lonmetadata::process_portfolio_metadata($dbh,$simulate,\%newnames,$url,$fullpath,$is_course,$dom,$uname,$group);
+            &portfolio_logging(%portmetalog);
+        }
+    }
+    my (%names_by_id,,%ids_by_name,%idstodelete,%idstoadd,%duplicates);
+    unless ($simulate || $oneuser) {
+        my $idshashref;
+        $idshashref = &tie_domain_hash($dom, "ids", &GDBM_WRCREAT());
+        if (ref($idshashref) eq 'HASH') {
+            %names_by_id = %{$idshashref};
+            while (my ($id,$uname) = each(%{$idshashref}) ) {
+                $id = &unescape($id);
+                $uname = &unescape($uname); 
+                $names_by_id{$id} = $uname;
+                push(@{$ids_by_name{$uname}},$id);
+            }
+            &untie_domain_hash($idshashref);
+        }
+    }
+    # Update allusers
+    foreach my $uname (keys(%allusers)) {
+        next if (exists($courses{$dom.'_'.$uname}));
+        my %userdata = 
+            &Apache::lonnet::get('environment',['firstname','lastname',
+                'middlename','generation','id','permanentemail'],$dom,$uname);
+        unless ($simulate || $oneuser) {
+            my $addid;
+            if ($userdata{'id'} ne '') {
+                $addid = $userdata{'id'};
+                $addid=~tr/A-Z/a-z/;
+            }
+            if (exists($ids_by_name{$uname})) {
+                if (ref($ids_by_name{$uname}) eq 'ARRAY') {
+                    if (scalar(@{$ids_by_name{$uname}}) > 1) {
+                        &log(0,"Multiple employee/student IDs found in ids.db for $uname:$dom -- ".join(', ',@{$ids_by_name{$uname}}));
+                    }
+                    foreach my $id (@{$ids_by_name{$uname}}) {
+                        if ($id eq $userdata{'id'}) {
+                            undef($addid);
+                        } else { 
+                            $idstodelete{$id} = $uname;
+                        }
+                    }
+                }
+            }
+            if ($addid ne '') {
+                if (exists($idstoadd{$addid})) {
+                    push(@{$duplicates{$addid}},$uname);
+                } else {
+                    if ((exists($names_by_id{$addid})) && ($names_by_id{$addid} ne $uname)) {
+                        &log(0,"In ids.db ($dom) $addid => $names_by_id{$addid} will be replaced by $addid => $uname");
+                    }
+                    $idstoadd{$addid} = $uname;
+                }
+            }
+        }
+        
+        $userdata{'username'} = $uname;
+        $userdata{'domain'} = $dom;
+        my %alluserslog = 
+            &LONCAPA::lonmetadata::process_allusers_data($dbh,$simulate,
+                \%newnames,$uname,$dom,\%userdata);
+        foreach my $item (keys(%alluserslog)) {
+            &log(0,$alluserslog{$item});
+        }
+    }
+    unless ($simulate || $oneuser) {
+        if (keys(%idstodelete) > 0) {
+            my %resulthash = &Apache::lonnet::iddel($dom,\%idstodelete,$hostid);
+            if ($resulthash{$hostid} eq 'ok') {
+                foreach my $id (sort(keys(%idstodelete))) {
+                    &log(0,"Record deleted from ids.db for $dom -- $id => ".$idstodelete{$id});
+                }
+            } else {
+                &log(0,"Error: '$resulthash{$hostid}' occurred when attempting to delete records from ids.db for $dom");
+            }
+        }
+        if (keys(%idstoadd) > 0) {
+            my $putresult = &Apache::lonnet::put_dom('ids',\%idstoadd,$dom,$hostid);
+            if ($putresult eq 'ok') {
+                foreach my $id (sort(keys(%idstoadd))) {
+                    &log(0,"Record added to ids.db for $dom -- $id => ".$idstoadd{$id});
+                }
+            } else {
+                &log(0,"Error: '$putresult' occurred when attempting to add records to ids.db for $dom"); 
+            }
+        }
+        if (keys(%duplicates) > 0) {
+            foreach my $id (sort(keys(%duplicates))) {
+                &log(0,"Duplicate IDs found for entries to add to ids.db in $dom -- $id => $idstodelete{$id}");
             }
-            &process_portfolio_metadata($url,$fullpath,$is_course,$dom,
-                                        $uname,$group);
         }
     }
 }
@@ -302,8 +407,19 @@ sub log {
     }
 }
 
+sub portfolio_logging {
+    my (%portlog) = @_;
+    foreach my $key (keys(%portlog)) {
+        if (ref($portlog{$key}) eq 'HASH') {
+            foreach my $item (keys(%{$portlog{$key}})) {
+                &log(0,$portlog{$key}{$item});
+            }
+        }
+    }
+}
+
 sub descend_tree {
-    my ($dir,$depth,$alldomusers) = @_;
+    my ($dom,$dir,$depth,$allportusers,$alldomusers) = @_;
     if (-d $dir) {
         opendir(DIR,$dir);
         my @contents = grep(!/^\./,readdir(DIR));
@@ -311,10 +427,12 @@ sub descend_tree {
         $depth ++;
         foreach my $item (@contents) {
             if ($depth < 4) {
-                &descend_tree($dir.'/'.$item,$depth,$alldomusers);
+                &descend_tree($dom,$dir.'/'.$item,$depth,$allportusers,$alldomusers);
             } else {
                 if (-e $dir.'/'.$item.'/file_permissions.db') {
-                 
+                    $$allportusers{$item} = '';
+                }
+                if (-e $dir.'/'.$item.'/passwd') {
                     $$alldomusers{$item} = '';
                 }
             }       
@@ -322,89 +440,6 @@ sub descend_tree {
     } 
 }
 
-sub process_portfolio_access_data {
-    my ($url,$access_hash) = @_;
-    foreach my $key (keys(%{$access_hash})) {
-        my $acc_data;
-        $acc_data->{url} = $url;
-        $acc_data->{keynum} = $key;
-        my ($num,$scope,$end,$start) =
-                        ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/);
-        next if (($scope ne 'public') && ($scope ne 'guest'));
-        $acc_data->{scope} = $scope;
-        if ($end != 0) {
-            $acc_data->{end} = &LONCAPA::lonmetadata::sqltime($end);
-        }
-        $acc_data->{start} = &LONCAPA::lonmetadata::sqltime($start);
-        if (! $simulate) {
-            my ($count,$err) =
-              &LONCAPA::lonmetadata::store_metadata($dbh,
-                                                $newnames{'access'},
-                                                'portfolio_access',$acc_data);
-            if ($err) {
-                &log(0,"MySQL Error Insert: ".$err);
-            }
-            if ($count < 1) {
-                &log(0,"Unable to insert record into MySQL database for $url");
-            }
-        }
-    }
-}
-
-sub process_portfolio_metadata {
-    my ($url,$fullpath,$is_course,$dom,$uname,$group) = @_;
-    my ($ref,$crs,$addedfields) = &portfolio_metadata($fullpath,$dom,$uname,
-                                                      $group);
-    &getfiledates($ref,$fullpath);
-    if ($is_course) {
-        $ref->{'groupname'} = $group;
-    }
-    my %Data;
-    if (ref($ref) eq 'HASH') {
-        %Data = %{$ref};
-    }
-    %Data = (
-             %Data,
-             'url'=>$url,
-             'version'=>'current',
-    );
-    if (! $simulate) {
-        my ($count,$err) =
-         &LONCAPA::lonmetadata::store_metadata($dbh,
-                                               $newnames{'portfolio'},
-                                               'portfolio_metadata',\%Data);
-        if ($err) {
-            &log(0,"MySQL Error Insert: ".$err);
-        }
-        if ($count < 1) {
-            &log(0,"Unable to insert record into MySQL portfolio_metadata database table for $url");
-        }
-        if (ref($addedfields) eq 'HASH') {
-            if (keys(%{$addedfields}) > 0) {
-                foreach my $key (keys(%{$addedfields})) {
-                    my $added_data = {
-                                'url'   => $url,
-                                'field' => $key,
-                                'value' => $addedfields->{$key},
-                                'courserestricted' => $crs,
-                    };
-                    ($count,$err) = &LONCAPA::lonmetadata::store_metadata($dbh,
-                                            $newnames{'addedfields'},
-                                            'portfolio_addedfields',
-                                            $added_data);
-                    if ($err) {
-                        &log(0,"MySQL Error Insert: ".$err);
-                    }
-                    if ($count < 1) {
-                        &log(0,"Unable to insert record into MySQL portfolio_addedfields database table for url = $url and field = $key");
-                    }
-                }
-            }
-        }
-    }
-    return;
-}
-
 ########################################################
 ########################################################
 ###                                                  ###
@@ -508,7 +543,7 @@ sub process_meta_file {
         %dyn=&get_dynamic_metadata($url);
         &count_type($url);
     }
-    &getfiledates($ref,$target);
+    &LONCAPA::lonmetadata::getfiledates($ref,$target);
     #
     my %Data = (
                 %$ref,
@@ -549,7 +584,8 @@ sub metadata {
     if ($filename !~ /\.meta$/) { 
         $filename.='.meta';
     }
-    my $metastring=&getfile($Apache::lonnet::perlvar{'lonDocRoot'}.'/res/'.$filename);
+    my $metastring = 
+        &LONCAPA::lonmetadata::getfile($Apache::lonnet::perlvar{'lonDocRoot'}.'/res/'.$filename);
     return undef if (! defined($metastring));
     my $parser=HTML::TokeParser->new(\$metastring);
     my $token;
@@ -580,125 +616,6 @@ sub metadata {
     return \%metacache;
 }
 
-###############################################################
-###############################################################
-###                                                         ###
-###  &portfolio_metadata($filepath,$dom,$uname,$group) ###
-###   Retrieve metadata for the given file                  ###
-###   Returns array -                                       ###
-###      contains reference to metadatahash and             ###
-###         optional reference to addedfields hash          ###
-###                                                         ###
-###############################################################
-###############################################################
-sub portfolio_metadata {
-    my ($fullpath,$dom,$uname,$group)=@_;
-    my ($mime) = ( $fullpath=~/\.(\w+)$/ );
-    my %metacache=();
-    if ($fullpath !~ /\.meta$/) {
-        $fullpath .= '.meta';
-    }
-    my (@standard_fields,%addedfields);
-    my $colsref = 
-       $LONCAPA::lonmetadata::Portfolio_metadata_table_description;
-    if (ref($colsref) eq 'ARRAY') {
-        my @columns = @{$colsref};
-        foreach my $coldata (@columns) {
-            push(@standard_fields,$coldata->{'name'});
-        }
-    }
-    my $metastring=&getfile($fullpath);
-    if (! defined($metastring)) {
-        $metacache{'keys'}= 'owner,domain,mime';
-        $metacache{'owner'} = $uname.':'.$dom;
-        $metacache{'domain'} = $dom;
-        $metacache{'mime'} = $mime;
-        if (defined($group)) {
-            $metacache{'keys'} .= ',courserestricted';
-            $metacache{'courserestricted'} = 'course.'.$dom.'_'.$uname;
-        } 
-    } else {
-        my $parser=HTML::TokeParser->new(\$metastring);
-        my $token;
-        while ($token=$parser->get_token) {
-            if ($token->[0] eq 'S') {
-                my $entry=$token->[1];
-                if ($metacache{'keys'}) {
-                    $metacache{'keys'}.=','.$entry;
-                } else {
-                    $metacache{'keys'}=$entry;
-                }
-                my $value = $parser->get_text('/'.$entry);
-                if (!grep(/^\Q$entry\E$/,@standard_fields)) {
-                    my $clean_value = lc($value);
-                    $clean_value =~ s/\s/_/g;
-                    if ($clean_value ne $entry) {
-                        if (defined($addedfields{$entry})) {
-                            $addedfields{$entry} .=','.$value;
-                        } else {
-                            $addedfields{$entry} = $value;
-                        }
-                    }
-                } else {
-                    $metacache{$entry} = $value;
-                }
-            }
-        } # End of ($token->[0] eq 'S')
-    }
-    if (keys(%addedfields) > 0) {
-        foreach my $key (sort keys(%addedfields)) {
-            $metacache{'addedfieldnames'} .= $key.',';
-            $metacache{'addedfieldvalues'} .= $addedfields{$key}.'&&&';
-        }
-        $metacache{'addedfieldnames'} =~ s/,$//;
-        $metacache{'addedfieldvalues'} =~ s/\&\&\&$//;
-        if ($metacache{'keys'}) {
-            $metacache{'keys'}.=',addedfieldnames';
-        } else {
-            $metacache{'keys'}='addedfieldnames';
-        }
-        $metacache{'keys'}.=',addedfieldvalues';
-    }
-    return (\%metacache,$metacache{'courserestricted'},\%addedfields);
-}
-
-##
-## &getfile($filename)
-##   Slurps up an entire file into a scalar.  
-##   Returns undef if the file does not exist
-sub getfile {
-    my $file = shift();
-    if (! -e $file ) { 
-        return undef; 
-    }
-    my $fh=IO::File->new($file);
-    my $contents = '';
-    while (<$fh>) { 
-        $contents .= $_;
-    }
-    return $contents;
-}
-
-##
-## &getfiledates() 
-## Converts creationdate and modifieddates to SQL format 
-## Applies stat() to file to retrieve dates if missing
-sub getfiledates {
-    my ($ref,$target) = @_;
-    if (! defined($ref->{'creationdate'}) ||
-        $ref->{'creationdate'} =~ /^\s*$/) {
-        $ref->{'creationdate'} = (stat($target))[9];
-    }
-    if (! defined($ref->{'lastrevisiondate'}) ||
-        $ref->{'lastrevisiondate'} =~ /^\s*$/) {
-        $ref->{'lastrevisiondate'} = (stat($target))[9];
-    }
-    $ref->{'creationdate'}     = 
-        &LONCAPA::lonmetadata::sqltime($ref->{'creationdate'});
-    $ref->{'lastrevisiondate'} = 
-        &LONCAPA::lonmetadata::sqltime($ref->{'lastrevisiondate'});
-}
-
 ########################################################
 ########################################################
 ###                                                  ###
@@ -866,7 +783,7 @@ sub write_copyright_count {
 ##   (copied from lond, modification of the return value)
 sub ishome {
     my $author=shift;
-    $author=~s/\/home\/httpd\/html\/res\/([^\/]*)\/([^\/]*).*/$1\/$2/;
+    $author=~s{/home/httpd/html/res/([^/]*)/([^/]*).*}{$1/$2};
     my ($udom,$uname)=split(/\//,$author);
     my $proname=propath($udom,$uname);
     if (-e $proname) {
@@ -877,20 +794,6 @@ sub ishome {
 }
 
 ##
-## &propath($udom,$uname)
-##   Returns the path to the users LON-CAPA directory
-##   (copied from lond)
-sub propath {
-    my ($udom,$uname)=@_;
-    $udom=~s/\W//g;
-    $uname=~s/\W//g;
-    my $subdir=$uname.'__';
-    $subdir =~ s/(.)(.)(.).*/$1\/$2\/$3/;
-    my $proname="$Apache::lonnet::perlvar{'lonUsersDir'}/$udom/$subdir/$uname";
-    return $proname;
-} 
-
-##
 ## &declutter($filename)
 ##   Given a filename, returns a url for the filename.
 sub declutter {
@@ -901,16 +804,3 @@ sub declutter {
     return $thisfn;
 }
 
-##
-## Escape / Unescape special characters
-sub unescape {
-    my $str=shift;
-    $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
-    return $str;
-}
-
-sub escape {
-    my $str=shift;
-    $str =~ s/(\W)/"%".unpack('H2',$1)/eg;
-    return $str;
-}