--- loncom/metadata_database/searchcat.pl 2006/01/27 15:53:49 1.66 +++ loncom/metadata_database/searchcat.pl 2016/01/31 21:25:49 1.84 @@ -2,7 +2,7 @@ # The LearningOnline Network # searchcat.pl "Search Catalog" batch script # -# $Id: searchcat.pl,v 1.66 2006/01/27 15:53:49 albertel Exp $ +# $Id: searchcat.pl,v 1.84 2016/01/31 21:25:49 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -65,18 +65,17 @@ and correct user experience. =cut use strict; -BEGIN { - eval "use Apache2::compat();"; -}; use DBI; use lib '/home/httpd/lib/perl/'; use LONCAPA::lonmetadata; - +use LONCAPA; use Getopt::Long; use IO::File; use HTML::TokeParser; use GDBM_File; use POSIX qw(strftime mktime); +use Mail::Send; +use Apache::loncommon(); use Apache::lonnet(); @@ -121,12 +120,25 @@ if (defined($oneuser)) { ## ## Use variables for table names so we can test this routine a little easier -my $oldname = 'metadata'; -my $newname = 'newmetadata'.$$; # append pid to have unique temporary table +my %oldnames = ( + 'metadata' => 'metadata', + 'portfolio' => 'portfolio_metadata', + 'access' => 'portfolio_access', + 'addedfields' => 'portfolio_addedfields', + 'allusers' => 'allusers', + ); + +my %newnames; +# new table names - append pid to have unique temporary tables +foreach my $key (keys(%oldnames)) { + $newnames{$key} = 'new'.$oldnames{$key}.$$; +} # # Only run if machine is a library server exit if ($Apache::lonnet::perlvar{'lonRole'} ne 'library'); +my $hostid = $Apache::lonnet::perlvar{'lonHostID'}; + # # Make sure this process is running from user=www my $wwwid=getpwnam('www'); @@ -158,16 +170,27 @@ if (! ($dbh = DBI->connect("DBI:mysql:lo } # This can return an error and still be okay, so we do not bother checking. # (perhaps it should be more robust and check for specific errors) -$dbh->do('DROP TABLE IF EXISTS '.$newname); +foreach my $key (keys(%newnames)) { + if ($newnames{$key} ne '') { + $dbh->do('DROP TABLE IF EXISTS '.$newnames{$key}); + } +} + # -# Create the new table -my $request = &LONCAPA::lonmetadata::create_metadata_storage($newname); -$dbh->do($request); -if ($dbh->err) { - $dbh->disconnect(); - &log(0,"MySQL Error Create: ".$dbh->errstr); - die $dbh->errstr; +# Create the new metadata, portfolio and allusers tables +foreach my $key (keys(%newnames)) { + if ($newnames{$key} ne '') { + my $request = + &LONCAPA::lonmetadata::create_metadata_storage($newnames{$key},$oldnames{$key}); + $dbh->do($request); + if ($dbh->err) { + $dbh->disconnect(); + &log(0,"MySQL Error Create: ".$dbh->errstr); + die $dbh->errstr; + } + } } + # # find out which users we need to examine my @domains = sort(&Apache::lonnet::current_machine_domains()); @@ -188,6 +211,7 @@ foreach my $dom (@domains) { if ($oneuser) { @homeusers=($oneuser); } + # # Loop through the users foreach my $user (@homeusers) { @@ -203,16 +227,262 @@ foreach my $dom (@domains) { no_chdir => 1, }, join('/',($Apache::lonnet::perlvar{'lonDocRoot'},'res',$dom,$user)) ); } + # Search for all users and public portfolio files + my (%allusers,%portusers,%courses); + if ($oneuser) { + %portusers = ( + $oneuser => '', + ); + %allusers = ( + $oneuser => '', + ); + %courses = &courseiddump($dom,'.',1,'.','.',$oneuser,undef, + undef,'.'); + } else { + # get courseIDs for domain on current machine + %courses=&Apache::lonnet::courseiddump($dom,'.',1,'.','.','.',1,[$hostid],'.'); + my $dir = $Apache::lonnet::perlvar{lonUsersDir}.'/'.$dom; + &descend_tree($dom,$dir,0,\%portusers,\%allusers); + } + foreach my $uname (keys(%portusers)) { + my $urlstart = '/uploaded/'.$dom.'/'.$uname; + my $pathstart = &propath($dom,$uname).'/userfiles'; + my $is_course = ''; + if (exists($courses{$dom.'_'.$uname})) { + $is_course = 1; + } + my $curr_perm = &Apache::lonnet::get_portfile_permissions($dom,$uname); + my %access = &Apache::lonnet::get_access_controls($curr_perm); + foreach my $file (keys(%access)) { + my ($group,$url,$fullpath); + if ($is_course) { + ($group, my ($path)) = ($file =~ /^(\w+)(\/.+)$/); + $fullpath = $pathstart.'/groups/'.$group.'/portfolio'.$path; + $url = $urlstart.'/groups/'.$group.'/portfolio'.$path; + } else { + $fullpath = $pathstart.'/portfolio'.$file; + $url = $urlstart.'/portfolio'.$file; + } + if (ref($access{$file}) eq 'HASH') { + my %portaccesslog = + &LONCAPA::lonmetadata::process_portfolio_access_data($dbh, + $simulate,\%newnames,$url,$fullpath,$access{$file}); + &portfolio_logging(%portaccesslog); + } + my %portmetalog = &LONCAPA::lonmetadata::process_portfolio_metadata($dbh,$simulate,\%newnames,$url,$fullpath,$is_course,$dom,$uname,$group); + &portfolio_logging(%portmetalog); + } + } + my %duplicates; + my %names_by_id = ( + id => {}, + clickers => {}, + ); + my %ids_by_name = ( + id => {}, + clickers => {}, + ); + my %idstodelete = ( + id => {}, + clickers => {}, + ); + my %idstoadd = ( + id => {}, + clickers => {}, + ); + my %namespace = ( + id => 'ids', + clickers => 'clickers', + ); + my %idtext = ( + id => 'employee/student IDs', + clickers => 'clicker IDs', + ); + unless ($simulate || $oneuser) { + foreach my $key ('id','clickers') { + my $hashref = &tie_domain_hash($dom,$namespace{$key},&GDBM_WRCREAT()); + if (ref($hashref) eq 'HASH') { + while (my ($id,$unamestr) = each(%{$hashref}) ) { + $id = &unescape($id); + $unamestr = &unescape($unamestr); + if ($key eq 'clickers') { + my @unames = split(/,/,$unamestr); + foreach my $uname (@unames) { + push(@{$ids_by_name{$key}{$uname}},$id); + } + $names_by_id{$key}{$id} = $unamestr; + } else { + $names_by_id{$key}{$id} = $unamestr; + push(@{$ids_by_name{$key}{$unamestr}},$id); + } + } + &untie_domain_hash($hashref); + } + } + } + # Update allusers + foreach my $uname (keys(%allusers)) { + next if (exists($courses{$dom.'_'.$uname})); + my %userdata = + &Apache::lonnet::get('environment',['firstname','lastname', + 'middlename','generation','id','permanentemail','clickers'], + $dom,$uname); + unless ($simulate || $oneuser) { + foreach my $key ('id','clickers') { + my %addid = (); + if ($userdata{$key} ne '') { + my $idfromenv = $userdata{$key}; + if ($key eq 'id') { + $idfromenv=~tr/A-Z/a-z/; + $addid{$idfromenv} = 1; + } else { + $idfromenv =~ s/^\s+//; + $idfromenv =~ s/\s+$//; + map { $addid{$_} = 1; } split(/,/,$idfromenv); + } + } + if (ref($ids_by_name{$key}{$uname}) eq 'ARRAY') { + if (scalar(@{$ids_by_name{$key}{$uname}}) > 1) { + &log(0,"Multiple $idtext{$key} found in $namespace{$key}.db for $uname:$dom -- ". + join(', ',@{$ids_by_name{$key}{$uname}})); + } + foreach my $id (@{$ids_by_name{$key}{$uname}}) { + if ($addid{$id}) { + delete($addid{$id}); + } else { + if ($key eq 'id') { + $idstodelete{$key}{$id} = $uname; + } else { + $idstodelete{$key}{$id} .= $uname.','; + } + } + } + } + if (keys(%addid)) { + foreach my $id (keys(%addid)) { + if ($key eq 'id') { + if (exists($idstoadd{$key}{$id})) { + push(@{$duplicates{$id}},$uname); + } else { + $idstoadd{$key}{$id} = $uname; + } + } else { + $idstoadd{$key}{$id} .= $uname.','; + } + } + } + } + } + + $userdata{'username'} = $uname; + $userdata{'domain'} = $dom; + my %alluserslog = + &LONCAPA::lonmetadata::process_allusers_data($dbh,$simulate, + \%newnames,$uname,$dom,\%userdata); + foreach my $item (keys(%alluserslog)) { + &log(0,$alluserslog{$item}); + } + } + unless ($simulate || $oneuser) { + foreach my $key ('id','clickers') { + if (keys(%{$idstodelete{$key}}) > 0) { + my %resulthash; + if ($key eq 'id') { + %resulthash = &Apache::lonnet::iddel($dom,$idstodelete{$key},$hostid,$namespace{$key}); + } else { + foreach my $delid (sort(keys(%{$idstodelete{$key}}))) { + $idstodelete{$key}{$delid} =~ s/,$//; + } + %resulthash = &Apache::lonnet::iddel($dom,$idstodelete{$key},$hostid,$namespace{$key}); + } + if ($resulthash{$hostid} eq 'ok') { + foreach my $id (sort(keys(%{$idstodelete{$key}}))) { + &log(0,"Record deleted from $namespace{$key}.db for $dom -- $id => ".$idstodelete{$key}{$id}); + } + } else { + &log(0,"Error: '$resulthash{$hostid}' occurred when attempting to delete records from $namespace{$key}.db for $dom"); + } + } + if (keys(%{$idstoadd{$key}}) > 0) { + my $idmessage = ''; + my %newids; + if ($key eq 'id') { + foreach my $addid (sort(keys(%{$idstoadd{$key}}))) { + if ((exists($names_by_id{$key}{$addid})) && ($names_by_id{$key}{$addid} ne $idstoadd{$key}{$addid}) && !($idstodelete{$key}{$addid})) { + &log(0,"Two usernames associated with a single ID $addid in domain: $dom: $names_by_id{$key}{$addid} (current) and $idstoadd{$key}{$addid}\n"); + $idmessage .= "$addid,$names_by_id{$key}{$addid},$idstoadd{$key}{$addid}\n"; + } else { + $newids{$addid} = $idstoadd{$key}{$addid}; + } + } + } else { + foreach my $addid (sort(keys(%{$idstoadd{$key}}))) { + $idstoadd{$key}{$addid} =~ s/,$//; + $newids{$addid} = $idstoadd{$key}{$addid}; + } + } + if (keys(%newids) > 0) { + my $putresult; + if ($key eq 'clickers') { + $putresult = &Apache::lonnet::updateclickers($dom,'add',\%newids,$hostid); + } else { + $putresult = &Apache::lonnet::put_dom($namespace{$key},\%newids,$dom,$hostid); + } + if ($putresult eq 'ok') { + foreach my $id (sort(keys(%newids))) { + &log(0,"Record added to $namespace{$key}.db for $dom -- $id => ".$newids{$id}); + } + } else { + &log(0,"Error: '$putresult' occurred when attempting to add records to $namespace{$key}.db for $dom"); + } + } + if ($idmessage) { + my $to = &Apache::loncommon::build_recipient_list(undef,'idconflictsmail',$dom); + if ($to ne '') { + my $msg = new Mail::Send; + $msg->to($to); + $msg->subject('LON-CAPA studentIDs conflict'); + my $lonhost = $Apache::lonnet::perlvar{'lonHostID'}; + my $hostname = &Apache::lonnet::hostname($lonhost); + my $replytoaddress = 'do-not-reply@'.$hostname; + $msg->add('Reply-to',$replytoaddress); + $msg->add('From','www@'.$hostname); + $msg->add('Content-type','text/plain; charset=UTF-8'); + if (my $fh = $msg->open()) { + print $fh + 'The following IDs are used for more than one user in your domain:'."\n". + 'Each row contains: Student/Employee ID, Current username in ids.db file, '. + 'Additional username'."\n\n". + $idmessage; + $fh->close; + } + } + } + } + } + if (keys(%duplicates) > 0) { + foreach my $id (sort(keys(%duplicates))) { + if (ref($duplicates{$id}) eq 'ARRAY') { + &log(0,"Duplicate IDs found for entries to add to ids.db in $dom -- $id => ".join(',',@{$duplicates{$id}})); + } + } + } + } } + # -# Rename the table +# Rename the tables if (! $simulate) { - $dbh->do('DROP TABLE IF EXISTS '.$oldname); - if (! $dbh->do('RENAME TABLE '.$newname.' TO '.$oldname)) { - &log(0,"MySQL Error Rename: ".$dbh->errstr); - die $dbh->errstr; - } else { - &log(1,"MySQL table rename successful."); + foreach my $key (keys(%oldnames)) { + if (($oldnames{$key} ne '') && ($newnames{$key} ne '')) { + $dbh->do('DROP TABLE IF EXISTS '.$oldnames{$key}); + if (! $dbh->do('RENAME TABLE '.$newnames{$key}.' TO '.$oldnames{$key})) { + &log(0,"MySQL Error Rename: ".$dbh->errstr); + die $dbh->errstr; + } else { + &log(1,"MySQL table rename successful for $key."); + } + } } } if (! $dbh->disconnect) { @@ -246,6 +516,39 @@ sub log { } } +sub portfolio_logging { + my (%portlog) = @_; + foreach my $key (keys(%portlog)) { + if (ref($portlog{$key}) eq 'HASH') { + foreach my $item (keys(%{$portlog{$key}})) { + &log(0,$portlog{$key}{$item}); + } + } + } +} + +sub descend_tree { + my ($dom,$dir,$depth,$allportusers,$alldomusers) = @_; + if (-d $dir) { + opendir(DIR,$dir); + my @contents = grep(!/^\./,readdir(DIR)); + closedir(DIR); + $depth ++; + foreach my $item (@contents) { + if (($depth < 4) && (length($item) == 1)) { + &descend_tree($dom,$dir.'/'.$item,$depth,$allportusers,$alldomusers); + } else { + if (-e $dir.'/'.$item.'/file_permissions.db') { + $$allportusers{$item} = ''; + } + if (-e $dir.'/'.$item.'/passwd') { + $$alldomusers{$item} = ''; + } + } + } + } +} + ######################################################## ######################################################## ### ### @@ -265,7 +568,7 @@ sub only_meta_files { foreach my $file (@PossibleFiles) { if ( ($file =~ /\.meta$/ && # Ends in meta $file !~ /\.\d+\.[^\.]+\.meta$/ # is not for a prior version - ) || (-d $file )) { # directories are okay + ) || (-d $File::Find::dir."/".$file )) { # directories are okay # but we do not want /. or /.. push(@ChosenFiles,$file); } @@ -296,7 +599,7 @@ sub log_metadata { return if (-d $fullfilename); # No need to do anything here for directories if ($debug) { &log(6,$fullfilename); - my $ref=&metadata($fullfilename); + my $ref = &metadata($fullfilename); if (! defined($ref)) { &log(6," No data"); return; @@ -320,7 +623,7 @@ sub process_meta_file { # &log(3,$filename) if ($debug); # - my $ref=&metadata($filename); + my $ref = &metadata($filename); # # $url is the original file url, not the metadata file my $target = $filename; @@ -349,25 +652,17 @@ sub process_meta_file { %dyn=&get_dynamic_metadata($url); &count_type($url); } + &LONCAPA::lonmetadata::getfiledates($ref,$target); # - if (! defined($ref->{'creationdate'}) || - $ref->{'creationdate'} =~ /^\s*$/) { - $ref->{'creationdate'} = (stat($target))[9]; - } - if (! defined($ref->{'lastrevisiondate'}) || - $ref->{'lastrevisiondate'} =~ /^\s*$/) { - $ref->{'lastrevisiondate'} = (stat($target))[9]; - } - $ref->{'creationdate'} = &sqltime($ref->{'creationdate'}); - $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'}); my %Data = ( %$ref, %dyn, 'url'=>$url, 'version'=>'current'); if (! $simulate) { - my ($count,$err) = &LONCAPA::lonmetadata::store_metadata($dbh,$newname, - \%Data); + my ($count,$err) = + &LONCAPA::lonmetadata::store_metadata($dbh,$newnames{'metadata'}, + 'metadata',\%Data); if ($err) { &log(0,"MySQL Error Insert: ".$err); } @@ -389,7 +684,7 @@ sub process_meta_file { ######################################################## ######################################################## sub metadata { - my ($uri)=@_; + my ($uri) = @_; my %metacache=(); $uri=&declutter($uri); my $filename=$uri; @@ -398,7 +693,8 @@ sub metadata { if ($filename !~ /\.meta$/) { $filename.='.meta'; } - my $metastring=&getfile($Apache::lonnet::perlvar{'lonDocRoot'}.'/res/'.$filename); + my $metastring = + &LONCAPA::lonmetadata::getfile($Apache::lonnet::perlvar{'lonDocRoot'}.'/res/'.$filename); return undef if (! defined($metastring)); my $parser=HTML::TokeParser->new(\$metastring); my $token; @@ -419,7 +715,7 @@ sub metadata { } foreach ( @{$token->[3]}) { $metacache{$uri.''.$unikey.'.'.$_}=$token->[2]->{$_}; - } + } if (! ($metacache{$uri.''.$unikey}=$parser->get_text('/'.$entry))){ $metacache{$uri.''.$unikey} = $metacache{$uri.''.$unikey.'.default'}; @@ -429,23 +725,6 @@ sub metadata { return \%metacache; } -## -## &getfile($filename) -## Slurps up an entire file into a scalar. -## Returns undef if the file does not exist -sub getfile { - my $file = shift(); - if (! -e $file ) { - return undef; - } - my $fh=IO::File->new($file); - my $contents = ''; - while (<$fh>) { - $contents .= $_; - } - return $contents; -} - ######################################################## ######################################################## ### ### @@ -613,7 +892,7 @@ sub write_copyright_count { ## (copied from lond, modification of the return value) sub ishome { my $author=shift; - $author=~s/\/home\/httpd\/html\/res\/([^\/]*)\/([^\/]*).*/$1\/$2/; + $author=~s{/home/httpd/html/res/([^/]*)/([^/]*).*}{$1/$2}; my ($udom,$uname)=split(/\//,$author); my $proname=propath($udom,$uname); if (-e $proname) { @@ -624,55 +903,6 @@ sub ishome { } ## -## &propath($udom,$uname) -## Returns the path to the users LON-CAPA directory -## (copied from lond) -sub propath { - my ($udom,$uname)=@_; - $udom=~s/\W//g; - $uname=~s/\W//g; - my $subdir=$uname.'__'; - $subdir =~ s/(.)(.)(.).*/$1\/$2\/$3/; - my $proname="$Apache::lonnet::perlvar{'lonUsersDir'}/$udom/$subdir/$uname"; - return $proname; -} - -## -## &sqltime($timestamp) -## -## Convert perl $timestamp to MySQL time. MySQL expects YYYY-MM-DD HH:MM:SS -## -sub sqltime { - my ($time) = @_; - my $mysqltime; - if ($time =~ - /(\d+)-(\d+)-(\d+) # YYYY-MM-DD - \s # a space - (\d+):(\d+):(\d+) # HH:MM::SS - /x ) { - # Some of the .meta files have the time in mysql - # format already, so just make sure they are 0 padded and - # pass them back. - $mysqltime = sprintf('%04d-%02d-%02d %02d:%02d:%02d', - $1,$2,$3,$4,$5,$6); - } elsif ($time =~ /^\d+$/) { - my @TimeData = gmtime($time); - # Alter the month to be 1-12 instead of 0-11 - $TimeData[4]++; - # Alter the year to be from 0 instead of from 1900 - $TimeData[5]+=1900; - $mysqltime = sprintf('%04d-%02d-%02d %02d:%02d:%02d', - @TimeData[5,4,3,2,1,0]); - } elsif (! defined($time) || $time == 0) { - $mysqltime = 0; - } else { - &log(0," sqltime:Unable to decode time ".$time); - $mysqltime = 0; - } - return $mysqltime; -} - -## ## &declutter($filename) ## Given a filename, returns a url for the filename. sub declutter { @@ -683,16 +913,3 @@ sub declutter { return $thisfn; } -## -## Escape / Unescape special characters -sub unescape { - my $str=shift; - $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg; - return $str; -} - -sub escape { - my $str=shift; - $str =~ s/(\W)/"%".unpack('H2',$1)/eg; - return $str; -}