--- loncom/metadata_database/searchcat.pl 2004/04/09 22:04:53 1.56 +++ loncom/metadata_database/searchcat.pl 2005/03/11 03:25:18 1.62 @@ -2,7 +2,7 @@ # The LearningOnline Network # searchcat.pl "Search Catalog" batch script # -# $Id: searchcat.pl,v 1.56 2004/04/09 22:04:53 matthew Exp $ +# $Id: searchcat.pl,v 1.62 2005/03/11 03:25:18 matthew Exp $ # # Copyright Michigan State University Board of Trustees # @@ -77,17 +77,20 @@ use HTML::TokeParser; use GDBM_File; use POSIX qw(strftime mktime); +use Sys::Hostname; + use File::Find; # # Set up configuration options -my ($simulate,$oneuser,$help,$verbose,$logfile,$debug); +my ($simulate,$oneuser,$help,$verbose,$logfile,$debug,$multidom); GetOptions ( 'help' => \$help, 'simulate' => \$simulate, 'only=s' => \$oneuser, 'verbose=s' => \$verbose, 'debug' => \$debug, + 'multi_domain' => \$multidom, ); if ($help) { @@ -100,6 +103,7 @@ Options: -only=user Only compute for the given user. Implies -simulate -verbose=val Sets logging level, val must be a number -debug Turns on debugging output + -multi_domain Parse the hosts.tab file domain(s) to use. ENDHELP exit 0; } @@ -119,7 +123,7 @@ if (defined($oneuser)) { ## ## Use variables for table names so we can test this routine a little easier my $oldname = 'metadata'; -my $newname = 'newmetadata'; +my $newname = 'newmetadata'.$$; # append pid to have unique temporary table # # Read loncapa_apache.conf and loncapa.conf @@ -142,8 +146,10 @@ if ($wwwid!=$<) { } # # Let people know we are running -open(LOG,'>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log'); +open(LOG,'>>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log'); &log(0,'==== Searchcat Run '.localtime()."===="); + + if ($debug) { &log(0,'simulating') if ($simulate); &log(0,'only processing user '.$oneuser) if ($oneuser); @@ -171,33 +177,60 @@ if ($dbh->err) { } # # find out which users we need to examine -my $dom = $perlvar{'lonDefDomain'}; -opendir(RESOURCES,"$perlvar{'lonDocRoot'}/res/$dom"); -my @homeusers = - grep { - &ishome("$perlvar{'lonDocRoot'}/res/$dom/$_"); - } grep { - !/^\.\.?$/; - } readdir(RESOURCES); -closedir RESOURCES; -# -if ($oneuser) { - @homeusers=($oneuser); -} -# -# Loop through the users -foreach my $user (@homeusers) { - &log(0,"=== User: ".$user); - &process_dynamic_metadata($user,$dom); - # - # Use File::Find to get the files we need to read/modify - find( - {preprocess => \&only_meta_files, -# wanted => \&print_filename, -# wanted => \&log_metadata, - wanted => \&process_meta_file, - }, - "$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$user"); +my @domains; +if (defined($multidom)) { + &log(1,'====multi domain setup===='); + # Peek into the hosts.tab and look for matches of our hostname + my $host = hostname(); + &log(9,'hostname = "'.$host.'"'); + open(HOSTFILE,$perlvar{'lonTabDir'}.'/hosts.tab') || + die ("Unable to determine domain(s) of multi-domain server"); + my %domains; + while () { + next if (/^\#/); + next if (!/:\Q$host\E/); + &log(9,$_); + $domains{(split(':',$_))[1]}++; + } + close HOSTFILE; + @domains = sort(keys(%domains)); + &log(9,join(',',@domains)); + if (! scalar(@domains)) { + die ("Unable to find any domains in the hosts.tab that match ".$host); + } +} else { + push(@domains,$perlvar{'lonDefDomain'}); +} + +foreach my $dom (@domains) { + &log(9,'domain = '.$dom); + opendir(RESOURCES,"$perlvar{'lonDocRoot'}/res/$dom"); + my @homeusers = + grep { + &ishome("$perlvar{'lonDocRoot'}/res/$dom/$_"); + } grep { + !/^\.\.?$/; + } readdir(RESOURCES); + closedir RESOURCES; + &log(5,'users = '.$dom.':'.join(',',@homeusers)); + # + if ($oneuser) { + @homeusers=($oneuser); + } + # + # Loop through the users + foreach my $user (@homeusers) { + &log(0,"=== User: ".$user); + &process_dynamic_metadata($user,$dom); + # + # Use File::Find to get the files we need to read/modify + find( + {preprocess => \&only_meta_files, + #wanted => \&print_filename, + #wanted => \&log_metadata, + wanted => \&process_meta_file, + }, join('/',($perlvar{'lonDocRoot'},'res',$dom,$user)) ); + } } # # Rename the table @@ -210,7 +243,6 @@ if (! $simulate) { &log(1,"MySQL table rename successful."); } } - if (! $dbh->disconnect) { &log(0,"MySQL Error Disconnect: ".$dbh->errstr); die $dbh->errstr; @@ -305,7 +337,6 @@ sub log_metadata { $_=$file; } - ## ## process_meta_file ## Called by File::Find. @@ -320,8 +351,9 @@ sub process_meta_file { my $ref=&metadata($filename); # # $url is the original file url, not the metadata file - my $url='/res/'.&declutter($filename); - $url=~s/\.meta$//; + my $target = $filename; + $target =~ s/\.meta$//; + my $url='/res/'.&declutter($target); &log(3," ".$url) if ($debug); # # Ignore some files based on their metadata @@ -346,6 +378,14 @@ sub process_meta_file { &count_type($url); } # + if (! defined($ref->{'creationdate'}) || + $ref->{'creationdate'} =~ /^\s*$/) { + $ref->{'creationdate'} = (stat($target))[9]; + } + if (! defined($ref->{'lastrevisiondate'}) || + $ref->{'lastrevisiondate'} =~ /^\s*$/) { + $ref->{'lastrevisiondate'} = (stat($target))[9]; + } $ref->{'creationdate'} = &sqltime($ref->{'creationdate'}); $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'}); my %Data = ( @@ -358,11 +398,9 @@ sub process_meta_file { \%Data); if ($err) { &log(0,"MySQL Error Insert: ".$err); - die $err; } if ($count < 1) { &log(0,"Unable to insert record into MySQL database for $url"); - die "Unable to insert record into MySQl database for $url"; } } # @@ -444,27 +482,30 @@ sub getfile { ######################################################## ######################################################## ## -## Dynamic metadata description +## Dynamic metadata description (incomplete) +## +## For a full description of all fields, +## see LONCAPA::lonmetadata ## ## Field Type ##----------------------------------------------------------- ## count integer ## course integer -## course_list comma seperated list of course ids +## course_list comma separated list of course ids ## avetries real -## avetries_list comma seperated list of real numbers +## avetries_list comma separated list of real numbers ## stdno real -## stdno_list comma seperated list of real numbers +## stdno_list comma separated list of real numbers ## usage integer -## usage_list comma seperated list of resources +## usage_list comma separated list of resources ## goto scalar -## goto_list comma seperated list of resources +## goto_list comma separated list of resources ## comefrom scalar -## comefrom_list comma seperated list of resources +## comefrom_list comma separated list of resources ## difficulty real -## difficulty_list comma seperated list of real numbers +## difficulty_list comma separated list of real numbers ## sequsage scalar -## sequsage_list comma seperated list of resources +## sequsage_list comma separated list of resources ## clear real ## technical real ## correct real @@ -491,34 +532,10 @@ sub process_dynamic_metadata { return 0; } # - # Process every stored element - while (my ($storedkey,$value) = each(%evaldata)) { - my ($source,$file,$type) = split('___',$storedkey); - $source = &unescape($source); - $file = &unescape($file); - $value = &unescape($value); - " got ".$file."\n ".$type." ".$source."\n"; - if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) { - # - # Statistics: $source is course id - $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value; - } elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){ - # - # Evaluation $source is username, check if they evaluated it - # more than once. If so, pad the entry with a space. - while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) { - $source .= ' '; - } - $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value; - } elsif ($type =~ /^(course|comefrom|goto|usage)$/) { - # - # Context $source is course id or resource - push(@{$DynamicData{$file}->{$type}},&unescape($source)); - } else { - &log(0," ".$user."@".$dom.":Process metadata: Unable to decode ".$type); - } - } + %DynamicData = &LONCAPA::lonmetadata::process_reseval_data(\%evaldata); untie(%evaldata); + $DynamicData{'domain'} = $dom; + print('user = '.$user.' domain = '.$dom.$/); # # Read in the access count data &log(7,'Reading access count data') if ($debug); @@ -547,75 +564,17 @@ sub process_dynamic_metadata { sub get_dynamic_metadata { my ($url) = @_; $url =~ s:^/res/::; - if (! exists($DynamicData{$url})) { - &log(7,' No dynamic data for '.$url) if ($debug); - return (); - } - my %data; - my $resdata = $DynamicData{$url}; - # - # Get the statistical data - foreach my $type (qw/avetries difficulty stdno/) { - my $count; - my $sum; - my @Values; - foreach my $coursedata (values(%{$resdata->{'statistics'}})) { - if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) { - $count++; - $sum += $coursedata->{$type}; - push(@Values,$coursedata->{$type}); - } - } - if ($count) { - $data{$type} = $sum/$count; - $data{$type.'_list'} = join(',',@Values); - } - } + my %data = &LONCAPA::lonmetadata::process_dynamic_metadata($url, + \%DynamicData); # find the count $data{'count'} = $Counts{$url}; # - # Get the context data - foreach my $type (qw/course goto comefrom/) { - if (defined($resdata->{$type}) && - ref($resdata->{$type}) eq 'ARRAY') { - $data{$type} = scalar(@{$resdata->{$type}}); - $data{$type.'_list'} = join(',',@{$resdata->{$type}}); - } - } - if (defined($resdata->{'usage'}) && - ref($resdata->{'usage'}) eq 'ARRAY') { - $data{'sequsage'} = scalar(@{$resdata->{'usage'}}); - $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}}); - } - # - # Get the evaluation data - foreach my $type (qw/clear technical correct helpful depth/) { - my $count; - my $sum; - foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){ - $sum += $resdata->{'evaluation'}->{$type}->{$evaluator}; - $count++; - } - if ($count > 0) { - $data{$type}=$sum/$count; - } - } - # - # put together comments - my $comments = '
'; - foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){ - $comments .= $evaluator.':'. - $resdata->{'evaluation'}->{'comments'}->{$evaluator}.'
'; - } - $comments .= '
'; - # # Log the dynamic metadata if ($debug) { while (my($k,$v)=each(%data)) { &log(8," ".$k." => ".$v); } } - # return %data; }