--- loncom/interface/lonsearchcat.pm 2002/07/12 21:02:27 1.142
+++ loncom/interface/lonsearchcat.pm 2002/07/26 16:37:58 1.144
@@ -1,7 +1,7 @@
# The LearningOnline Network with CAPA
# Search Catalog
#
-# $Id: lonsearchcat.pm,v 1.142 2002/07/12 21:02:27 matthew Exp $
+# $Id: lonsearchcat.pm,v 1.144 2002/07/26 16:37:58 matthew Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -87,8 +87,10 @@ use Apache::lonnet();
use Apache::File();
use CGI qw(:standard);
use Text::Query;
+use DBI;
use GDBM_File;
use Apache::loncommon();
+use Apache::lonmysql();
# ---------------------------------------- variables used throughout the module
@@ -125,6 +127,10 @@ that produces it. Adding a new view typ
adding a line to the definition of this hash and making sure the function
takes the proper parameters.
+=item $results_db
+
+The name of the database results from searches are put in.
+
=back
=cut
@@ -140,6 +146,7 @@ my $importbutton; # button to take the s
my %groupsearch_db; # database hash
my $diropendb = ""; # db file
+my $results_db = "";
# View Description Function Pointer
my %Views = ("Detailed Citation View" => \&detailed_citation_view,
"Summary View" => \&summary_view,
@@ -178,16 +185,19 @@ sub handler {
$r->content_type('text/html');
$r->send_http_header;
return OK if $r->header_only;
-
+ ##
+ ## Initialize global variables
+ ##
my $domain = $r->dir_config('lonDefDomain');
$diropendb= "/home/httpd/perl/tmp/".&Apache::lonnet::escape($domain).
"\_".&Apache::lonnet::escape($ENV{'user.name'})."_searchcat.db";
-
+ $results_db = "/home/httpd/perl/tmp/".&Apache::lonnet::escape($domain).
+ '_'.&Apache::lonnet::escape($ENV{'user.name'})."_searchresults.db";
&Apache::loncommon::get_unprocessed_cgi($ENV{'QUERY_STRING'},
['catalogmode','launch','acts','mode','form','element',
'reqinterface']);
##
- ## Clear out old values from database
+ ## Clear out old values from groupsearch database
##
if ($ENV{'form.launch'} eq '1') {
if (tie(%groupsearch_db,'GDBM_File',$diropendb,&GDBM_WRCREAT,0640)) {
@@ -239,24 +249,23 @@ END
# We are running a search
my ($query,$customquery,$customshow,$libraries) =
(undef,undef,undef,undef);
+ my $pretty_string;
if ($searchtype eq 'Basic') {
- $query = &parse_basic_search($r);
+ ($query,$pretty_string) = &parse_basic_search($r);
} elsif ($ENV{'form.advancedsubmit'} eq 'SEARCH') {
- ($query,$customquery,$customshow,$libraries)
+ ($query,$customquery,$customshow,$libraries,$pretty_string)
= &parse_advanced_search($r);
return OK if (! defined($query));
}
# Output some information to the user.
- $r->print(&search_results_header($searchtype));
+ $r->print(&search_results_header($searchtype,$pretty_string));
$r->print("Sending search request to LON-CAPA servers.
\n");
$r->rflush();
- # Send query statements over the network to be processed by
- # either the SQL database or a recursive scheme of 'grep'-like
- # actions (for custom metadata).
- my $reply=&Apache::lonnet::metadata_query($query,$customquery,
- $customshow,$libraries);
+ &run_search($r,$query,$customquery,$customshow,$libraries);
+ &display_results($r,$searchtype,$hidden,$importbutton,
+ $closebutton);
+
$r->rflush();
- &output_results($searchtype,$r,$reply,$hidden);
} else {
#
# We need to get information to search on
@@ -561,13 +570,13 @@ to be somewhat persistent.
sub make_persistent {
my %save = %{shift()};
my $persistent='';
- foreach (keys %save) {
- if (/^form\./ && !/submit/) {
- my $name=$_;
+ foreach my $name (keys %save) {
+ if ($name =~ /^form\./ && $name !~ /submit/) {
my @values = (ref($save{$name}) ? @{$save{$name}} : ($save{$name}));
$name=~s/^form\.//;
foreach (@values) {
s/\"/\'/g; # do not mess with html field syntax
+ next if (! $_ );
$persistent.=<
END
@@ -886,6 +895,7 @@ Parse advanced search form and return th
sub parse_advanced_search {
my ($r)=@_;
my $fillflag=0;
+ my $pretty_search_string = "
\n";
# Clean up fields for safety
for my $field ('title','author','subject','keywords','url','version',
'creationdatestart_month','creationdatestart_day',
@@ -925,16 +935,25 @@ sub parse_advanced_search {
# Turn the form input into a SQL-based query
my $query='';
my @queries;
+ my $font = '';
# Evaluate logical expression AND/OR/NOT phrase fields.
foreach my $field ('title','author','subject','notes','abstract','url',
'keywords','version','owner','mime') {
if ($ENV{'form.'.$field}) {
my $searchphrase = $ENV{'form.'.$field};
+ $pretty_search_string .= $font."$field contains ".
+ $searchphrase."";
if ($ENV{'form.'.$field.'_related'}) {
- $searchphrase = &related_version($searchphrase);
- $ENV{'form.'.$field} = $searchphrase;
- $ENV{'form.'.$field.'_related'} = undef;
+ my @New_Words;
+ ($searchphrase,@New_Words) = &related_version($searchphrase);
+ if (@New_Words) {
+ $pretty_search_string .= " with related words: ".
+ "@New_Words.";
+ } else {
+ $pretty_search_string .= " with no related words.";
+ }
}
+ $pretty_search_string .= "
\n";
push @queries,&build_SQL_query($field,$searchphrase);
}
}
@@ -945,10 +964,17 @@ sub parse_advanced_search {
# Evaluate option lists
if ($ENV{'form.language'} and $ENV{'form.language'} ne 'any') {
push @queries,"(language like \"$ENV{'form.language'}\")";
+ $pretty_search_string.=$font."language= ".
+ &Apache::loncommon::languagedescription($ENV{'form.language'}).
+ "
\n";
}
if ($ENV{'form.copyright'} and $ENV{'form.copyright'} ne 'any') {
push @queries,"(copyright like \"$ENV{'form.copyright'}\")";
+ $pretty_search_string.=$font."copyright = ".
+ &Apache::loncommon::copyrightdescription($ENV{'form.copyright'}).
+ "
\n";
}
+ #
# Evaluate date windows
my $datequery=&build_date_queries(
$ENV{'form.creationdatestart_month'},
@@ -968,18 +994,23 @@ sub parse_advanced_search {
if ($datequery=~/^Incorrect/) {
&output_date_error($r,$datequery);
return ;
- }
- elsif ($datequery) {
+ } elsif ($datequery) {
+ # Here is where you would set up pretty_search_string to output
+ # date query information.
push @queries,$datequery;
}
# Process form information for custom metadata querying
my $customquery=undef;
if ($ENV{'form.custommetadata'}) {
+ $pretty_search_string .=$font."Custom Metadata Search: ".
+ $ENV{'form.custommetadata'}."
\n";
$customquery=&build_custommetadata_query('custommetadata',
$ENV{'form.custommetadata'});
}
my $customshow=undef;
if ($ENV{'form.customshow'}) {
+ $pretty_search_string .=$font."Custom Metadata Display: ".
+ $ENV{'form.customshow'}."
\n";
$customshow=$ENV{'form.customshow'};
$customshow=~s/[^\w\s]//g;
my @fields=split(/\s+/,$customshow);
@@ -994,18 +1025,28 @@ sub parse_advanced_search {
my @allowed_domains = (ref($ENV{'form.domains'}) ? @{$ENV{'form.domains'}}
: ($ENV{'form.domains'}) );
my %domain_hash = ();
+ my $pretty_domains_string;
foreach (@allowed_domains) {
$domain_hash{$_}++;
}
- foreach (keys(%Apache::lonnet::libserv)) {
- if ($_ eq 'any') {
- $libraries_to_query = undef;
- last;
+ if ($domain_hash{'any'}) {
+ $pretty_domains_string = "Searching all domains.";
+ } else {
+ if (@allowed_domains > 1) {
+ $pretty_domains_string = "Searching domains:";
+ } else {
+ $pretty_domains_string = "Searching domain ";
}
- if (exists($domain_hash{$Apache::lonnet::hostdom{$_}})) {
- push @$libraries_to_query,$_;
+ foreach (sort @allowed_domains) {
+ $pretty_domains_string .= "$_ ";
+ }
+ foreach (keys(%Apache::lonnet::libserv)) {
+ if (exists($domain_hash{$Apache::lonnet::hostdom{$_}})) {
+ push @$libraries_to_query,$_;
+ }
}
}
+ $pretty_search_string .= $pretty_domains_string."
\n";
#
if (@queries) {
$query=join(" AND ",@queries);
@@ -1013,7 +1054,8 @@ sub parse_advanced_search {
} elsif ($customquery) {
$query = '';
}
- return ($query,$customquery,$customshow,$libraries_to_query);
+ return ($query,$customquery,$customshow,$libraries_to_query,
+ $pretty_search_string);
}
######################################################################
@@ -1047,9 +1089,16 @@ sub parse_basic_search {
&output_blank_field_error($r);
return OK;
}
+ my $pretty_search_string = ''.$ENV{'form.basicexp'}.'';
my $search_string = $ENV{'form.basicexp'};
if ($ENV{'form.related'}) {
- $search_string = &related_version($ENV{'form.basicexp'});
+ my @New_Words;
+ ($search_string,@New_Words) = &related_version($ENV{'form.basicexp'});
+ if (@New_Words) {
+ $pretty_search_string .= " with related words: @New_Words.";
+ } else {
+ $pretty_search_string .= " with no related words.";
+ }
}
# Build SQL query string based on form page
my $query='';
@@ -1058,7 +1107,8 @@ sub parse_basic_search {
'keywords'));
$concatarg='title' if $ENV{'form.titleonly'};
$query=&build_SQL_query('concat('.$concatarg.')',$search_string);
- return 'select * from metadata where '.$query;
+ $pretty_search_string .= "
\n";
+ return 'select * from metadata where '.$query,$pretty_search_string;
}
@@ -1082,16 +1132,17 @@ Note: Using this twice on a string is pr
sub related_version {
my $search_string = shift;
my $result = $search_string;
+ my %New_Words = ();
while ($search_string =~ /(\w+)/cg) {
my $word = $1;
next if (lc($word) =~ /\b(or|and|not)\b/);
my @Words = &Apache::loncommon::get_related_words($word);
- my $replacement = join " OR ", ($word,
- ($#Words>4? @Words[0..4] : @Words)
- );
+ @Words = ($#Words>4? @Words[0..4] : @Words);
+ foreach (@Words) { $New_Words{$_}++;}
+ my $replacement = join " OR ", ($word,@Words);
$result =~ s/(\b)$word(\b)/$1($replacement)$2/g;
}
- return $result;
+ return $result,sort(keys(%New_Words));
}
######################################################################
@@ -1263,6 +1314,400 @@ sub build_date_queries {
######################################################################
######################################################################
+=pod
+
+=item ©right_check()
+
+=cut
+
+######################################################################
+######################################################################
+
+sub copyright_check {
+ my $Metadata = shift;
+ # Check copyright tags and skip results the user cannot use
+ my (undef,undef,$resdom,$resname) = split('/',
+ $Metadata->{'url'});
+ # Check for priv
+ if (($Metadata->{'copyright'} eq 'priv') &&
+ (($ENV{'user.name'} ne $resname) &&
+ ($ENV{'user.domain'} ne $resdom))) {
+ return 0;
+ }
+ # Check for domain
+ if (($Metadata->{'copyright'} eq 'domain') &&
+ ($ENV{'user.domain'} ne $resdom)) {
+ return 0;
+ }
+ return 1;
+}
+
+#####################################################################
+#####################################################################
+
+=pod
+
+=item MySQL Table Description
+
+MySQL table creation requires a precise description of the data to be
+stored. The use of the correct types to hold data is vital to efficient
+storage and quick retrieval of records. The columns must be described in
+the following format:
+
+=cut
+
+##
+## Restrictions:
+## columns of type 'text' and 'blob' cannot have defaults.
+## columns of type 'enum' cannot be used for FULLTEXT.
+##
+my @DataOrder = qw/id title author subject url keywords version notes
+ abstract mime lang owner copyright creationdate lastrevisiondate hostname
+ idx_title idx_author idx_subject idx_abstract idx_mime idx_language
+ idx_owner idx_copyright/;
+
+my %Datatypes =
+ ( id =>{ type => 'INT',
+ restrictions => 'NOT NULL',
+ primary_key => 'yes',
+ auto_inc => 'yes'
+ },
+ title =>{ type=>'TEXT'},
+ author =>{ type=>'TEXT'},
+ subject =>{ type=>'TEXT'},
+ url =>{ type=>'TEXT',
+ restrictions => 'NOT NULL' },
+ keywords =>{ type=>'TEXT'},
+ version =>{ type=>'TEXT'},
+ notes =>{ type=>'TEXT'},
+ abstract =>{ type=>'TEXT'},
+ mime =>{ type=>'TEXT'},
+ lang =>{ type=>'TEXT'},
+ owner =>{ type=>'TEXT'},
+ copyright =>{ type=>'TEXT'},
+ hostname =>{ type=>'TEXT'},
+ #--------------------------------------------------
+ creationdate =>{ type=>'DATETIME'},
+ lastrevisiondate =>{ type=>'DATETIME'},
+ #--------------------------------------------------
+ idx_title =>{ type=>'FULLTEXT', target=>'title'},
+ idx_author =>{ type=>'FULLTEXT', target=>'author'},
+ idx_subject =>{ type=>'FULLTEXT', target=>'subject'},
+ idx_abstract =>{ type=>'FULLTEXT', target=>'abstract'},
+ idx_mime =>{ type=>'FULLTEXT', target=>'mime'},
+ idx_language =>{ type=>'FULLTEXT', target=>'lang'},
+ idx_owner =>{ type=>'FULLTEXT', target=>'owner'},
+ idx_copyright =>{ type=>'FULLTEXT', target=>'copyright'},
+ );
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &write_status()
+
+=cut
+
+######################################################################
+######################################################################
+sub write_status {
+ my ($r,$string) = @_;
+ $r->print("".$string."
\n");
+ $r->rflush();
+ return;
+}
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &run_search
+
+=cut
+
+######################################################################
+######################################################################
+sub run_search {
+ my ($r,$query,$customquery,$customshow,$serverlist) = @_;
+ #
+ my @Servers_to_contact;
+ if (defined($serverlist)) {
+ @Servers_to_contact = @$serverlist;
+ } else {
+ @Servers_to_contact = sort(keys(%Apache::lonnet::libserv));
+ }
+ my %Server_status;
+ #
+ # Timing variables
+ my $starttime = time;
+ my $max_time = 120; # seconds for the search to complete
+ #
+ # Create Table
+ #####################################
+ my $table = &Apache::lonmysql::create_table
+ ( { columns => \%Datatypes,
+ column_order => \@DataOrder,
+ } );
+ if (! defined($table)) {
+ # What do I do now? Print out an error page.
+ &Apache::lonnet::logthis("lonmysql attempted to create a table ".
+ "and this was the result:".
+ &Apache::lonmysql::get_error());
+ $r->print("An internal error occured with the database.
".
+ "The error has been logged, but you should probably alert".
+ " your system administrator.");
+ return;
+ }
+ $ENV{'form.table'}=$table;
+ #
+ #####################################
+ my $hitcountsum;
+ my $server;
+ my $status;
+ while ((time - $starttime < $max_time) &&
+ ((@Servers_to_contact) || keys(%Server_status))) {
+ # Send out a search request if it needs to be done.
+ if (@Servers_to_contact) {
+ # Contact one server
+ my $server = shift(@Servers_to_contact);
+ my $reply=&Apache::lonnet::metadata_query($query,$customquery,
+ $customshow,[$server]);
+ # We should let the user know we have contacted another server
+ ($server) = keys(%$reply);
+ $Server_status{$server} = $reply->{$server};
+ # &write_status($r,"Contacted:$server:reply:".
+ # $Server_status{$server});
+ # Hmmm, should we add to $max_time if we contact a server?
+ } else {
+ sleep(1); # wait a sec. to give time for files to be written
+ }
+ while (my ($server,$status) = each(%Server_status)) {
+ if ($status eq 'con_lost') {
+ delete ($Server_status{$server});
+ # &write_status($r,"Removing $server");
+ next;
+ }
+ $status=~/^([\.\w]+)$/;
+ my $datafile=$r->dir_config('lonDaemons').'/tmp/'.$1;
+ if (-e $datafile && ! -e "$datafile.end") {
+ # Let the user know we are receiving data from the server
+ &write_status($r,"$server:Receiving file");
+ next;
+ }
+ if (-e "$datafile.end") {
+ if (-z "$datafile") {
+ delete($Server_status{$server});
+ next;
+ }
+ my $fh;
+ if (!($fh=Apache::File->new($datafile))) {
+ # Error opening file...
+ # Tell the user and exit...?
+ # Should I give up on opening it?
+ &write_status("Unable to open $datafile");
+ next;
+ }
+ # Read in the whole file.
+ while (my $result = <$fh>) {
+ # handle custom fields? Someday we will!
+ chomp($result);
+ next unless $result;
+ # Parse the result.
+ my %Fields = &parse_raw_result($result,$server);
+ $Fields{'hostname'} = $server;
+ next if (! ©right_check(\%Fields));
+ # Store the result in the mysql database
+ my $result = &Apache::lonmysql::store_row($table,\%Fields);
+ if (! defined($result)) {
+ &write_status($r,&Apache::lonmysql::get_error());
+ }
+ # &write_status($r,&Apache::lonmysql::get_debug());
+ $hitcountsum ++;
+ } # End of foreach (@results)
+ $fh->close();
+ # $server is only deleted if the results file has been
+ # found and (successfully) opened. This may be a bad idea.
+ delete($Server_status{$server});
+ }
+ }
+ # Finished looping through the servers
+ }
+ &Apache::lonmysql::disconnect_from_db();
+ # We have run out of time or run out of servers to talk to and
+ # results to get.
+ if ($hitcountsum > 0) {
+ $r->print("Total results = $hitcountsum