--- loncom/interface/lonsearchcat.pm 2002/07/08 14:28:10 1.137 +++ loncom/interface/lonsearchcat.pm 2006/03/19 22:08:38 1.258 @@ -1,7 +1,7 @@ # The LearningOnline Network with CAPA # Search Catalog # -# $Id: lonsearchcat.pm,v 1.137 2002/07/08 14:28:10 matthew Exp $ +# $Id: lonsearchcat.pm,v 1.258 2006/03/19 22:08:38 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -25,14 +25,6 @@ # # http://www.lon-capa.org/ # -# YEAR=2001 -# 3/8, 3/12, 3/13, 3/14, 3/15, 3/19 Scott Harrison -# 3/20, 3/21, 3/22, 3/26, 3/27, 4/2, 8/15, 8/24, 8/25 Scott Harrison -# 10/12,10/14,10/15,10/16,11/28,11/29,12/10,12/12,12/16 Scott Harrison -# YEAR=2002 -# 1/17 Scott Harrison -# 6/17 Matthew Hall -# ############################################################################### ############################################################################### @@ -40,7 +32,7 @@ =head1 NAME -lonsearchcat +lonsearchcat - LONCAPA Search Interface =head1 SYNOPSIS @@ -56,7 +48,7 @@ described at http://www.lon-capa.org. lonsearchcat presents the user with an interface to search the LON-CAPA digital library. lonsearchcat also initiates the execution of a search by sending the search parameters to LON-CAPA servers. The progress of -search (on a server basis) is displayed to the user in a seperate window. +search (on a server basis) is displayed to the user in a separate window. =head1 Internals @@ -67,255 +59,724 @@ search (on a server basis) is displayed ############################################################################### ############################################################################### -############################################################################### -## ## -## ORGANIZATION OF THIS PERL MODULE ## -## ## -## 1. Modules used by this module ## -## 2. Variables used throughout the module ## -## 3. handler subroutine called via Apache and mod_perl ## -## 4. Other subroutines ## -## ## -############################################################################### - package Apache::lonsearchcat; -# ------------------------------------------------- modules used by this module use strict; -use Apache::Constants qw(:common); -use Apache::lonnet(); +use Apache::Constants qw(:common :http); +use Apache::lonnet; use Apache::File(); use CGI qw(:standard); use Text::Query; use GDBM_File; use Apache::loncommon(); - -# ---------------------------------------- variables used throughout the module - -###################################################################### -###################################################################### - -=pod - -=item Global variables - -=over 4 - -=item $closebutton - -button that closes the search window - -=item $importbutton - -button to take the select results and go to group sorting - -=item %hash - -The ubiquitous database hash - -=item $diropendb - -The full path to the (temporary) search database file. This is set and -used in &handler() and is also used in &output_results(). - -=back - -=cut +use Apache::lonmysql(); +use Apache::lonmeta; +use Apache::lonhtmlcommon; +use Apache::lonlocal; +use LONCAPA::lonmetadata(); +use HTML::Entities(); +use Parse::RecDescent; +use Apache::lonnavmaps; +use Apache::lonindexer(); ###################################################################### ###################################################################### - -# -- dynamically rendered interface components -my $closebutton; # button that closes the search window -my $importbutton; # button to take the selected results and go to group sorting - -# -- miscellaneous variables -my %hash; # database hash -my $diropendb = ""; # db file - -###################################################################### -###################################################################### - -=pod - -=item &handler() - main handler invoked by httpd child - -=item Variables - -=over 4 - -=item $hidden - -holds 'hidden' html forms - -=item $scrout - -string that holds portions of the screen output - -=back - -=cut +## +## Global variables +## +###################################################################### +###################################################################### +my %groupsearch_db; # Database hash used to save values for the + # groupsearch RAT interface. +my %persistent_db; # gdbm hash which holds data which is supposed to + # persist across calls to lonsearchcat.pm + +# The different view modes and associated functions + +my %Views = ("detailed" => \&detailed_citation_view, + "detailedpreview" => \&detailed_citation_preview, + "summary" => \&summary_view, + "summarypreview" => \&summary_preview, + "fielded" => \&fielded_format_view, + "xml" => \&xml_sgml_view, + "compact" => \&compact_view); ###################################################################### ###################################################################### sub handler { my $r = shift; - untie %hash; +# &set_defaults(); + # + # set form defaults + # + my $hidden_fields;# Hold all the hidden fields used to keep track + # of the search system state + my $importbutton; # button to take the selected results and go to group + # sorting + my $diropendb; # The full path to the (temporary) search database file. + # This is set and used in &handler() and is also used in + # &output_results(). - $r->content_type('text/html'); + my $loaderror=&Apache::lonnet::overloaderror($r); + if ($loaderror) { return $loaderror; } + # + my $closebutton; # button that closes the search window + # This button is different for the RAT compared to + # normal invocation. + # + &Apache::loncommon::content_type($r,'text/html'); $r->send_http_header; return OK if $r->header_only; - + ## + ## Prevent caching of the search interface window. Hopefully this means + ## we will get the launch=1 passed in a little more. + &Apache::loncommon::no_cache($r); + ## + ## Pick up form fields passed in the links. + ## + &Apache::loncommon::get_unprocessed_cgi($ENV{'QUERY_STRING'}, + ['catalogmode','launch','acts','mode','form','element','pause', + 'phase','persistent_db_id','table','start','show', + 'cleargroupsort','titleelement']); + ## + ## The following is a trick - we wait a few seconds if asked to so + ## the daemon running the search can get ahead of the daemon + ## printing the results. We only need (theoretically) to do + ## this once, so the pause indicator is deleted + ## + if (exists($env{'form.pause'})) { + sleep(1); + delete($env{'form.pause'}); + } + ## + ## Initialize global variables + ## my $domain = $r->dir_config('lonDefDomain'); - $diropendb= "/home/httpd/perl/tmp/".&Apache::lonnet::escape($domain). - "\_".&Apache::lonnet::escape($ENV{'user.name'})."_searchcat.db"; + $diropendb= "/home/httpd/perl/tmp/". + "$env{'user.domain'}_$env{'user.name'}_searchcat.db"; + # + # set the name of the persistent database + # $env{'form.persistent_db_id'} can only have digits in it. + if (! exists($env{'form.persistent_db_id'}) || + ($env{'form.persistent_db_id'} =~ /\D/) || + ($env{'form.launch'} eq '1')) { + $env{'form.persistent_db_id'} = time; + } - &Apache::loncommon::get_unprocessed_cgi($ENV{'QUERY_STRING'}, - ['catalogmode','launch','acts','mode','form','element', - 'reqinterface']); + my $persistent_db_file = "/home/httpd/perl/tmp/". + &Apache::lonnet::escape($domain). + '_'.&Apache::lonnet::escape($env{'user.name'}). + '_'.$env{'form.persistent_db_id'}.'_persistent_search.db'; ## - ## Clear out old values from database + &Apache::lonhtmlcommon::clear_breadcrumbs(); + if (exists($env{'request.course.id'}) && $env{'request.course.id'} ne '') { + &Apache::lonhtmlcommon::add_breadcrumb + ({href=>'/adm/searchcat?'. + 'catalogmode='.$env{'form.catalogmode'}. + '&launch='.$env{'form.launch'}. + '&mode='.$env{'form.mode'}, + text=>"Course and Catalog Search", + target=>'_top', + bug=>'Searching',}); + } else { + &Apache::lonhtmlcommon::add_breadcrumb + ({href=>'/adm/searchcat?'. + 'catalogmode='.$env{'form.catalogmode'}. + '&launch='.$env{'form.launch'}. + '&mode='.$env{'form.mode'}, + text=>"Catalog Search", + target=>'_top', + bug=>'Searching',}); + } + # + if ($env{'form.phase'} !~ m/(basic|adv|course)_search/) { + if (! &get_persistent_form_data($persistent_db_file)) { + if ($env{'form.phase'} =~ /(run_search|results)/) { + &Apache::lonnet::logthis('lonsearchcat:'. + 'Unable to recover data from '. + $persistent_db_file); + my $msg = + 'We were unable to retrieve data describing your search. '. + 'This is a serious error and has been logged. '. + 'Please alert your LON-CAPA administrator.'; + return &error_page($r,$msg); + } + } + } else { + &clean_up_environment(); + } + ## + ## Clear out old values from groupsearch database ## - if ($ENV{'form.launch'} eq '1') { - if (tie(%hash,'GDBM_File',$diropendb,&GDBM_WRCREAT,0640)) { + untie %groupsearch_db if (tied(%groupsearch_db)); + if (($env{'form.cleargroupsort'} eq '1') || + (($env{'form.launch'} eq '1') && + ($env{'form.catalogmode'} eq 'groupsearch'))) { + if (tie(%groupsearch_db,'GDBM_File',$diropendb,&GDBM_WRCREAT(),0640)) { &start_fresh_session(); - untie %hash; + untie %groupsearch_db; + delete($env{'form.cleargroupsort'}); } else { - $r->print('
Unable to tie hash to db '. - 'file'); - return OK; + # This is a stupid error to give to the user. + # It really tells them nothing. + my $msg = 'Unable to tie hash to db file.'; + return &error_page($r,$msg); } } ## - ## Produce some output, so people know it is working + ## Configure hidden fields ## - $r->print("\n"); - $r->rflush; + $hidden_fields = ''."\n"; + if (exists($env{'form.catalogmode'})) { + $hidden_fields .= &hidden_field('catalogmode'); + } + if (exists($env{'form.form'})) { + $hidden_fields .= &hidden_field('form'); + } + if (exists($env{'form.element'})) { + $hidden_fields .= &hidden_field('element'); + } + if (exists($env{'form.titleelement'})) { + $hidden_fields .= &hidden_field('titleelement'); + } + if (exists($env{'form.mode'})) { + $hidden_fields .= &hidden_field('mode'); + } ## ## Configure dynamic components of interface ## - my $hidden; # Holds 'hidden' html forms - if ($ENV{'form.catalogmode'} eq 'interactive') { - $hidden="". - "\n"; - $closebutton=""."\n"; - } elsif ($ENV{'form.catalogmode'} eq 'groupsearch') { - $hidden=<'.&mt('No matches found in resources').'.
'); + } + +# Check discussions if requested + if ($discuss) { + my $totaldiscussions = 0; + $r->print(''.&mt('No matches found in postings').'.
'); + } + } + +# =================================================== Done going through course + $r->print(&Apache::loncommon::end_page()); +} + +# =============================== This pulls up a resource and its dependencies + +sub checkonthis { + my ($r,$id,$url,$level,$title,$fulltext,$symb,@allwords)=@_; + $alreadyseen{$id}=1; + if (&Apache::loncommon::connection_aborted($r)) { return; } + $r->rflush(); + + my $result=$title.' '; + if ($env{'request.role.adv'} || !$hash{'encrypted_'.$id}) { + $result.=&Apache::lonnet::metadata($url,'title').' '. + &Apache::lonnet::metadata($url,'subject').' '. + &Apache::lonnet::metadata($url,'abstract').' '. + &Apache::lonnet::metadata($url,'keywords'); + } + my ($extension)=($url=~/\.(\w+)$/); + if (&Apache::loncommon::fileembstyle($extension) eq 'ssi' && + ($url) && ($fulltext)) { + $result.=&Apache::lonnet::ssi_body($url.'?symb='.&Apache::lonnet::escape($symb)); + } + $result=~s/\s+/ /gs; + my $applies = 0; + $applies = &checkwords($result,$applies,@allwords); +# Does this resource apply? + if ($applies) { + $r->print('-Enter terms or phrases separated by AND, OR, or NOT -then press SEARCH below. +$lt{'note'}.
-ENDDOCUMENT
- $scrout.=' '.&simpletextfield('basicexp',$ENV{'form.basicexp'},40).
- ' ';
-# $scrout.=&simplecheckbox('allversions',$ENV{'form.allversions'});
-# $scrout.='Search historic archives';
- $scrout.=<Advanced Search | |
-
-$closebutton
-
-
-
-
+ENDCOURSESEARCH
+ $scrout.=' '.
+ &Apache::lonhtmlcommon::textbox('courseexp',
+ $env{'form.courseexp'},40);
+ my $crscheckbox =
+ &Apache::lonhtmlcommon::checkbox('crsfulltext',
+ $env{'form.crsfulltext'});
+ my $relcheckbox =
+ &Apache::lonhtmlcommon::checkbox('crsrelated',
+ $env{'form.crsrelated'});
+ my $discheckbox =
+ &Apache::lonhtmlcommon::checkbox('crsdiscuss',
+ $env{'form.crsrelated'});
+ $scrout.=(<
+
+
+
+
- - +sub print_advanced_search_form{ + my ($r,$closebutton,$hidden_fields) = @_; + my $bread_crumb = + &Apache::lonhtmlcommon::breadcrumbs(undef,'Searching', + 'Search_Advanced', + undef,undef, + $env{'form.catalogmode'} ne 'groupsearch'); + my %lt=&Apache::lonlocal::texthash('srch' => 'Search', + 'reset' => 'Reset', + 'help' => 'Help'); + my $advanced_buttons=<<"END"; + + $closebutton - -
END - my $scrout=<<"ENDHEADER"; - - -+Search:$pretty_query_string +
+ +END + $r->print($result.&Apache::loncommon::end_page()); + return; +} + +##################################################################### +##################################################################### + +=pod + +=item MySQL Table Description + +MySQL table creation requires a precise description of the data to be +stored. The use of the correct types to hold data is vital to efficient +storage and quick retrieval of records. The columns must be described in +the following format: + +=cut + +##################################################################### +##################################################################### +# +# These should probably be scoped but I don't have time right now... +# +my @Datatypes; +my @Fullindicies; + +###################################################################### +###################################################################### + +=pod + +=item &create_results_table() + +Creates the table of search results by calling lonmysql. Stores the +table id in $env{'form.table'} + +Inputs: none. + +Returns: the identifier of the table on success, undef on error. + +=cut + +###################################################################### +###################################################################### +sub set_up_table_structure { + my ($datatypes,$fullindicies) = + &LONCAPA::lonmetadata::describe_metadata_storage(); + # Copy the table description before modifying it... + @Datatypes = @{$datatypes}; + unshift(@Datatypes,{name => 'id', + type => 'MEDIUMINT', + restrictions => 'UNSIGNED NOT NULL', + primary_key => 'yes', + auto_inc => 'yes' }); + @Fullindicies = @{$fullindicies}; + return; +} + +sub create_results_table { + &set_up_table_structure(); + my $table = &Apache::lonmysql::create_table + ( { columns => \@Datatypes, + FULLTEXT => [{'columns' => \@Fullindicies},], + } ); + if (defined($table)) { + $env{'form.table'} = $table; + return $table; + } + return undef; # Error... +} + +###################################################################### +###################################################################### + +=pod + +=item Search Status update functions + +Each of the following functions changes the values of one of the +input fields used to display the search status to the user. The names +should be explanatory. + +Inputs: Apache request handler ($r), text to display. + +Returns: Nothing. + +=over 4 + +=item &update_count_status() + +=item &update_status() + +=item &update_seconds() + +=back + +=cut + +###################################################################### +###################################################################### +sub update_count_status { + my ($r,$text) = @_; + $text =~ s/\'/\\\'/g; + $r->print + ("\n"); + $r->rflush(); +} + +sub update_status { + my ($r,$text) = @_; + $text =~ s/\'/\\\'/g; + $r->print + ("\n"); + $r->rflush(); +} + +{ + my $max_time = 300; # seconds for the search to complete + my $start_time = 0; + my $last_time = 0; + +sub reset_timing { + $start_time = 0; + $last_time = 0; +} + +sub time_left { + if ($start_time == 0) { + $start_time = time; + } + my $time_left = $max_time - (time - $start_time); + $time_left = 0 if ($time_left < 0); + return $time_left; +} + +sub update_seconds { + my ($r) = @_; + my $time = &time_left(); + if (($last_time-$time) > 0) { + $r->print("\n"); + $r->rflush(); + } + $last_time = $time; +} + +} + +###################################################################### +###################################################################### + +=pod + +=item &revise_button() + +Inputs: None + +Returns: html string for a 'revise search' button. + +=cut + +###################################################################### +###################################################################### +sub revise_button { + my $revise_phase = 'disp_basic'; + $revise_phase = 'disp_adv' if ($env{'form.searchmode'} eq 'advanced'); + my $newloc = '/adm/searchcat'. + '?persistent_db_id='.$env{'form.persistent_db_id'}. + '&cleargroupsort=1'. + '&phase='.$revise_phase; + my $result = qq{ }; + return $result; +} + +###################################################################### +###################################################################### + +=pod + +=item &run_search() + +Executes a search query by sending it the the other servers and putting the +results into MySQL. + +=cut + +###################################################################### +###################################################################### +sub run_search { + my ($r,$query,$customquery,$customshow,$serverlist,$pretty_string) = @_; + + my $connection = $r->connection; # - # make query information persistent to allow for subsequent revision - my $persistent=&make_persistent(\%ENV); + # Print run_search header # - # Begin producing output - $r->print(&search_results_header($mode)); + my $start_page = &Apache::loncommon::start_page('Search Status',undef, + {'only_body' => 1}); + my $breadcrumbs = &Apache::lonhtmlcommon::breadcrumbs + (undef,'Searching','Searching',undef,undef, + $env{'form.catalogmode'} ne 'groupsearch'); + $r->print(<copyright = 'priv' -> Skipping $Fields{'url'}\n"); + # wait a sec. to give time for files to be written + # This sleep statement is here instead of outside the else + # block because we do not want to pause if we have servers + # left to contact. + if (scalar (keys(%Server_status))) { + &update_status($r, + &mt('waiting on [_1]',join(' ',keys(%Server_status)))); + } + sleep(1); + } + # + # Loop through the servers we have contacted but do not + # have results from yet, looking for results. + foreach my $server (keys(%Server_status)) { + last if ($connection->aborted()); + &update_seconds($r); + my $status = $Server_status{$server}; + if ($status eq 'con_lost') { + delete ($Server_status{$server}); + next; + } + $status=~s|/||g; + my $datafile=$r->dir_config('lonDaemons').'/tmp/'.$status; + if (-e $datafile && ! -e "$datafile.end") { + &update_status($r,&mt('Receiving results from [_1]',$server)); + next; + } + last if ($connection->aborted()); + if (-e "$datafile.end") { + &update_status($r,&mt('Reading results from [_1]',$server)); + if (-z "$datafile") { + delete($Server_status{$server}); next; } - # Check for domain - if (($Fields{'copyright'} eq 'domain') && - ($ENV{'user.domain'} ne $resdom)) { -$r->print("
copyright = 'domain' -> Skipping $Fields{'url'}\n"); - next; + my $fh; + if (!($fh=Apache::File->new($datafile))) { + $r->print("Unable to open search results file for ". + "server $server. Omitting from search"); + delete($Server_status{$server}); + next; } - # - $Fields{'extrashow'}=$extrashow; - if ($extrashow) { - foreach my $field (@customfields) { - my $value=''; - $value = $1 if ($customhash{$Fields{'url'}}=~/\<{$field}[^\>]*\>(.*?)\<\/{$field}[^\>]*\>/s); - $Fields{'extrashow'}=~s/\<\!\-\- $field \-\-\>/ $value/g; + # Read in the whole file. + while (my $result = <$fh>) { + last if ($connection->aborted()); + # + # Records are stored one per line + chomp($result); + next if (! $result); + # + # Parse the result. + my %Fields = &parse_raw_result($result,$server); + $Fields{'hostname'} = $server; + # + # Skip if external and we did not want that + next if ((! $env{'form.inclext'}) && ($Fields{'url'}=~/^\/ext\//)); + # Skip based on copyright + next if (! ©right_check(\%Fields)); + + # + # Store the result in the mysql database + my $result = &Apache::lonmysql::store_row($table,\%Fields); + if (! defined($result)) { + $r->print(&Apache::lonmysql::get_error()); + } + # + $hitcountsum ++; + &update_seconds($r); + if ($hitcountsum % 50 == 0) { + &update_count_status($r,$hitcountsum); } } - $compiledresult.="\n
\n";
- if ($ENV{'form.catalogmode'} eq 'interactive') {
- my $titleesc=$Fields{'title'};
- $titleesc=~s/\'/\\'/; # '
- $compiledresult.=< \n";
+ if (! defined($Fields{'title'}) || $Fields{'title'} eq '') {
+ $Fields{'title'} = 'Untitled';
+ }
+ my $prefix=&catalogmode_output($Fields{'title'},$Fields{'url'},
+ $Fields{'id'},$checkbox_num++);
+ # Render the result into html
+ $output.= &$viewfunction($prefix,%Fields);
+ # Print them out as they come in.
+ $r->print($output);
+ $r->rflush();
+ }
+ if (@Results < 1) {
+ $r->print(&mt("There were no results matching your query"));
+ } else {
+ $r->print
+ ('
'
+ );
+ if ($total_results == 0) {
+ $r->print(''.
+ ''.
+ ' '.
+ &prev_next_buttons($min,$env{'form.show'},$total_results).
+ ' '.
+ &viewoptions().' '.&mt('There are currently no results').'.
'.
+ "".
+ &Apache::loncommon::end_page());
+ return;
+ } else {
+ $r->print('
END
- }
- if ($ENV{'form.catalogmode'} eq 'groupsearch') {
- $fnum+=0;
- $hash{"pre_${fnum}_link"}=$Fields{'url'};
- $hash{"pre_${fnum}_title"}=$Fields{'title'};
- $compiledresult.=<
END
-#
-#
- $fnum++;
- }
- # Render the result into html
- $compiledresult.= &$viewfunction(%Fields, hostname => $rkey );
- if ($compiledresult or $servercount!=$servernum) {
- $compiledresult.="
";
- }
- }
- untie %hash;
- }
- if ($compiledresult) {
- $resultflag=1;
- $r->print($compiledresult);
- }
- } # End of foreach loop over servers remaining
- } # End of big loop - while($serversleft && $timeremain)
- unless ($resultflag) {
- $r->print("\nThere were no results that matched your query\n");
- }
- $r->print(''.
- "\n");
- $r->print("