[BACK]Return to searchcat.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/searchcat.pl between versions 1.23 and 1.32

version 1.23, 2002/10/08 18:45:33 version 1.32, 2003/03/26 20:15:57
Line 26 Line 26
 #  #
 # http://www.lon-capa.org/  # http://www.lon-capa.org/
 #  #
 # YEAR=2001  
 # 04/14/2001, 04/16/2001 Scott Harrison  
 #  
 # YEAR=2002  
 # 05/11/2002 Scott Harrison  
 #  
 ###  ###
   =pod
   
   =head1 NAME
   
   B<searchcat.pl> - put authoritative filesystem data into sql database.
   
   =head1 SYNOPSIS
   
   Ordinarily this script is to be called from a loncapa cron job
   (CVS source location: F<loncapa/loncom/cron/loncapa>; typical
   filesystem installation location: F</etc/cron.d/loncapa>).
   
   Here is the cron job entry.
   
   C<# Repopulate and refresh the metadata database used for the search catalog.>
   C<10 1 * * 7    www    /home/httpd/perl/searchcat.pl>
   
   This script only allows itself to be run as the user C<www>.
   
   =head1 DESCRIPTION
   
 # This script goes through a LON-CAPA resource  This script goes through a loncapa resource directory and gathers metadata.
 # directory and gathers metadata.  The metadata is entered into a SQL database.
 # The metadata is entered into a SQL database.  
   This script also does general database maintenance such as reformatting
   the C<loncapa:metadata> table if it is deprecated.
   
   This script evaluates dynamic metadata from the authors'
   F<nohist_resevaldata.db> database file in order to store it in MySQL, as
   well as to compress the filesize (add up all "count"-type metadata).
   
   This script is playing an increasingly important role for a loncapa
   library server.  The proper operation of this script is critical for a smooth
   and correct user experience.
   
   =cut
   
 use lib '/home/httpd/lib/perl/';  use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;  use LONCAPA::Configuration;
Line 45  use IO::File; Line 71  use IO::File;
 use HTML::TokeParser;  use HTML::TokeParser;
 use DBI;  use DBI;
 use GDBM_File;  use GDBM_File;
   use POSIX qw(strftime mktime);
   
 my @metalist;  my @metalist;
   
Line 69  sub escape { Line 96  sub escape {
 # ------------------------------------------- Code to evaluate dynamic metadata  # ------------------------------------------- Code to evaluate dynamic metadata
   
 sub dynamicmeta {  sub dynamicmeta {
 #  
 #  
 # Do nothing for now ...  
 #  
 #  
     return;  
 #  
 # ..., but stuff below already works  
 #  
     my $url=&declutter(shift);      my $url=&declutter(shift);
     $url=~s/\.meta$//;      $url=~s/\.meta$//;
     my %returnhash=();      my %returnhash=();
     my ($adomain,$aauthor)=($url=~/^(\w+)\/(\w+)\//);      my ($adomain,$aauthor)=($url=~/^(\w+)\/(\w+)\//);
     my $prodir=&propath($adomain,$aauthor);      my $prodir=&propath($adomain,$aauthor);
     if (tie(%evaldata,'GDBM_File',      if ((tie(%evaldata,'GDBM_File',
             $prodir.'/nohist_resevaldata.db',&GDBM_WRCREAT(),0640)) {              $prodir.'/nohist_resevaldata.db',&GDBM_READER(),0640)) &&
           (tie(%newevaldata,'GDBM_File',
               $prodir.'/nohist_new_resevaldata.db',&GDBM_WRCREAT(),0640))) {
        my %sum=();         my %sum=();
        my %cnt=();         my %cnt=();
        my %listitems=('count'        => 'add',         my %listitems=('count'        => 'add',
Line 127  sub dynamicmeta { Line 148  sub dynamicmeta {
              $sum{$ctype}=''.$evaldata{$_};               $sum{$ctype}=''.$evaldata{$_};
        }         }
     }      }
     if ($ctype eq 'count') {      if ($ctype ne 'count') {
        delete($evaldata{$_});         $newevaldata{$_}=$evaldata{$_};
             }     }
  }   }
       }        }
       foreach (keys %cnt) {        foreach (keys %cnt) {
Line 143  sub dynamicmeta { Line 164  sub dynamicmeta {
      }       }
      if ($returnhash{'count'}) {       if ($returnhash{'count'}) {
          my $newkey=$$.'_'.time.'_searchcat___'.&escape($url).'___count';           my $newkey=$$.'_'.time.'_searchcat___'.&escape($url).'___count';
          $evaldata{$newkey}=$returnhash{'count'};           $newevaldata{$newkey}=$returnhash{'count'};
      }       }
      untie(%evaldata);       untie(%evaldata);
        untie(%newevaldata);
    }     }
    return %returnhash;     return %returnhash;
 }  }
Line 168  delete $perlvar{'lonReceipt'}; # remove Line 190  delete $perlvar{'lonReceipt'}; # remove
 # ------------------------------------- Only run if machine is a library server  # ------------------------------------- Only run if machine is a library server
 exit unless $perlvar{'lonRole'} eq 'library';  exit unless $perlvar{'lonRole'} eq 'library';
   
   # ----------------------------- Make sure this process is running from user=www
   
   my $wwwid=getpwnam('www');
   if ($wwwid!=$<) {
      $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
      $subj="LON: $perlvar{'lonHostID'} User ID mismatch";
      system("echo 'User ID mismatch. searchcat.pl must be run as user www.' |\
    mailto $emailto -s '$subj' > /dev/null");
      exit 1;
   }
   
   
   # ---------------------------------------------------------- We are in business
   
   open(LOG,'>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log');
   print LOG '==== Searchcat Run '.localtime()."====\n\n";
 my $dbh;  my $dbh;
 # ------------------------------------- Make sure that database can be accessed  # ------------------------------------- Make sure that database can be accessed
 {  {
     unless (      unless (
     $dbh = DBI->connect("DBI:mysql:loncapa","www",$perlvar{'lonSqlAccess'},{ RaiseError =>0,PrintError=>0})      $dbh = DBI->connect("DBI:mysql:loncapa","www",$perlvar{'lonSqlAccess'},{ RaiseError =>0,PrintError=>0})
     ) {       ) { 
  print "Cannot connect to database!\n";   print LOG "Cannot connect to database!\n";
  exit;   exit;
     }      }
     my $make_metadata_table = "CREATE TABLE IF NOT EXISTS metadata (".      my $make_metadata_table = "CREATE TABLE IF NOT EXISTS metadata (".
Line 199  my @homeusers=grep Line 237  my @homeusers=grep
           grep {!/^\.\.?$/} readdir(RESOURCES);            grep {!/^\.\.?$/} readdir(RESOURCES);
 closedir RESOURCES;  closedir RESOURCES;
 foreach my $user (@homeusers) {  foreach my $user (@homeusers) {
       print LOG "\n=== User: ".$user."\n\n";
   # Remove left-over db-files from potentially crashed searchcat run
       my $prodir=&propath($perlvar{'lonDefDomain'},$user);
       unlink($prodir.'/nohist_new_resevaldata.db');
   # Use find.pl
       undef @metalist;
       @metalist=();
     &find("$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$user");      &find("$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$user");
 }  
   
 # -- process each file to get metadata and put into search catalog SQL database  # -- process each file to get metadata and put into search catalog SQL database
 # Also, check to see if already there.  # Also, check to see if already there.
 # I could just delete (without searching first), but this works for now.  # I could just delete (without searching first), but this works for now.
 foreach my $m (@metalist) {  foreach my $m (@metalist) {
       print LOG "- ".$m."\n";
     my $ref=&metadata($m);      my $ref=&metadata($m);
     my $m2='/res/'.&declutter($m);      my $m2='/res/'.&declutter($m);
     $m2=~s/\.meta$//;      $m2=~s/\.meta$//;
Line 240  foreach my $m (@metalist) { Line 285  foreach my $m (@metalist) {
 # Need to, perhaps, remove stale SQL database records.  # Need to, perhaps, remove stale SQL database records.
 # ... not yet implemented  # ... not yet implemented
   
   
   # -------------------------------------------------- Copy over the new db-files
       system('mv '.$prodir.'/nohist_new_resevaldata.db '.
            $prodir.'/nohist_resevaldata.db');
   }
 # --------------------------------------------------- Close database connection  # --------------------------------------------------- Close database connection
 $dbh->disconnect;  $dbh->disconnect;
   print LOG "\n==== Searchcat completed ".localtime()." ====\n";
   close(LOG);
   exit 0;
   # =============================================================================
   
 # ---------------------------------------------------------------- Get metadata  # ---------------------------------------------------------------- Get metadata
 # significantly altered from subroutine present in lonnet  # significantly altered from subroutine present in lonnet
Line 335  sub propath { Line 389  sub propath {
 # ---------------------------- convert 'time' format into a datetime sql format  # ---------------------------- convert 'time' format into a datetime sql format
 sub sqltime {  sub sqltime {
     my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =      my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
  localtime(@_[0]);   localtime(&unsqltime(@_[0]));
     $mon++; $year+=1900;      $mon++; $year+=1900;
     return "$year-$mon-$mday $hour:$min:$sec";      return "$year-$mon-$mday $hour:$min:$sec";
 }  }
   
   sub maketime {
       my %th=@_;
       return POSIX::mktime(
           ($th{'seconds'},$th{'minutes'},$th{'hours'},
            $th{'day'},$th{'month'}-1,$th{'year'}-1900,0,0,$th{'dlsav'}));
   }
   
   
   #########################################
   #
   # Retro-fixing of un-backward-compatible time format
   
   sub unsqltime {
       my $timestamp=shift;
       if ($timestamp=~/^(\d+)\-(\d+)\-(\d+)\s+(\d+)\:(\d+)\:(\d+)$/) {
          $timestamp=&maketime(
      'year'=>$1,'month'=>$2,'day'=>$3,
              'hours'=>$4,'minutes'=>$5,'seconds'=>$6);
       }
       return $timestamp;
   }
   
Removed from v.1.23  
changed lines
  Added in v.1.32

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>