[BACK]Return to searchcat.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/searchcat.pl between versions 1.24 and 1.46

version 1.24, 2002/10/18 13:54:31 version 1.46, 2003/12/24 20:41:32
Line 26 Line 26
 #  #
 # http://www.lon-capa.org/  # http://www.lon-capa.org/
 #  #
 # YEAR=2001  
 # 04/14/2001, 04/16/2001 Scott Harrison  
 #  
 # YEAR=2002  
 # 05/11/2002 Scott Harrison  
 #  
 ###  ###
   
 # This script goes through a LON-CAPA resource  =pod
 # directory and gathers metadata.  
 # The metadata is entered into a SQL database.  =head1 NAME
   
   B<searchcat.pl> - put authoritative filesystem data into sql database.
   
   =head1 SYNOPSIS
   
   Ordinarily this script is to be called from a loncapa cron job
   (CVS source location: F<loncapa/loncom/cron/loncapa>; typical
   filesystem installation location: F</etc/cron.d/loncapa>).
   
   Here is the cron job entry.
   
   C<# Repopulate and refresh the metadata database used for the search catalog.>
   C<10 1 * * 7    www    /home/httpd/perl/searchcat.pl>
   
   This script only allows itself to be run as the user C<www>.
   
   =head1 DESCRIPTION
   
   This script goes through a loncapa resource directory and gathers metadata.
   The metadata is entered into a SQL database.
   
   This script also does general database maintenance such as reformatting
   the C<loncapa:metadata> table if it is deprecated.
   
   This script evaluates dynamic metadata from the authors'
   F<nohist_resevaldata.db> database file in order to store it in MySQL, as
   well as to compress the filesize (add up all "count"-type metadata).
   
   This script is playing an increasingly important role for a loncapa
   library server.  The proper operation of this script is critical for a smooth
   and correct user experience.
   
   =cut
   
   use strict;
   
 use lib '/home/httpd/lib/perl/';  use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;  use LONCAPA::Configuration;
Line 47  use DBI; Line 76  use DBI;
 use GDBM_File;  use GDBM_File;
 use POSIX qw(strftime mktime);  use POSIX qw(strftime mktime);
   
   require "find.pl";
   
 my @metalist;  my @metalist;
   
   my $simplestatus='';
   my %countext=();
   
   # ----------------------------------------------------- write out simple status
   sub writesimple {
       open(SMP,'>/home/httpd/html/lon-status/mysql.txt');
       print SMP $simplestatus."\n";
       close(SMP);
   }
   
   sub writecount {
       open(RSMP,'>/home/httpd/html/lon-status/rescount.txt');
       foreach (keys %countext) {
    print RSMP $_.'='.$countext{$_}.'&';
       }
       print RSMP 'time='.time."\n";
       close(RSMP);
   }
   
   # -------------------------------------- counts files with different extensions
   sub count {
       my $file=shift;
       $file=~/\.(\w+)$/;
       my $ext=lc($1);
       if (defined($countext{$ext})) {
    $countext{$ext}++;
       } else {
    $countext{$ext}=1;
       }
   }
 # ----------------------------------------------------- Un-Escape Special Chars  # ----------------------------------------------------- Un-Escape Special Chars
   
 sub unescape {  sub unescape {
Line 66  sub escape { Line 126  sub escape {
     return $str;      return $str;
 }  }
   
   
 # ------------------------------------------- Code to evaluate dynamic metadata  # ------------------------------------------- Code to evaluate dynamic metadata
   
 sub dynamicmeta {  sub dynamicmeta {
 #  
 #  
 # Do nothing for now ...  
 #  
 #  
     return;  
 #  
 # ..., but stuff below already works  
 #  
     my $url=&declutter(shift);      my $url=&declutter(shift);
     $url=~s/\.meta$//;      $url=~s/\.meta$//;
     my %returnhash=();      my %returnhash=();
     my ($adomain,$aauthor)=($url=~/^(\w+)\/(\w+)\//);      my ($adomain,$aauthor)=($url=~/^(\w+)\/(\w+)\//);
     my $prodir=&propath($adomain,$aauthor);      my $prodir=&propath($adomain,$aauthor);
     if (tie(%evaldata,'GDBM_File',  
             $prodir.'/nohist_resevaldata.db',&GDBM_WRCREAT(),0640)) {  # Get metadata except counts
        my %sum=();      if (tie(my %evaldata,'GDBM_File',
        my %cnt=();              $prodir.'/nohist_resevaldata.db',&GDBM_READER(),0640)) {
        my %listitems=('count'        => 'add',   my %sum=();
                       'course'       => 'add',   my %cnt=();
                       'avetries'     => 'avg',   my %concat=();
                       'stdno'        => 'add',   my %listitems=(
                       'difficulty'   => 'avg',         'course'       => 'add',
                       'clear'        => 'avg',         'goto'         => 'add',
                       'technical'    => 'avg',         'comefrom'     => 'add',
                       'helpful'      => 'avg',         'avetries'     => 'avg',
                       'correct'      => 'avg',         'stdno'        => 'add',
                       'depth'        => 'avg',         'difficulty'   => 'avg',
                       'comments'     => 'app',         'clear'        => 'avg',
                       'usage'        => 'cnt'         'technical'    => 'avg',
                       );         'helpful'      => 'avg',
        my $regexp=$url;         'correct'      => 'avg',
        $regexp=~s/(\W)/\\$1/g;         'depth'        => 'avg',
        $regexp='___'.$regexp.'___([a-z]+)$';         'comments'     => 'app',
        foreach (keys %evaldata) {         'usage'        => 'cnt'
  my $key=&unescape($_);         );
  if ($key=~/$regexp/) {  
     my $ctype=$1;   my $regexp=$url;
             if (defined($cnt{$ctype})) {    $regexp=~s/(\W)/\\$1/g;
                $cnt{$ctype}++;    $regexp='___'.$regexp.'___([a-z]+)$';
             } else {    while (my ($esckey,$value)=each %evaldata) {
                $cnt{$ctype}=1;       my $key=&unescape($esckey);
             }      if ($key=~/$regexp/) {
             unless ($listitems{$ctype} eq 'app') {   my ($item,$purl,$cat)=split(/___/,$key);
                if (defined($sum{$ctype})) {   if (defined($cnt{$cat})) { $cnt{$cat}++; } else { $cnt{$cat}=1; }
                   $sum{$ctype}+=$evaldata{$_};   unless ($listitems{$cat} eq 'app') {
           } else {      if (defined($sum{$cat})) {
                   $sum{$ctype}=$evaldata{$_};   $sum{$cat}+=$evaldata{$esckey};
        }   $concat{$cat}.=','.$item;
             } else {      } else {
                if (defined($sum{$ctype})) {   $sum{$cat}=$evaldata{$esckey};
                   if ($evaldata{$_}) {   $concat{$cat}=$item;
                      $sum{$ctype}.='<hr>'.$evaldata{$_};      }
           }   } else {
         } else {      if (defined($sum{$cat})) {
              $sum{$ctype}=''.$evaldata{$_};   if ($evaldata{$esckey}=~/\w/) {
        }      $sum{$cat}.='<hr>'.$evaldata{$esckey};
    }
       } else {
    $sum{$cat}=''.$evaldata{$esckey};
       }
    }
     }      }
     if ($ctype eq 'count') {   }
        delete($evaldata{$_});   untie(%evaldata);
             }  # transfer gathered data to returnhash, calculate averages where applicable
  }   while (my $cat=each(%cnt)) {
       }      if ($listitems{$cat} eq 'avg') {
       foreach (keys %cnt) {   $returnhash{$cat}=int(($sum{$cat}/$cnt{$cat})*100.0+0.5)/100.0;
          if ($listitems{$_} eq 'avg') {      } elsif ($listitems{$cat} eq 'cnt') {
      $returnhash{$_}=int(($sum{$_}/$cnt{$_})*100.0+0.5)/100.0;   $returnhash{$cat}=$cnt{$cat};
          } elsif ($listitems{$_} eq 'cnt') {      } else {
              $returnhash{$_}=$cnt{$_};   $returnhash{$cat}=$sum{$cat};
          } else {      }
              $returnhash{$_}=$sum{$_};      $returnhash{$cat.'_list'}=$concat{$cat};
          }   }
      }      }
      if ($returnhash{'count'}) {  # get count
          my $newkey=$$.'_'.time.'_searchcat___'.&escape($url).'___count';      if (tie(my %evaldata,'GDBM_File',
          $evaldata{$newkey}=$returnhash{'count'};              $prodir.'/nohist_accesscount.db',&GDBM_READER(),0640)) {
      }   my $escurl=&escape($url);
      untie(%evaldata);   if (! exists($evaldata{$escurl})) {
    }      $returnhash{'count'}=0;
    return %returnhash;   } else {
       $returnhash{'count'}=$evaldata{$escurl};
    }
    untie %evaldata;
       }
       return %returnhash;
 }  }
       
 # ----------------- Code to enable 'find' subroutine listing of the .meta files  
 require "find.pl";  
 sub wanted {  
     (($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_)) &&  
     -f _ &&  
     /^.*\.meta$/ && !/^.+\.\d+\.[^\.]+\.meta$/ &&  
     push(@metalist,"$dir/$_");  
 }  
   
 # ---------------  Read loncapa_apache.conf and loncapa.conf and get variables  # ---------------  Read loncapa_apache.conf and loncapa.conf and get variables
 my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');  my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
 my %perlvar=%{$perlvarref};  my %perlvar=%{$perlvarref};
Line 169  delete $perlvar{'lonReceipt'}; # remove Line 220  delete $perlvar{'lonReceipt'}; # remove
 # ------------------------------------- Only run if machine is a library server  # ------------------------------------- Only run if machine is a library server
 exit unless $perlvar{'lonRole'} eq 'library';  exit unless $perlvar{'lonRole'} eq 'library';
   
   # ----------------------------- Make sure this process is running from user=www
   
   my $wwwid=getpwnam('www');
   if ($wwwid!=$<) {
       my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
       my $subj="LON: $perlvar{'lonHostID'} User ID mismatch";
       system("echo 'User ID mismatch. searchcat.pl must be run as user www.' |\
    mailto $emailto -s '$subj' > /dev/null");
       exit 1;
   }
   
   
   # ---------------------------------------------------------- We are in business
   
   open(LOG,'>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log');
   print LOG '==== Searchcat Run '.localtime()."====\n\n";
   $simplestatus='time='.time.'&';
 my $dbh;  my $dbh;
 # ------------------------------------- Make sure that database can be accessed  # ------------------------------------- Make sure that database can be accessed
 {  {
     unless (      unless (
     $dbh = DBI->connect("DBI:mysql:loncapa","www",$perlvar{'lonSqlAccess'},{ RaiseError =>0,PrintError=>0})      $dbh = DBI->connect("DBI:mysql:loncapa","www",$perlvar{'lonSqlAccess'},{ RaiseError =>0,PrintError=>0})
     ) {       ) { 
  print "Cannot connect to database!\n";   print LOG "Cannot connect to database!\n";
    $simplestatus.='mysql=defunct';
    &writesimple();
  exit;   exit;
     }      }
   
   # Create table for static metadata, unless exists
     my $make_metadata_table = "CREATE TABLE IF NOT EXISTS metadata (".      my $make_metadata_table = "CREATE TABLE IF NOT EXISTS metadata (".
         "title TEXT, author TEXT, subject TEXT, url TEXT, keywords TEXT, ".          "title TEXT, author TEXT, subject TEXT, url TEXT, keywords TEXT, ".
         "version TEXT, notes TEXT, abstract TEXT, mime TEXT, language TEXT, ".          "version TEXT, notes TEXT, abstract TEXT, mime TEXT, language TEXT, ".
Line 191  my $dbh; Line 263  my $dbh;
         "FULLTEXT idx_copyright (copyright)) TYPE=MYISAM";          "FULLTEXT idx_copyright (copyright)) TYPE=MYISAM";
     # It would sure be nice to have some logging mechanism.      # It would sure be nice to have some logging mechanism.
     $dbh->do($make_metadata_table);      $dbh->do($make_metadata_table);
   
   # Create table for dynamic metadata, unless exists
       my $make_dynmetadata_table = "CREATE TABLE IF NOT EXISTS dynmetadata (".
           "url TEXT, count INTEGER UNSIGNED, ".
           "course INTEGER UNSIGNED, course_list TEXT, ".
           "goto INTEGER UNSIGNED, goto_list TEXT, ".
           "comefrom INTEGER UNSIGNED, comefrom_list TEXT, ".
           "usage INTEGER UNSIGNED, usage_list TEXT, ".
           "stdno INTEGER UNSIGNED, stdno_list TEXT, ".
    "avetries FLOAT, avetries_list TEXT, ".
           "difficulty FLOAT, difficulty_list TEXT ".
           "TYPE=MYISAM";
       # It would sure be nice to have some logging mechanism.
   ####    $dbh->do($make_dynmetadata_table);
   
 }  }
   
 # ------------------------------------------------------------- get .meta files  # ------------------------------------------------------------- get .meta files
 opendir(RESOURCES,"$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}");  opendir(RESOURCES,"$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}");
 my @homeusers=grep  my @homeusers = grep {
           {&ishome("$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$_")}      &ishome("$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$_")
           grep {!/^\.\.?$/} readdir(RESOURCES);      } grep {!/^\.\.?$/} readdir(RESOURCES);
 closedir RESOURCES;  closedir RESOURCES;
   
   #
   # Create the statement handlers we need
   my $delete_sth = $dbh->prepare
       ("DELETE FROM metadata WHERE url LIKE BINARY ?");
   
   my $insert_sth = $dbh->prepare
       ("INSERT INTO metadata VALUES (".
        "?,".   # title
        "?,".   # author
        "?,".   # subject
        "?,".   # m2???
        "?,".   # version
        "?,".   # current
        "?,".   # notes
        "?,".   # abstract
        "?,".   # mime
        "?,".   # language
        "?,".   # creationdate
        "?,".   # revisiondate
        "?,".   # owner
        "?)"    # copyright
        );
   
 foreach my $user (@homeusers) {  foreach my $user (@homeusers) {
     &find("$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$user");      print LOG "\n=== User: ".$user."\n\n";
 }  
   
 # -- process each file to get metadata and put into search catalog SQL database      my $prodir=&propath($perlvar{'lonDefDomain'},$user);
 # Also, check to see if already there.      # Use find.pl
 # I could just delete (without searching first), but this works for now.      undef @metalist;
 foreach my $m (@metalist) {      @metalist=();
     my $ref=&metadata($m);      &find("$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$user");
     my $m2='/res/'.&declutter($m);      # -- process each file to get metadata and put into search catalog SQL
     $m2=~s/\.meta$//;      # database.  Also, check to see if already there.
     &dynamicmeta($m2);      # I could just delete (without searching first), but this works for now.
     my $q2="select * from metadata where url like binary '$m2'";      foreach my $m (@metalist) {
     my $sth = $dbh->prepare($q2);          print LOG "- ".$m."\n";
     $sth->execute();          my $ref=&metadata($m);
     my $r1=$sth->fetchall_arrayref;          my $m2='/res/'.&declutter($m);
     if (@$r1) {          $m2=~s/\.meta$//;
  $sth=$dbh->prepare("delete from metadata where url like binary '$m2'");    if ($ref->{'obsolete'}) { print LOG "obsolete\n"; next; }
         $sth->execute();   if ($ref->{'copyright'} eq 'private') { print LOG "private\n"; next; }
           &dynamicmeta($m2);
    &count($m2);
           $delete_sth->execute($m2);
           $insert_sth->execute($ref->{'title'},
                                $ref->{'author'},
                                $ref->{'subject'},
                                $m2,
                                $ref->{'keywords'},
                                'current',
                                $ref->{'notes'},
                                $ref->{'abstract'},
                                $ref->{'mime'},
                                $ref->{'language'},
                                sqltime($ref->{'creationdate'}),
                                sqltime($ref->{'lastrevisiondate'}),
                                $ref->{'owner'},
                                $ref->{'copyright'});
   #        if ($dbh->err()) {
   #            print STDERR "Error:".$dbh->errstr()."\n";
   #        }
           $ref = undef;
     }      }
     $sth=$dbh->prepare('insert into metadata values ('.      
   '"'.delete($ref->{'title'}).'"'.','.      # --------------------------------------------------- Clean up database
   '"'.delete($ref->{'author'}).'"'.','.      # Need to, perhaps, remove stale SQL database records.
   '"'.delete($ref->{'subject'}).'"'.','.      # ... not yet implemented
   '"'.$m2.'"'.','.          
   '"'.delete($ref->{'keywords'}).'"'.','.  }
   '"'.'current'.'"'.','.  
   '"'.delete($ref->{'notes'}).'"'.','.  
   '"'.delete($ref->{'abstract'}).'"'.','.  
   '"'.delete($ref->{'mime'}).'"'.','.  
   '"'.delete($ref->{'language'}).'"'.','.  
   '"'.sqltime(delete($ref->{'creationdate'})).'"'.','.  
   '"'.sqltime(delete($ref->{'lastrevisiondate'})).'"'.','.  
   '"'.delete($ref->{'owner'}).'"'.','.  
   '"'.delete($ref->{'copyright'}).'"'.')');  
     $sth->execute();  
 }  
   
 # ----------------------------------------------------------- Clean up database  
 # Need to, perhaps, remove stale SQL database records.  
 # ... not yet implemented  
   
 # --------------------------------------------------- Close database connection  # --------------------------------------------------- Close database connection
 $dbh->disconnect;  $dbh->disconnect;
   print LOG "\n==== Searchcat completed ".localtime()." ====\n";
   close(LOG);
   &writesimple();
   &writecount();
   exit 0;
   
   
   
   # =============================================================================
   
 # ---------------------------------------------------------------- Get metadata  # ---------------------------------------------------------------- Get metadata
 # significantly altered from subroutine present in lonnet  # significantly altered from subroutine present in lonnet
Line 259  sub metadata { Line 383  sub metadata {
         my $parser=HTML::TokeParser->new(\$metastring);          my $parser=HTML::TokeParser->new(\$metastring);
         my $token;          my $token;
         while ($token=$parser->get_token) {          while ($token=$parser->get_token) {
            if ($token->[0] eq 'S') {              if ($token->[0] eq 'S') {
       my $entry=$token->[1];                  my $entry=$token->[1];
               my $unikey=$entry;                  my $unikey=$entry;
               if (defined($token->[2]->{'part'})) {                   if (defined($token->[2]->{'part'})) { 
                  $unikey.='_'.$token->[2]->{'part'};                       $unikey.='_'.$token->[2]->{'part'}; 
       }                  }
               if (defined($token->[2]->{'name'})) {                   if (defined($token->[2]->{'name'})) { 
                  $unikey.='_'.$token->[2]->{'name'};                       $unikey.='_'.$token->[2]->{'name'}; 
       }                  }
               if ($metacache{$uri.'keys'}) {                  if ($metacache{$uri.'keys'}) {
                  $metacache{$uri.'keys'}.=','.$unikey;                      $metacache{$uri.'keys'}.=','.$unikey;
               } else {                  } else {
                  $metacache{$uri.'keys'}=$unikey;                      $metacache{$uri.'keys'}=$unikey;
       }                  }
               map {                  map {
   $metacache{$uri.''.$unikey.'.'.$_}=$token->[2]->{$_};                      $metacache{$uri.''.$unikey.'.'.$_}=$token->[2]->{$_};
               } @{$token->[3]};                  } @{$token->[3]};
               unless (                  unless (
                  $metacache{$uri.''.$unikey}=$parser->get_text('/'.$entry)                          $metacache{$uri.''.$unikey}=$parser->get_text('/'.$entry)
       ) { $metacache{$uri.''.$unikey}=                          ) { $metacache{$uri.''.$unikey}=
       $metacache{$uri.''.$unikey.'.default'};                                  $metacache{$uri.''.$unikey.'.default'};
       }                          }
           }              }
        }          }
     }      }
     return \%metacache;      return \%metacache;
 }  }
Line 290  sub metadata { Line 414  sub metadata {
 # ------------------------------------------------------------ Serves up a file  # ------------------------------------------------------------ Serves up a file
 # returns either the contents of the file or a -1  # returns either the contents of the file or a -1
 sub getfile {  sub getfile {
   my $file=shift;      my $file=shift;
   if (! -e $file ) { return -1; };      if (! -e $file ) { return -1; };
   my $fh=IO::File->new($file);      my $fh=IO::File->new($file);
   my $a='';      my $a='';
   while (<$fh>) { $a .=$_; }      while (<$fh>) { $a .=$_; }
   return $a      return $a;
 }  }
   
 # ------------------------------------------------------------- Declutters URLs  # ------------------------------------------------------------- Declutters URLs
Line 343  sub sqltime { Line 467  sub sqltime {
   
 sub maketime {  sub maketime {
     my %th=@_;      my %th=@_;
     return POSIX::mktime(      return POSIX::mktime(($th{'seconds'},$th{'minutes'},$th{'hours'},
         ($th{'seconds'},$th{'minutes'},$th{'hours'},                            $th{'day'},$th{'month'}-1,
          $th{'day'},$th{'month'}-1,$th{'year'}-1900,0,0,$th{'dlsav'}));                            $th{'year'}-1900,0,0,$th{'dlsav'}));
 }  }
   
   
Line 356  sub maketime { Line 480  sub maketime {
 sub unsqltime {  sub unsqltime {
     my $timestamp=shift;      my $timestamp=shift;
     if ($timestamp=~/^(\d+)\-(\d+)\-(\d+)\s+(\d+)\:(\d+)\:(\d+)$/) {      if ($timestamp=~/^(\d+)\-(\d+)\-(\d+)\s+(\d+)\:(\d+)\:(\d+)$/) {
        $timestamp=&maketime(          $timestamp=&maketime('year'=>$1,'month'=>$2,'day'=>$3,
    'year'=>$1,'month'=>$2,'day'=>$3,                               'hours'=>$4,'minutes'=>$5,'seconds'=>$6);
            'hours'=>$4,'minutes'=>$5,'seconds'=>$6);  
     }      }
     return $timestamp;      return $timestamp;
 }  }
   
   # ----------------- Code to enable 'find' subroutine listing of the .meta files
   
   no strict "vars";
   
   sub wanted {
       (($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_)) &&
            -f _ &&
            /^.*\.meta$/ && !/^.+\.\d+\.[^\.]+\.meta$/ &&
            push(@metalist,"$dir/$_");
   }
Removed from v.1.24  
changed lines
  Added in v.1.46

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>