--- loncom/lonnet/perl/lonnet.pm 2003/10/23 21:01:54 1.432 +++ loncom/lonnet/perl/lonnet.pm 2003/11/10 21:50:21 1.444 @@ -1,7 +1,7 @@ # The LearningOnline Network # TCP networking package # -# $Id: lonnet.pm,v 1.432 2003/10/23 21:01:54 albertel Exp $ +# $Id: lonnet.pm,v 1.444 2003/11/10 21:50:21 www Exp $ # # Copyright Michigan State University Board of Trustees # @@ -25,44 +25,6 @@ # # http://www.lon-capa.org/ # -# 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, -# 7/1,7/2,7/9,7/10,7/12,7/14,7/15,7/19, -# 11/8,11/16,11/18,11/22,11/23,12/22, -# 01/06,01/13,02/24,02/28,02/29, -# 03/01,03/02,03/06,03/07,03/13, -# 04/05,05/29,05/31,06/01, -# 06/05,06/26 Gerd Kortemeyer -# 06/26 Ben Tyszka -# 06/30,07/15,07/17,07/18,07/20,07/21,07/22,07/25 Gerd Kortemeyer -# 08/14 Ben Tyszka -# 08/22,08/28,08/31,09/01,09/02,09/04,09/05,09/25,09/28,09/30 Gerd Kortemeyer -# 10/04 Gerd Kortemeyer -# 10/04 Guy Albertelli -# 10/06,10/09,10/10,10/11,10/14,10/20,10/23,10/25,10/26,10/27,10/28,10/29, -# 10/30,10/31, -# 11/2,11/14,11/15,11/16,11/20,11/21,11/22,11/25,11/27, -# 12/02,12/12,12/13,12/14,12/28,12/29 Gerd Kortemeyer -# 05/01/01 Guy Albertelli -# 05/01,06/01,09/01 Gerd Kortemeyer -# 09/01 Guy Albertelli -# 09/01,10/01,11/01 Gerd Kortemeyer -# YEAR=2001 -# 3/2 Gerd Kortemeyer -# 3/19,3/20 Gerd Kortemeyer -# 5/26,5/28 Gerd Kortemeyer -# 5/30 H. K. Ng -# 6/1 Gerd Kortemeyer -# July Guy Albertelli -# 8/4,8/7,8/8,8/9,8/11,8/16,8/17,8/18,8/20,8/23,9/20,9/21,9/26, -# 10/2 Gerd Kortemeyer -# 11/17,11/20,11/22,11/29 Gerd Kortemeyer -# 12/5 Matthew Hall -# 12/5 Guy Albertelli -# 12/6,12/7,12/12 Gerd Kortemeyer -# 12/21,12/22,12/27,12/28 Gerd Kortemeyer -# YEAR=2002 -# 1/4,2/4,2/7 Gerd Kortemeyer -# ### package Apache::lonnet; @@ -73,7 +35,7 @@ use LWP::UserAgent(); use HTTP::Headers; use vars qw(%perlvar %hostname %homecache %badServerCache %hostip %iphost %spareid %hostdom - %libserv %pr %prp %metacache %packagetab %titlecache + %libserv %pr %prp %metacache %packagetab %titlecache %courseresversioncache %resversioncache %courselogs %accesshash %userrolehash $processmarker $dumpcount %coursedombuf %coursenumbuf %coursehombuf %coursedescrbuf %courseresdatacache %userresdatacache %usectioncache %domaindescription %domain_auth_def %domain_auth_arg_def @@ -274,10 +236,19 @@ sub transfer_profile_to_env { $idf->close(); } my $envi; + my %Remove; for ($envi=0;$envi<=$#profile;$envi++) { chomp($profile[$envi]); my ($envname,$envvalue)=split(/=/,$profile[$envi]); $ENV{$envname} = $envvalue; + if (my ($key,$time) = ($envname =~ /^(cgi\.(\d+)_\d+\.)/)) { + if ($time < time-300) { + $Remove{$key}++; + } + } + } + foreach my $expired_key (keys(%Remove)) { + &delenv($expired_key); } $ENV{'user.environment'} = "$lonidsdir/$handle.id"; } @@ -396,7 +367,7 @@ sub userload { while ($filename=readdir(LONIDS)) { if ($filename eq '.' || $filename eq '..') {next;} my ($mtime)=(stat($perlvar{'lonIDsDir'}.'/'.$filename))[9]; - if ($curtime-$mtime < 3600) { $numusers++; } + if ($curtime-$mtime < 1800) { $numusers++; } } closedir(LONIDS); } @@ -852,15 +823,24 @@ sub devalidate_cache { my ($cache,$id,$name) = @_; delete $$cache{$id.'.time'}; delete $$cache{$id}; - my $filename=$perlvar{'lonDaemons'}.'/tmp/'.$name.".db"; + my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; open(DB,"$filename.lock"); flock(DB,LOCK_EX); my %hash; if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { - delete($hash{$id}); - delete($hash{$id.'.time'}); + eval <<'EVALBLOCK'; + delete($hash{$id}); + delete($hash{$id.'.time'}); +EVALBLOCK + if ($@) { + &logthis("devalidate_cache blew up :$@:$name"); + unlink($filename); + } } else { - &logthis("Unable to tie hash"); + if (-e $filename) { + &logthis("Unable to tie hash (devalidate cache): $name"); + unlink($filename); + } } untie(%hash); flock(DB,LOCK_UN); @@ -878,7 +858,7 @@ sub is_cached { return (undef,undef); } else { if (time-($$cache{$id.'.time'})>$time) { -# &logthis("Devailidating $id - ".time-($$cache{$id.'.time'})); +# &logthis("Devalidating $id - ".time-($$cache{$id.'.time'})); &devalidate_cache($cache,$id,$name); return (undef,undef); } @@ -896,69 +876,28 @@ sub do_cache { $$cache{$id}; } -sub save_cache { - my ($cache,$name)=@_; - my $starttime=&Time::HiRes::time(); -# &logthis("Saving :$name:"); - eval lock_store($cache,$perlvar{'lonDaemons'}.'/tmp/'.$name.".storable"); - if ($@) { &logthis("lock_store threw a die ".$@); } -# &logthis("save_cache took ".(&Time::HiRes::time()-$starttime)); -} - -sub load_cache { - my ($cache,$name)=@_; - my $starttime=&Time::HiRes::time(); -# &logthis("Before Loading $name size is ".scalar(%$cache)); - my $tmpcache; - eval { - $tmpcache=lock_retrieve($perlvar{'lonDaemons'}.'/tmp/'.$name.".storable"); - }; - if ($@) { &logthis("lock_retreive threw a die ".$@); return; } - if (!%$cache) { - my $count; - while (my ($key,$value)=each(%$tmpcache)) { - $count++; - $$cache{$key}=$value; - } -# &logthis("Initial load: $count"); - } else { - my $key; - my $count; - while ($key=each(%$tmpcache)) { - if ($key !~/^(.*)\.time$/) { next; } - my $name=$1; - if (exists($$cache{$key})) { - if ($$tmpcache{$key} >= $$cache{$key}) { - $$cache{$key}=$$tmpcache{$key}; - $$cache{$name}=$$tmpcache{$name}; - } else { -# &logthis("Would have overwritten $name with is set to expire at ".$$cache{$key}." with ".$$tmpcache{$key}." Whew!"); - } - } else { - $count++; - $$cache{$key}=$$tmpcache{$key}; - $$cache{$name}=$$tmpcache{$name}; - } - } -# &logthis("Additional load: $count"); - } -# &logthis("After Loading $name size is ".scalar(%$cache)); -# &logthis("load_cache took ".(&Time::HiRes::time()-$starttime)); -} - sub save_cache_item { my ($cache,$name,$id)=@_; my $starttime=&Time::HiRes::time(); - # &logthis("Saving :$name:$id"); +# &logthis("Saving :$name:$id"); my %hash; - my $filename=$perlvar{'lonDaemons'}.'/tmp/'.$name.".db"; + my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; open(DB,"$filename.lock"); flock(DB,LOCK_EX); if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { - $hash{$id.'.time'}=$$cache{$id.'.time'}; - $hash{$id}=freeze({'item'=>$$cache{$id}}); + eval <<'EVALBLOCK'; + $hash{$id.'.time'}=$$cache{$id.'.time'}; + $hash{$id}=freeze({'item'=>$$cache{$id}}); +EVALBLOCK + if ($@) { + &logthis("save_cache blew up :$@:$name"); + unlink($filename); + } } else { - &logthis("Unable to tie hash"); + if (-e $filename) { + &logthis("Unable to tie hash (save cache item): $name"); + unlink($filename); + } } untie(%hash); flock(DB,LOCK_UN); @@ -971,29 +910,38 @@ sub load_cache_item { my $starttime=&Time::HiRes::time(); # &logthis("Before Loading $name for $id size is ".scalar(%$cache)); my %hash; - my $filename=$perlvar{'lonDaemons'}.'/tmp/'.$name.".db"; + my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; open(DB,"$filename.lock"); flock(DB,LOCK_SH); if (tie(%hash,'GDBM_File',$filename,&GDBM_READER(),0640)) { - if (!%$cache) { - my $count; - while (my ($key,$value)=each(%hash)) { - $count++; - if ($key =~ /\.time$/) { - $$cache{$key}=$value; - } else { - my $hashref=thaw($value); - $$cache{$key}=$hashref->{'item'}; + eval <<'EVALBLOCK'; + if (!%$cache) { + my $count; + while (my ($key,$value)=each(%hash)) { + $count++; + if ($key =~ /\.time$/) { + $$cache{$key}=$value; + } else { + my $hashref=thaw($value); + $$cache{$key}=$hashref->{'item'}; + } } - } # &logthis("Initial load: $count"); - } else { - my $hashref=thaw($hash{$id}); - $$cache{$id}=$hashref->{'item'}; - $$cache{$id.'.time'}=$hash{$id.'.time'}; - } + } else { + my $hashref=thaw($hash{$id}); + $$cache{$id}=$hashref->{'item'}; + $$cache{$id.'.time'}=$hash{$id.'.time'}; + } +EVALBLOCK + if ($@) { + &logthis("load_cache blew up :$@:$name"); + unlink($filename); + } } else { - &logthis("Unable to tie hash"); + if (-e $filename) { + &logthis("Unable to tie hash (load cache item): $name"); + unlink($filename); + } } untie(%hash); flock(DB,LOCK_UN); @@ -1071,6 +1019,8 @@ sub getversion { sub currentversion { my $fname=shift; + my ($result,$cached)=&is_cached(\%resversioncache,$fname,'resversion',600); + if (defined($cached)) { return $result; } my $author=$fname; $author=~s/\/home\/httpd\/html\/res\/([^\/]*)\/([^\/]*).*/$1\/$2/; my ($udom,$uname)=split(/\//,$author); @@ -1082,7 +1032,7 @@ sub currentversion { if (($answer eq 'con_lost') || ($answer eq 'rejected')) { return -1; } - return $answer; + return &do_cache(\%resversioncache,$fname,$answer,'resversion'); } # ----------------------------- Subscribe to a resource, return URL if possible @@ -1117,7 +1067,7 @@ sub repcopy { &logthis("Subscribe returned $remoteurl: $filename"); return HTTP_SERVICE_UNAVAILABLE; } elsif ($remoteurl eq 'not_found') { - &logthis("Subscribe returned not_found: $filename"); + #&logthis("Subscribe returned not_found: $filename"); return HTTP_NOT_FOUND; } elsif ($remoteurl =~ /^rejected by/) { &logthis("Subscribe returned $remoteurl: $filename"); @@ -2478,7 +2428,7 @@ sub customaccess { sub allowed { my ($priv,$uri)=@_; - + $uri=&deversion($uri); my $orguri=$uri; $uri=&declutter($uri); @@ -2763,6 +2713,7 @@ sub allowed { sub is_on_map { my $uri=&declutter(shift); + $uri=~s/\.\d+\.(\w+)$/\.$1/; my @uriparts=split(/\//,$uri); my $filename=$uriparts[$#uriparts]; my $pathname=$uri; @@ -2774,10 +2725,7 @@ sub is_on_map { if ($match) { return (1,$1); } else { - my ($name,$ext)=($filename=~/^(.+)\.(\w+)$/); - $ENV{'acc.res.'.$ENV{'request.course.id'}.'.'.$pathname}=~ - /\&(\Q$name\E\.\d+\.$ext)\:([\d\|]+)\&/; - return (0,$2,$pathname.'/'.$1); + return (0,0); } } @@ -4024,13 +3972,13 @@ sub gettitle { sub symblist { my ($mapname,%newhash)=@_; - $mapname=declutter($mapname); + $mapname=&deversion(&declutter($mapname)); my %hash; if (($ENV{'request.course.fn'}) && (%newhash)) { if (tie(%hash,'GDBM_File',$ENV{'request.course.fn'}.'_symb.db', &GDBM_WRCREAT(),0640)) { foreach (keys %newhash) { - $hash{declutter($_)}=$mapname.'___'.$newhash{$_}; + $hash{declutter($_)}=$mapname.'___'.&deversion($newhash{$_}); } if (untie(%hash)) { return 'ok'; @@ -4044,14 +3992,16 @@ sub symblist { sub symbverify { my ($symb,$thisfn)=@_; - $thisfn=&symbclean(&declutter($thisfn)); + $thisfn=&declutter($thisfn); # direct jump to resource in page or to a sequence - will construct own symbs if ($thisfn=~/\.(page|sequence)$/) { return 1; } # check URL part my ($map,$resid,$url)=&decode_symb($symb); + unless ($url eq $thisfn) { return 0; } $symb=&symbclean($symb); + $thisfn=&deversion($thisfn); my %bighash; my $okay=0; @@ -4107,11 +4057,33 @@ sub decode_symb { sub fixversion { my $fn=shift; if ($fn=~/^(adm|uploaded|public)/) { return $fn; } - my ($match,$cond,$versioned)=&is_on_map($fn); - unless ($match) { - $fn=$versioned; - } - return $fn; + my %bighash; + my $uri=&clutter($fn); + my $key=$ENV{'request.course.id'}.'_'.$uri; +# is this cached? + my ($result,$cached)=&is_cached(\%courseresversioncache,$key, + 'courseresversion',600); + if (defined($cached)) { return $result; } +# unfortunately not cached, or expired + if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', + &GDBM_READER(),0640)) { + if ($bighash{'version_'.$uri}) { + my $version=$bighash{'version_'.$uri}; + unless (($version eq 'mostrecent') || + ($version==&getversion($uri))) { + $uri=~s/\.(\w+)$/\.$version\.$1/; + } + } + untie %bighash; + } + return &do_cache + (\%courseresversioncache,$key,&declutter($uri),'courseresversion'); +} + +sub deversion { + my $url=shift; + $url=~s/\.\d+\.(\w+)$/\.$1/; + return $url; } # ------------------------------------------------------ Return symb list entry @@ -4211,7 +4183,7 @@ sub numval { } sub latest_rnd_algorithm_id { - return '64bit'; + return '64bit2'; } sub rndseed { @@ -4228,6 +4200,8 @@ sub rndseed { my $CODE=$ENV{'scantron.CODE'}; if (defined($CODE)) { &rndseed_CODE_64bit($symb,$courseid,$domain,$username); + } elsif ($which eq '64bit2') { + return &rndseed_64bit2($symb,$courseid,$domain,$username); } elsif ($which eq '64bit') { return &rndseed_64bit($symb,$courseid,$domain,$username); } @@ -4271,14 +4245,36 @@ sub rndseed_64bit { } } +sub rndseed_64bit2 { + my ($symb,$courseid,$domain,$username)=@_; + { + use integer; + # strings need to be an even # of cahracters long, it it is odd the + # last characters gets thrown away + my $symbchck=unpack("%32S*",$symb.' ') << 21; + my $symbseed=numval($symb) << 10; + my $namechck=unpack("%32S*",$username.' '); + + my $nameseed=numval($username) << 21; + my $domainseed=unpack("%32S*",$domain.' ') << 10; + my $courseseed=unpack("%32S*",$courseid.' '); + + my $num1=$symbchck+$symbseed+$namechck; + my $num2=$nameseed+$domainseed+$courseseed; + #&Apache::lonxml::debug("$symbseed:$nameseed;$domainseed|$courseseed;$namechck:$symbchck"); + #&Apache::lonxml::debug("rndseed :$num:$symb"); + return "$num1,$num2"; + } +} + sub rndseed_CODE_64bit { my ($symb,$courseid,$domain,$username)=@_; { use integer; - my $symbchck=unpack("%32S*",$symb) << 16; + my $symbchck=unpack("%32S*",$symb.' ') << 16; my $symbseed=numval($symb); my $CODEseed=numval($ENV{'scantron.CODE'}) << 16; - my $courseseed=unpack("%32S*",$courseid); + my $courseseed=unpack("%32S*",$courseid.' '); my $num1=$symbseed+$CODEseed; my $num2=$courseseed+$symbchck; #&Apache::lonxml::debug("$symbseed:$CODEseed|$courseseed:$symbchck"); @@ -4419,20 +4415,28 @@ sub mod_perl_version { } return 1; } + +sub correct_line_ends { + my ($result)=@_; + $$result =~s/\r\n/\n/mg; + $$result =~s/\r/\n/mg; +} # ================================================================ Main Program sub goodbye { &logthis("Starting Shut down"); -#not converted to using infrastruture - &logthis(sprintf("%-20s is %s",'%homecache',scalar(%homecache))); +#not converted to using infrastruture and probably shouldn't be &logthis(sprintf("%-20s is %s",'%badServerCache',scalar(%badServerCache))); - &logthis(sprintf("%-20s is %s",'%metacache',scalar(%metacache))); #converted + &logthis(sprintf("%-20s is %s",'%metacache',scalar(%metacache))); + &logthis(sprintf("%-20s is %s",'%homecache',scalar(%homecache))); &logthis(sprintf("%-20s is %s",'%titlecache',scalar(%titlecache))); &logthis(sprintf("%-20s is %s",'%courseresdatacache',scalar(%courseresdatacache))); #1.1 only &logthis(sprintf("%-20s is %s",'%userresdatacache',scalar(%userresdatacache))); &logthis(sprintf("%-20s is %s",'%usectioncache',scalar(%usectioncache))); + &logthis(sprintf("%-20s is %s",'%courseresversioncache',scalar(%courseresversioncache))); + &logthis(sprintf("%-20s is %s",'%resversioncache',scalar(%resversioncache))); &flushcourselogs(); &logthis("Shutting down"); return DONE;