--- loncom/lonnet/perl/lonnet.pm	2018/09/02 23:22:47	1.1172.2.96
+++ loncom/lonnet/perl/lonnet.pm	2019/07/26 19:09:16	1.1172.2.106
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1172.2.96 2018/09/02 23:22:47 raeburn Exp $
+# $Id: lonnet.pm,v 1.1172.2.106 2019/07/26 19:09:16 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -75,7 +75,7 @@ use LWP::UserAgent();
 use HTTP::Date;
 use Image::Magick;
 
-use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir
+use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir $deftex
             $_64bit %env %protocol %loncaparevs %serverhomeIDs %needsrelease
             %managerstab);
 
@@ -2193,6 +2193,9 @@ sub get_domain_defaults {
         } elsif ($domconfig{'coursedefaults'}{'canclone'}) {
             $domdefaults{'canclone'}=$domconfig{'coursedefaults'}{'canclone'};
         }
+        if ($domconfig{'coursedefaults'}{'texengine'}) {
+            $domdefaults{'texengine'} = $domconfig{'coursedefaults'}{'texengine'};
+        }
     }
     if (ref($domconfig{'usersessions'}) eq 'HASH') {
         if (ref($domconfig{'usersessions'}{'remote'}) eq 'HASH') {
@@ -2256,6 +2259,65 @@ sub get_domain_defaults {
     return %domdefaults;
 }
 
+sub get_dom_cats {
+    my ($dom) = @_;
+    return unless (&domain($dom));
+    my ($cats,$cached)=&is_cached_new('cats',$dom);
+    unless (defined($cached)) {
+        my %domconfig = &get_dom('configuration',['coursecategories'],$dom);
+        if (ref($domconfig{'coursecategories'}) eq 'HASH') {
+            if (ref($domconfig{'coursecategories'}{'cats'}) eq 'HASH') {
+                %{$cats} = %{$domconfig{'coursecategories'}{'cats'}};
+            } else {
+                $cats = {};
+            }
+        } else {
+            $cats = {};
+        }
+        &Apache::lonnet::do_cache_new('cats',$dom,$cats,3600);
+    }
+    return $cats;
+}
+
+sub get_dom_instcats {
+    my ($dom) = @_;
+    return unless (&domain($dom));
+    my ($instcats,$cached)=&is_cached_new('instcats',$dom);
+    unless (defined($cached)) {
+        my (%coursecodes,%codes,@codetitles,%cat_titles,%cat_order);
+        my $totcodes = &retrieve_instcodes(\%coursecodes,$dom);
+        if ($totcodes > 0) {
+            my $caller = 'global';
+            if (&auto_instcode_format($caller,$dom,\%coursecodes,\%codes,
+                                      \@codetitles,\%cat_titles,\%cat_order) eq 'ok') {
+                $instcats = {
+                                codes => \%codes,
+                                codetitles => \@codetitles,
+                                cat_titles => \%cat_titles,
+                                cat_order => \%cat_order,
+                            };
+                &do_cache_new('instcats',$dom,$instcats,3600);
+            }
+        }
+    }
+    return $instcats;
+}
+
+sub retrieve_instcodes {
+    my ($coursecodes,$dom) = @_;
+    my $totcodes;
+    my %courses = &courseiddump($dom,'.',1,'.','.','.',undef,undef,'Course');
+    foreach my $course (keys(%courses)) {
+        if (ref($courses{$course}) eq 'HASH') {
+            if ($courses{$course}{'inst_code'} ne '') {
+                $$coursecodes{$course} = $courses{$course}{'inst_code'};
+                $totcodes ++;
+            }
+        }
+    }
+    return $totcodes;
+}
+
 # --------------------------------------------------- Assign a key to a student
 
 sub assign_access_key {
@@ -2840,8 +2902,7 @@ sub absolute_url {
 sub ssi {
 
     my ($fn,%form)=@_;
-    my $ua=new LWP::UserAgent;
-    my $request;
+    my ($request,$response);
 
     $form{'no_update_last_known'}=1;
     &Apache::lonenc::check_encrypt(\$fn);
@@ -2858,7 +2919,30 @@ sub ssi {
     }
 
     $request->header(Cookie => $ENV{'HTTP_COOKIE'});
-    my $response= $ua->request($request);
+
+    if (($env{'request.course.id'}) &&
+        ($form{'grade_courseid'} eq $env{'request.course.id'}) &&
+        ($form{'grade_username'} ne '') && ($form{'grade_domain'} ne '') &&
+        ($form{'grade_symb'} ne '') &&
+        (&Apache::lonnet::allowed('mgr',$env{'request.course.id'}.
+                                 ($env{'request.course.sec'}?'/'.$env{'request.course.sec'}:'')))) {
+        if (LWP::UserAgent->VERSION >= 5.834) {
+            my $ua=new LWP::UserAgent;
+            $ua->local_address('127.0.0.1');
+            $response = $ua->request($request);
+        } else {
+            {
+                require LWP::Protocol::http;
+                local @LWP::Protocol::http::EXTRA_SOCK_OPTS = (LocalAddr => '127.0.0.1');
+                my $ua=new LWP::UserAgent;
+                $response = $ua->request($request);
+                @LWP::Protocol::http::EXTRA_SOCK_OPTS = ();
+            }
+        }
+    } else {
+        my $ua=new LWP::UserAgent;
+        $response = $ua->request($request);
+    }
     if (wantarray) {
 	return ($response->content, $response);
     } else {
@@ -2878,6 +2962,72 @@ sub externalssi {
     }
 }
 
+# If the local copy of a replicated resource is outdated, trigger a
+# connection from the homeserver to flush the delayed queue. If no update
+# happens, remove local copies of outdated resource (and corresponding
+# metadata file).
+
+sub remove_stale_resfile {
+    my ($url) = @_;
+    my $removed;
+    if ($url=~m{^/res/($match_domain)/($match_username)/}) {
+        my $audom = $1;
+        my $auname = $2;
+        unless (($url =~ /\.\d+\.\w+$/) || ($url =~ m{^/res/lib/templates/})) {
+            my $homeserver = &homeserver($auname,$audom);
+            unless (($homeserver eq 'no_host') ||
+                    (grep { $_ eq $homeserver } &current_machine_ids())) {
+                my $fname = &filelocation('',$url);
+                if (-e $fname) {
+                    my $ua=new LWP::UserAgent;
+                    $ua->timeout(5);
+                    my $protocol = $protocol{$homeserver};
+                    $protocol = 'http' if ($protocol ne 'https');
+                    my $hostname = &hostname($homeserver);
+                    if ($hostname) {
+                        my $uri = $protocol.'://'.$hostname.'/raw/'.&declutter($url);
+                        my $request=new HTTP::Request('HEAD',$uri);
+                        my $response=$ua->request($request);
+                        if ($response->is_success()) {
+                            my $remmodtime = &HTTP::Date::str2time( $response->header('Last-modified') );
+                            my $locmodtime = (stat($fname))[9];
+                            if ($locmodtime < $remmodtime) {
+                                my $stale;
+                                my $answer = &reply('pong',$homeserver);
+                                if ($answer eq $homeserver.':'.$perlvar{'lonHostID'}) {
+                                    sleep(0.2);
+                                    $locmodtime = (stat($fname))[9];
+                                    if ($locmodtime < $remmodtime) {
+                                        my $posstransfer = $fname.'.in.transfer';
+                                        if ((-e $posstransfer) && ($remmodtime < (stat($posstransfer))[9])) {
+                                            $removed = 1;
+                                        } else {
+                                            $stale = 1;
+                                        }
+                                    } else {
+                                        $removed = 1;
+                                    }
+                                } else {
+                                    $stale = 1;
+                                }
+                                if ($stale) {
+                                    unlink($fname);
+                                    if ($uri!~/\.meta$/) {
+                                        unlink($fname.'.meta');
+                                    }
+                                    &reply("unsub:$fname",$homeserver);
+                                    $removed = 1;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return $removed;
+}
+
 # -------------------------------- Allow a /uploaded/ URI to be vouched for
 
 sub allowuploaded {
@@ -4805,7 +4955,12 @@ sub set_first_access {
     }
     $cachedkey='';
     my $firstaccess=&get_first_access($type,$symb,$map);
-    if (!$firstaccess) {
+    if ($firstaccess) {
+        &logthis("First access time already set ($firstaccess) when attempting ".
+                 "to set new value (type: $type, extent: $res) for $uname:$udom ".
+                 "in $courseid");
+        return 'already_set';
+    } else {
         my $start = time;
 	my $putres = &put('firstaccesstimes',{"$courseid\0$res"=>$start},
                           $udom,$uname);
@@ -4818,6 +4973,12 @@ sub set_first_access {
                         'course.'.$courseid.'.timerinterval.'.$res => $interval,
                      }
                   );
+            if (($cachedtime) && (abs($start-$cachedtime) < 5)) {
+                $cachedtimes{"$courseid\0$res"} = $start;
+            }
+        } elsif ($putres ne 'refused') {
+            &logthis("Result: $putres when attempting to set first access time ".
+                     "(type: $type, extent: $res) for $uname:$udom in $courseid");
         }
         return $putres;
     }
@@ -10999,6 +11160,7 @@ sub add_prefix_and_part {
 
 my %metaentry;
 my %importedpartids;
+my %importedrespids;
 sub metadata {
     my ($uri,$what,$liburi,$prefix,$depthcount)=@_;
     $uri=&declutter($uri);
@@ -11026,9 +11188,11 @@ sub metadata {
     }
     {
 # Imported parts would go here
-        my %importedids=();
-        my @origfileimportpartids=();
+        my @origfiletagids=();
         my $importedparts=0;
+
+# Imported responseids would go here
+        my $importedresponses=0;
 #
 # Is this a recursive call for a library?
 #
@@ -11123,8 +11287,37 @@ sub metadata {
                         my $dir=$filename;
                         $dir=~s|[^/]*$||;
                         $location=&filelocation($dir,$location);
-                       
+
+                        my $importid=$token->[2]->{'id'};
                         my $importmode=$token->[2]->{'importmode'};
+#
+# Check metadata for imported file to
+# see if it contained response items
+#
+                        my %currmetaentry = %metaentry;
+                        my $libresponseorder = &metadata($location,'responseorder');
+                        my $origfile;
+                        if ($libresponseorder ne '') {
+                            if ($#origfiletagids<0) {
+                                undef(%importedrespids);
+                                undef(%importedpartids);
+                            }
+                            @{$importedrespids{$importid}} = split(/\s*,\s*/,$libresponseorder);
+                            if (@{$importedrespids{$importid}} > 0) {
+                                $importedresponses = 1;
+# We need to get the original file and the imported file to get the response order correct
+# Load and inspect original file
+                                if ($#origfiletagids<0) {
+                                    my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                    $origfile=&getfile($origfilelocation);
+                                    @origfiletagids=($origfile=~/<((?:\w+)response|import|part)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                                }
+                            }
+                        }
+# Do not overwrite contents of %metaentry hash for resource itself with 
+# hash populated for imported library file
+                        %metaentry = %currmetaentry;
+                        undef(%currmetaentry);
                         if ($importmode eq 'problem') {
 # Import as problem/response
                            $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'});
@@ -11133,12 +11326,15 @@ sub metadata {
                            $importedparts=1;
 # We need to get the original file and the imported file to get the part order correct
 # Good news: we do not need to worry about nested libraries, since parts cannot be nested
-# Load and inspect original file
-                           if ($#origfileimportpartids<0) {
-                              undef(%importedpartids);
-                              my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
-                              my $origfile=&getfile($origfilelocation);
-                              @origfileimportpartids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+# Load and inspect original file if we didn't do that already
+                           if ($#origfiletagids<0) {
+                               undef(%importedrespids);
+                               undef(%importedpartids);
+                               if ($origfile eq '') {
+                                   my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                   $origfile=&getfile($origfilelocation);
+                                   @origfiletagids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                               }
                            }
 
 # Load and inspect imported file
@@ -11252,20 +11448,48 @@ sub metadata {
 	    grep { ! $seen{$_} ++ } (split(',',$metaentry{':packages'}));
 	$metaentry{':packages'} = join(',',@uniq_packages);
 
-        if ($importedparts) {
+        if (($importedresponses) || ($importedparts)) {
+            if ($importedparts) {
 # We had imported parts and need to rebuild partorder
-           $metaentry{':partorder'}='';
-           $metathesekeys{'partorder'}=1;
-           for (my $index=0;$index<$#origfileimportpartids;$index+=2) {
-               if ($origfileimportpartids[$index] eq 'part') {
-# original part, part of the problem
-                  $metaentry{':partorder'}.=','.$origfileimportpartids[$index+1];
-               } else {
-# we have imported parts at this position
-                  $metaentry{':partorder'}.=','.$importedpartids{$origfileimportpartids[$index+1]};
-               }
-           }
-           $metaentry{':partorder'}=~s/^\,//;
+                $metaentry{':partorder'}='';
+                $metathesekeys{'partorder'}=1;
+            }
+            if ($importedresponses) {
+# We had imported responses and need to rebuild responseorder
+                $metaentry{':responseorder'}='';
+                $metathesekeys{'responseorder'}=1;
+            }
+            for (my $index=0;$index<$#origfiletagids;$index+=2) {
+                my $origid = $origfiletagids[$index+1];
+                if ($origfiletagids[$index] eq 'part') {
+# Original part, part of the problem
+                    if ($importedparts) {
+                        $metaentry{':partorder'}.=','.$origid;
+                    }
+                } elsif ($origfiletagids[$index] eq 'import') {
+                    if ($importedparts) {
+# We have imported parts at this position
+                        $metaentry{':partorder'}.=','.$importedpartids{$origid};
+                    }
+                    if ($importedresponses) {
+# We have imported responses at this position
+                        if (ref($importedrespids{$origid}) eq 'ARRAY') {
+                            $metaentry{':responseorder'}.=','.join(',',map { $origid.'_'.$_ } @{$importedrespids{$origid}});
+                        }
+                    }
+                } else {
+# Original response item, part of the problem
+                    if ($importedresponses) {
+                        $metaentry{':responseorder'}.=','.$origid;
+                    }
+                }
+            }
+            if ($importedparts) {
+                $metaentry{':partorder'}=~s/^\,//;
+            }
+            if ($importedresponses) {
+                $metaentry{':responseorder'}=~s/^\,//;
+            }
         }
 
 	$metaentry{':keys'} = join(',',keys(%metathesekeys));
@@ -12487,6 +12711,27 @@ sub default_login_domain {
     return $domain;
 }
 
+sub shared_institution {
+    my ($dom) = @_;
+    my $same_intdom;
+    my $hostintdom = &internet_dom($perlvar{'lonHostID'});
+    if ($hostintdom ne '') {
+        my %iphost = &get_iphost();
+        my $primary_id = &domain($dom,'primary');
+        my $primary_ip = &get_host_ip($primary_id);
+        if (ref($iphost{$primary_ip}) eq 'ARRAY') {
+            foreach my $id (@{$iphost{$primary_ip}}) {
+                my $intdom = &internet_dom($id);
+                if ($intdom eq $hostintdom) {
+                    $same_intdom = 1;
+                    last;
+                }
+            }
+        }
+    }
+    return $same_intdom;
+}
+
 # ------------------------------------------------------------- Declutters URLs
 
 sub declutter {
@@ -12637,12 +12882,12 @@ sub get_dns {
 	&$func(\@content,$hashref);
 	return;
     }
-    close($config);
     my $which = (split('/',$url))[3];
     &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
-    open($config,"<","$perlvar{'lonTabDir'}/dns_$which.tab");
-    my @content = <$config>;
-    &$func(\@content,$hashref);
+    if (open(my $config,"<","$perlvar{'lonTabDir'}/dns_$which.tab")) {
+        my @content = <$config>;
+        &$func(\@content,$hashref);
+    }
     return;
 }
 
@@ -13254,6 +13499,11 @@ BEGIN {
 
 }
 
+# ------------- set default texengine (domain default overrides this)
+{
+    $deftex = LONCAPA::texengine();
+}
+
 $memcache=new Cache::Memcached({'servers'           => ['127.0.0.1:11211'],
 				'compress_threshold'=> 20_000,
  			        });