--- loncom/homework/grades.pm	2017/12/31 14:00:41	1.749
+++ loncom/homework/grades.pm	2020/08/26 18:13:40	1.771
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.749 2017/12/31 14:00:41 raeburn Exp $
+# $Id: grades.pm,v 1.771 2020/08/26 18:13:40 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -46,7 +46,10 @@ use Apache::lonenc;
 use Apache::lonstathelpers;
 use Apache::lonquickgrades;
 use Apache::bridgetask();
+use Apache::lontexconvert();
 use String::Similarity;
+use HTML::Parser();
+use File::MMagic;
 use LONCAPA;
 
 use POSIX qw(floor);
@@ -312,7 +315,7 @@ sub reset_caches {
                     $add_to_form = { 'code_for_randomlist' => $scancode,};
                 }
             }
-            my $analyze = 
+            my $analyze =
                 &get_analyze($symb,$uname,$udom,undef,$add_to_form,
                              undef,undef,undef,$bubbles_per_row);
             if (ref($analyze) eq 'HASH') {
@@ -342,7 +345,7 @@ sub cleanRecord {
     if ($response =~ /^(option|rank)$/) {
 	my %answer=&Apache::lonnet::str2hash($answer);
         my @answer = %answer;
-        %answer = map {&HTML::Entities::encode($_, '"<>&')}  @answer;
+        %answer = map {&HTML::Entities::encode($_, '"<>&')} @answer;
 	my %grading=&Apache::lonnet::str2hash($record->{$version."resource.$partid.$respid.submissiongrading"});
 	my ($toprow,$bottomrow);
 	foreach my $foil (@$order) {
@@ -360,7 +363,7 @@ sub cleanRecord {
     } elsif ($response eq 'match') {
 	my %answer=&Apache::lonnet::str2hash($answer);
         my @answer = %answer;
-        %answer = map {&HTML::Entities::encode($_, '"<>&')}  @answer;
+        %answer = map {&HTML::Entities::encode($_, '"<>&')} @answer;
 	my %grading=&Apache::lonnet::str2hash($record->{$version."resource.$partid.$respid.submissiongrading"});
 	my @items=&Apache::lonnet::str2array($record->{$version."resource.$partid.$respid.submissionitems"});
 	my ($toprow,$middlerow,$bottomrow);
@@ -417,8 +420,8 @@ sub cleanRecord {
 	    $env{'form.kwstyle'}  = $keyhash{$loginuser.'_kwstyle'} ne '' ? $keyhash{$loginuser.'_kwstyle'} : '';
 	    $env{'form.'.$symb} = 1; # so that we don't have to read it from disk for multiple sub of the same prob.
 	}
+        $answer = &Apache::lontexconvert::msgtexconverted($answer);
 	return '<br /><br /><blockquote><tt>'.&keywords_highlight($answer).'</tt></blockquote>';
-
     } elsif ( $response eq 'organic') {
         my $result=&mt('Smile representation: [_1]',
                            '"<tt>'.&HTML::Entities::encode($answer, '"<>&').'</tt>"');
@@ -502,7 +505,7 @@ COMMONJSFUNCTIONS
 #--- Dumps the class list with usernames,list of sections,
 #--- section, ids and fullnames for each user.
 sub getclasslist {
-    my ($getsec,$filterlist,$getgroup) = @_;
+    my ($getsec,$filterbyaccstatus,$getgroup,$symb,$submitonly,$filterbysubmstatus) = @_;
     my @getsec;
     my @getgroup;
     my $stu_status = join(':',&Apache::loncommon::get_env_multiple('form.Status'));
@@ -530,6 +533,13 @@ sub getclasslist {
     #
     my %sections;
     my %fullnames;
+    my ($cdom,$cnum,$partlist);
+    if (($filterbysubmstatus) && ($submitonly ne 'all') && ($symb ne '')) {
+        $cdom = $env{"course.$env{'request.course.id'}.domain"};
+        $cnum = $env{"course.$env{'request.course.id'}.num"};
+        my $res_error;
+        ($partlist,my $handgrade,my $responseType) = &response_type($symb,\$res_error);
+    }
     foreach my $student (keys(%$classlist)) {
         my $end      = 
             $classlist->{$student}->[&Apache::loncoursedata::CL_END()];
@@ -546,7 +556,7 @@ sub getclasslist {
         my $group   = 
             $classlist->{$student}->[&Apache::loncoursedata::CL_GROUP()];
 	# filter students according to status selected
-	if ($filterlist && (!($stu_status =~ /Any/))) {
+	if ($filterbyaccstatus && (!($stu_status =~ /Any/))) {
 	    if (!($stu_status =~ $status)) {
 		delete($classlist->{$student});
 		next;
@@ -563,13 +573,58 @@ sub getclasslist {
     	            } 
 	        }
     	        if (($grp eq 'none') && !$group) {
-        	        $exclude = 0;
+        	    $exclude = 0;
         	}
 	    }
 	    if ($exclude) {
 	        delete($classlist->{$student});
+		next;
 	    }
 	}
+        if (($filterbysubmstatus) && ($submitonly ne 'all') && ($symb ne '')) {
+            my $udom =
+                $classlist->{$student}->[&Apache::loncoursedata::CL_SDOM()];
+            my $uname =
+                $classlist->{$student}->[&Apache::loncoursedata::CL_SNAME()];
+            if (($symb ne '') && ($udom ne '') && ($uname ne '')) {
+                if ($submitonly eq 'queued') {
+                    my %queue_status =
+                        &Apache::bridgetask::get_student_status($symb,$cdom,$cnum,
+                                                                $udom,$uname);
+                    if (!defined($queue_status{'gradingqueue'})) {
+                        delete($classlist->{$student});
+                        next;
+                    }
+                } else {
+                    my (%status) =&student_gradeStatus($symb,$udom,$uname,$partlist);
+                    my $submitted = 0;
+                    my $graded = 0;
+                    my $incorrect = 0;
+                    foreach (keys(%status)) {
+                        $submitted = 1 if ($status{$_} ne 'nothing');
+                        $graded = 1 if ($status{$_} =~ /^ungraded/);
+                        $incorrect = 1 if ($status{$_} =~ /^incorrect/);
+
+                        my ($foo,$partid,$foo1) = split(/\./,$_);
+                        if ($status{'resource.'.$partid.'.submitted_by'} ne '') {
+                            $submitted = 0;
+                        }
+                    }
+                    if (!$submitted && ($submitonly eq 'yes' ||
+                                        $submitonly eq 'incorrect' ||
+                                        $submitonly eq 'graded')) {
+                        delete($classlist->{$student});
+                        next;
+                    } elsif (!$graded && ($submitonly eq 'graded')) {
+                        delete($classlist->{$student});
+                        next;
+                    } elsif (!$incorrect && $submitonly eq 'incorrect') {
+                        delete($classlist->{$student});
+                        next;
+                    }
+                }
+            }
+        }
 	$section = ($section ne '' ? $section : 'none');
 	if (&canview($section)) {
 	    if (!@getsec || grep(/^\Q$section\E$/,@getsec)) {
@@ -584,7 +639,6 @@ sub getclasslist {
 	    delete($classlist->{$student});
 	}
     }
-    my %seen = ();
     my @sections = sort(keys(%sections));
     return ($classlist,\@sections,\%fullnames);
 }
@@ -600,7 +654,7 @@ sub canmodify {
 		#can modify the requested section
 		return 1;
 	    } else {
-		# can't modify the request section
+		# can't modify the requested section
 		return 0;
 	    }
 	}
@@ -613,19 +667,19 @@ sub canview {
     my ($sec)=@_;
     if ($perm{'vgr'}) {
 	if (!defined($perm{'vgr_section'})) {
-	    # can modify whole class
+	    # can view whole class
 	    return 1;
 	} else {
 	    if ($sec eq $perm{'vgr_section'}) {
-		#can modify the requested section
+		#can view the requested section
 		return 1;
 	    } else {
-		# can't modify the request section
+		# can't view the requested section
 		return 0;
 	    }
 	}
     }
-    #can't modify
+    #can't view
     return 0;
 }
 
@@ -766,14 +820,14 @@ sub initialverifyreceipt {
 
 #--- Check whether a receipt number is valid.---
 sub verifyreceipt {
-    my ($request,$symb)  = @_;
+    my ($request,$symb) = @_;
 
     my $courseid = $env{'request.course.id'};
     my $receipt  = &Apache::lonnet::recprefix($courseid).'-'.
 	$env{'form.receipt'};
     $receipt     =~ s/[^\-\d]//g;
 
-    my $title.=
+    my $title =
 	'<h3><span class="LC_info">'.
 	&mt('Verifying Receipt Number [_1]',$receipt).
 	'</span></h3>'."\n";
@@ -862,7 +916,7 @@ sub listStudents {
     my $getsec    = $env{'form.section'} eq '' ? 'all' : $env{'form.section'};
     my $getgroup  = $env{'form.group'} eq '' ? 'all' : $env{'form.group'};
     unless ($submitonly) {
-       $submitonly= $env{'form.submitonly'} eq '' ? 'all' : $env{'form.submitonly'};
+        $submitonly = $env{'form.submitonly'} eq '' ? 'all' : $env{'form.submitonly'};
     }
 
     my $result='';
@@ -1159,8 +1213,8 @@ LISTJAVASCRIPT
 #---- Called from the listStudents routine
 
 sub check_script {
-    my ($form, $type)=@_;
-    my $chkallscript= &Apache::lonhtmlcommon::scripttag('
+    my ($form,$type) = @_;
+    my $chkallscript = &Apache::lonhtmlcommon::scripttag('
     function checkall() {
         for (i=0; i<document.forms.'.$form.'.elements.length; i++) {
             ele = document.forms.'.$form.'.elements[i];
@@ -1205,7 +1259,7 @@ sub check_buttons {
 
 #     Displays the submissions for one student or a group of students
 sub processGroup {
-    my ($request,$symb)  = @_;
+    my ($request,$symb) = @_;
     my $ctr        = 0;
     my @stuchecked = &Apache::loncommon::get_env_multiple('form.stuinfo');
     my $total      = scalar(@stuchecked)-1;
@@ -1952,7 +2006,6 @@ sub show_problem {
 sub files_exist {
     my ($r, $symb) = @_;
     my @students = &Apache::loncommon::get_env_multiple('form.stuinfo');
-
     foreach my $student (@students) {
         my ($uname,$udom,$fullname) = split(/:/,$student);
         my %record = &Apache::lonnet::restore($symb,$env{'request.course.id'},
@@ -1972,10 +2025,9 @@ sub files_exist {
 sub download_all_link {
     my ($r,$symb) = @_;
     unless (&files_exist($r, $symb)) {
-       $r->print(&mt('There are currently no submitted documents.'));
-       return;
+        $r->print(&mt('There are currently no submitted documents.'));
+        return;
     }
-
     my $all_students = 
 	join("\n", &Apache::loncommon::get_env_multiple('form.stuinfo'));
 
@@ -1995,7 +2047,55 @@ sub submit_download_link {
     my ($request,$symb) = @_;
     if (!$symb) { return ''; }
 #FIXME: Figure out which type of problem this is and provide appropriate download
-    &download_all_link($request,$symb);
+    my $res_error;
+    my ($partlist,$handgrade,$responseType) = &response_type($symb,$res_error);
+    if (ref($res_error)) {
+        if ($$res_error) {
+            $request->print(&mt('An error occurred retrieving response types'));
+            return;
+        }
+    }
+    my ($numupload,$numessay) = (0,0);
+    if (ref($responseType) eq 'HASH') {
+        foreach my $part (sort(keys(%$responseType))) {
+            foreach my $id (sort(keys(%{ $responseType->{$part} }))) {
+                my $responsetype = $responseType->{$part}->{$id};
+                if ($responsetype eq 'essay') {
+                    my $uploadedfiletypes =
+                        &Apache::lonnet::EXT("resource.$part".'_'."$id.uploadedfiletypes",$symb);
+                    if ($uploadedfiletypes) {
+                        $numupload++;
+                    } else {
+                        $numessay++;
+                    }
+                }
+            }
+        }
+    }
+    if (($numupload) || ($numessay)) {
+        my $submitonly= $env{'form.submitonly'} eq '' ? 'all' : $env{'form.submitonly'};
+        my $getsec    = $env{'form.section'} eq '' ? 'all' : $env{'form.section'};
+        my $getgroup  = $env{'form.group'} eq '' ? 'all' : $env{'form.group'};
+        (undef,undef,my $fullname) = &getclasslist($getsec,1,$getgroup,$symb,$submitonly,1);
+        if (ref($fullname) eq 'HASH') {
+            my @students = map { $_.':'.$fullname->{$_} } (keys(%{$fullname}));
+            if (@students) {
+                @{$env{'form.stuinfo'}} = @students;
+                if ($numupload) {
+                    &download_all_link($request,$symb);
+                }
+# FIXME Need to provide a mechanism to download essays, i.e., if $numessay > 0
+# Needs to omit user's identity if resource instance is for an anonymous survey.
+            } else {
+                $request->print(&mt('No students match the criteria you selected'));
+            }
+        } else {
+            $request->print(&mt('Could not retrieve student information'));
+        }
+    } else {
+        $request->print(&mt('No essayresponse items found'));
+    }
+    return;
 }
 
 sub build_section_inputs {
@@ -2022,6 +2122,7 @@ sub submission {
     my $probtitle=&Apache::lonnet::gettitle($symb); 
     if ($symb eq '') { $request->print("Unable to handle ambiguous references:."); return ''; }
     my $is_tool = ($symb =~ /ext\.tool$/);
+    my ($essayurl,%coursedesc_by_cid);
 
     if (!&canview($usec)) {
         $request->print(
@@ -2152,11 +2253,24 @@ sub submission {
 #
 # Load the other essays for similarity check
 #
-            my (undef,undef,$essayurl) = &Apache::lonnet::decode_symb($symb);
-	    my ($adom,$aname,$apath)=($essayurl=~/^($LONCAPA::domain_re)\/($LONCAPA::username_re)\/(.*)$/);
-	    $apath=&escape($apath);
-	    $apath=~s/\W/\_/gs;
-            &init_old_essays($symb,$apath,$adom,$aname);
+            (undef,undef,$essayurl) = &Apache::lonnet::decode_symb($symb);
+            if ($essayurl eq 'lib/templates/simpleproblem.problem') {
+                my $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+                my $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+                if ($cdom ne '' && $cnum ne '') {
+                    my ($map,$id,$res) = &Apache::lonnet::decode_symb($symb);
+                    if ($map =~ m{^\Quploaded/$cdom/$cnum/\E(default(?:|_\d+)\.(?:sequence|page))$}) {
+                        my $apath = $1.'_'.$id;
+                        $apath=~s/\W/\_/gs;
+                        &init_old_essays($symb,$apath,$cdom,$cnum);
+                    }
+                }
+            } else {
+	        my ($adom,$aname,$apath)=($essayurl=~/^($LONCAPA::domain_re)\/($LONCAPA::username_re)\/(.*)$/);
+	        $apath=&escape($apath);
+	        $apath=~s/\W/\_/gs;
+                &init_old_essays($symb,$apath,$adom,$aname);
+            }
         }
     }
 
@@ -2307,24 +2421,52 @@ sub submission {
 		        &most_similar($uname,$udom,$symb,$subval);
 		    if ($osim) {
 			$osim=int($osim*100.0);
-			my %old_course_desc = 
-			    &Apache::lonnet::coursedescription($ocrsid,
-							{'one_time' => 1});
-
                         if ($hide eq 'anon') {
                             $similar='<hr /><span class="LC_warning">'.&mt("Essay was found to be similar to another essay submitted for this assignment.").'<br />'.
                                      &mt('As the current submission is for an anonymous survey, no other details are available.').'</span><hr />';
                         } else {
-			    $similar="<hr /><h3><span class=\"LC_warning\">".
-				&mt('Essay is [_1]% similar to an essay by [_2] in course [_3] (course id [_4]:[_5])',
-				    $osim,
-				    &Apache::loncommon::plainname($oname,$odom).' ('.$oname.':'.$odom.')',
-				        $old_course_desc{'description'},
-				        $old_course_desc{'num'},
-				        $old_course_desc{'domain'}).
-				    '</span></h3><blockquote><i>'.
-				    &keywords_highlight($oessay).
-				    '</i></blockquote><hr />';
+			    $similar='<hr />';
+                            if ($essayurl eq 'lib/templates/simpleproblem.problem') {
+                                $similar .= '<h3><span class="LC_warning">'.
+                                            &mt('Essay is [_1]% similar to an essay by [_2]',
+                                                $osim,
+                                                &Apache::loncommon::plainname($oname,$odom).' ('.$oname.':'.$odom.')').
+                                            '</span></h3>';
+                            } else {
+                                my %old_course_desc;
+                                if ($ocrsid ne '') {
+                                    if (ref($coursedesc_by_cid{$ocrsid}) eq 'HASH') {
+                                        %old_course_desc = %{$coursedesc_by_cid{$ocrsid}};
+                                    } else {
+                                        my $args;
+                                        if ($ocrsid ne $env{'request.course.id'}) {
+                                            $args = {'one_time' => 1};
+                                        }
+                                        %old_course_desc =
+                                            &Apache::lonnet::coursedescription($ocrsid,$args);
+                                        $coursedesc_by_cid{$ocrsid} = \%old_course_desc;
+                                    }
+                                    $similar .=
+                                        '<h3><span class="LC_warning">'.
+                                        &mt('Essay is [_1]% similar to an essay by [_2] in course [_3] (course id [_4]:[_5])',
+                                            $osim,
+                                            &Apache::loncommon::plainname($oname,$odom).' ('.$oname.':'.$odom.')',
+                                            $old_course_desc{'description'},
+                                            $old_course_desc{'num'},
+                                            $old_course_desc{'domain'}).
+                                        '</span></h3>';
+                                } else {
+                                    $similar .=
+                                        '<h3><span class="LC_warning">'.
+                                        &mt('Essay is [_1]% similar to an essay by [_2] in an unknown course',
+                                            $osim,
+                                            &Apache::loncommon::plainname($oname,$odom).' ('.$oname.':'.$odom.')').
+                                        '</span></h3>';
+                                }
+                            }
+                            $similar .= '<blockquote><i>'.
+                                        &keywords_highlight($oessay).
+                                        '</i></blockquote><hr />';
                         }
 	            }
 		}
@@ -4056,7 +4198,7 @@ sub editgrades {
 
     my $section_display = join (", ",&Apache::loncommon::get_env_multiple('form.section'));
     my $title='<h2>'.&mt('Current Grade Status').'</h2>';
-    $title.='<h4>'.&mt('<b>Section: </b>[_1]',$section_display).'</h4>'."\n";
+    $title.='<h4><b>'.&mt('Section:').'</b> '.$section_display.'</h4>'."\n";
 
     my $result= &Apache::loncommon::start_data_table().
 	&Apache::loncommon::start_data_table_header_row().
@@ -4499,7 +4641,7 @@ ENDUPFORM
 
 
 sub csvuploadmap {
-    my ($request,$symb)= @_;
+    my ($request,$symb) = @_;
     if (!$symb) {return '';}
 
     my $datatoken;
@@ -4595,7 +4737,7 @@ sub get_fields {
 }
 
 sub csvuploadassign {
-    my ($request,$symb)= @_;
+    my ($request,$symb) = @_;
     if (!$symb) {return '';}
     my $error_msg = '';
     my $datatoken = &Apache::loncommon::valid_datatoken($env{'form.datatoken'});
@@ -4711,7 +4853,7 @@ sub csvuploadassign {
 		$grades{$store_key}=$entries{$fields{$dest}};
 	    }
 	}
-	if (! %grades) { 
+	if (! %grades) {
            push(@skipped,&mt("[_1]: no data to save","$username:$domain")); 
         } else {
 	   $grades{"resource.regrader"}="$env{'user.name'}:$env{'user.domain'}";
@@ -4782,6 +4924,7 @@ LISTJAVASCRIPT
     my $cdom      = $env{"course.$env{'request.course.id'}.domain"};
     my $cnum      = $env{"course.$env{'request.course.id'}.num"};
     my $getsec    = $env{'form.section'} eq '' ? 'all' : $env{'form.section'};
+    my $getgroup  = $env{'form.group'} eq '' ? 'all' : $env{'form.group'};
 
     my $result='<h3><span class="LC_info">&nbsp;'.
 	&mt('Manual Grading by Page or Sequence').'</span></h3>';
@@ -4871,7 +5014,7 @@ LISTJAVASCRIPT
 	'<th>'.&nameUserString('header').'</th>'.
 	&Apache::loncommon::end_data_table_header_row();
  
-    my (undef,undef,$fullname) = &getclasslist($getsec,'1');
+    my (undef,undef,$fullname) = &getclasslist($getsec,'1',$getgroup);
     my $ptr = 1;
     foreach my $student (sort 
 			 {
@@ -5167,11 +5310,11 @@ sub displaySubByDates {
             }
             my @matchKey;
             if ($isTask) {
-                @matchKey = sort(grep /^resource\.\d+\.\Q$partid\E\.award$/,@versionKeys);
+                @matchKey = sort(grep(/^resource\.\d+\.\Q$partid\E\.award$/,@versionKeys));
             } elsif ($is_tool) {
-                @matchKey = sort(grep /^resource\.\Q$partid\E\.awarded$/,@versionKeys);
+                @matchKey = sort(grep(/^resource\.\Q$partid\E\.awarded$/,@versionKeys));
             } else {
-                @matchKey = sort(grep /^resource\.\Q$partid\E\..*?\.submission$/,@versionKeys);
+                @matchKey = sort(grep(/^resource\.\Q$partid\E\..*?\.submission$/,@versionKeys));
             }
 #	    next if ($$record{"$version:resource.$partid.solved"} eq '');
 	    my $display_part=&get_display_part($partid,$symb);
@@ -5511,7 +5654,7 @@ the homework problem.
 
 sub defaultFormData {
     my ($symb)=@_;
-    return '<input type="hidden" name="symb"    value="'.&Apache::lonenc::check_encrypt($symb).'" />';
+    return '<input type="hidden" name="symb" value="'.&Apache::lonenc::check_encrypt($symb).'" />';
 }
 
 
@@ -5664,7 +5807,7 @@ sub scantron_uploads {
 sub scantron_scantab {
     my $result='<select name="scantron_format">'."\n";
     $result.='<option></option>'."\n";
-    my @lines = &get_scantronformat_file();
+    my @lines = &Apache::lonnet::get_scantronformat_file();
     if (@lines > 0) {
         foreach my $line (@lines) {
             next if (($line =~ /^\#/) || ($line eq ''));
@@ -5676,62 +5819,6 @@ sub scantron_scantab {
     return $result;
 }
 
-=pod
-
-=item get_scantronformat_file
-
-  Returns an array containing lines from the scantron format file for
-  the domain of the course.
-
-  If a url for a custom.tab file is listed in domain's configuration.db, 
-  lines are from this file.
-
-  Otherwise, if a default.tab has been published in RES space by the 
-  domainconfig user, lines are from this file.
-
-  Otherwise, fall back to getting lines from the legacy file on the
-  local server:  /home/httpd/lonTabs/default_scantronformat.tab    
-
-=cut
-
-sub get_scantronformat_file {
-    my $cdom= $env{'course.'.$env{'request.course.id'}.'.domain'};
-    my %domconfig = &Apache::lonnet::get_dom('configuration',['scantron'],$cdom);
-    my $gottab = 0;
-    my @lines;
-    if (ref($domconfig{'scantron'}) eq 'HASH') {
-        if ($domconfig{'scantron'}{'scantronformat'} ne '') {
-            my $formatfile = &Apache::lonnet::getfile($Apache::lonnet::perlvar{'lonDocRoot'}.$domconfig{'scantron'}{'scantronformat'});
-            if ($formatfile ne '-1') {
-                @lines = split("\n",$formatfile,-1);
-                $gottab = 1;
-            }
-        }
-    }
-    if (!$gottab) {
-        my $confname = $cdom.'-domainconfig';
-        my $default = $Apache::lonnet::perlvar{'lonDocRoot'}.'/res/'.$cdom.'/'.$confname.'/default.tab';
-        my $formatfile =  &Apache::lonnet::getfile($default);
-        if ($formatfile ne '-1') {
-            @lines = split("\n",$formatfile,-1);
-            $gottab = 1;
-        }
-    }
-    if (!$gottab) {
-        my @domains = &Apache::lonnet::current_machine_domains();
-        if (grep(/^\Q$cdom\E$/,@domains)) {
-            my $fh=Apache::File->new($Apache::lonnet::perlvar{'lonTabDir'}.'/scantronformat.tab');
-            @lines = <$fh>;
-            close($fh);
-        } else {
-            my $fh=Apache::File->new($Apache::lonnet::perlvar{'lonTabDir'}.'/default_scantronformat.tab');
-            @lines = <$fh>;
-            close($fh);
-        }
-    }
-    return @lines;
-}
-
 =pod 
 
 =item scantron_CODElist
@@ -5810,27 +5897,18 @@ sub scantron_selectphase {
 
     $ssi_error = 0;
 
-    if (&Apache::lonnet::allowed('usc',$env{'request.role.domain'}) ||
-        &Apache::lonnet::allowed('usc',$env{'request.course.id'})) {
+    if (&Apache::lonnet::allowed('usc',$env{'request.role.domain'}) || $perm{'usc'}) {
 
 	# Chunk of form to prompt for a scantron file upload.
 
         $r->print('
-    <br />
-    '.&Apache::loncommon::start_data_table('LC_scantron_action').'
-       '.&Apache::loncommon::start_data_table_header_row().'
-            <th>
-              &nbsp;'.&mt('Specify a bubblesheet data file to upload.').'
-            </th>
-       '.&Apache::loncommon::end_data_table_header_row().'
-       '.&Apache::loncommon::start_data_table_row().'
-            <td>
-');
-    my $default_form_data=&defaultFormData($symb);
+    <br />');
     my $cdom= $env{'course.'.$env{'request.course.id'}.'.domain'};
     my $cnum= $env{'course.'.$env{'request.course.id'}.'.num'};
+    my $csec= $env{'request.course.sec'};
     my $alertmsg = &mt('Please use the browse button to select a file from your local directory.');
     &js_escape(\$alertmsg);
+    my ($formatoptions,$formattitle,$formatjs) = &scantron_upload_dataformat($cdom);
     $r->print(&Apache::lonhtmlcommon::scripttag('
     function checkUpload(formname) {
 	if (formname.upfile.value == "") {
@@ -5838,24 +5916,43 @@ sub scantron_selectphase {
 	    return false;
 	}
 	formname.submit();
-    }'));
+    }'."\n".$formatjs));
     $r->print('
               <form enctype="multipart/form-data" action="/adm/grades" name="rules" method="post">
                 '.$default_form_data.'
                 <input name="courseid" type="hidden" value="'.$cnum.'" />
+                <input name="coursesec" type="hidden" value="'.$csec.'" />
                 <input name="domainid" type="hidden" value="'.$cdom.'" />
                 <input name="command" value="scantronupload_save" type="hidden" />
-                '.&mt('File to upload: [_1]','<input type="file" name="upfile" size="50" />').'
-                <br />
-                <input type="button" onclick="javascript:checkUpload(this.form);" value="'.&mt('Upload Bubblesheet Data').'" />
-              </form>
-');
+              '.&Apache::loncommon::start_data_table('LC_scantron_action').'
+              '.&Apache::loncommon::start_data_table_header_row().'
+                <th>
+                &nbsp;'.&mt('Specify a bubblesheet data file to upload.').'
+                </th>
+              '.&Apache::loncommon::end_data_table_header_row().'
+              '.&Apache::loncommon::start_data_table_row().'
+            <td>
+                '.&mt('File to upload: [_1]','<input type="file" name="upfile" size="50" />').'<br />'."\n");
+    if ($formatoptions) {
+        $r->print('</td>
+                 '.&Apache::loncommon::end_data_table_row().'
+                 '.&Apache::loncommon::start_data_table_row().'
+                 <td>'.$formattitle.('&nbsp;'x2).$formatoptions.'
+                 </td>
+                 '.&Apache::loncommon::end_data_table_row().'
+                 '.&Apache::loncommon::start_data_table_row().'
+                 <td>'
+        );
+    } else {
+        $r->print(' <br />');
+    }
+    $r->print('<input type="button" onclick="javascript:checkUpload(this.form);" value="'.&mt('Upload Bubblesheet Data').'" />
+              </td>
+             '.&Apache::loncommon::end_data_table_row().'
+             '.&Apache::loncommon::end_data_table().'
+             </form>'
+    );
 
-        $r->print('
-            </td>
-       '.&Apache::loncommon::end_data_table_row().'
-       '.&Apache::loncommon::end_data_table().'
-');
     }
 
     # Chunk of form to prompt for a file to grade and how:
@@ -5905,8 +6002,6 @@ sub scantron_selectphase {
    
     $r->print($result);
 
-
-
     # Chunk of the form that prompts to view a scoring office file,
     # corrected file, skipped records in a file.
 
@@ -5968,98 +6063,6 @@ sub scantron_selectphase {
     return;
 }
 
-=pod
-
-=item get_scantron_config
-
-   Parse and return the bubblesheet configuration line selected as a
-   hash of configuration file fields.
-
- Arguments:
-    which - the name of the configuration to parse from the file.
-
-
- Returns:
-            If the named configuration is not in the file, an empty
-            hash is returned.
-    a hash with the fields
-      name         - internal name for the this configuration setup
-      description  - text to display to operator that describes this config
-      CODElocation - if 0 or the string 'none'
-                          - no CODE exists for this config
-                     if -1 || the string 'letter'
-                          - a CODE exists for this config and is
-                            a string of letters
-                     Unsupported value (but planned for future support)
-                          if a positive integer
-                               - The CODE exists as the first n items from
-                                 the question section of the form
-                          if the string 'number'
-                               - The CODE exists for this config and is
-                                 a string of numbers
-      CODEstart   - (only matter if a CODE exists) column in the line where
-                     the CODE starts
-      CODElength  - length of the CODE
-      IDstart     - column where the student/employee ID starts
-      IDlength    - length of the student/employee ID info
-      Qstart      - column where the information from the bubbled
-                    'questions' start
-      Qlength     - number of columns comprising a single bubble line from
-                    the sheet. (usually either 1 or 10)
-      Qon         - either a single character representing the character used
-                    to signal a bubble was chosen in the positional setup, or
-                    the string 'letter' if the letter of the chosen bubble is
-                    in the final, or 'number' if a number representing the
-                    chosen bubble is in the file (1->A 0->J)
-      Qoff        - the character used to represent that a bubble was
-                    left blank
-      PaperID     - if the scanning process generates a unique number for each
-                    sheet scanned the column that this ID number starts in
-      PaperIDlength - number of columns that comprise the unique ID number
-                      for the sheet of paper
-      FirstName   - column that the first name starts in
-      FirstNameLength - number of columns that the first name spans
- 
-      LastName    - column that the last name starts in
-      LastNameLength - number of columns that the last name spans
-      BubblesPerRow - number of bubbles available in each row used to 
-                      bubble an answer. (If not specified, 10 assumed).
-
-=cut
-
-sub get_scantron_config {
-    my ($which) = @_;
-    my @lines = &get_scantronformat_file();
-    my %config;
-    #FIXME probably should move to XML it has already gotten a bit much now
-    foreach my $line (@lines) {
-	my ($name,$descrip)=split(/:/,$line);
-	if ($name ne $which ) { next; }
-	chomp($line);
-	my @config=split(/:/,$line);
-	$config{'name'}=$config[0];
-	$config{'description'}=$config[1];
-	$config{'CODElocation'}=$config[2];
-	$config{'CODEstart'}=$config[3];
-	$config{'CODElength'}=$config[4];
-	$config{'IDstart'}=$config[5];
-	$config{'IDlength'}=$config[6];
-	$config{'Qstart'}=$config[7];
- 	$config{'Qlength'}=$config[8];
-	$config{'Qoff'}=$config[9];
-	$config{'Qon'}=$config[10];
-	$config{'PaperID'}=$config[11];
-	$config{'PaperIDlength'}=$config[12];
-	$config{'FirstName'}=$config[13];
-	$config{'FirstNamelength'}=$config[14];
-	$config{'LastName'}=$config[15];
-	$config{'LastNamelength'}=$config[16];
-        $config{'BubblesPerRow'}=$config[17];
-	last;
-    }
-    return %config;
-}
-
 =pod 
 
 =item username_to_idmap
@@ -6107,7 +6110,7 @@ sub username_to_idmap {
    Process a requested correction to a scanline.
 
   Arguments:
-    $scantron_config   - hash from &get_scantron_config()
+    $scantron_config   - hash from &Apache::lonnet::get_scantron_config()
     $scan_data         - hash of correction information 
                           (see &scantron_getfile())
     $line              - existing scanline
@@ -6790,7 +6793,7 @@ sub scantron_filter {
 
 sub scantron_process_corrections {
     my ($r) = @_;
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
     my $classlist=&Apache::loncoursedata::get_classlist();
     my $which=$env{'form.scantron_line'};
@@ -6959,7 +6962,7 @@ sub check_for_error {
 sub scantron_warning_screen {
     my ($button_text,$symb)=@_;
     my $title=&Apache::lonnet::gettitle($env{'form.selectpage'});
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my $CODElist;
     if ($scantron_config{'CODElocation'} &&
 	$scantron_config{'CODEstart'} &&
@@ -6976,7 +6979,7 @@ sub scantron_warning_screen {
             '<tr><td><b>'.&mt('Hand-graded items: points from last bubble in row').'</b></td><td><tt>'.
             $env{'form.scantron_lastbubblepoints'}.'</tt></td></tr>';
     }
-    return ('
+    return '
 <p>
 <span class="LC_warning">
 '.&mt("Please double check the information below before clicking on '[_1]'",&mt($button_text)).'</span>
@@ -6988,9 +6991,7 @@ sub scantron_warning_screen {
 </table>
 <p> '.&mt("If this information is correct, please click on '[_1]'.",&mt($button_text)).'<br />
 '.&mt('If something is incorrect, please return to [_1]Grade/Manage/Review Bubblesheets[_2] to start over.','<a href="/adm/grades?symb='.$symb.'&command=scantron_selectphase" class="LC_info">','</a>').'</p>
-
-<br />
-');
+';
 }
 
 =pod
@@ -7016,15 +7017,58 @@ sub scantron_do_warning {
 	} 
 	if ( $env{'form.scantron_selectfile'} eq '') {
 	    $r->print('<p><span class="LC_error">'.&mt("You have not selected a file that contains the student's response data.").'</span></p>');
-	} 
+	}
 	if ( $env{'form.scantron_format'} eq '') {
 	    $r->print('<p><span class="LC_error">'.&mt("You have not selected the format of the student's response data.").'</span></p>');
-	} 
+	}
     } else {
 	my $warning=&scantron_warning_screen('Grading: Validate Records',$symb);
+        my ($checksec,@possibles) = &gradable_sections();
+        my $gradesections;
+        if ($checksec) {
+            my $file=$env{'form.scantron_selectfile'};
+            if (&valid_file($file)) {
+                my %bysec = &scantron_get_sections();
+                my $table;
+                if ((keys(%bysec) > 1) || ((keys(%bysec) == 1) && ((keys(%bysec))[0] ne $checksec))) {
+                    $gradesections = &mt('Your current role is for section [_1].','<i>'.$checksec.'</i>').'<br />';
+                    $table = &Apache::loncommon::start_data_table()."\n".
+                             &Apache::loncommon::start_data_table_header_row().
+                             '<th>'.&mt('Section').'</th><th>'.&mt('Number of records').'</th>'.
+                              &Apache::loncommon::end_data_table_header_row()."\n";
+                    if ($bysec{'none'}) {
+                        $table .= &Apache::loncommon::start_data_table_row().
+                                  '<td>'.&mt('None').'</td><td>'.$bysec{'none'}.'</td>'.
+                                  &Apache::loncommon::end_data_table_row()."\n";
+                    }
+                    foreach my $sec (sort { $a <=> $b } keys(%bysec)) {
+                        next if ($sec eq 'none');
+                        $table .= &Apache::loncommon::start_data_table_row().
+                                  '<td>'.$sec.'</td><td>'.$bysec{$sec}.'</td>'.
+                                  &Apache::loncommon::end_data_table_row()."\n";
+                    }
+                    $table .= &Apache::loncommon::end_data_table()."\n";
+                    $gradesections .= &mt('Sections represented in the bubblesheet data file (based on bubbled student IDs) are as follows:').
+                                      '<p>'.$table.'</p>';
+                    if (@possibles) {
+                        $gradesections .= '<p>'.
+                                          &mt('You have role(s) in [quant,_1,other section,other sections] with privileges to manage grades.',
+                                              scalar(@possibles)).'<br />'.
+                                          &mt('Check which of those section(s), in addition to section [_1], you wish to grade using this bubblesheet file:',
+                                              '<i>'.$checksec.'</i>').' ';
+                        foreach my $sec (sort {$a <=> $b } @possibles) {
+                            $gradesections .= '<label><input type="checkbox" name="scantron_othersections" value="'.$sec.'" />'.$sec.'</label>'.('&nbsp;'x2);
+                        }
+                        $gradesections .= '</p>';
+                    }
+                }
+            } else {
+                $gradesections = '<p class="LC_error">'.&mt('The selected file is unavailable').'</p>';
+            }
+        }
         my $bubbledbyhand=&hand_bubble_option();
 	$r->print('
-'.$warning.$bubbledbyhand.'
+'.$warning.$gradesections.$bubbledbyhand.'
 <input type="submit" name="submit" value="'.&mt('Grading: Validate Records').'" />
 <input type="hidden" name="command" value="scantron_validate" />
 ');
@@ -7111,11 +7155,42 @@ sub scantron_validate_file {
     if ($env{'form.scantron_corrections'}) {
 	&scantron_process_corrections($r);
     }
-    $r->print('<p>'.&mt('Gathering necessary information.').'</p>');$r->rflush();
+
+    $r->print('<p>'.&mt('Gathering necessary information.').'</p>');
+    my ($checksec,@gradable);
+    if ($env{'request.course.sec'}) {
+        ($checksec,my @possibles) = &gradable_sections();
+        if ($checksec) {
+            if (@possibles) {
+                my @chosensecs = &Apache::loncommon::get_env_multiple('form.scantron_othersections');
+                if (@chosensecs) {
+                    foreach my $sec (@chosensecs) {
+                        if (grep(/^\Q$sec\E$/,@possibles)) {
+                            unless (grep(/^\Q$sec\E$/,@gradable)) {
+                                push(@gradable,$sec);
+                            }
+                        }
+                    }
+                }
+            }
+            $r->print('<p><table>');
+            if (@gradable) {
+                my @showsections = sort { $a <=> $b } (@gradable,$checksec);
+                $r->print(
+                    '<tr><td><b>'.&mt('Sections to be Graded:').'</b></td><td>'.join(', ',@showsections).'</td></tr>');
+            } else {
+                $r->print(
+                    '<tr><td><b>'.&mt('Section to be Graded:').'</b></td><td>'.$checksec.'</td></tr>');
+            }
+            $r->print('</table></p>');
+        }
+    }
+    $r->rflush();
+
     #get the student pick code ready
     $r->print(&Apache::loncommon::studentbrowser_javascript());
     my $nav_error;
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my $max_bubble=&scantron_get_maxbubble(\$nav_error,\%scantron_config);
     if ($nav_error) {
         $r->print(&navmap_errormsg());
@@ -7136,7 +7211,7 @@ sub scantron_validate_file {
 	$env{'form.validatepass'} = 0;
     }
     my $currentphase=$env{'form.validatepass'};
-
+    my %skipbysec=();
 
     my $stop=0;
     while (!$stop && $currentphase < scalar(@validate_phases)) {
@@ -7146,13 +7221,29 @@ sub scantron_validate_file {
 	my $which="scantron_validate_".$validate_phases[$currentphase];
 	{
 	    no strict 'refs';
-	    ($stop,$currentphase)=&$which($r,$currentphase);
+            my @extras=();
+            if ($validate_phases[$currentphase] eq 'ID') {
+                @extras = (\%skipbysec,$checksec,@gradable);
+            }
+	    ($stop,$currentphase)=&$which($r,$currentphase,@extras);
 	}
     }
     if (!$stop) {
 	my $warning=&scantron_warning_screen('Start Grading',$symb);
+        my $secinfo;
+        if (keys(%skipbysec) > 0) {
+            my $seclist = '<ul>';
+            foreach my $sec (sort { $a <=> $b } keys(%skipbysec)) {
+                $seclist .= '<li>'.&mt('section [_1]: [_2]',$sec,$skipbysec{$sec}).'</li>';
+            }
+            $seclist .= '</ul>';
+            $secinfo = '<p class="LC_info">'.
+                       &mt('Numbers of records for students in sections not being graded [_1]',
+                           $seclist).
+                       '</p>';
+        }
 	$r->print(&mt('Validation process complete.').'<br />'.
-                  $warning.
+                  $secinfo.$warning.
                   &mt('Perform verification for each student after storage of submissions?').
                   '&nbsp;<span class="LC_nobreak"><label>'.
                   '<input type="radio" name="verifyrecord" value="1" />'.&mt('Yes').'</label>'.
@@ -7568,14 +7659,15 @@ sub scantron_validate_sequence {
 
 
 sub scantron_validate_ID {
-    my ($r,$currentphase) = @_;
+    my ($r,$currentphase,$skipbysec,$checksec,@gradable) = @_;
     
     #get student info
     my $classlist=&Apache::loncoursedata::get_classlist();
     my %idmap=&username_to_idmap($classlist);
+    my $secidx = &Apache::loncoursedata::CL_SECTION();
 
     #get scantron line setup
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
 
     my $nav_error;
@@ -7586,6 +7678,7 @@ sub scantron_validate_ID {
     }
 
     my %found=('ids'=>{},'usernames'=>{});
+    my $unsavedskips = 0;
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
 	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
@@ -7598,13 +7691,41 @@ sub scantron_validate_ID {
 	}
 	if ($found) {
 	    my $username=$idmap{$found};
+            if ($checksec) {
+                if (ref($classlist->{$username}) eq 'ARRAY') {
+                    my $stusec = $classlist->{$username}->[$secidx];
+                    if ($stusec ne $checksec) {
+                        unless ((@gradable > 0) && (grep(/^\Q$stusec\E$/,@gradable))) {
+                            my $skip=1;
+                            &scantron_put_line($scanlines,$scan_data,$i,$line,$skip);
+                            if (ref($skipbysec) eq 'HASH') {
+                                if ($stusec eq '') {
+                                    $skipbysec->{'none'} ++;
+                                } else {
+                                    $skipbysec->{$stusec} ++;
+                                }
+                            }
+                            $unsavedskips ++;
+                            next;
+                        }
+                    }
+                }
+            }
 	    if ($found{'ids'}{$found}) {
 		&scantron_get_correction($r,$i,$scan_record,\%scantron_config,
 					 $line,'duplicateID',$found);
+                if ($unsavedskips) {
+                    &scantron_putfile($scanlines,$scan_data);
+                    $unsavedskips = 0;
+                }
 		return(1,$currentphase);
 	    } elsif ($found{'usernames'}{$username}) {
 		&scantron_get_correction($r,$i,$scan_record,\%scantron_config,
 					 $line,'duplicateID',$username);
+                if ($unsavedskips) {
+                    &scantron_putfile($scanlines,$scan_data);
+                    $unsavedskips = 0;
+                }
 		return(1,$currentphase);
 	    }
 	    #FIXME store away line we previously saw the ID on to use above
@@ -7613,29 +7734,95 @@ sub scantron_validate_ID {
 	} else {
 	    if ($id =~ /^\s*$/) {
 		my $username=&scan_data($scan_data,"$i.user");
-		if (defined($username) && $found{'usernames'}{$username}) {
+                if (($checksec && $username ne '')) {
+                    if (ref($classlist->{$username}) eq 'ARRAY') {
+                        my $stusec = $classlist->{$username}->[$secidx];
+                        if ($stusec ne $checksec) {
+                            unless ((@gradable > 0) && (grep(/^\Q$stusec\E$/,@gradable))) {
+                                my $skip=1;
+                                &scantron_put_line($scanlines,$scan_data,$i,$line,$skip);
+                                if (ref($skipbysec) eq 'HASH') {
+                                    if ($stusec eq '') {
+                                        $skipbysec->{'none'} ++;
+                                    } else {
+                                        $skipbysec->{$stusec} ++;
+                                    }
+                                }
+                                $unsavedskips ++;
+                                next;
+                            }
+                        }
+                    }
+		} elsif (defined($username) && $found{'usernames'}{$username}) {
 		    &scantron_get_correction($r,$i,$scan_record,
 					     \%scantron_config,
 					     $line,'duplicateID',$username);
+                    if ($unsavedskips) {
+                        &scantron_putfile($scanlines,$scan_data);
+                        $unsavedskips = 0;
+                    }
 		    return(1,$currentphase);
 		} elsif (!defined($username)) {
 		    &scantron_get_correction($r,$i,$scan_record,
 					     \%scantron_config,
 					     $line,'incorrectID');
+                    if ($unsavedskips) {
+                        &scantron_putfile($scanlines,$scan_data);
+                        $unsavedskips = 0;
+                    }
 		    return(1,$currentphase);
 		}
 		$found{'usernames'}{$username}++;
 	    } else {
 		&scantron_get_correction($r,$i,$scan_record,\%scantron_config,
 					 $line,'incorrectID');
+                if ($unsavedskips) {
+                    &scantron_putfile($scanlines,$scan_data);
+                    $unsavedskips = 0;
+                }
 		return(1,$currentphase);
 	    }
 	}
     }
-
+    if ($unsavedskips) {
+        &scantron_putfile($scanlines,$scan_data);
+        $unsavedskips = 0;
+    }
     return (0,$currentphase+1);
 }
 
+sub scantron_get_sections {
+    my %bysec;
+    if ($env{'form.scantron_format'} ne '') {
+        my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
+        my ($scanlines,$scan_data)=&scantron_getfile();
+        my $classlist=&Apache::loncoursedata::get_classlist();
+        my %idmap=&username_to_idmap($classlist);
+        foreach my $key (keys(%idmap)) {
+            my $lckey = lc($key);
+            $idmap{$lckey} = $idmap{$key};
+        }
+        my $secidx = &Apache::loncoursedata::CL_SECTION();
+        for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+            my $line=&scantron_get_line($scanlines,$scan_data,$i);
+            if ($line=~/^[\s\cz]*$/) { next; }
+            my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
+                                                     $scan_data);
+            my $id=lc($$scan_record{'scantron.ID'});
+            if (exists($idmap{$id})) {
+                if (ref($classlist->{$idmap{$id}}) eq 'ARRAY') {
+                    my $stusec = $classlist->{$idmap{$id}}->[$secidx];
+                    if ($stusec eq '') {
+                        $bysec{'none'} ++;
+                    } else {
+                        $bysec{$stusec} ++;
+                    }
+                }
+            }
+        }
+    }
+    return %bysec;
+}
 
 sub scantron_get_correction {
     my ($r,$i,$scan_record,$scan_config,$line,$error,$arg,
@@ -7822,7 +8009,7 @@ sub verify_bubbles_checked {
     my $ansnumstr = join('","',@ansnums);
     my $warning = &mt("A bubble or 'No bubble' selection has not been made for one or more lines.");
     &js_escape(\$warning);
-    my $output = &Apache::lonhtmlcommon::scripttag((<<ENDSCRIPT));
+    my $output = &Apache::lonhtmlcommon::scripttag(<<ENDSCRIPT);
 function verify_bubble_radio(form) {
     var ansnumArray = new Array ("$ansnumstr");
     var need_bubble_count = 0;
@@ -8045,7 +8232,7 @@ sub prompt_for_corrections {
 
  Arguments:
     $r           - Apache request object
-    $scan_config - hash from &get_scantron_config()
+    $scan_config - hash from &Apache::lonnet::get_scantron_config()
     $line        - Number of the line being displayed.
     $questionnum - Question number (may include subquestion)
     $error       - Type of error.
@@ -8209,7 +8396,7 @@ sub get_codes {
 
 sub scantron_validate_CODE {
     my ($r,$currentphase) = @_;
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     if ($scantron_config{'CODElocation'} &&
 	$scantron_config{'CODEstart'} &&
 	$scantron_config{'CODElength'}) {
@@ -8283,7 +8470,7 @@ sub scantron_validate_doublebubble {
         &Apache::lonnet::decode_symb($env{'form.selectpage'});
 
     #get scantron line setup
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
 
     my $navmap = Apache::lonnavmaps::navmap->new();
@@ -8465,7 +8652,7 @@ sub scantron_validate_missingbubbles {
         &Apache::lonnet::decode_symb($env{'form.selectpage'});
 
     #get scantron line setup
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
 
     my $navmap = Apache::lonnavmaps::navmap->new();
@@ -8594,7 +8781,7 @@ sub hand_bubble_option {
         }
     }
     if ($needs_hand_bubbles) {
-        my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+        my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
         my $bubbles_per_row = &bubblesheet_bubbles_per_row(\%scantron_config);
         return &mt('The sequence to be graded contains response types which are handgraded.').'<p>'.
                &mt('If you have already graded these by bubbling sheets to indicate points awarded, [_1]what point value is assigned to a filled last bubble in each row?','<br />').
@@ -8613,7 +8800,7 @@ sub scantron_process_students {
     }
     my $default_form_data=&defaultFormData($symb);
 
-    my %scantron_config=&get_scantron_config($env{'form.scantron_format'});
+    my %scantron_config=&Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my $bubbles_per_row = &bubblesheet_bubbles_per_row(\%scantron_config); 
     my ($scanlines,$scan_data)=&scantron_getfile();
     my $classlist=&Apache::loncoursedata::get_classlist();
@@ -8653,9 +8840,10 @@ sub scantron_process_students {
 SCANTRONFORM
     $r->print($result);
 
+    my ($checksec,@possibles)=&gradable_sections();
     my @delayqueue;
     my (%completedstudents,%scandata);
-    
+
     my $lock=&Apache::lonnet::set_lock(&mt('Grading bubblesheet exam'));
     my $count=&get_todo_count($scanlines,$scan_data);
     my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r,$count);
@@ -8681,7 +8869,7 @@ SCANTRONFORM
 	return '';		# Dunno why the other returns return '' rather than just returning.
     }
 
-    my %lettdig = &letter_to_digits();
+    my %lettdig = &Apache::lonnet::letter_to_digits();
     my $numletts = scalar(keys(%lettdig));
     my %orderedforcode;
 
@@ -8715,6 +8903,13 @@ SCANTRONFORM
  	    next;
  	}
         my $usec = $classlist->{$uname}->[&Apache::loncoursedata::CL_SECTION];
+        if (($checksec ne '') && ($checksec ne $usec)) {
+            unless (grep(/^\Q$usec\E$/,@possibles)) {
+                &scantron_add_delay(\@delayqueue,$line,
+                                    "No role with manage grades privilege in student's section ($usec)",3);
+                next;
+            }
+        }
         my $user = $uname.':'.$usec;
   	($uname,$udom)=split(/:/,$uname);
 
@@ -9006,8 +9201,9 @@ sub grade_student_bubbles {
 }
 
 sub scantron_upload_scantron_data {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     my $dom = $env{'request.role.domain'};
+    my ($formatoptions,$formattitle,$formatjs) = &scantron_upload_dataformat($dom);
     my $domdesc = &Apache::lonnet::domain($dom,'description');
     $r->print(&Apache::loncommon::coursebrowser_javascript($dom));
     my $select_link=&Apache::loncommon::selectcourse_link('rules','courseid',
@@ -9047,6 +9243,7 @@ sub scantron_upload_scantron_data {
         return;
     }
 
+    '.$formatjs.'
 '));
     $r->print('
 <h3>'.&mt('Send bubblesheet data to a course').'</h3>
@@ -9062,7 +9259,12 @@ sub scantron_upload_scantron_data {
   &Apache::lonhtmlcommon::row_closure().
   &Apache::lonhtmlcommon::row_title(&mt('Domain')).
   '<input name="domainid" type="hidden" />'.$domdesc.
-  &Apache::lonhtmlcommon::row_closure().
+  &Apache::lonhtmlcommon::row_closure());
+    if ($formatoptions) {
+        $r->print(&Apache::lonhtmlcommon::row_title($formattitle).$formatoptions.
+                  &Apache::lonhtmlcommon::row_closure());
+    }
+    $r->print(
   &Apache::lonhtmlcommon::row_title(&mt('File to upload')).
   '<input type="file" name="upfile" size="50" />'.
   &Apache::lonhtmlcommon::row_closure(1).
@@ -9075,9 +9277,87 @@ sub scantron_upload_scantron_data {
     return '';
 }
 
+sub scantron_upload_dataformat {
+    my ($dom) = @_;
+    my ($formatoptions,$formattitle,$formatjs);
+    $formatjs = <<'END';
+function toggleScantab(form) {
+   return;
+}
+END
+    my %domconfig = &Apache::lonnet::get_dom('configuration',['scantron'],$dom);
+    if (ref($domconfig{'scantron'}) eq 'HASH') {
+        if (ref($domconfig{'scantron'}{'config'}) eq 'HASH') {
+            if (keys(%{$domconfig{'scantron'}{'config'}}) > 1) {
+                if (($domconfig{'scantron'}{'config'}{'dat'}) &&
+                    (ref($domconfig{'scantron'}{'config'}{'csv'}) eq 'HASH')) {
+                    if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {  
+                        if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}})) {
+                            my ($onclick,$formatextra,$singleline);
+                            my @lines = &Apache::lonnet::get_scantronformat_file();
+                            my $count = 0;
+                            foreach my $line (@lines) {
+                                next if ($line =~ /^#/);
+                                $singleline = $line;
+                                $count ++;
+                            }
+                            if ($count > 1) {
+                                $formatextra = '<div style="display:none" id="bubbletype">'.
+                                               '<span class="LC_nobreak">'.
+                                               &mt('Bubblesheet type:').'&nbsp;'.
+                                               &scantron_scantab().'</span></div>';
+                                $onclick = ' onclick="toggleScantab(this.form);"';
+                                $formatjs = <<"END";
+function toggleScantab(form) {
+    var divid = 'bubbletype';
+    if (document.getElementById(divid)) {
+        var radioname = 'fileformat';
+        var num = form.elements[radioname].length;
+        if (num) {
+            for (var i=0; i<num; i++) {
+                if (form.elements[radioname][i].checked) {
+                    var chosen = form.elements[radioname][i].value;
+                    if (chosen == 'dat') {
+                        document.getElementById(divid).style.display = 'none';
+                    } else if (chosen == 'csv') {
+                        document.getElementById(divid).style.display = 'block';
+                    }
+                }
+            }
+        }
+    }
+    return;
+}
+
+END
+                            } elsif ($count == 1) {
+                                my $formatname = (split(/:/,$singleline,2))[0];
+                                $formatextra = '<input type="hidden" name="scantron_format" value="'.$formatname.'" />';
+                            }
+                            $formattitle = &mt('File format');
+                            $formatoptions = '<label><input name="fileformat" type="radio" value="dat" checked="checked"'.$onclick.' />'.
+                                             &mt('Plain Text (no delimiters)').
+                                             '</label>'.('&nbsp;'x2).
+                                             '<label><input name="fileformat" type="radio" value="csv"'.$onclick.' />'.
+                                             &mt('Comma separated values').'</label>'.$formatextra;
+                        }
+                    }
+                }
+            } elsif (keys(%{$domconfig{'scantron'}{'config'}}) == 1) {
+                if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
+                    if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}})) {
+                        $formattitle = &mt('Bubblesheet type');
+                        $formatoptions = &scantron_scantab();
+                    }
+                }
+            }
+        }
+    }
+    return ($formatoptions,$formattitle,$formatjs);
+}
 
 sub scantron_upload_scantron_data_save {
-    my($r,$symb)=@_;
+    my ($r,$symb) = @_;
     my $doanotherupload=
 	'<br /><form action="/adm/grades" method="post">'."\n".
 	'<input type="hidden" name="command" value="scantronupload" />'."\n".
@@ -9085,7 +9365,9 @@ sub scantron_upload_scantron_data_save {
 	'</form>'."\n";
     if (!&Apache::lonnet::allowed('usc',$env{'form.domainid'}) &&
 	!&Apache::lonnet::allowed('usc',
-			    $env{'form.domainid'}.'_'.$env{'form.courseid'})) {
+			    $env{'form.domainid'}.'_'.$env{'form.courseid'}) &&
+        !&Apache::lonnet::allowed('usc',
+                            $env{'form.domainid'}.'_'.$env{'form.courseid'}.'/'.$env{'form.coursesec'})) {
 	$r->print(&mt("You are not allowed to upload bubblesheet data to the requested course.")."<br />");
 	unless ($symb) {
 	    $r->print($doanotherupload);
@@ -9101,8 +9383,38 @@ sub scantron_upload_scantron_data_save {
                 &mt('The file: [_1] you attempted to upload contained no information. Please check that you entered the correct filename.',
                         '<span class="LC_filename">'.&HTML::Entities::encode($env{'form.upfile.filename'},'<>&"').'</span>'),1));
     } else {
-        my $result = 
-            &Apache::lonnet::userfileupload('upfile','','scantron','','','',
+        my %domconfig = &Apache::lonnet::get_dom('configuration',['scantron'],$env{'form.domainid'});
+        my $parser;
+        if (ref($domconfig{'scantron'}) eq 'HASH') {
+            if (ref($domconfig{'scantron'}{'config'}) eq 'HASH') {
+                my $is_csv;
+                my @possibles = keys(%{$domconfig{'scantron'}{'config'}});
+                if (@possibles > 1) {
+                    if ($env{'form.fileformat'} eq 'csv') {
+                        if (ref($domconfig{'scantron'}{'config'}{'csv'}) eq 'HASH') {
+                            if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
+                                if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}}) > 1) {
+                                    $is_csv = 1;
+                                }
+                            }
+                        }
+                    }
+                } elsif (@possibles == 1) {
+                    if (ref($domconfig{'scantron'}{'config'}{'csv'}) eq 'HASH') {
+                        if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
+                            if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}}) > 1) {
+                                $is_csv = 1;
+                            }
+                        }
+                    }
+                }
+                if ($is_csv) {
+                   $parser = $domconfig{'scantron'}{'config'}{'csv'};
+                }
+            }
+        }
+        my $result =
+            &Apache::lonnet::userfileupload('upfile','scantron','scantron',$parser,'','',
                                             $env{'form.courseid'},$env{'form.domainid'});
         if ($result =~ m{^/uploaded/}) {
             $r->print(
@@ -9111,8 +9423,17 @@ sub scantron_upload_scantron_data_save {
                         (length($env{'form.upfile'})-1),
                         '<span class="LC_filename">'.$result.'</span>'));
             ($uploadedfile) = ($result =~ m{/([^/]+)$});
+            if ($uploadedfile =~ /^scantron_orig_/) {
+                my $logname = $uploadedfile;
+                $logname =~ s/^scantron_orig_//;
+                if ($logname ne '') {
+                    my $now = time;
+                    my %info = ($logname => { $now => $env{'user.name'}.':'.$env{'user.domain'} });  
+                    &Apache::lonnet::put('scantronupload',\%info,$env{'form.domainid'},$env{'form.courseid'});
+                }
+            }
             $r->print(&validate_uploaded_scantron_file($env{'form.domainid'},
-                                                       $env{'form.courseid'},$uploadedfile));
+                                                       $env{'form.courseid'},$symb,$uploadedfile));
         } else {
             $r->print(
                 &Apache::lonhtmlcommon::confirm_success(&mt('Upload failed'),1).'<br />'.
@@ -9130,13 +9451,34 @@ sub scantron_upload_scantron_data_save {
 }
 
 sub validate_uploaded_scantron_file {
-    my ($cdom,$cname,$fname) = @_;
+    my ($cdom,$cname,$symb,$fname,$context,$countsref) = @_;
+
     my $scanlines=&Apache::lonnet::getfile('/uploaded/'.$cdom.'/'.$cname.'/'.$fname);
     my @lines;
     if ($scanlines ne '-1') {
         @lines=split("\n",$scanlines,-1);
     }
-    my $output;
+    my ($output,$secidx,$checksec,$priv,%crsroleshash,@possibles);
+    $secidx = &Apache::loncoursedata::CL_SECTION();
+    if ($context eq 'download') {
+        $priv = 'mgr';
+    } else {
+        $priv = 'usc';
+    }
+    unless ((&Apache::lonnet::allowed($priv,$env{'request.role.domain'})) ||
+            (($env{'request.course.id'}) &&
+             (&Apache::lonnet::allowed($priv,$env{'request.course.id'})))) {
+        if ($env{'request.course.sec'} ne '') {
+            unless (&Apache::lonnet::allowed($priv,
+                                         "$env{'request.course.id'}/$env{'request.course.sec'}")) {
+                unless ($context eq 'download') {
+                    $output = '<p class="LC_warning">'.&mt('You do not have permission to upload bubblesheet data').'</p>';
+                }
+                return $output;
+            }
+            ($checksec,@possibles)=&gradable_sections();
+        }
+    }
     if (@lines) {
         my (%counts,$max_match_format);
         my ($found_match_count,$max_match_count,$max_match_pct) = (0,0,0);
@@ -9147,7 +9489,7 @@ sub validate_uploaded_scantron_file {
             $idmap{$lckey} = $idmap{$key};
         }
         my %unique_formats;
-        my @formatlines = &get_scantronformat_file();
+        my @formatlines = &Apache::lonnet::get_scantronformat_file();
         foreach my $line (@formatlines) {
             chomp($line);
             my @config = split(/:/,$line);
@@ -9166,6 +9508,8 @@ sub validate_uploaded_scantron_file {
             %{$counts{$key}} = (
                                'found'   => 0,
                                'total'   => 0,
+                               'totalanysec' => 0,
+                               'othersec' => 0,
                               );
             foreach my $line (@lines) {
                 next if ($line =~ /^#/);
@@ -9173,6 +9517,23 @@ sub validate_uploaded_scantron_file {
                 my $id = substr($line,$idstart-1,$idlength);
                 $id = lc($id);
                 if (exists($idmap{$id})) {
+                    if ($checksec ne '') {
+                        $counts{$key}{'totalanysec'} ++;
+                        if (ref($classlist->{$idmap{$id}}) eq 'ARRAY') {
+                            my $stusec = $classlist->{$idmap{$id}}->[$secidx];
+                            if ($stusec ne $checksec) {
+                                if (@possibles) {
+                                    unless (grep(/^\Q$stusec\E$/,@possibles)) {
+                                        $counts{$key}{'othersec'} ++;
+                                        next;
+                                    }
+                                } else {
+                                    $counts{$key}{'othersec'} ++;
+                                    next;
+                                }
+                            }
+                        }
+                    }
                     $counts{$key}{'found'} ++;
                 }
                 $counts{$key}{'total'} ++;
@@ -9187,7 +9548,7 @@ sub validate_uploaded_scantron_file {
                 }
             }
         }
-        if (ref($unique_formats{$max_match_format}) eq 'ARRAY') {
+        if ((ref($unique_formats{$max_match_format}) eq 'ARRAY') && ($context ne 'download')) {
             my $format_descs;
             my $numwithformat = @{$unique_formats{$max_match_format}};
             for (my $i=0; $i<$numwithformat; $i++) {
@@ -9232,13 +9593,179 @@ sub validate_uploaded_scantron_file {
                     '<li>'.&mt('The course roster is not up to date.').'</li>'.
                     '</ul>';
             }
+            if (($checksec ne '') && (ref($counts{$max_match_format}) eq 'HASH')) {
+                if ($counts{$max_match_format}{'othersec'}) {
+                    my $percent_nongrade = (100*$counts{$max_match_format}{'othersec'})/($counts{$max_match_format}{'totalanysec'});
+                    my $showpct = sprintf("%.0f",$percent_nongrade).'%';
+                    my $confirmdel = &mt('Are you sure you want to permanently delete this file?');
+                    &js_escape(\$confirmdel);
+                    $output .= '<p class="LC_warning">'.
+                               &mt('Comparison of student IDs in the uploaded file with the course roster found [_1][quant,_2,match,matches][_3] for students in section(s) for which none of your role(s) have privileges to modify grades',
+                                   '<b>',$counts{$max_match_format}{'othersec'},'</b>').
+                               '<br />'.
+                               &mt('Unless you are assigned role(s) which allow modification of grades in additional sections, [_1] of the records in this file will be automatically excluded when you perform bubblesheet grading.','<b>'.$showpct.'</b>').
+                               '</p><p>'.
+                               &mt('If you prefer to delete the file now, use: [_1]').
+                               '<form method="post" name="delupload" action="/adm/grades">'.
+                               '<input type="hidden" name="symb" value="'.$symb.'" />'.
+                               '<input type="hidden" name="domainid" value="'.$cdom.'" />'.
+                               '<input type="hidden" name="courseid" value="'.$cname.'" />'.
+                               '<input type="hidden" name="coursesec" value="'.$env{'request.course.sec'}.'" />'. 
+                               '<input type="hidden" name="uploadedfile" value="'.$fname.'" />'. 
+                               '<input type="hidden" name="command" value="scantronupload_delete" />'.
+                               '<input type="button" name="delbutton" value="'.&mt('Delete Uploaded File').'" onclick="javascript:if (confirm('."'$confirmdel'".')) { document.delupload.submit(); }" />'.
+                               '</form></p>';
+                }
+            }
         }
-    } else {
+        if (($context eq 'download') && ($checksec ne '')) {
+            if ((ref($countsref) eq 'HASH') && (ref($counts{$max_match_format}) eq 'HASH')) {
+                $countsref->{'totalanysec'} = $counts{$max_match_format}{'totalanysec'};
+                $countsref->{'othersec'} = $counts{$max_match_format}{'othersec'};
+            }
+        } 
+    } elsif ($context ne 'download') {
         $output = '<p class="LC_warning">'.&mt('Uploaded file contained no data').'</p>';
     }
     return $output;
 }
 
+sub gradable_sections {
+    my $checksec = $env{'request.course.sec'};
+    my @oksecs;
+    if ($checksec) {
+        my %availablesecs = &sections_grade_privs();
+        if (ref($availablesecs{'mgr'}) eq 'ARRAY') {
+            foreach my $sec (@{$availablesecs{'mgr'}}) {
+                unless (grep(/^\Q$sec\E$/,@oksecs)) {
+                    push(@oksecs,$sec);
+                }
+            }
+            if (grep(/^all$/,@oksecs)) {
+                undef($checksec);
+            }
+        }
+    }
+    return($checksec,@oksecs);
+}
+
+sub sections_grade_privs {
+    my $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+    my $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+    my %availablesecs = (
+                          mgr => [],
+                          vgr => [],
+                          usc => [],
+                        );
+    my $ccrole = 'cc';
+    if ($env{'course.'.$env{'request.course.id'}.'.type'} eq 'Community') {
+        $ccrole = 'co';
+    }
+    my %crsroleshash = &Apache::lonnet::get_my_roles($env{'user.name'},$env{'user.domain'},
+                                                     'userroles',['active'],
+                                                     [$ccrole,'in','cr'],$cdom,1);
+    my $crsid = $cnum.':'.$cdom;
+    foreach my $item (keys(%crsroleshash)) {
+        next unless ($item =~ /^$crsid\:/);
+        my ($crsnum,$crsdom,$role,$sec) = split(/\:/,$item);
+        my $suffix = "/$cdom/$cnum./$cdom/$cnum";
+        if ($sec ne '') {
+            $suffix = "/$cdom/$cnum/$sec./$cdom/$cnum/$sec";
+        }
+        if (($role eq $ccrole) || ($role eq 'in')) {
+            foreach my $priv ('mgr','vgr','usc') { 
+                unless (grep(/^all$/,@{$availablesecs{$priv}})) {
+                    if ($sec eq '') {
+                        $availablesecs{$priv} = ['all'];
+                    } elsif ($sec ne $env{'request.course.sec'}) {
+                        unless (grep(/^\Q$sec\E$/,@{$availablesecs{$priv}})) {
+                            push(@{$availablesecs{$priv}},$sec);
+                        }
+                    }
+                }
+            }
+        } elsif ($role =~ m{^cr/}) {
+            foreach my $priv ('mgr','vgr','usc') {
+                unless (grep(/^all$/,@{$availablesecs{$priv}})) {
+                    if ($env{"user.priv.$role.$suffix"} =~ /:$priv&/) {
+                        if ($sec eq '') {
+                            $availablesecs{$priv} = ['all'];
+                        } elsif ($sec ne $env{'request.course.sec'}) {
+                            unless (grep(/^\Q$sec\E$/,@{$availablesecs{$priv}})) {
+                                push(@{$availablesecs{$priv}},$sec);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return %availablesecs;
+}
+
+sub scantron_upload_delete {
+    my ($r,$symb) = @_;
+    my $filename = $env{'form.uploadedfile'};
+    if ($filename =~ /^scantron_orig_/) {
+        if (&Apache::lonnet::allowed('usc',$env{'form.domainid'}) ||
+            &Apache::lonnet::allowed('usc',
+                                     $env{'form.domainid'}.'_'.$env{'form.courseid'}) ||
+            &Apache::lonnet::allowed('usc',
+                                     $env{'form.domainid'}.'_'.$env{'form.courseid'}.'/'.$env{'form.coursesec'})) {
+            my $uploadurl = '/uploaded/'.$env{'form.domainid'}.'/'.$env{'form.courseid'}.'/'.$env{'form.uploadedfile'};
+            my $retrieval = &Apache::lonnet::getfile($uploadurl);
+            if ($retrieval eq '-1') {
+                $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion failed'),1).'<br />'.
+                          &mt('File requested for deletion not found.'));
+            } else {
+                $filename =~ s/^scantron_orig_//;
+                if ($filename ne '') {
+                    my ($is_valid,$numleft);
+                    my %info = &Apache::lonnet::get('scantronupload',[$filename],$env{'form.domainid'},$env{'form.courseid'});
+                    if (keys(%info)) {
+                        if (ref($info{$filename}) eq 'HASH') {
+                            foreach my $timestamp (sort(keys(%{$info{$filename}}))) {
+                                if ($info{$filename}{$timestamp} eq $env{'user.name'}.':'.$env{'user.domain'}) {
+                                    $is_valid = 1;
+                                    delete($info{$filename}{$timestamp}); 
+                                }
+                            }
+                            $numleft = scalar(keys(%{$info{$filename}}));
+                        }
+                    }
+                    if ($is_valid) {
+                        my $result = &Apache::lonnet::removeuploadedurl($uploadurl);
+                        if ($result eq 'ok') {
+                            $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion successful')).'<br />');
+                            if ($numleft) {
+                                &Apache::lonnet::put('scantronupload',\%info,$env{'form.domainid'},$env{'form.courseid'});
+                            } else {
+                                &Apache::lonnet::del('scantronupload',[$filename],$env{'form.domainid'},$env{'form.courseid'});
+                            }
+                        } else {
+                            $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion failed'),1).'<br />'.
+                                      &mt('Result was [_1]',$result));
+                        }
+                    } else {
+                        $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion failed'),1).'<br />'.
+                                  &mt('File requested for deletion was uploaded by a different user.'));
+                    }
+                } else {
+                    $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion failed'),1).'<br />'.
+                              &mt('Filename of bubblesheet data file requested for deletion is invalid.'));
+                }
+            }
+        } else {
+            $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion failed'),1).'<br />'. 
+                      &mt('You are not permitted to delete bubblesheet data files from the requested course.'));
+        }
+    } else {
+        $r->print(&Apache::lonhtmlcommon::confirm_success(&mt('File deletion failed'),1).'<br />'.
+                          &mt('Filename of bubblesheet data file requested for deletion is invalid.'));
+    }
+    return;
+}
+
 sub valid_file {
     my ($requested_file)=@_;
     foreach my $filename (sort(&scantron_filenames())) {
@@ -9248,7 +9775,7 @@ sub valid_file {
 }
 
 sub scantron_download_scantron_data {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     my $default_form_data=&defaultFormData($symb);
     my $cname=$env{'course.'.$env{'request.course.id'}.'.num'};
     my $cdom=$env{'course.'.$env{'request.course.id'}.'.domain'};
@@ -9261,6 +9788,29 @@ sub scantron_download_scantron_data {
 ');
 	return;
     }
+    my (%uploader,$is_owner,%counts,$percent);
+    my %uploader = &Apache::lonnet::get('scantronupload',[$file],$cdom,$cname);
+    if (ref($uploader{$file}) eq 'HASH') {
+        foreach my $timestamp (sort { $a <=> $b } keys(%{$uploader{$file}})) {
+            if ($uploader{$file}{$timestamp} eq $env{'user.name'}.':'.$env{'user.domain'}) {
+                $is_owner = 1;
+                last;
+            }
+        }
+    }
+    unless ($is_owner) {
+        &validate_uploaded_scantron_file($cdom,$cname,$symb,'scantron_orig_'.$file,'download',\%counts);
+        if ($counts{'totalanysec'}) {
+            my $percent_othersec = (100*$counts{'othersec'})/($counts{'totalanysec'});
+            if ($percent_othersec >= 10) {
+                my $showpct = sprintf("%.0f",$percent_othersec).'%';
+                $r->print('<p class="LC_warning">'.
+                          &mt('The original uploaded file includes [_1] or more of records for students for which none of your roles have rights to modify grades, so files are unavailable for download.',$showpct).
+                          '</p>');
+                return;
+            }
+        }
+    }
     my $orig='/uploaded/'.$cdom.'/'.$cname.'/scantron_orig_'.$file;
     my $corrected='/uploaded/'.$cdom.'/'.$cname.'/scantron_corrected_'.$file;
     my $skipped='/uploaded/'.$cdom.'/'.$cname.'/scantron_skipped_'.$file;
@@ -9288,16 +9838,16 @@ sub checkscantron_results {
     my ($r,$symb) = @_;
     if (!$symb) {return '';}
     my $cid = $env{'request.course.id'};
-    my %lettdig = &letter_to_digits();
+    my %lettdig = &Apache::lonnet::letter_to_digits();
     my $numletts = scalar(keys(%lettdig));
     my $cnum = $env{'course.'.$cid.'.num'};
     my $cdom = $env{'course.'.$cid.'.domain'};
     my (undef, undef, $sequence) = &Apache::lonnet::decode_symb($env{'form.selectpage'});
     my %record;
     my %scantron_config =
-        &Apache::grades::get_scantron_config($env{'form.scantron_format'});
+        &Apache::lonnet::get_scantron_config($env{'form.scantron_format'});
     my $bubbles_per_row = &bubblesheet_bubbles_per_row(\%scantron_config);
-    my ($scanlines,$scan_data)=&Apache::grades::scantron_getfile();
+    my ($scanlines,$scan_data)=&scantron_getfile();
     my $classlist=&Apache::loncoursedata::get_classlist();
     my %idmap=&Apache::grades::username_to_idmap($classlist);
     my $navmap=Apache::lonnavmaps::navmap->new();
@@ -9619,23 +10169,6 @@ sub verify_scantron_grading {
     return ($counter,$record);
 }
 
-sub letter_to_digits {
-    my %lettdig = (
-                    A => 1,
-                    B => 2,
-                    C => 3,
-                    D => 4,
-                    E => 5,
-                    F => 6,
-                    G => 7,
-                    H => 8,
-                    I => 9,
-                    J => 0,
-                  );
-    return %lettdig;
-}
-
-
 #-------- end of section for handling grading scantron forms -------
 #
 #-------------------------------------------------------------------
@@ -9690,7 +10223,7 @@ sub grading_menu {
                     		icon => 'grade_students.png',
                     		linktitle => 'Grade current resource for a selection of students.'
                         }, 
-                        {       linktext => 'Grade ungraded submissions.',
+                        {       linktext => 'Grade ungraded submissions',
                                 url => $url1b,
                                 permission => 'F',
                                 icon => 'ungrade_sub.png',
@@ -9756,7 +10289,6 @@ sub grading_menu {
     return $Str;    
 }
 
-
 sub ungraded {
     my ($request)=@_;
     &submit_options($request);
@@ -9811,7 +10343,7 @@ sub submit_options_download {
         '<input type="hidden" name="symb"        value="'.&Apache::lonenc::check_encrypt($symb).'" />'."\n";
     $result.='
 <h2>
-  '.&mt('Select Students for Which to Download Submissions').'
+  '.&mt('Select Students for whom to Download Submissions').'
 </h2>'.&selectfield(1,$is_tool).'
                 <input type="hidden" name="command" value="downloadfileslink" /> 
               <input type="submit" value="'.&mt('Next').' &rarr;" />
@@ -9839,8 +10371,6 @@ sub submit_options {
 	      <input type="submit" value="'.&mt('Next').' &rarr;" />
             </div>
           </div>
-
-
   </form>';
     return $result;
 }
@@ -9920,7 +10450,7 @@ sub reset_perm {
 
 sub init_perm {
     &reset_perm();
-    foreach my $test_perm ('vgr','mgr','opa') {
+    foreach my $test_perm ('vgr','mgr','opa','usc') {
 
 	my $scope = $env{'request.course.id'};
 	if (!($perm{$test_perm}=&Apache::lonnet::allowed($test_perm,$scope))) {
@@ -10108,12 +10638,12 @@ ENDUPFORM
 <input type="text" name="givenanswer" size="50" />
 <input type="hidden" name="waschecked" value="$env{'form.gradingmechanism'}" />
 ENDGRADINGFORM
-         $result.='</td>'.&Apache::loncommon::end_data_table_row().
+    $result.='</td>'.&Apache::loncommon::end_data_table_row().
                      &Apache::loncommon::start_data_table_row().'<td>'.(<<ENDPERCFORM);
       <label>$pcorrect: <input type="text" name="pcorrect" size="4" value="$env{'form.pcorrect'}" onchange="sanitycheck()" /></label>
 <br /><label>$pincorrect: <input type="text" name="pincorrect" size="4" value="$env{'form.pincorrect'}" onchange="sanitycheck()" /></label>
 <br /><input type="button" onclick="javascript:checkUpload(this.form);" value="$upload" />
-</form>'
+</form>
 ENDPERCFORM
     $result.='</td>'.
              &Apache::loncommon::end_data_table_row().
@@ -10122,7 +10652,7 @@ ENDPERCFORM
 }
 
 sub process_clicker_file {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     if (!$symb) {return '';}
 
     my %Saveable_Parameters=&clicker_grading_parameters();
@@ -10194,6 +10724,22 @@ sub process_clicker_file {
                         '<span class="LC_filename">'.&HTML::Entities::encode($env{'form.upfile.filename'},'<>&"').'</span>'),1);
         return $result;
     }
+    my $mimetype;
+    if ($env{'form.upfiletype'} eq 'iclicker') {
+        my $mm = new File::MMagic;
+        $mimetype = $mm->checktype_contents($env{'form.upfile'});
+        unless (($mimetype eq 'text/plain') || ($mimetype eq 'text/html')) {
+            $result.= '<p>'.
+                &Apache::lonhtmlcommon::confirm_success(
+                    &mt('File format is neither csv (iclicker 6) nor xml (iclicker 7)'),1).'</p>';
+            return $result;
+        }
+    } elsif (($env{'form.upfiletype'} ne 'interwrite') && ($env{'form.upfiletype'} ne 'turning')) {
+        $result .= '<p>'.
+            &Apache::lonhtmlcommon::confirm_success(
+                &mt('Invalid clicker type: choose one of: i>clicker, Interwrite PRS, or Turning Technologies.'),1).'</p>';
+        return $result;
+    }
 
 # Were able to get all the info needed, now analyze the file
 
@@ -10220,12 +10766,14 @@ ENDHEADER
     my $errormsg='';
     my $number=0;
     if ($env{'form.upfiletype'} eq 'iclicker') {
-	($errormsg,$number)=&iclicker_eval(\@questiontitles,\%responses);
-    }
-    if ($env{'form.upfiletype'} eq 'interwrite') {
+        if ($mimetype eq 'text/plain') {
+            ($errormsg,$number)=&iclicker_eval(\@questiontitles,\%responses);
+        } elsif ($mimetype eq 'text/html') {
+            ($errormsg,$number)=&iclickerxml_eval(\@questiontitles,\%responses);
+        }
+    } elsif ($env{'form.upfiletype'} eq 'interwrite') {
         ($errormsg,$number)=&interwrite_eval(\@questiontitles,\%responses);
-    }
-    if ($env{'form.upfiletype'} eq 'turning') {
+    } elsif ($env{'form.upfiletype'} eq 'turning') {
         ($errormsg,$number)=&turning_eval(\@questiontitles,\%responses);
     }
     $result.='<br />'.&mt('Found [_1] question(s)',$number).'<br />'.
@@ -10278,7 +10826,7 @@ ENDHEADER
                    "\n".&mt("Username").": <input type='text' name='uname".$id."' />&nbsp;".
                    "\n".&mt("Domain").": ".
                    &Apache::loncommon::select_dom_form($env{'course.'.$env{'request.course.id'}.'.domain'},'udom'.$id).'&nbsp;'.
-                   &Apache::loncommon::selectstudent_link('clickeranalysis','uname'.$id,'udom'.$id,0,$id);
+                   &Apache::loncommon::selectstudent_link('clickeranalysis','uname'.$id,'udom'.$id,'',$id);
           $unknown_count++;
        }
     }
@@ -10333,6 +10881,49 @@ sub iclicker_eval {
     return ($errormsg,$number);
 }
 
+sub iclickerxml_eval {
+    my ($questiontitles,$responses)=@_;
+    my $number=0;
+    my $errormsg='';
+    my @state;
+    my %respbyid;
+    my $p = HTML::Parser->new
+    (
+        xml_mode => 1,
+        start_h =>
+            [sub {
+                 my ($tagname,$attr) = @_;
+                 push(@state,$tagname);
+                 if ("@state" eq "ssn p") {
+                     my $title = $attr->{qn};
+                     $title =~ s/(^\s+|\s+$)//g;
+                     $questiontitles->[$number]=$title;
+                 } elsif ("@state" eq "ssn p v") {
+                     my $id = $attr->{id};
+                     my $entry = $attr->{ans};
+                     $id=~s/^[\#0]+//;
+                     $entry =~s/[^a-zA-Z0-9\.\*\-\+]+//g;
+                     $respbyid{$id}[$number] = $entry;
+                 }
+            }, "tagname, attr"],
+         end_h =>
+               [sub {
+                   my ($tagname) = @_;
+                   if ("@state" eq "ssn p") {
+                       $number++;
+                   }
+                   pop(@state);
+                }, "tagname"],
+    );
+
+    $p->parse($env{'form.upfile'});
+    $p->eof;
+    foreach my $id (keys(%respbyid)) {
+        $responses->{$id}=join(',',@{$respbyid{$id}});
+    }
+    return ($errormsg,$number);
+}
+
 sub interwrite_eval {
     my ($questiontitles,$responses)=@_;
     my $number=0;
@@ -10391,7 +10982,7 @@ sub turning_eval {
 
 
 sub assign_clicker_grades {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     if (!$symb) {return '';}
 # See which part we are saving to
     my $res_error;
@@ -10402,11 +10993,11 @@ sub assign_clicker_grades {
 # FIXME: This should probably look for the first handgradeable part
     my $part=$$partlist[0];
 # Start screen output
-    my $result=&Apache::loncommon::start_data_table().
-             &Apache::loncommon::start_data_table_header_row().
-             '<th>'.&mt('Assigning grades based on clicker file').'</th>'.
-             &Apache::loncommon::end_data_table_header_row().
-             &Apache::loncommon::start_data_table_row().'<td>';
+    my $result = &Apache::loncommon::start_data_table().
+                 &Apache::loncommon::start_data_table_header_row().
+                 '<th>'.&mt('Assigning grades based on clicker file').'</th>'.
+                 &Apache::loncommon::end_data_table_header_row().
+                 &Apache::loncommon::start_data_table_row().'<td>';
 # Get correct result
 # FIXME: Possibly need delimiter other than ":"
     my @correct=();
@@ -10468,7 +11059,7 @@ sub assign_clicker_grades {
           for (my $i=0;$i<$number;$i++) {
              if  ($correct[$i] eq '-') {
                 $realnumber--;
-             } elsif (($answer[$i]) || ($answer[$i]=~/^[0\.]+$/))  {
+             } elsif (($answer[$i]) || ($answer[$i]=~/^[0\.]+$/)) {
                 if ($gradingmechanism eq 'attendance') {
                    $sum+=$pcorrect;
                 } elsif ($correct[$i] eq '*') {
@@ -10527,24 +11118,34 @@ sub navmap_errormsg {
 }
 
 sub startpage {
-    my ($r,$symb,$crumbs,$onlyfolderflag,$nodisplayflag,$stuvcurrent,$stuvdisp,$nomenu,$js) = @_;
+    my ($r,$symb,$crumbs,$onlyfolderflag,$nodisplayflag,$stuvcurrent,$stuvdisp,$nomenu,$js,$onload) = @_;
+    my %args;
+    if ($onload) {
+         my %loaditems = (
+                        'onload' => $onload,
+                      );
+         $args{'add_entries'} = \%loaditems;
+    }
     if ($nomenu) {
-        $r->print(&Apache::loncommon::start_page("Student's Version",$js,{'only_body' => '1'}));
+        $args{'only_body'} = 1; 
+        $r->print(&Apache::loncommon::start_page("Student's Version",$js,\%args));
     } else {
         unshift(@$crumbs,{href=>&href_symb_cmd($symb,'gradingmenu'),text=>"Grading"});
-        $r->print(&Apache::loncommon::start_page('Grading',$js,
-                                                 {'bread_crumbs' => $crumbs}));
-        &Apache::lonquickgrades::startGradeScreen($r,($env{'form.symb'}?'probgrading':'grading'));
+        $args{'bread_crumbs'} = $crumbs;
+        $r->print(&Apache::loncommon::start_page('Grading',$js,\%args));
+        if ($env{'request.course.id'}) {
+            &Apache::lonquickgrades::startGradeScreen($r,($env{'form.symb'}?'probgrading':'grading'));
+        }
     }
     unless ($nodisplayflag) {
-       $r->print(&Apache::lonhtmlcommon::resource_info_box($symb,$onlyfolderflag,$stuvcurrent,$stuvdisp));
+        $r->print(&Apache::lonhtmlcommon::resource_info_box($symb,$onlyfolderflag,$stuvcurrent,$stuvdisp));
     }
 }
 
 sub select_problem {
     my ($r)=@_;
     $r->print('<h3>'.&mt('Select the problem or one of the problems you want to grade').'</h3><form action="/adm/grades">');
-    $r->print(&Apache::lonstathelpers::problem_selector('.',undef,1,undef,undef,undef,undef,1));
+    $r->print(&Apache::lonstathelpers::problem_selector('.',undef,1,undef,undef,1,1));
     $r->print('<input type="hidden" name="command" value="gradingmenu" />');
     $r->print('<input type="submit" value="'.&mt('Next').' &rarr;" /></form>');
 }
@@ -10717,7 +11318,8 @@ sub handler {
             &startpage($request,$symb,[{href=>'', text=>'Upload Scores'}],1,1);
 	    $request->print(&csvuploadassign($request,$symb));
 	} elsif ($command eq 'scantron_selectphase' && $perm{'mgr'}) {
-            &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
+            &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1,
+                       undef,undef,undef,undef,'toggleScantab(document.rules);');
 	    $request->print(&scantron_selectphase($request,undef,$symb));
  	} elsif ($command eq 'scantron_warning' && $perm{'mgr'}) {
             &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
@@ -10729,19 +11331,21 @@ sub handler {
             &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
 	    $request->print(&scantron_process_students($request,$symb));
  	} elsif ($command eq 'scantronupload' && 
- 		 (&Apache::lonnet::allowed('usc',$env{'request.role.domain'})||
-		  &Apache::lonnet::allowed('usc',$env{'request.course.id'}))) {
-            &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
+ 		 (&Apache::lonnet::allowed('usc',$env{'request.role.domain'}) || $perm{'usc'})) {
+            &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1,
+                       undef,undef,undef,undef,'toggleScantab(document.rules);');
  	    $request->print(&scantron_upload_scantron_data($request,$symb)); 
  	} elsif ($command eq 'scantronupload_save' &&
- 		 (&Apache::lonnet::allowed('usc',$env{'request.role.domain'})||
-		  &Apache::lonnet::allowed('usc',$env{'request.course.id'}))) {
+ 		 (&Apache::lonnet::allowed('usc',$env{'request.role.domain'}) || $perm{'usc'})) {
             &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
  	    $request->print(&scantron_upload_scantron_data_save($request,$symb));
- 	} elsif ($command eq 'scantron_download' &&
-		 &Apache::lonnet::allowed('usc',$env{'request.course.id'})) {
+ 	} elsif ($command eq 'scantron_download' && ($perm{'usc'} || $perm{'mgr'})) {
             &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
  	    $request->print(&scantron_download_scantron_data($request,$symb));
+        } elsif ($command eq 'scantronupload_delete' &&
+                 (&Apache::lonnet::allowed('usc',$env{'request.role.domain'}) || $perm{'usc'})) {
+            &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
+            &scantron_upload_delete($request,$symb);
         } elsif ($command eq 'checksubmissions' && $perm{'vgr'}) {
             &startpage($request,$symb,[{href=>'', text=>'Grade/Manage/Review Bubblesheets'}],1,1);
             $request->print(&checkscantron_results($request,$symb));
@@ -10751,7 +11355,7 @@ sub handler {
          } elsif ($command eq 'downloadfileslink' && $perm{'vgr'}) {
             &startpage($request,$symb,
    [{href=>&href_symb_cmd($symb,'downloadfilesselect'), text=>'Select which submissions to download'},
-    {href=>'', text=>'Download submissions'}]);
+    {href=>'', text=>'Download submitted files'}]);
             &submit_download_link($request,$symb);
 	} elsif ($command) {
             &startpage($request,$symb,[{href=>'', text=>'Access denied'}]);
@@ -10763,7 +11367,7 @@ sub handler {
     }
     if ($env{'form.inhibitmenu'}) {
         $request->print(&Apache::loncommon::end_page());
-    } else {
+    } elsif ($env{'request.course.id'}) {
         &Apache::lonquickgrades::endGradeScreen($request);
     }
     &reset_caches();
@@ -10914,7 +11518,7 @@ Side Effects: None.
     $r           - Apache request object
     $i           - number of the current scanline
     $scan_record - hash ref as returned from &scantron_parse_scanline()
-    $scan_config - hash ref as returned from &get_scantron_config()
+    $scan_config - hash ref as returned from &Apache::lonnet::get_scantron_config()
     $line        - full contents of the current scanline
     $error       - error condition, valid values are
                    'incorrectCODE', 'duplicateCODE',
@@ -11001,7 +11605,12 @@ Side Effects: None.
 =item  scantron_upload_scantron_data_save() : 
 
    Adds a provided bubble information data file to the course if user
-   has the correct privileges to do so. 
+   has the correct privileges to do so.
+
+= item scantron_upload_delete() :
+
+   Deletes a previously uploaded bubble information data file, if user
+   was the one who uploaded the file, and has the privileges to do so.
 
 =item  valid_file() :