--- loncom/homework/grades.pm	2004/05/10 23:18:27	1.199
+++ loncom/homework/grades.pm	2004/05/14 21:30:27	1.202
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.199 2004/05/10 23:18:27 albertel Exp $
+# $Id: grades.pm,v 1.202 2004/05/14 21:30:27 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -3420,18 +3420,25 @@ sub getSequenceDropDown {
     return $result;
 }
 
-sub scantron_uploads {
-    if (!-e $Apache::lonnet::perlvar{'lonScansDir'}) { return ''};
-    my $result=	'<select name="scantron_selectfile">';
+sub scantron_filenames {
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my @files=&Apache::lonnet::dirlist('userfiles',$cdom,$cname,
 				    &Apache::loncommon::propath($cdom,$cname));
-    $result.="<option></option>";
-    foreach my $filename (@files) {
+    my @possiblenames;
+    foreach my $filename (sort(@files)) {
 	($filename)=split(/&/,$filename);
 	if ($filename!~/^scantron_orig_/) { next ; }
 	$filename=~s/^scantron_orig_//;
+	push(@possiblenames,$filename);
+    }
+    return @possiblenames;
+}
+
+sub scantron_uploads {
+    my $result=	'<select name="scantron_selectfile">';
+    $result.="<option></option>";
+    foreach my $filename (sort(&scantron_filenames())) {
 	$result.="<option>$filename</option>\n";
     }
     $result.="</select>";
@@ -3457,7 +3464,7 @@ sub scantron_CODElist {
     my $cnum = $ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my @names=&Apache::lonnet::getkeys('CODEs',$cdom,$cnum);
     my $namechoice='<option></option>';
-    foreach my $name (@names) {
+    foreach my $name (sort(@names)) {
 	if ($name =~ /^error: 2 /) { next; }
 	$namechoice.='<option value="'.$name.'">'.$name.'</option>';
     }
@@ -3522,8 +3529,8 @@ sub scantron_selectphase {
           <tr bgcolor="#ffffe6">
 	    <td> Options: </td>
             <td>
-                <input type="checkbox" name="scantron_options_redo" value="redo_skipped"/> Do only skipped records <br />
-                <input type="checkbox" name="scantron_options_ignore" value="ignore_corrections"/> Remove any exisiting corrections
+                <input type="checkbox" name="scantron_options_redo" value="redo_skipped"/> Do only previously skipped records <br />
+                <input type="checkbox" name="scantron_options_ignore" value="ignore_corrections"/> Remove all exisiting corrections
 	    </td>
           </tr>
           <tr bgcolor="#ffffe6">
@@ -3602,18 +3609,8 @@ SCANTRONFORM
               <td> Filename of scoring office file: </td><td> $file_selector </td>
             </tr>
             <tr bgcolor="#ffffe6">
-	      <td>
-                Records to download
-              </td>
-              <td>
-                  <input type="radio" name="scantron_options" value="download_skipped"/> Skipped Records <br />
-                  <input type="radio" name="scantron_options" value="download_corrected"/> Corrected Records <br />
-                  <input checked="on" type="radio" name="scantron_options" value="dowload_orig"/> Original Records
-              </td>
-            </tr>
-            <tr bgcolor="#ffffe6">
               <td colspan="2">
-                <input type="submit" value="Validate Scantron Records" />
+                <input type="submit" value="Show List of Files" />
               </td>
             </tr>
           </table>
@@ -3833,7 +3830,7 @@ sub scantron_process_corrections {
     my ($scanlines,$scan_data)=&scantron_getfile();
     my $classlist=&Apache::loncoursedata::get_classlist();
     my $which=$ENV{'form.scantron_line'};
-    my $line=&scantron_get_line($scanlines,$which);
+    my $line=&scantron_get_line($scanlines,$scan_data,$which);
     my ($skip,$err,$errmsg);
     if ($ENV{'form.scantron_skip_record'}) {
 	$skip=1;
@@ -3879,24 +3876,79 @@ sub scantron_process_corrections {
     if ($err) {
 	$r->print("Unable to accept last correction, an error occurred :$errmsg:");
     } else {
-	&scantron_put_line($scanlines,$which,$line,$skip);
+	&scantron_put_line($scanlines,$scan_data,$which,$line,$skip);
 	&scantron_putfile($scanlines,$scan_data);
     }
 }
 
+sub reset_skipping_status {
+    my ($scanlines,$scan_data)=&scantron_getfile();
+    &scan_data($scan_data,'remember_skipping',undef,1);
+    &scantron_putfile(undef,$scan_data);
+}
+
+sub allow_skipping {
+    my ($scan_data,$i)=@_;
+    my %remembered=split(':',&scan_data($scan_data,'remember_skipping'));
+    delete($remembered{$i});
+    &scan_data($scan_data,'remember_skipping',join(':',%remembered));
+}
+
+sub should_be_skipped {
+    my ($scan_data,$i)=@_;
+    if ($ENV{'form.scantron_options_redo'} !~ /^redo_/) {
+	# not redoing old skips
+	return 0;
+    }
+    my %remembered=split(':',&scan_data($scan_data,'remember_skipping'));
+    if (exists($remembered{$i})) { return 0; }
+    return 1;
+}
+
+sub remember_current_skipped {
+    my ($scanlines,$scan_data)=&scantron_getfile();
+    my %to_remember;
+    for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+	if ($scanlines->{'skipped'}[$i]) {
+	    $to_remember{$i}=1;
+	}
+    }
+    &Apache::lonnet::logthis('remembering '.join(':',%to_remember));
+    &scan_data($scan_data,'remember_skipping',join(':',%to_remember));
+    &scantron_putfile(undef,$scan_data);
+}
+
+sub check_for_error {
+    my ($r,$result)=@_;
+    if ($result ne 'ok' && $result ne 'not_found' ) {
+	$r->print("An error occured ($result) when trying to Remove the existing corrections.");
+    }
+}
 
 sub scantron_validate_file {
     my ($r) = @_;
     my ($symb,$url)=&get_symb_and_url($r);
     if (!$symb) {return '';}
     my $default_form_data=&defaultFormData($symb,$url);
+    
+    # do the detection of only doing skipped records first befroe we delete
+    # them  when doing the corrections reset
+    if ($ENV{'form.scantron_options_redo'} ne 'redo_skipped_ready') {
+	&reset_skipping_status();
+    }
+    if ($ENV{'form.scantron_options_redo'} eq 'redo_skipped') {
+	&remember_current_skipped();
+	&scantron_remove_file('skipped');
+	$ENV{'form.scantron_options_redo'}='redo_skipped_ready';
+    }
+
     if ($ENV{'form.scantron_options_ignore'} eq 'ignore_corrections') {
-	my $result=&scantron_remove('corrected');
-	if ($result ne 'ok' && $result ne 'not_found' ) {
-	    $r->print("An error occured ($result) when trying to Remove the existing corrections.");
-	}
+	&check_for_error($r,&scantron_remove_file('corrected'));
+	&check_for_error($r,&scantron_remove_file('skipped'));
+	&check_for_error($r,&scantron_remove_scan_data());
 	$ENV{'form.scantron_options_ignore'}='done';
     }
+
     if ($ENV{'form.scantron_corrections'}) {
 	&scantron_process_corrections($r);
     }
@@ -3956,35 +4008,44 @@ SCANTRONFORM
     return '';
 }
 
-sub scantron_remove {
+sub scantron_remove_file {
     my ($which)=@_;
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $file='scantron_';
-    if ($which eq 'corrected') {
-	$file.='corrected_';
+    if ($which eq 'corrected' || $which eq 'skipped') {
+	$file.=$which.'_';
     } else {
 	return 'refused';
     }
     $file.=$ENV{'form.scantron_selectfile'};
-    my $result=&Apache::lonnet::removeuserfile($cname,$cdom,$file);
+    return &Apache::lonnet::removeuserfile($cname,$cdom,$file);
+}
+
+sub scantron_remove_scan_data {
+    my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
+    my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my @keys=&Apache::lonnet::getkeys('nohist_scantrondata',$cdom,$cname);
     my @todelete;
     my $filename=$ENV{'form.scantron_selectfile'};
     foreach my $key (@keys) {
 	if ($key=~/^\Q$filename\E_/) {
+	    if ($ENV{'form.scantron_options_redo'} eq 'redo_skipped_ready' &&
+		$key=~/remember_skipping/) {
+		next;
+	    }
 	    push(@todelete,$key);
 	}
     }
+    my $result;
     if (@todelete) {
-	&Apache::lonnet::del('nohist_scantrondata',\@todelete,$cdom,$cname);
+	$result=&Apache::lonnet::del('nohist_scantrondata',\@todelete,$cdom,$cname);
     }
     return $result;
 }
 
 sub scantron_getfile {
-    #FIXME really would prefer a scantron directory but tokenwrapper
-    # doesn't allow access to subdirs of userfiles
+    #FIXME really would prefer a scantron directory
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $lines;
@@ -4027,34 +4088,48 @@ sub lonnet_putfile {
 
 sub scantron_putfile {
     my ($scanlines,$scan_data) = @_;
-    #FIXME really would prefer a scantron directory but tokenwrapper
-    # doesn't allow access to subdirs of userfiles
+    #FIXME really would prefer a scantron directory
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
-    my $prefix='scantron_';
+    if ($scanlines) {
+	my $prefix='scantron_';
 # no need to update orig, shouldn't change
 #   &lonnet_putfile(join("\n",@{$scanlines->{'orig'}}),$prefix.'orig_'.
 #		    $ENV{'form.scantron_selectfile'});
-    &lonnet_putfile(join("\n",@{$scanlines->{'corrected'}}),
-		    $prefix.'corrected_'.
-		    $ENV{'form.scantron_selectfile'});
-    &lonnet_putfile(join("\n",@{$scanlines->{'skipped'}}),
-		    $prefix.'skipped_'.
-		    $ENV{'form.scantron_selectfile'});
+	&lonnet_putfile(join("\n",@{$scanlines->{'corrected'}}),
+			$prefix.'corrected_'.
+			$ENV{'form.scantron_selectfile'});
+	&lonnet_putfile(join("\n",@{$scanlines->{'skipped'}}),
+			$prefix.'skipped_'.
+			$ENV{'form.scantron_selectfile'});
+    }
     &Apache::lonnet::put('nohist_scantrondata',$scan_data,$cdom,$cname);
 }
 
 sub scantron_get_line {
-    my ($scanlines,$i)=@_;
-    if ($scanlines->{'skipped'}[$i]) {return undef;}
+    my ($scanlines,$scan_data,$i)=@_;
+    if (&should_be_skipped($scan_data,$i)) { return undef; }
+    if ($scanlines->{'skipped'}[$i]) { return undef; }
     if ($scanlines->{'corrected'}[$i]) {return $scanlines->{'corrected'}[$i];}
     return $scanlines->{'orig'}[$i]; 
 }
 
+sub get_todo_count {
+    my ($scanlines,$scan_data)=@_;
+    my $count=0;
+    for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
+	if ($line=~/^[\s\cz]*$/) { next; }
+	$count++;
+    }
+    return $count;
+}
+
 sub scantron_put_line {
-    my ($scanlines,$i,$newline,$skip)=@_;
+    my ($scanlines,$scan_data,$i,$newline,$skip)=@_;
     if ($skip) {
 	$scanlines->{'skipped'}[$i]=$newline;
+	&allow_skipping($scan_data,$i);
 	return;
     }
     $scanlines->{'corrected'}[$i]=$newline;
@@ -4073,7 +4148,7 @@ sub scantron_validate_ID {
 
     my %found=('ids'=>{},'usernames'=>{});
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4308,7 +4383,7 @@ sub scantron_validate_CODE {
 
     my ($scanlines,$scan_data)=&scantron_getfile();
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4342,7 +4417,7 @@ sub scantron_validate_doublebubble {
     my %scantron_config=&get_scantron_config($ENV{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4392,7 +4467,7 @@ sub scantron_validate_missingbubbles {
     my $max_bubble=&scantron_get_maxbubble();
     if (!$max_bubble) { $max_bubble=2**31; }
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4437,19 +4512,25 @@ SCANTRONFORM
     my @delayqueue;
     my %completedstudents;
     
+    my $count=&get_todo_count($scanlines,$scan_data);
     my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r,'Scantron Status',
- 				    'Scantron Progress',$scanlines->{'count'},
+ 				    'Scantron Progress',$count,
 				    'inline',undef,'scantronupload');
     &Apache::lonhtmlcommon::Update_PrgWin($r,\%prog_state,
 					  'Processing first student');
     my $start=&Time::HiRes::time();
     my $i=-1;
-    my ($uname,$udom);
+    my ($uname,$udom,$started);
     while ($i<$scanlines->{'count'}) {
  	($uname,$udom)=('','');
  	$i++;
- 	my $line=&scantron_get_line($scanlines,$i);
+ 	my $line=&scantron_get_line($scanlines,$scan_data,$i);
  	if ($line=~/^[\s\cz]*$/) { next; }
+	if ($started) {
+	    &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+						     'last student');
+	}
+	$started=1;
  	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
  						 $scan_data);
  	unless ($uname=&scantron_find_student($scan_record,$scan_data,
@@ -4487,15 +4568,13 @@ SCANTRONFORM
     } continue {
 	&Apache::lonnet::delenv('form.counter');
 	&Apache::lonnet::delenv('scantron\.');
-	&Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
-						 'last student');
     }
     &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
 #    my $lasttime = &Time::HiRes::time()-$start;
 #    $r->print("<p>took $lasttime</p>");
 
     $navmap->untieHashes();
-    $r->print("</form><p>Done</p>");
+    $r->print("</form>");
     $r->print(&show_grading_menu_form($symb,$url));
     return '';
 }
@@ -4591,6 +4670,50 @@ sub scantron_upload_scantron_data_save {
     return '';
 }
 
+sub valid_file {
+    my ($requested_file)=@_;
+    foreach my $filename (sort(&scantron_filenames())) {
+	&Apache::lonnet::logthis("$requested_file  $filename");
+	if ($requested_file eq $filename) { return 1; }
+    }
+    return 0;
+}
+
+sub scantron_download_scantron_data {
+    my ($r)=@_;
+    my $default_form_data=&defaultFormData(&get_symb_and_url($r,1));
+    my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
+    my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
+    my $file=$ENV{'form.scantron_selectfile'};
+    if (! &valid_file($file)) {
+	$r->print(<<ERROR);
+	<p>
+	    The requested file name was invalid.
+        </p>
+ERROR
+	$r->print(&show_grading_menu_form(&get_symb_and_url($r,1)));
+	return;
+    }
+    my $orig='/uploaded/'.$cdom.'/'.$cname.'/scantron_orig_'.$file;
+    my $corrected='/uploaded/'.$cdom.'/'.$cname.'/scantron_corrected_'.$file;
+    my $skipped='/uploaded/'.$cdom.'/'.$cname.'/scantron_skipped_'.$file;
+    &Apache::lonnet::allowuploaded('/adm/grades',$orig);
+    &Apache::lonnet::allowuploaded('/adm/grades',$corrected);
+    &Apache::lonnet::allowuploaded('/adm/grades',$skipped);
+    $r->print(<<DOWNLOAD);
+    <p>
+	<a href="$orig">Original</a> file as uploaded by the scantron office.
+    </p>
+    <p>
+	<a href="$corrected">Corrections</a>, a file of corrected records that were used in grading.
+    </p>
+    <p>
+	<a href="$skipped">Skipped</a>, a file of records that were skipped.
+    </p>
+DOWNLOAD
+    $r->print(&show_grading_menu_form(&get_symb_and_url($r,1)));
+    return '';
+}
 
 #-------- end of section for handling grading scantron forms -------
 #
@@ -4882,7 +5005,7 @@ sub handler {
  		 (&Apache::lonnet::allowed('usc',$ENV{'request.role.domain'})||
 		  &Apache::lonnet::allowed('usc',$ENV{'request.course.id'}))) {
  	    $request->print(&scantron_upload_scantron_data_save($request));
- 	} elsif ($command eq 'scantrondownload' &&
+ 	} elsif ($command eq 'scantron_download' &&
 		 &Apache::lonnet::allowed('usc',$ENV{'request.course.id'})) {
  	    $request->print(&scantron_download_scantron_data($request));
 	} elsif ($command) {