--- loncom/homework/grades.pm	2019/01/28 21:36:53	1.756
+++ loncom/homework/grades.pm	2020/05/08 13:49:02	1.767
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.756 2019/01/28 21:36:53 raeburn Exp $
+# $Id: grades.pm,v 1.767 2020/05/08 13:49:02 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -48,6 +48,8 @@ use Apache::lonquickgrades;
 use Apache::bridgetask();
 use Apache::lontexconvert();
 use String::Similarity;
+use HTML::Parser();
+use File::MMagic;
 use LONCAPA;
 
 use POSIX qw(floor);
@@ -313,7 +315,7 @@ sub reset_caches {
                     $add_to_form = { 'code_for_randomlist' => $scancode,};
                 }
             }
-            my $analyze = 
+            my $analyze =
                 &get_analyze($symb,$uname,$udom,undef,$add_to_form,
                              undef,undef,undef,$bubbles_per_row);
             if (ref($analyze) eq 'HASH') {
@@ -343,7 +345,7 @@ sub cleanRecord {
     if ($response =~ /^(option|rank)$/) {
 	my %answer=&Apache::lonnet::str2hash($answer);
         my @answer = %answer;
-        %answer = map {&HTML::Entities::encode($_, '"<>&')}  @answer;
+        %answer = map {&HTML::Entities::encode($_, '"<>&')} @answer;
 	my %grading=&Apache::lonnet::str2hash($record->{$version."resource.$partid.$respid.submissiongrading"});
 	my ($toprow,$bottomrow);
 	foreach my $foil (@$order) {
@@ -361,7 +363,7 @@ sub cleanRecord {
     } elsif ($response eq 'match') {
 	my %answer=&Apache::lonnet::str2hash($answer);
         my @answer = %answer;
-        %answer = map {&HTML::Entities::encode($_, '"<>&')}  @answer;
+        %answer = map {&HTML::Entities::encode($_, '"<>&')} @answer;
 	my %grading=&Apache::lonnet::str2hash($record->{$version."resource.$partid.$respid.submissiongrading"});
 	my @items=&Apache::lonnet::str2array($record->{$version."resource.$partid.$respid.submissionitems"});
 	my ($toprow,$middlerow,$bottomrow);
@@ -420,7 +422,6 @@ sub cleanRecord {
 	}
         $answer = &Apache::lontexconvert::msgtexconverted($answer);
 	return '<br /><br /><blockquote><tt>'.&keywords_highlight($answer).'</tt></blockquote>';
-
     } elsif ( $response eq 'organic') {
         my $result=&mt('Smile representation: [_1]',
                            '"<tt>'.&HTML::Entities::encode($answer, '"<>&').'</tt>"');
@@ -653,7 +654,7 @@ sub canmodify {
 		#can modify the requested section
 		return 1;
 	    } else {
-		# can't modify the request section
+		# can't modify the requested section
 		return 0;
 	    }
 	}
@@ -666,19 +667,19 @@ sub canview {
     my ($sec)=@_;
     if ($perm{'vgr'}) {
 	if (!defined($perm{'vgr_section'})) {
-	    # can modify whole class
+	    # can view whole class
 	    return 1;
 	} else {
 	    if ($sec eq $perm{'vgr_section'}) {
-		#can modify the requested section
+		#can view the requested section
 		return 1;
 	    } else {
-		# can't modify the request section
+		# can't view the requested section
 		return 0;
 	    }
 	}
     }
-    #can't modify
+    #can't view
     return 0;
 }
 
@@ -819,14 +820,14 @@ sub initialverifyreceipt {
 
 #--- Check whether a receipt number is valid.---
 sub verifyreceipt {
-    my ($request,$symb)  = @_;
+    my ($request,$symb) = @_;
 
     my $courseid = $env{'request.course.id'};
     my $receipt  = &Apache::lonnet::recprefix($courseid).'-'.
 	$env{'form.receipt'};
     $receipt     =~ s/[^\-\d]//g;
 
-    my $title.=
+    my $title =
 	'<h3><span class="LC_info">'.
 	&mt('Verifying Receipt Number [_1]',$receipt).
 	'</span></h3>'."\n";
@@ -915,7 +916,7 @@ sub listStudents {
     my $getsec    = $env{'form.section'} eq '' ? 'all' : $env{'form.section'};
     my $getgroup  = $env{'form.group'} eq '' ? 'all' : $env{'form.group'};
     unless ($submitonly) {
-       $submitonly= $env{'form.submitonly'} eq '' ? 'all' : $env{'form.submitonly'};
+        $submitonly = $env{'form.submitonly'} eq '' ? 'all' : $env{'form.submitonly'};
     }
 
     my $result='';
@@ -1212,8 +1213,8 @@ LISTJAVASCRIPT
 #---- Called from the listStudents routine
 
 sub check_script {
-    my ($form, $type)=@_;
-    my $chkallscript= &Apache::lonhtmlcommon::scripttag('
+    my ($form,$type) = @_;
+    my $chkallscript = &Apache::lonhtmlcommon::scripttag('
     function checkall() {
         for (i=0; i<document.forms.'.$form.'.elements.length; i++) {
             ele = document.forms.'.$form.'.elements[i];
@@ -1258,7 +1259,7 @@ sub check_buttons {
 
 #     Displays the submissions for one student or a group of students
 sub processGroup {
-    my ($request,$symb)  = @_;
+    my ($request,$symb) = @_;
     my $ctr        = 0;
     my @stuchecked = &Apache::loncommon::get_env_multiple('form.stuinfo');
     my $total      = scalar(@stuchecked)-1;
@@ -2024,8 +2025,8 @@ sub files_exist {
 sub download_all_link {
     my ($r,$symb) = @_;
     unless (&files_exist($r, $symb)) {
-       $r->print(&mt('There are currently no submitted documents.'));
-       return;
+        $r->print(&mt('There are currently no submitted documents.'));
+        return;
     }
     my $all_students = 
 	join("\n", &Apache::loncommon::get_env_multiple('form.stuinfo'));
@@ -4736,7 +4737,7 @@ sub get_fields {
 }
 
 sub csvuploadassign {
-    my ($request,$symb)= @_;
+    my ($request,$symb) = @_;
     if (!$symb) {return '';}
     my $error_msg = '';
     my $datatoken = &Apache::loncommon::valid_datatoken($env{'form.datatoken'});
@@ -4852,7 +4853,7 @@ sub csvuploadassign {
 		$grades{$store_key}=$entries{$fields{$dest}};
 	    }
 	}
-	if (! %grades) { 
+	if (! %grades) {
            push(@skipped,&mt("[_1]: no data to save","$username:$domain")); 
         } else {
 	   $grades{"resource.regrader"}="$env{'user.name'}:$env{'user.domain'}";
@@ -4923,6 +4924,7 @@ LISTJAVASCRIPT
     my $cdom      = $env{"course.$env{'request.course.id'}.domain"};
     my $cnum      = $env{"course.$env{'request.course.id'}.num"};
     my $getsec    = $env{'form.section'} eq '' ? 'all' : $env{'form.section'};
+    my $getgroup  = $env{'form.group'} eq '' ? 'all' : $env{'form.group'};
 
     my $result='<h3><span class="LC_info">&nbsp;'.
 	&mt('Manual Grading by Page or Sequence').'</span></h3>';
@@ -5012,7 +5014,7 @@ LISTJAVASCRIPT
 	'<th>'.&nameUserString('header').'</th>'.
 	&Apache::loncommon::end_data_table_header_row();
  
-    my (undef,undef,$fullname) = &getclasslist($getsec,'1');
+    my (undef,undef,$fullname) = &getclasslist($getsec,'1',$getgroup);
     my $ptr = 1;
     foreach my $student (sort 
 			 {
@@ -5652,7 +5654,7 @@ the homework problem.
 
 sub defaultFormData {
     my ($symb)=@_;
-    return '<input type="hidden" name="symb"    value="'.&Apache::lonenc::check_encrypt($symb).'" />';
+    return '<input type="hidden" name="symb" value="'.&Apache::lonenc::check_encrypt($symb).'" />';
 }
 
 
@@ -5902,7 +5904,6 @@ sub scantron_selectphase {
 
         $r->print('
     <br />');
-    my $default_form_data=&defaultFormData($symb);
     my $cdom= $env{'course.'.$env{'request.course.id'}.'.domain'};
     my $cnum= $env{'course.'.$env{'request.course.id'}.'.num'};
     my $alertmsg = &mt('Please use the browse button to select a file from your local directory.');
@@ -7825,7 +7826,7 @@ sub verify_bubbles_checked {
     my $ansnumstr = join('","',@ansnums);
     my $warning = &mt("A bubble or 'No bubble' selection has not been made for one or more lines.");
     &js_escape(\$warning);
-    my $output = &Apache::lonhtmlcommon::scripttag((<<ENDSCRIPT));
+    my $output = &Apache::lonhtmlcommon::scripttag(<<ENDSCRIPT);
 function verify_bubble_radio(form) {
     var ansnumArray = new Array ("$ansnumstr");
     var need_bubble_count = 0;
@@ -9009,7 +9010,7 @@ sub grade_student_bubbles {
 }
 
 sub scantron_upload_scantron_data {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     my $dom = $env{'request.role.domain'};
     my ($formatoptions,$formattitle,$formatjs) = &scantron_upload_dataformat($dom);
     my $domdesc = &Apache::lonnet::domain($dom,'description');
@@ -9111,7 +9112,9 @@ END
                             }
                             if ($count > 1) {
                                 $formatextra = '<div style="display:none" id="bubbletype">'.
-                                               &scantron_scantab().'</div>';
+                                               '<span class="LC_nobreak">'.
+                                               &mt('Bubblesheet type:').'&nbsp;'.
+                                               &scantron_scantab().'</span></div>';
                                 $onclick = ' onclick="toggleScantab(this.form);"';
                                 $formatjs = <<"END";
 function toggleScantab(form) {
@@ -9126,7 +9129,7 @@ function toggleScantab(form) {
                     if (chosen == 'dat') {
                         document.getElementById(divid).style.display = 'none';
                     } else if (chosen == 'csv') {
-                        document.getElementById(divid).style.display = 'inline-block';
+                        document.getElementById(divid).style.display = 'block';
                     }
                 }
             }
@@ -9152,7 +9155,7 @@ END
             } elsif (keys(%{$domconfig{'scantron'}{'config'}}) == 1) {
                 if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
                     if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}})) {
-                        $formattitle = &mt('Format of bubblesheet data file:');
+                        $formattitle = &mt('Bubblesheet type');
                         $formatoptions = &scantron_scantab();
                     }
                 }
@@ -9163,7 +9166,7 @@ END
 }
 
 sub scantron_upload_scantron_data_save {
-    my($r,$symb)=@_;
+    my ($r,$symb) = @_;
     my $doanotherupload=
 	'<br /><form action="/adm/grades" method="post">'."\n".
 	'<input type="hidden" name="command" value="scantronupload" />'."\n".
@@ -9364,7 +9367,7 @@ sub valid_file {
 }
 
 sub scantron_download_scantron_data {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     my $default_form_data=&defaultFormData($symb);
     my $cname=$env{'course.'.$env{'request.course.id'}.'.num'};
     my $cdom=$env{'course.'.$env{'request.course.id'}.'.domain'};
@@ -9790,7 +9793,7 @@ sub grading_menu {
                     		icon => 'grade_students.png',
                     		linktitle => 'Grade current resource for a selection of students.'
                         }, 
-                        {       linktext => 'Grade ungraded submissions.',
+                        {       linktext => 'Grade ungraded submissions',
                                 url => $url1b,
                                 permission => 'F',
                                 icon => 'ungrade_sub.png',
@@ -9939,8 +9942,6 @@ sub submit_options {
 	      <input type="submit" value="'.&mt('Next').' &rarr;" />
             </div>
           </div>
-
-
   </form>';
     return $result;
 }
@@ -10208,12 +10209,12 @@ ENDUPFORM
 <input type="text" name="givenanswer" size="50" />
 <input type="hidden" name="waschecked" value="$env{'form.gradingmechanism'}" />
 ENDGRADINGFORM
-         $result.='</td>'.&Apache::loncommon::end_data_table_row().
+    $result.='</td>'.&Apache::loncommon::end_data_table_row().
                      &Apache::loncommon::start_data_table_row().'<td>'.(<<ENDPERCFORM);
       <label>$pcorrect: <input type="text" name="pcorrect" size="4" value="$env{'form.pcorrect'}" onchange="sanitycheck()" /></label>
 <br /><label>$pincorrect: <input type="text" name="pincorrect" size="4" value="$env{'form.pincorrect'}" onchange="sanitycheck()" /></label>
 <br /><input type="button" onclick="javascript:checkUpload(this.form);" value="$upload" />
-</form>'
+</form>
 ENDPERCFORM
     $result.='</td>'.
              &Apache::loncommon::end_data_table_row().
@@ -10222,7 +10223,7 @@ ENDPERCFORM
 }
 
 sub process_clicker_file {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     if (!$symb) {return '';}
 
     my %Saveable_Parameters=&clicker_grading_parameters();
@@ -10294,6 +10295,22 @@ sub process_clicker_file {
                         '<span class="LC_filename">'.&HTML::Entities::encode($env{'form.upfile.filename'},'<>&"').'</span>'),1);
         return $result;
     }
+    my $mimetype;
+    if ($env{'form.upfiletype'} eq 'iclicker') {
+        my $mm = new File::MMagic;
+        $mimetype = $mm->checktype_contents($env{'form.upfile'});
+        unless (($mimetype eq 'text/plain') || ($mimetype eq 'text/html')) {
+            $result.= '<p>'.
+                &Apache::lonhtmlcommon::confirm_success(
+                    &mt('File format is neither csv (iclicker 6) nor xml (iclicker 7)'),1).'</p>';
+            return $result;
+        }
+    } elsif (($env{'form.upfiletype'} ne 'interwrite') && ($env{'form.upfiletype'} ne 'turning')) {
+        $result .= '<p>'.
+            &Apache::lonhtmlcommon::confirm_success(
+                &mt('Invalid clicker type: choose one of: i>clicker, Interwrite PRS, or Turning Technologies.'),1).'</p>';
+        return $result;
+    }
 
 # Were able to get all the info needed, now analyze the file
 
@@ -10320,12 +10337,14 @@ ENDHEADER
     my $errormsg='';
     my $number=0;
     if ($env{'form.upfiletype'} eq 'iclicker') {
-	($errormsg,$number)=&iclicker_eval(\@questiontitles,\%responses);
-    }
-    if ($env{'form.upfiletype'} eq 'interwrite') {
+        if ($mimetype eq 'text/plain') {
+            ($errormsg,$number)=&iclicker_eval(\@questiontitles,\%responses);
+        } elsif ($mimetype eq 'text/html') {
+            ($errormsg,$number)=&iclickerxml_eval(\@questiontitles,\%responses);
+        }
+    } elsif ($env{'form.upfiletype'} eq 'interwrite') {
         ($errormsg,$number)=&interwrite_eval(\@questiontitles,\%responses);
-    }
-    if ($env{'form.upfiletype'} eq 'turning') {
+    } elsif ($env{'form.upfiletype'} eq 'turning') {
         ($errormsg,$number)=&turning_eval(\@questiontitles,\%responses);
     }
     $result.='<br />'.&mt('Found [_1] question(s)',$number).'<br />'.
@@ -10378,7 +10397,7 @@ ENDHEADER
                    "\n".&mt("Username").": <input type='text' name='uname".$id."' />&nbsp;".
                    "\n".&mt("Domain").": ".
                    &Apache::loncommon::select_dom_form($env{'course.'.$env{'request.course.id'}.'.domain'},'udom'.$id).'&nbsp;'.
-                   &Apache::loncommon::selectstudent_link('clickeranalysis','uname'.$id,'udom'.$id,0,$id);
+                   &Apache::loncommon::selectstudent_link('clickeranalysis','uname'.$id,'udom'.$id,'',$id);
           $unknown_count++;
        }
     }
@@ -10433,6 +10452,49 @@ sub iclicker_eval {
     return ($errormsg,$number);
 }
 
+sub iclickerxml_eval {
+    my ($questiontitles,$responses)=@_;
+    my $number=0;
+    my $errormsg='';
+    my @state;
+    my %respbyid;
+    my $p = HTML::Parser->new
+    (
+        xml_mode => 1,
+        start_h =>
+            [sub {
+                 my ($tagname,$attr) = @_;
+                 push(@state,$tagname);
+                 if ("@state" eq "ssn p") {
+                     my $title = $attr->{qn};
+                     $title =~ s/(^\s+|\s+$)//g;
+                     $questiontitles->[$number]=$title;
+                 } elsif ("@state" eq "ssn p v") {
+                     my $id = $attr->{id};
+                     my $entry = $attr->{ans};
+                     $id=~s/^[\#0]+//;
+                     $entry =~s/[^a-zA-Z0-9\.\*\-\+]+//g;
+                     $respbyid{$id}[$number] = $entry;
+                 }
+            }, "tagname, attr"],
+         end_h =>
+               [sub {
+                   my ($tagname) = @_;
+                   if ("@state" eq "ssn p") {
+                       $number++;
+                   }
+                   pop(@state);
+                }, "tagname"],
+    );
+
+    $p->parse($env{'form.upfile'});
+    $p->eof;
+    foreach my $id (keys(%respbyid)) {
+        $responses->{$id}=join(',',@{$respbyid{$id}});
+    }
+    return ($errormsg,$number);
+}
+
 sub interwrite_eval {
     my ($questiontitles,$responses)=@_;
     my $number=0;
@@ -10491,7 +10553,7 @@ sub turning_eval {
 
 
 sub assign_clicker_grades {
-    my ($r,$symb)=@_;
+    my ($r,$symb) = @_;
     if (!$symb) {return '';}
 # See which part we are saving to
     my $res_error;
@@ -10502,11 +10564,11 @@ sub assign_clicker_grades {
 # FIXME: This should probably look for the first handgradeable part
     my $part=$$partlist[0];
 # Start screen output
-    my $result=&Apache::loncommon::start_data_table().
-             &Apache::loncommon::start_data_table_header_row().
-             '<th>'.&mt('Assigning grades based on clicker file').'</th>'.
-             &Apache::loncommon::end_data_table_header_row().
-             &Apache::loncommon::start_data_table_row().'<td>';
+    my $result = &Apache::loncommon::start_data_table().
+                 &Apache::loncommon::start_data_table_header_row().
+                 '<th>'.&mt('Assigning grades based on clicker file').'</th>'.
+                 &Apache::loncommon::end_data_table_header_row().
+                 &Apache::loncommon::start_data_table_row().'<td>';
 # Get correct result
 # FIXME: Possibly need delimiter other than ":"
     my @correct=();
@@ -10568,7 +10630,7 @@ sub assign_clicker_grades {
           for (my $i=0;$i<$number;$i++) {
              if  ($correct[$i] eq '-') {
                 $realnumber--;
-             } elsif (($answer[$i]) || ($answer[$i]=~/^[0\.]+$/))  {
+             } elsif (($answer[$i]) || ($answer[$i]=~/^[0\.]+$/)) {
                 if ($gradingmechanism eq 'attendance') {
                    $sum+=$pcorrect;
                 } elsif ($correct[$i] eq '*') {
@@ -10637,15 +10699,17 @@ sub startpage {
     }
     if ($nomenu) {
         $args{'only_body'} = 1; 
-        $r->print(&Apache::loncommon::start_page("Student's Version",$js,\%args);
+        $r->print(&Apache::loncommon::start_page("Student's Version",$js,\%args));
     } else {
         unshift(@$crumbs,{href=>&href_symb_cmd($symb,'gradingmenu'),text=>"Grading"});
         $args{'bread_crumbs'} = $crumbs;
         $r->print(&Apache::loncommon::start_page('Grading',$js,\%args));
-        &Apache::lonquickgrades::startGradeScreen($r,($env{'form.symb'}?'probgrading':'grading'));
+        if ($env{'request.course.id'}) {
+            &Apache::lonquickgrades::startGradeScreen($r,($env{'form.symb'}?'probgrading':'grading'));
+        }
     }
     unless ($nodisplayflag) {
-       $r->print(&Apache::lonhtmlcommon::resource_info_box($symb,$onlyfolderflag,$stuvcurrent,$stuvdisp));
+        $r->print(&Apache::lonhtmlcommon::resource_info_box($symb,$onlyfolderflag,$stuvcurrent,$stuvdisp));
     }
 }
 
@@ -10873,7 +10937,7 @@ sub handler {
     }
     if ($env{'form.inhibitmenu'}) {
         $request->print(&Apache::loncommon::end_page());
-    } else {
+    } elsif ($env{'request.course.id'}) {
         &Apache::lonquickgrades::endGradeScreen($request);
     }
     &reset_caches();
@@ -11024,7 +11088,7 @@ Side Effects: None.
     $r           - Apache request object
     $i           - number of the current scanline
     $scan_record - hash ref as returned from &scantron_parse_scanline()
-    $scan_config - hash ref as returned from &get_scantron_config()
+    $scan_config - hash ref as returned from &Apache::lonnet::get_scantron_config()
     $line        - full contents of the current scanline
     $error       - error condition, valid values are
                    'incorrectCODE', 'duplicateCODE',