--- loncom/homework/grades.pm	2019/01/27 23:16:25	1.755
+++ loncom/homework/grades.pm	2020/02/12 16:25:56	1.762
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.755 2019/01/27 23:16:25 raeburn Exp $
+# $Id: grades.pm,v 1.762 2020/02/12 16:25:56 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -48,6 +48,8 @@ use Apache::lonquickgrades;
 use Apache::bridgetask();
 use Apache::lontexconvert();
 use String::Similarity;
+use HTML::Parser();
+use File::MMagic;
 use LONCAPA;
 
 use POSIX qw(floor);
@@ -4923,6 +4925,7 @@ LISTJAVASCRIPT
     my $cdom      = $env{"course.$env{'request.course.id'}.domain"};
     my $cnum      = $env{"course.$env{'request.course.id'}.num"};
     my $getsec    = $env{'form.section'} eq '' ? 'all' : $env{'form.section'};
+    my $getgroup  = $env{'form.group'} eq '' ? 'all' : $env{'form.group'};
 
     my $result='<h3><span class="LC_info">&nbsp;'.
 	&mt('Manual Grading by Page or Sequence').'</span></h3>';
@@ -5012,7 +5015,7 @@ LISTJAVASCRIPT
 	'<th>'.&nameUserString('header').'</th>'.
 	&Apache::loncommon::end_data_table_header_row();
  
-    my (undef,undef,$fullname) = &getclasslist($getsec,'1');
+    my (undef,undef,$fullname) = &getclasslist($getsec,'1',$getgroup);
     my $ptr = 1;
     foreach my $student (sort 
 			 {
@@ -5902,7 +5905,6 @@ sub scantron_selectphase {
 
         $r->print('
     <br />');
-    my $default_form_data=&defaultFormData($symb);
     my $cdom= $env{'course.'.$env{'request.course.id'}.'.domain'};
     my $cnum= $env{'course.'.$env{'request.course.id'}.'.num'};
     my $alertmsg = &mt('Please use the browse button to select a file from your local directory.');
@@ -5915,7 +5917,7 @@ sub scantron_selectphase {
 	    return false;
 	}
 	formname.submit();
-    }'.$formatjs));
+    }'."\n".$formatjs));
     $r->print('
               <form enctype="multipart/form-data" action="/adm/grades" name="rules" method="post">
                 '.$default_form_data.'
@@ -9099,20 +9101,23 @@ END
             if (keys(%{$domconfig{'scantron'}{'config'}}) > 1) {
                 if (($domconfig{'scantron'}{'config'}{'dat'}) &&
                     (ref($domconfig{'scantron'}{'config'}{'csv'}) eq 'HASH')) {
-                    if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}})) {
-                        my ($onclick,$formatextra,$singleline);
-                        my @lines = &Apache::lonnet::get_scantronformat_file();
-                        my $count = 0;
-                        foreach my $line (@lines) {
-                            next if ($line =~ /^#/);
-                            $singleline = $line;
-                            $count ++;
-                        }
-                        if ($count > 1) {
-                            $formatextra = '<div style="display:none" id="bubbletype">'.
-                                           &scantron_scantab().'</div>';
-                            $onclick = ' onclick="toggleScantab(this.form);"';
-                            $formatjs = <<"END";
+                    if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {  
+                        if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}})) {
+                            my ($onclick,$formatextra,$singleline);
+                            my @lines = &Apache::lonnet::get_scantronformat_file();
+                            my $count = 0;
+                            foreach my $line (@lines) {
+                                next if ($line =~ /^#/);
+                                $singleline = $line;
+                                $count ++;
+                            }
+                            if ($count > 1) {
+                                $formatextra = '<div style="display:none" id="bubbletype">'.
+                                               '<span class="LC_nobreak">'.
+                                               &mt('Bubblesheet type:').'&nbsp;'.
+                                               &scantron_scantab().'</span></div>';
+                                $onclick = ' onclick="toggleScantab(this.form);"';
+                                $formatjs = <<"END";
 function toggleScantab(form) {
     var divid = 'bubbletype';
     if (document.getElementById(divid)) {
@@ -9125,7 +9130,7 @@ function toggleScantab(form) {
                     if (chosen == 'dat') {
                         document.getElementById(divid).style.display = 'none';
                     } else if (chosen == 'csv') {
-                        document.getElementById(divid).style.display = 'inline-block';
+                        document.getElementById(divid).style.display = 'block';
                     }
                 }
             }
@@ -9135,22 +9140,25 @@ function toggleScantab(form) {
 }
 
 END
-                        } elsif ($count == 1) {
-                            my $formatname = (split(/:/,$singleline,2))[0];
-                            $formatextra = '<input type="hidden" name="scantron_format" value="'.$formatname.'" />';
+                            } elsif ($count == 1) {
+                                my $formatname = (split(/:/,$singleline,2))[0];
+                                $formatextra = '<input type="hidden" name="scantron_format" value="'.$formatname.'" />';
+                            }
+                            $formattitle = &mt('File format');
+                            $formatoptions = '<label><input name="fileformat" type="radio" value="dat" checked="checked"'.$onclick.' />'.
+                                             &mt('Plain Text (no delimiters)').
+                                             '</label>'.('&nbsp;'x2).
+                                             '<label><input name="fileformat" type="radio" value="csv"'.$onclick.' />'.
+                                             &mt('Comma separated values').'</label>'.$formatextra;
                         }
-                        $formattitle = &mt('File format');
-                        $formatoptions = '<label><input name="fileformat" type="radio" value="dat" checked="checked"'.$onclick.' />'.
-                                         &mt('Plain Text (no delimiters)').
-                                         '</label>'.('&nbsp;'x2).
-                                         '<label><input name="fileformat" type="radio" value="csv"'.$onclick.' />'.
-                                         &mt('Comma separated values').'</label>'.$formatextra;
                     }
                 }
             } elsif (keys(%{$domconfig{'scantron'}{'config'}}) == 1) {
-                if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}})) {
-                    $formattitle = &mt('Format of bubblesheet data file:');
-                    $formatoptions = &scantron_scantab();
+                if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
+                    if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}})) {
+                        $formattitle = &mt('Bubblesheet type');
+                        $formatoptions = &scantron_scantab();
+                    }
                 }
             }
         }
@@ -9192,15 +9200,19 @@ sub scantron_upload_scantron_data_save {
                 if (@possibles > 1) {
                     if ($env{'form.fileformat'} eq 'csv') {
                         if (ref($domconfig{'scantron'}{'config'}{'csv'}) eq 'HASH') {
-                            if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}}) > 1) {
-                                $is_csv = 1;
+                            if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
+                                if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}}) > 1) {
+                                    $is_csv = 1;
+                                }
                             }
                         }
                     }
                 } elsif (@possibles == 1) {
                     if (ref($domconfig{'scantron'}{'config'}{'csv'}) eq 'HASH') {
-                        if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}}) > 1) {
-                            $is_csv = 1;
+                        if (ref($domconfig{'scantron'}{'config'}{'csv'}{'fields'}) eq 'HASH') {
+                            if (keys(%{$domconfig{'scantron'}{'config'}{'csv'}{'fields'}}) > 1) {
+                                $is_csv = 1;
+                            }
                         }
                     }
                 }
@@ -10286,6 +10298,22 @@ sub process_clicker_file {
                         '<span class="LC_filename">'.&HTML::Entities::encode($env{'form.upfile.filename'},'<>&"').'</span>'),1);
         return $result;
     }
+    my $mimetype;
+    if ($env{'form.upfiletype'} eq 'iclicker') {
+        my $mm = new File::MMagic;
+        $mimetype = $mm->checktype_contents($env{'form.upfile'});
+        unless (($mimetype eq 'text/plain') || ($mimetype eq 'text/html')) {
+            $result.= '<p>'.
+                &Apache::lonhtmlcommon::confirm_success(
+                    &mt('File format is neither csv (iclicker 6) nor xml (iclicker 7)'),1).'</p>';
+            return $result;
+        }
+    } elsif (($env{'form.upfiletype'} ne 'interwrite') && ($env{'form.upfiletype'} ne 'turning')) {
+        $result .= '<p>'.
+            &Apache::lonhtmlcommon::confirm_success(
+                &mt('Invalid clicker type: choose one of: i>clicker, Interwrite PRS, or Turning Technologies.'),1).'</p>';
+        return $result;
+    }
 
 # Were able to get all the info needed, now analyze the file
 
@@ -10312,12 +10340,14 @@ ENDHEADER
     my $errormsg='';
     my $number=0;
     if ($env{'form.upfiletype'} eq 'iclicker') {
-	($errormsg,$number)=&iclicker_eval(\@questiontitles,\%responses);
-    }
-    if ($env{'form.upfiletype'} eq 'interwrite') {
+        if ($mimetype eq 'text/plain') {
+            ($errormsg,$number)=&iclicker_eval(\@questiontitles,\%responses);
+        } elsif ($mimetype eq 'text/html') {
+            ($errormsg,$number)=&iclickerxml_eval(\@questiontitles,\%responses);
+        }
+    } elsif ($env{'form.upfiletype'} eq 'interwrite') {
         ($errormsg,$number)=&interwrite_eval(\@questiontitles,\%responses);
-    }
-    if ($env{'form.upfiletype'} eq 'turning') {
+    } elsif ($env{'form.upfiletype'} eq 'turning') {
         ($errormsg,$number)=&turning_eval(\@questiontitles,\%responses);
     }
     $result.='<br />'.&mt('Found [_1] question(s)',$number).'<br />'.
@@ -10370,7 +10400,7 @@ ENDHEADER
                    "\n".&mt("Username").": <input type='text' name='uname".$id."' />&nbsp;".
                    "\n".&mt("Domain").": ".
                    &Apache::loncommon::select_dom_form($env{'course.'.$env{'request.course.id'}.'.domain'},'udom'.$id).'&nbsp;'.
-                   &Apache::loncommon::selectstudent_link('clickeranalysis','uname'.$id,'udom'.$id,0,$id);
+                   &Apache::loncommon::selectstudent_link('clickeranalysis','uname'.$id,'udom'.$id,'',$id);
           $unknown_count++;
        }
     }
@@ -10425,6 +10455,49 @@ sub iclicker_eval {
     return ($errormsg,$number);
 }
 
+sub iclickerxml_eval {
+    my ($questiontitles,$responses)=@_;
+    my $number=0;
+    my $errormsg='';
+    my @state;
+    my %respbyid;
+    my $p = HTML::Parser->new
+    (
+        xml_mode => 1,
+        start_h =>
+            [sub {
+                 my ($tagname,$attr) = @_;
+                 push(@state,$tagname);
+                 if ("@state" eq "ssn p") {
+                     my $title = $attr->{qn};
+                     $title =~ s/(^\s+|\s+$)//g;
+                     $questiontitles->[$number]=$title;
+                 } elsif ("@state" eq "ssn p v") {
+                     my $id = $attr->{id};
+                     my $entry = $attr->{ans};
+                     $id=~s/^[\#0]+//;
+                     $entry =~s/[^a-zA-Z0-9\.\*\-\+]+//g;
+                     $respbyid{$id}[$number] = $entry;
+                 }
+            }, "tagname, attr"],
+         end_h =>
+               [sub {
+                   my ($tagname) = @_;
+                   if ("@state" eq "ssn p") {
+                       $number++;
+                   }
+                   pop(@state);
+                }, "tagname"],
+    );
+
+    $p->parse($env{'form.upfile'});
+    $p->eof;
+    foreach my $id (keys(%respbyid)) {
+        $responses->{$id}=join(',',@{$respbyid{$id}});
+    }
+    return ($errormsg,$number);
+}
+
 sub interwrite_eval {
     my ($questiontitles,$responses)=@_;
     my $number=0;
@@ -10629,7 +10702,7 @@ sub startpage {
     }
     if ($nomenu) {
         $args{'only_body'} = 1; 
-        $r->print(&Apache::loncommon::start_page("Student's Version",$js,\%args);
+        $r->print(&Apache::loncommon::start_page("Student's Version",$js,\%args));
     } else {
         unshift(@$crumbs,{href=>&href_symb_cmd($symb,'gradingmenu'),text=>"Grading"});
         $args{'bread_crumbs'} = $crumbs;
@@ -10637,7 +10710,7 @@ sub startpage {
         &Apache::lonquickgrades::startGradeScreen($r,($env{'form.symb'}?'probgrading':'grading'));
     }
     unless ($nodisplayflag) {
-       $r->print(&Apache::lonhtmlcommon::resource_info_box($symb,$onlyfolderflag,$stuvcurrent,$stuvdisp));
+        $r->print(&Apache::lonhtmlcommon::resource_info_box($symb,$onlyfolderflag,$stuvcurrent,$stuvdisp));
     }
 }
 
@@ -11016,7 +11089,7 @@ Side Effects: None.
     $r           - Apache request object
     $i           - number of the current scanline
     $scan_record - hash ref as returned from &scantron_parse_scanline()
-    $scan_config - hash ref as returned from &get_scantron_config()
+    $scan_config - hash ref as returned from &Apache::lonnet::get_scantron_config()
     $line        - full contents of the current scanline
     $error       - error condition, valid values are
                    'incorrectCODE', 'duplicateCODE',