--- loncom/homework/cleanxml/post_xml.pm 2015/12/15 15:33:30 1.3 +++ loncom/homework/cleanxml/post_xml.pm 2017/01/17 20:29:06 1.12 @@ -1,7 +1,7 @@ # The LearningOnline Network # Third step to clean a file. # -# $Id: post_xml.pm,v 1.3 2015/12/15 15:33:30 damieng Exp $ +# $Id: post_xml.pm,v 1.12 2017/01/17 20:29:06 damieng Exp $ # # Copyright Michigan State University Board of Trustees # @@ -41,16 +41,18 @@ use Cwd 'abs_path'; use XML::LibXML; use HTML::TokeParser; # used to parse sty files use Tie::IxHash; # for ordered hashes +use tth; +use Apache::html_to_xml; no warnings 'recursion'; # yes, fix_paragraph is using heavy recursion, I know # these are constants -my @block_elements = ('parameter','location','answer','foil','image','polygon','rectangle','text','conceptgroup','itemgroup','item','label','data','function','array','unit','answergroup','functionplotresponse','functionplotruleset','functionplotelements','functionplotcustomrule','essayresponse','hintpart','formulahint','numericalhint','reactionhint','organichint','optionhint','radiobuttonhint','stringhint','customhint','mathhint','formulahintcondition','numericalhintcondition','reactionhintcondition','organichintcondition','optionhintcondition','radiobuttonhintcondition','stringhintcondition','customhintcondition','mathhintcondition','imageresponse','foilgroup','datasubmission','textfield','hiddensubmission','radiobuttonresponse','rankresponse','matchresponse','import','style','script','window','block','library','notsolved','part','postanswerdate','preduedate','problem','problemtype','randomlabel','bgimg','labelgroup','randomlist','solved','while','tex','print','web','gnuplot','curve','Task','IntroParagraph','ClosingParagraph','Question','QuestionText','Setup','Instance','InstanceText','Criteria','CriteriaText','GraderNote','languageblock','translated','lang','instructorcomment','dataresponse','togglebox','standalone','comment','drawimage','allow','displayduedate','displaytitle','responseparam','organicstructure','scriptlib','parserlib','drawoptionlist','spline','backgroundplot','plotobject','plotvector','drawvectorsum','functionplotrule','functionplotvectorrule','functionplotvectorsumrule','axis','key','xtics','ytics','title','xlabel','ylabel','hiddenline','dtm'); -my @inline_like_block = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse', 'hint', 'hintgroup'); # inline elements treated like blocks for pretty print and some other things +my @block_elements = ('parameter','location','answer','foil','image','polygon','rectangle','text','conceptgroup','itemgroup','item','label','data','function','array','unit','answergroup','functionplotresponse','functionplotruleset','functionplotelements','functionplotcustomrule','essayresponse','hintpart','formulahint','numericalhint','reactionhint','organichint','optionhint','radiobuttonhint','stringhint','customhint','mathhint','formulahintcondition','numericalhintcondition','reactionhintcondition','organichintcondition','optionhintcondition','radiobuttonhintcondition','stringhintcondition','customhintcondition','mathhintcondition','imageresponse','foilgroup','datasubmission','textfield','hiddensubmission','radiobuttonresponse','rankresponse','matchresponse','import','style','script','window','block','library','notsolved','part','postanswerdate','preduedate','problem','problemtype','randomlabel','bgimg','labelgroup','randomlist','solved','while','tex','print','web','gnuplot','curve','Task','IntroParagraph','ClosingParagraph','Question','QuestionText','Setup','Instance','InstanceText','Criteria','CriteriaText','GraderNote','languageblock','instructorcomment','dataresponse','togglebox','standalone','comment','drawimage','allow','displayduedate','displaytitle','responseparam','organicstructure','scriptlib','parserlib','drawoptionlist','spline','backgroundplot','plotobject','plotvector','drawvectorsum','functionplotrule','functionplotvectorrule','functionplotvectorsumrule','axis','key','xtics','ytics','title','xlabel','ylabel','hiddenline','dtm'); +my @inline_like_block = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse', 'hint', 'hintgroup','translated','lang'); # inline elements treated like blocks for pretty print and some other things my @responses = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse','essayresponse','radiobuttonresponse','matchresponse','rankresponse','imageresponse','functionplotresponse'); my @block_html = ('html','head','body','section','h1','h2','h3','h4','h5','h6','div','p','ul','ol','li','table','tbody','tr','td','th','dl','dt','dd','pre','noscript','hr','address','blockquote','object','applet','embed','map','form','fieldset','iframe','center','frameset'); my @no_newline_inside = ('import','parserlib','scriptlib','data','function','label','xlabel','ylabel','tic','text','rectangle','image','title','h1','h2','h3','h4','h5','h6','li','td','p'); -my @preserve_elements = ('script','answer','pre'); +my @preserve_elements = ('script','answer','pre','style'); my @accepting_style = ('section','h1','h2','h3','h4','h5','h6','div','p','li','td','th','dt','dd','pre','blockquote'); my @latex_math = ('\alpha', '\theta', '\omicron', '\tau', '\beta', '\vartheta', '\pi', '\upsilon', '\gamma', '\gamma', '\varpi', '\phi', '\delta', '\kappa', '\rho', '\varphi', '\epsilon', '\lambda', '\varrho', '\chi', '\varepsilon', '\mu', '\sigma', '\psi', '\zeta', '\nu', '\varsigma', '\omega', '\eta', '\xi', '\Gamma', '\Lambda', '\Sigma', '\Psi', '\Delta', '\Xi', '\Upsilon', '\Omega', '\Theta', '\Pi', '\Phi', @@ -126,12 +128,16 @@ sub post_xml { remove_useless_notsolved($root); + fix_comments($root); + fix_paragraphs_inside($root, \@all_block); remove_empty_style($root); fix_empty_lc_elements($root); + reduce_empty_p($root); + lowercase_attribute_values($root); replace_numericalresponse_unit_attribute($root); @@ -150,7 +156,7 @@ sub fix_structure { # the root element has already been added in pre_xml my $root = $doc->documentElement; # inside the root, replace html, problem and library elements by their content - my @toreplace = ('html','problem','library'); + my @toreplace = ('html','problem','library','Task'); foreach my $name (@toreplace) { my @elements = $root->getElementsByTagName($name); foreach my $element (@elements) { @@ -424,6 +430,15 @@ sub replace_m { my $var_key1 = 'dfhg3df54hg65hg4'; my $var_key2 = 'dfhg654d6f5g4h5f'; my $eval = defined $m->getAttribute('eval') && $m->getAttribute('eval') eq 'on'; + my $display = $m->getAttribute('display'); + if (defined $display) { + if ($display eq '') { + $display = undef; + } + if (lc($display) eq 'jsmath') { + $display = 'mathjax'; + } + } if ($eval) { # replace variables foreach my $variable (@variables) { @@ -461,6 +476,9 @@ sub replace_m { if ($eval) { $new_node->setAttribute('eval', 'on'); } + if (defined $display) { + $new_node->setAttribute('display', $display); + } $new_node->appendChild($doc->createTextNode($new_text)); $m->parentNode->replaceChild($new_node, $m); next; @@ -491,7 +509,7 @@ sub replace_m { # there are math separators inside, even after hiding variables, or there was no math symbol # hide math parts inside before running tth - my $math_key1 = '#ghjgdh5hg45gf'; + my $math_key1 = '#5752398247516385'; my $math_key2 = '#'; my @maths = (); my @separators = (['$$','$$'], ['\\(','\\)'], ['\\[','\\]'], ['$','$']); @@ -522,14 +540,30 @@ sub replace_m { $math =~ s/&/&/g; $math =~ s//>/g; + my ($mel, $inside); if ($math =~ /^\$\$(.*)\$\$$/s) { - $math = ''.$1.''; + $mel = 'dtm'; + $inside = $1; } elsif ($math =~ /^\\\[(.*)\\\]$/s) { - $math = ''.$1.''; + $mel = 'dtm'; + $inside = $1; } elsif ($math =~ /^\\\((.*)\\\)$/s) { - $math = ''.$1.''; + $mel = 'tm'; + $inside = $1; } elsif ($math =~ /^\$(.*)\$$/s) { - $math = ''.$1.''; + $mel = 'tm'; + $inside = $1; + } + if (defined $inside) { + if ($inside =~ /^\s*$/) { + $math = ''; + } else { + $math = '<'.$mel; + if ($eval && $inside =~ /$var_key1/) { + $math .= ' eval="on"'; + } + $math .= '>'.$inside.''; + } } my $replace = $math_key1.($i+1).$math_key2; $html_text =~ s/$replace/$math/; @@ -552,11 +586,11 @@ sub replace_m { # Returns the HTML equivalent of LaTeX input, using tth sub tth { my ($text) = @_; - my ($fh, $tmp_path) = tempfile(); - binmode($fh, ':utf8'); - print $fh $text; - close $fh; - my $output = `tth -r -w2 -u -y0 < $tmp_path 2>/dev/null`; + my $output = &tth::tth($text); + my $errorstring = &tth::ttherror(); + if ($errorstring) { + die $errorstring; + } # hopefully the temp file will not be removed before this point (otherwise we should use unlink_on_destroy 0) $output =~ s/^\s*|\s*$//; $output =~ s/
<\/div>//; # why is tth using such ugly markup for \newline ? @@ -567,7 +601,7 @@ sub tth { sub html_to_dom { my ($text) = @_; $text = ''.$text.''; - my $textref = html_to_xml::html_to_xml(\$text); + my $textref = Apache::html_to_xml::html_to_xml(\$text); utf8::upgrade($$textref); # otherwise the XML parser fails when the HTML parser turns   into a character my $dom_doc = XML::LibXML->load_xml(string => $textref); my $root = $dom_doc->documentElement; @@ -1808,12 +1842,31 @@ sub remove_useless_notsolved { } } +# Use
 for multi-line comments without elements.
+sub fix_comments {
+  my ($root) = @_;
+  my $doc = $root->ownerDocument;
+  my @comments = $root->getElementsByTagName('comment');
+  foreach my $comment (@comments) {
+    my $first = $comment->firstChild;
+    if (defined $first) {
+      if ($first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /\n/ &&
+          !defined $first->nextSibling) {
+        my $pre = $doc->createElement('pre');
+        $comment->removeChild($first);
+        $comment->appendChild($pre);
+        $pre->appendChild($first);
+      }
+    }
+  }
+}
+
 # adds a paragraph inside if needed and calls fix_paragraph for all paragraphs (including new ones)
 sub fix_paragraphs_inside {
   my ($node, $all_block) = @_;
   # blocks in which paragrahs will be added:
-  my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','solved','notsolved','languageblock','translated','lang','instructorcomment','togglebox','standalone','form');
-  my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote');
+  my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','languageblock','instructorcomment','togglebox','standalone','body','form');
+  my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote','solved','notsolved');
   if ((string_in_array(\@blocks_with_p, $node->nodeName) && paragraph_needed($node)) ||
       (string_in_array(\@fix_p_if_br_or_p, $node->nodeName) && paragraph_inside($node))) {
     # if non-empty, add paragraphs where needed between all br and remove br
@@ -1833,6 +1886,18 @@ sub fix_paragraphs_inside {
           push(@new_children, $doc->createElement('p'));
         }
         $p = undef;
+        # ignore the next node if it is a br (the paragraph default margin will take as much space)
+        # (ignoring whitespace)
+        while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) {
+          my $next2 = $next->nextSibling;
+          $node->removeChild($next);
+          $next = $next2;
+        }
+        if (defined $next && $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'br') {
+          my $next2 = $next->nextSibling;
+          $node->removeChild($next);
+          $next = $next2;
+        }
       } elsif ($child->nodeType == XML_ELEMENT_NODE && string_in_array(\@inline_like_block, $child->nodeName)) {
         # inline_like_block: use the paragraph if there is one, otherwise do not create one
         if (defined $p) {
@@ -2009,6 +2074,20 @@ sub fix_paragraph {
               if (!defined $left || !$left_needs_p) {
                 $replacement->appendChild($middle);
               }
+              # ignore the next node if it is a br (the paragraph default margin will take as much space)
+              my $first_right;
+              if (defined $right) {
+                $first_right = $right->firstChild;
+                # ignore non-nbsp whitespace
+                while (defined $first_right && $first_right->nodeType == XML_TEXT_NODE &&
+                    $first_right->nodeValue =~ /^[ \t\f\n\r]*$/) {
+                  $first_right = $first_right->nextSibling;
+                }
+              }
+              if (defined $first_right && $first_right->nodeType == XML_ELEMENT_NODE &&
+                  $first_right->nodeName eq 'br') {
+                $right->removeChild($first_right);
+              }
             } else {
               fix_paragraphs_inside($n, $all_block);
               $replacement->appendChild($n);
@@ -2246,6 +2325,33 @@ sub fix_empty_lc_elements {
   }
 }
 
+# remove consecutive empty paragraphs (they will not show anyway)
+sub reduce_empty_p {
+  my ($node) = @_;
+  my $next;
+  for (my $child=$node->firstChild; defined $child; $child=$next) {
+    $next = $child->nextSibling;
+    while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) {
+      $next = $next->nextSibling;
+    }
+    if ($child->nodeType == XML_ELEMENT_NODE && $child->nodeName eq 'p' && defined $next &&
+        $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'p') {
+      my $first = $child->firstChild;
+      if (!defined $first || (!defined $first->nextSibling &&
+          $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) {
+        $first = $next->firstChild;
+        if (!defined $first || (!defined $first->nextSibling &&
+            $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) {
+          $node->removeChild($child);
+        }
+      }
+    }
+    if ($child->nodeType == XML_ELEMENT_NODE) {
+      reduce_empty_p($child);
+    }
+  }
+}
+
 # turn some attribute values into lowercase when they should be
 sub lowercase_attribute_values {
   my ($root) = @_;
@@ -2394,8 +2500,29 @@ sub pretty {
   my $type = $node->nodeType;
   if ($type == XML_ELEMENT_NODE) {
     my $name = $node->nodeName;
-    if ((string_in_array($all_block, $name) || string_in_array(\@inline_like_block, $name)) &&
-        !string_in_array(\@preserve_elements, $name)) {
+    if (string_in_array(\@preserve_elements, $name)) {
+      # remove newlines at the beginning and the end of preserve elements
+      if (defined $node->firstChild && ($node->firstChild->nodeType == XML_TEXT_NODE ||
+          $node->firstChild->nodeType == XML_CDATA_SECTION_NODE)) {
+        my $text = $node->firstChild->nodeValue;
+        $text =~ s/^\n+//;
+        if ($text eq '') {
+          $node->removeChild($node->firstChild);
+        } else {
+          $node->firstChild->setData($text);
+        }
+      }
+      if (defined $node->lastChild && ($node->lastChild->nodeType == XML_TEXT_NODE ||
+          $node->lastChild->nodeType == XML_CDATA_SECTION_NODE)) {
+        my $text = $node->lastChild->nodeValue;
+        $text =~ s/\n+$//;
+        if ($text eq '') {
+          $node->removeChild($node->lastChild);
+        } else {
+          $node->lastChild->setData($text);
+        }
+      }
+    } elsif (string_in_array($all_block, $name) || string_in_array(\@inline_like_block, $name)) {
       # make sure there is a newline at the beginning and at the end if there is anything inside
       if (defined $node->firstChild && !string_in_array(\@no_newline_inside, $name)) {
         my $first = $node->firstChild;
@@ -2478,26 +2605,6 @@ sub pretty {
         if ($text eq '') {
           $node->removeChild($node->lastChild);
         } else {
-          $node->lastChild->setData($text);
-        }
-      }
-    } elsif (string_in_array(\@preserve_elements, $name)) {
-      # collapse newlines at the beginning and the end of scripts
-      if (defined $node->firstChild && $node->firstChild->nodeType == XML_TEXT_NODE) {
-        my $text = $node->firstChild->nodeValue;
-        $text =~ s/^\n( *\n)+/\n/;
-        if ($text eq '') {
-          $node->removeChild($node->firstChild);
-        } else {
-          $node->firstChild->setData($text);
-        }
-      }
-      if (defined $node->lastChild && $node->lastChild->nodeType == XML_TEXT_NODE) {
-        my $text = $node->lastChild->nodeValue;
-        $text =~ s/\n( *\n)+$/\n/;
-        if ($text eq '') {
-          $node->removeChild($node->lastChild);
-        } else {
           $node->lastChild->setData($text);
         }
       }