--- loncom/homework/cleanxml/post_xml.pm	2016/01/08 20:32:42	1.6
+++ loncom/homework/cleanxml/post_xml.pm	2016/11/10 19:48:22	1.10
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # Third step to clean a file.
 #
-# $Id: post_xml.pm,v 1.6 2016/01/08 20:32:42 damieng Exp $
+# $Id: post_xml.pm,v 1.10 2016/11/10 19:48:22 damieng Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -41,6 +41,8 @@ use Cwd 'abs_path';
 use XML::LibXML;
 use HTML::TokeParser; # used to parse sty files
 use Tie::IxHash; # for ordered hashes
+use tth;
+use Apache::html_to_xml;
 
 no warnings 'recursion'; # yes, fix_paragraph is using heavy recursion, I know
 
@@ -126,12 +128,16 @@ sub post_xml {
   
   remove_useless_notsolved($root);
   
+  fix_comments($root);
+  
   fix_paragraphs_inside($root, \@all_block);
 
   remove_empty_style($root);
   
   fix_empty_lc_elements($root);
   
+  reduce_empty_p($root);
+  
   lowercase_attribute_values($root);
   
   replace_numericalresponse_unit_attribute($root);
@@ -552,11 +558,11 @@ sub replace_m {
 # Returns the HTML equivalent of LaTeX input, using tth
 sub tth {
   my ($text) = @_;
-  my ($fh, $tmp_path) = tempfile();
-  binmode($fh, ':utf8');
-  print $fh $text;
-  close $fh;
-  my $output = `tth -r -w2 -u -y0 < $tmp_path 2>/dev/null`;
+  my $output = &tth::tth($text);
+  my $errorstring = &tth::ttherror();
+  if ($errorstring) {
+    die $errorstring;
+  }
   # hopefully the temp file will not be removed before this point (otherwise we should use unlink_on_destroy 0)
   $output =~ s/^\s*|\s*$//;
   $output =~ s/<div class="p"><!----><\/div>/<br\/>/; # why is tth using such ugly markup for \newline ?
@@ -567,7 +573,7 @@ sub tth {
 sub html_to_dom {
   my ($text) = @_;
   $text = '<root>'.$text.'</root>';
-  my $textref = html_to_xml::html_to_xml(\$text);
+  my $textref = Apache::html_to_xml::html_to_xml(\$text);
   utf8::upgrade($$textref); # otherwise the XML parser fails when the HTML parser turns &nbsp; into a character
   my $dom_doc = XML::LibXML->load_xml(string => $textref);
   my $root = $dom_doc->documentElement;
@@ -1808,12 +1814,31 @@ sub remove_useless_notsolved {
   }
 }
 
+# Use <pre> for multi-line comments without elements.
+sub fix_comments {
+  my ($root) = @_;
+  my $doc = $root->ownerDocument;
+  my @comments = $root->getElementsByTagName('comment');
+  foreach my $comment (@comments) {
+    my $first = $comment->firstChild;
+    if (defined $first) {
+      if ($first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /\n/ &&
+          !defined $first->nextSibling) {
+        my $pre = $doc->createElement('pre');
+        $comment->removeChild($first);
+        $comment->appendChild($pre);
+        $pre->appendChild($first);
+      }
+    }
+  }
+}
+
 # adds a paragraph inside if needed and calls fix_paragraph for all paragraphs (including new ones)
 sub fix_paragraphs_inside {
   my ($node, $all_block) = @_;
   # blocks in which paragrahs will be added:
-  my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','solved','notsolved','languageblock','instructorcomment','togglebox','standalone','body','form');
-  my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote');
+  my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','languageblock','instructorcomment','togglebox','standalone','body','form');
+  my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote','solved','notsolved');
   if ((string_in_array(\@blocks_with_p, $node->nodeName) && paragraph_needed($node)) ||
       (string_in_array(\@fix_p_if_br_or_p, $node->nodeName) && paragraph_inside($node))) {
     # if non-empty, add paragraphs where needed between all br and remove br
@@ -2245,6 +2270,33 @@ sub fix_empty_lc_elements {
     }
   }
 }
+
+# remove consecutive empty paragraphs (they will not show anyway)
+sub reduce_empty_p {
+  my ($node) = @_;
+  my $next;
+  for (my $child=$node->firstChild; defined $child; $child=$next) {
+    $next = $child->nextSibling;
+    while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) {
+      $next = $next->nextSibling;
+    }
+    if ($child->nodeType == XML_ELEMENT_NODE && $child->nodeName eq 'p' && defined $next &&
+        $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'p') {
+      my $first = $child->firstChild;
+      if (!defined $first || (!defined $first->nextSibling &&
+          $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) {
+        $first = $next->firstChild;
+        if (!defined $first || (!defined $first->nextSibling &&
+            $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) {
+          $node->removeChild($child);
+        }
+      }
+    }
+    if ($child->nodeType == XML_ELEMENT_NODE) {
+      reduce_empty_p($child);
+    }
+  }
+}
 
 # turn some attribute values into lowercase when they should be
 sub lowercase_attribute_values {