--- loncom/homework/cleanxml/post_xml.pm 2015/12/15 15:33:30 1.3
+++ loncom/homework/cleanxml/post_xml.pm 2017/01/17 20:29:06 1.12
@@ -1,7 +1,7 @@
# The LearningOnline Network
# Third step to clean a file.
#
-# $Id: post_xml.pm,v 1.3 2015/12/15 15:33:30 damieng Exp $
+# $Id: post_xml.pm,v 1.12 2017/01/17 20:29:06 damieng Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -41,16 +41,18 @@ use Cwd 'abs_path';
use XML::LibXML;
use HTML::TokeParser; # used to parse sty files
use Tie::IxHash; # for ordered hashes
+use tth;
+use Apache::html_to_xml;
no warnings 'recursion'; # yes, fix_paragraph is using heavy recursion, I know
# these are constants
-my @block_elements = ('parameter','location','answer','foil','image','polygon','rectangle','text','conceptgroup','itemgroup','item','label','data','function','array','unit','answergroup','functionplotresponse','functionplotruleset','functionplotelements','functionplotcustomrule','essayresponse','hintpart','formulahint','numericalhint','reactionhint','organichint','optionhint','radiobuttonhint','stringhint','customhint','mathhint','formulahintcondition','numericalhintcondition','reactionhintcondition','organichintcondition','optionhintcondition','radiobuttonhintcondition','stringhintcondition','customhintcondition','mathhintcondition','imageresponse','foilgroup','datasubmission','textfield','hiddensubmission','radiobuttonresponse','rankresponse','matchresponse','import','style','script','window','block','library','notsolved','part','postanswerdate','preduedate','problem','problemtype','randomlabel','bgimg','labelgroup','randomlist','solved','while','tex','print','web','gnuplot','curve','Task','IntroParagraph','ClosingParagraph','Question','QuestionText','Setup','Instance','InstanceText','Criteria','CriteriaText','GraderNote','languageblock','translated','lang','instructorcomment','dataresponse','togglebox','standalone','comment','drawimage','allow','displayduedate','displaytitle','responseparam','organicstructure','scriptlib','parserlib','drawoptionlist','spline','backgroundplot','plotobject','plotvector','drawvectorsum','functionplotrule','functionplotvectorrule','functionplotvectorsumrule','axis','key','xtics','ytics','title','xlabel','ylabel','hiddenline','dtm');
-my @inline_like_block = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse', 'hint', 'hintgroup'); # inline elements treated like blocks for pretty print and some other things
+my @block_elements = ('parameter','location','answer','foil','image','polygon','rectangle','text','conceptgroup','itemgroup','item','label','data','function','array','unit','answergroup','functionplotresponse','functionplotruleset','functionplotelements','functionplotcustomrule','essayresponse','hintpart','formulahint','numericalhint','reactionhint','organichint','optionhint','radiobuttonhint','stringhint','customhint','mathhint','formulahintcondition','numericalhintcondition','reactionhintcondition','organichintcondition','optionhintcondition','radiobuttonhintcondition','stringhintcondition','customhintcondition','mathhintcondition','imageresponse','foilgroup','datasubmission','textfield','hiddensubmission','radiobuttonresponse','rankresponse','matchresponse','import','style','script','window','block','library','notsolved','part','postanswerdate','preduedate','problem','problemtype','randomlabel','bgimg','labelgroup','randomlist','solved','while','tex','print','web','gnuplot','curve','Task','IntroParagraph','ClosingParagraph','Question','QuestionText','Setup','Instance','InstanceText','Criteria','CriteriaText','GraderNote','languageblock','instructorcomment','dataresponse','togglebox','standalone','comment','drawimage','allow','displayduedate','displaytitle','responseparam','organicstructure','scriptlib','parserlib','drawoptionlist','spline','backgroundplot','plotobject','plotvector','drawvectorsum','functionplotrule','functionplotvectorrule','functionplotvectorsumrule','axis','key','xtics','ytics','title','xlabel','ylabel','hiddenline','dtm');
+my @inline_like_block = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse', 'hint', 'hintgroup','translated','lang'); # inline elements treated like blocks for pretty print and some other things
my @responses = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse','essayresponse','radiobuttonresponse','matchresponse','rankresponse','imageresponse','functionplotresponse');
my @block_html = ('html','head','body','section','h1','h2','h3','h4','h5','h6','div','p','ul','ol','li','table','tbody','tr','td','th','dl','dt','dd','pre','noscript','hr','address','blockquote','object','applet','embed','map','form','fieldset','iframe','center','frameset');
my @no_newline_inside = ('import','parserlib','scriptlib','data','function','label','xlabel','ylabel','tic','text','rectangle','image','title','h1','h2','h3','h4','h5','h6','li','td','p');
-my @preserve_elements = ('script','answer','pre');
+my @preserve_elements = ('script','answer','pre','style');
my @accepting_style = ('section','h1','h2','h3','h4','h5','h6','div','p','li','td','th','dt','dd','pre','blockquote');
my @latex_math = ('\alpha', '\theta', '\omicron', '\tau', '\beta', '\vartheta', '\pi', '\upsilon', '\gamma', '\gamma', '\varpi', '\phi', '\delta', '\kappa', '\rho', '\varphi', '\epsilon', '\lambda', '\varrho', '\chi', '\varepsilon', '\mu', '\sigma', '\psi', '\zeta', '\nu', '\varsigma', '\omega', '\eta', '\xi',
'\Gamma', '\Lambda', '\Sigma', '\Psi', '\Delta', '\Xi', '\Upsilon', '\Omega', '\Theta', '\Pi', '\Phi',
@@ -126,12 +128,16 @@ sub post_xml {
remove_useless_notsolved($root);
+ fix_comments($root);
+
fix_paragraphs_inside($root, \@all_block);
remove_empty_style($root);
fix_empty_lc_elements($root);
+ reduce_empty_p($root);
+
lowercase_attribute_values($root);
replace_numericalresponse_unit_attribute($root);
@@ -150,7 +156,7 @@ sub fix_structure {
# the root element has already been added in pre_xml
my $root = $doc->documentElement;
# inside the root, replace html, problem and library elements by their content
- my @toreplace = ('html','problem','library');
+ my @toreplace = ('html','problem','library','Task');
foreach my $name (@toreplace) {
my @elements = $root->getElementsByTagName($name);
foreach my $element (@elements) {
@@ -424,6 +430,15 @@ sub replace_m {
my $var_key1 = 'dfhg3df54hg65hg4';
my $var_key2 = 'dfhg654d6f5g4h5f';
my $eval = defined $m->getAttribute('eval') && $m->getAttribute('eval') eq 'on';
+ my $display = $m->getAttribute('display');
+ if (defined $display) {
+ if ($display eq '') {
+ $display = undef;
+ }
+ if (lc($display) eq 'jsmath') {
+ $display = 'mathjax';
+ }
+ }
if ($eval) {
# replace variables
foreach my $variable (@variables) {
@@ -461,6 +476,9 @@ sub replace_m {
if ($eval) {
$new_node->setAttribute('eval', 'on');
}
+ if (defined $display) {
+ $new_node->setAttribute('display', $display);
+ }
$new_node->appendChild($doc->createTextNode($new_text));
$m->parentNode->replaceChild($new_node, $m);
next;
@@ -491,7 +509,7 @@ sub replace_m {
# there are math separators inside, even after hiding variables, or there was no math symbol
# hide math parts inside before running tth
- my $math_key1 = '#ghjgdh5hg45gf';
+ my $math_key1 = '#5752398247516385';
my $math_key2 = '#';
my @maths = ();
my @separators = (['$$','$$'], ['\\(','\\)'], ['\\[','\\]'], ['$','$']);
@@ -522,14 +540,30 @@ sub replace_m {
$math =~ s/&/&/g;
$math =~ s/</g;
$math =~ s/>/>/g;
+ my ($mel, $inside);
if ($math =~ /^\$\$(.*)\$\$$/s) {
- $math = '
for multi-line comments without elements. +sub fix_comments { + my ($root) = @_; + my $doc = $root->ownerDocument; + my @comments = $root->getElementsByTagName('comment'); + foreach my $comment (@comments) { + my $first = $comment->firstChild; + if (defined $first) { + if ($first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /\n/ && + !defined $first->nextSibling) { + my $pre = $doc->createElement('pre'); + $comment->removeChild($first); + $comment->appendChild($pre); + $pre->appendChild($first); + } + } + } +} + # adds a paragraph inside if needed and calls fix_paragraph for all paragraphs (including new ones) sub fix_paragraphs_inside { my ($node, $all_block) = @_; # blocks in which paragrahs will be added: - my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','solved','notsolved','languageblock','translated','lang','instructorcomment','togglebox','standalone','form'); - my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote'); + my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','languageblock','instructorcomment','togglebox','standalone','body','form'); + my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote','solved','notsolved'); if ((string_in_array(\@blocks_with_p, $node->nodeName) && paragraph_needed($node)) || (string_in_array(\@fix_p_if_br_or_p, $node->nodeName) && paragraph_inside($node))) { # if non-empty, add paragraphs where needed between all br and remove br @@ -1833,6 +1886,18 @@ sub fix_paragraphs_inside { push(@new_children, $doc->createElement('p')); } $p = undef; + # ignore the next node if it is a br (the paragraph default margin will take as much space) + # (ignoring whitespace) + while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) { + my $next2 = $next->nextSibling; + $node->removeChild($next); + $next = $next2; + } + if (defined $next && $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'br') { + my $next2 = $next->nextSibling; + $node->removeChild($next); + $next = $next2; + } } elsif ($child->nodeType == XML_ELEMENT_NODE && string_in_array(\@inline_like_block, $child->nodeName)) { # inline_like_block: use the paragraph if there is one, otherwise do not create one if (defined $p) { @@ -2009,6 +2074,20 @@ sub fix_paragraph { if (!defined $left || !$left_needs_p) { $replacement->appendChild($middle); } + # ignore the next node if it is a br (the paragraph default margin will take as much space) + my $first_right; + if (defined $right) { + $first_right = $right->firstChild; + # ignore non-nbsp whitespace + while (defined $first_right && $first_right->nodeType == XML_TEXT_NODE && + $first_right->nodeValue =~ /^[ \t\f\n\r]*$/) { + $first_right = $first_right->nextSibling; + } + } + if (defined $first_right && $first_right->nodeType == XML_ELEMENT_NODE && + $first_right->nodeName eq 'br') { + $right->removeChild($first_right); + } } else { fix_paragraphs_inside($n, $all_block); $replacement->appendChild($n); @@ -2246,6 +2325,33 @@ sub fix_empty_lc_elements { } } +# remove consecutive empty paragraphs (they will not show anyway) +sub reduce_empty_p { + my ($node) = @_; + my $next; + for (my $child=$node->firstChild; defined $child; $child=$next) { + $next = $child->nextSibling; + while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) { + $next = $next->nextSibling; + } + if ($child->nodeType == XML_ELEMENT_NODE && $child->nodeName eq 'p' && defined $next && + $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'p') { + my $first = $child->firstChild; + if (!defined $first || (!defined $first->nextSibling && + $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) { + $first = $next->firstChild; + if (!defined $first || (!defined $first->nextSibling && + $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) { + $node->removeChild($child); + } + } + } + if ($child->nodeType == XML_ELEMENT_NODE) { + reduce_empty_p($child); + } + } +} + # turn some attribute values into lowercase when they should be sub lowercase_attribute_values { my ($root) = @_; @@ -2394,8 +2500,29 @@ sub pretty { my $type = $node->nodeType; if ($type == XML_ELEMENT_NODE) { my $name = $node->nodeName; - if ((string_in_array($all_block, $name) || string_in_array(\@inline_like_block, $name)) && - !string_in_array(\@preserve_elements, $name)) { + if (string_in_array(\@preserve_elements, $name)) { + # remove newlines at the beginning and the end of preserve elements + if (defined $node->firstChild && ($node->firstChild->nodeType == XML_TEXT_NODE || + $node->firstChild->nodeType == XML_CDATA_SECTION_NODE)) { + my $text = $node->firstChild->nodeValue; + $text =~ s/^\n+//; + if ($text eq '') { + $node->removeChild($node->firstChild); + } else { + $node->firstChild->setData($text); + } + } + if (defined $node->lastChild && ($node->lastChild->nodeType == XML_TEXT_NODE || + $node->lastChild->nodeType == XML_CDATA_SECTION_NODE)) { + my $text = $node->lastChild->nodeValue; + $text =~ s/\n+$//; + if ($text eq '') { + $node->removeChild($node->lastChild); + } else { + $node->lastChild->setData($text); + } + } + } elsif (string_in_array($all_block, $name) || string_in_array(\@inline_like_block, $name)) { # make sure there is a newline at the beginning and at the end if there is anything inside if (defined $node->firstChild && !string_in_array(\@no_newline_inside, $name)) { my $first = $node->firstChild; @@ -2478,26 +2605,6 @@ sub pretty { if ($text eq '') { $node->removeChild($node->lastChild); } else { - $node->lastChild->setData($text); - } - } - } elsif (string_in_array(\@preserve_elements, $name)) { - # collapse newlines at the beginning and the end of scripts - if (defined $node->firstChild && $node->firstChild->nodeType == XML_TEXT_NODE) { - my $text = $node->firstChild->nodeValue; - $text =~ s/^\n( *\n)+/\n/; - if ($text eq '') { - $node->removeChild($node->firstChild); - } else { - $node->firstChild->setData($text); - } - } - if (defined $node->lastChild && $node->lastChild->nodeType == XML_TEXT_NODE) { - my $text = $node->lastChild->nodeValue; - $text =~ s/\n( *\n)+$/\n/; - if ($text eq '') { - $node->removeChild($node->lastChild); - } else { $node->lastChild->setData($text); } }