RFC T157418: Trim whitespace in table cells, list items, headings

* Matmarex had implemented this for wikitext headings in b3dd3881.
* This patch extends this to wikitext list items and wikitext table cells.
* Updated RELEASE NOTES.

tests/parser/parserTests.txt:
* All whitespace removed in output of list items, table cells, and
  headings. Removed corresponding whitespace in the input wikitext
  except for a few tests where the whitespace is significant "| +"
  or "| -", for example.
* Updated output of html/parsoid sections as well.
* Added new tests to spec white-space trimming behavior.

tests/phpunit/*:
* Fixed a few tests that used whitespace in list items and table cells.

Bug: T157418
Change-Id: I8ea34c7ab893c0c125c81d810feeb3c581e4bba1

RELEASE-NOTES-1.31

@@ -293,6 +293,10 @@ changes to languages because of Phabricator reports.
   can use ApiOpenSearch::getOpenSearchTemplate() instead.
 * The global function wfBaseConvert, deprecated in 1.27, has been removed. Use
   Wikimedia\base_convert() directly.
+* RFC 157418: Whitespace is trimmed from wikitext headings, wikitext list items,
+  wikitext table captions, wikitext table headings, wikitext table cells. HTML
+  headings, HTML list items, HTML table captions, HTML table headings, HTML table cells
+  will not have this trimming behavior.
 
 == Compatibility ==
 MediaWiki 1.31 requires PHP 5.5.9 or later. Although HHVM 3.18.5 or later is supported,

includes/parser/BlockLevelPass.php

@@ -236,7 +236,8 @@ class BlockLevelPass {
 					$term = $t2 = '';
 					if ( $this->findColonNoLinks( $t, $term, $t2 ) !== false ) {
 						$t = $t2;
-						$output .= $term . $this->nextItem( ':' );
+						// Trim whitespace in list items
+						$output .= trim( $term ) . $this->nextItem( ':' );
 					}
 				}
 			} elseif ( $prefixLength || $lastPrefixLength ) {
@@ -274,7 +275,8 @@ class BlockLevelPass {
 						# @todo FIXME: This is dupe of code above
 						if ( $this->findColonNoLinks( $t, $term, $t2 ) !== false ) {
 							$t = $t2;
-							$output .= $term . $this->nextItem( ':' );
+							// Trim whitespace in list items
+							$output .= trim( $term ) . $this->nextItem( ':' );
 						}
 					}
 					++$commonPrefixLength;
@@ -371,9 +373,12 @@ class BlockLevelPass {
 				$this->inPre = false;
 			}
 			if ( $pendingPTag === false ) {
-				$output .= $t;
 				if ( $prefixLength === 0 ) {
+					$output .= $t;
 					$output .= "\n";
+				} else {
+					// Trim whitespace in list items
+					$output .= trim( $t );
 				}
 			}
 		}

includes/parser/Parser.php

@@ -1113,7 +1113,11 @@ class Parser {
 					$line = "</{$last_tag}>{$line}";
 				}
 				array_pop( $tr_attributes );
-				$outLine = $line . str_repeat( '</dd></dl>', $indent_level );
+				if ( $indent_level > 0 ) {
+					$outLine = rtrim( $line ) . str_repeat( '</dd></dl>', $indent_level );
+				} else {
+					$outLine = $line;
+				}
 			} elseif ( $first_two === '|-' ) {
 				# Now we have a table row
 				$line = preg_replace( '#^\|-+#', '', $line );
@@ -1204,13 +1208,15 @@ class Parser {
 					# be mistaken as delimiting cell parameters
 					# Bug T153140: Neither should language converter markup.
 					if ( preg_match( '/\[\[|-\{/', $cell_data[0] ) === 1 ) {
-						$cell = "{$previous}<{$last_tag}>{$cell}";
+						$cell = "{$previous}<{$last_tag}>" . trim( $cell );
 					} elseif ( count( $cell_data ) == 1 ) {
-						$cell = "{$previous}<{$last_tag}>{$cell_data[0]}";
+						// Whitespace in cells is trimmed
+						$cell = "{$previous}<{$last_tag}>" . trim( $cell_data[0] );
 					} else {
 						$attributes = $this->mStripState->unstripBoth( $cell_data[0] );
 						$attributes = Sanitizer::fixTagAttributes( $attributes, $last_tag );
-						$cell = "{$previous}<{$last_tag}{$attributes}>{$cell_data[1]}";
+						// Whitespace in cells is trimmed
+						$cell = "{$previous}<{$last_tag}{$attributes}>" . trim( $cell_data[1] );
 					}
 
 					$outLine .= $cell;
@@ -4050,6 +4056,7 @@ class Parser {
 
 		# Get all headlines for numbering them and adding funky stuff like [edit]
 		# links - this is for later, but we need the number of headlines right now
+		# This regexp also trims whitespace in the heading's content
 		$matches = [];
 		$numMatches = preg_match_all(
 			'/<H(?P<level>[1-6])(?P<attrib>.*?>)\s*(?P<header>[\s\S]*?)\s*<\/H[1-6] *>/i',

tests/phpunit/includes/StatusTest.php

@@ -401,8 +401,8 @@ class StatusTest extends MediaWikiLangTestCase {
 			$status,
 			"* ⧼fooBar!⧽\n* ⧼fooBar2!⧽\n",
 			"(wrap-long: * (fooBar!)\n* (fooBar2!)\n)",
-			"<ul><li> ⧼fooBar!⧽</li>\n<li> ⧼fooBar2!⧽</li></ul>\n",
-			"<p>(wrap-long: * (fooBar!)\n</p>\n<ul><li> (fooBar2!)</li></ul>\n<p>)\n</p>",
+			"<ul><li>⧼fooBar!⧽</li>\n<li>⧼fooBar2!⧽</li></ul>\n",
+			"<p>(wrap-long: * (fooBar!)\n</p>\n<ul><li>(fooBar2!)</li></ul>\n<p>)\n</p>",
 		];
 
 		$status = new Status();
@@ -422,8 +422,8 @@ class StatusTest extends MediaWikiLangTestCase {
 			$status,
 			"* ⧼fooBar!⧽\n* ⧼fooBar2!⧽\n",
 			"(wrap-long: * (fooBar!: foo, bar)\n* (fooBar2!)\n)",
-			"<ul><li> ⧼fooBar!⧽</li>\n<li> ⧼fooBar2!⧽</li></ul>\n",
-			"<p>(wrap-long: * (fooBar!: foo, bar)\n</p>\n<ul><li> (fooBar2!)</li></ul>\n<p>)\n</p>",
+			"<ul><li>⧼fooBar!⧽</li>\n<li>⧼fooBar2!⧽</li></ul>\n",
+			"<p>(wrap-long: * (fooBar!: foo, bar)\n</p>\n<ul><li>(fooBar2!)</li></ul>\n<p>)\n</p>",
 		];
 
 		return $testCases;

tests/phpunit/includes/content/WikitextStructureTest.php

@@ -104,7 +104,7 @@ END;
 		$this->assertEquals( "Opening text is opening.", $struct->getOpeningText() );
 		$this->assertEquals( "Opening text is opening.   Then we got more text",
 			$struct->getMainText() );
-		$this->assertEquals( [ "Header table  row in table  another row in table" ],
+		$this->assertEquals( [ "Header table row in table another row in table" ],
 			$struct->getAuxiliaryText() );
 	}
 }