Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generate TOC HTML on demand in ParserOutput::getText()

Browse source 
* Rather than computing TOC HTML in Parser and setting it in
  ParserOutput, compute it on demand based on section metadata.

  This will let Parsoid set section metadata in ParserOutput
  and have the TOC generated automatically.

* This required fixing some "bugs" in Linker's generateTOC
  which didn't properly close tags and relied on Tidy to fix
  up unclosed li and ul tags.

* This patch relies on converting section metadata objects to
  array objects, but Linker::generateTOC could be converted to
  use TOC data instead.

* Since TOC generation is now moved to getText(), this is done
  post-PC load and this eliminates the parser cache split on
  user language for TOC heading localization.

Bug: T293513
Change-Id: Ief1bba326d3612b40930440c872a61abadffab10
This commit is contained in:
C. Scott Ananian 2023-01-19 15:37:16 -05:00
parent 90079c7499
commit 439656e019
4 changed files with 147 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
includes/linker/Linker.php
View file

@ -49,6 +49,7 @@ use TitleValue;
use User;
use WatchedItem;
use Wikimedia\IPUtils;
use Wikimedia\Parsoid\Core\SectionMetadata;
use Wikimedia\Rdbms\SelectQueryBuilder;
use Xml;
@ -1685,12 +1686,16 @@ class Linker {
* @param string $tocline
* @param string $tocnumber
* @param int $level
* @param int|false $sectionIndex
* @param string|false $sectionIndex
* @return string
*/
public static function tocLine( $linkAnchor, $tocline, $tocnumber, $level, $sectionIndex = false ) {
$classes = "toclevel-$level";
if ( $sectionIndex !== false ) {
// Parser.php used to suppress tocLine by setting $sectionindex to false.
// In those circumstances, we can now encounter '' or a "T-" prefixed index
// for when the section comes from templates.
if ( $sectionIndex !== false && $sectionIndex !== '' && !str_starts_with( $sectionIndex, "T-" ) ) {
$classes .= " tocsection-$sectionIndex";
}
@ -1758,29 +1763,43 @@ class Linker {
* Generate a table of contents from a section tree.
*
* @since 1.16.3. $lang added in 1.17
* @param array[] $tree Return value of ParserOutput::getSections()
* @param array[]|SectionMetadata[] $tree Return value of ParserOutput::getSections()
* @param Language|null $lang Language for the toc title, defaults to user language
* @param array $options FIXME: Document
* @return string HTML fragment
*/
public static function generateTOC( $tree, Language $lang = null ) {
public static function generateTOC( $tree, Language $lang = null, array $options = [] ): string {
$toc = '';
$lastLevel = 0;
$maxTocLevel = $options['maxtoclevel'] ?? null;
foreach ( $tree as $section ) {
if ( $section['toclevel'] > $lastLevel ) {
$toc .= self::tocIndent();
} elseif ( $section['toclevel'] < $lastLevel ) {
$toc .= self::tocUnindent(
$lastLevel - $section['toclevel'] );
} else {
$toc .= self::tocLineEnd();
if ( $section instanceof SectionMetadata ) {
$section = $section->toLegacy();
}
$tocLevel = $section['toclevel'];
if ( $maxTocLevel !== null && $tocLevel < $maxTocLevel ) {
if ( $tocLevel > $lastLevel ) {
$toc .= self::tocIndent();
} elseif ( $tocLevel < $lastLevel ) {
if ( $lastLevel < $maxTocLevel ) {
$toc .= self::tocUnindent(
$lastLevel - $tocLevel );
} else {
$toc .= self::tocLineEnd();
}
} else {
$toc .= self::tocLineEnd();
}
$toc .= self::tocLine( $section['linkAnchor'],
$section['line'], $section['number'],
$section['toclevel'], $section['index'] );
$lastLevel = $section['toclevel'];
$toc .= self::tocLine( $section['linkAnchor'],
$section['line'], $section['number'],
$tocLevel, $section['index'] );
$lastLevel = $tocLevel;
}
}
if ( $lastLevel < $maxTocLevel && $lastLevel > 0 ) {
$toc .= self::tocUnindent( $lastLevel - 1 );
}
$toc .= self::tocLineEnd();
return self::tocList( $toc, $lang );
}

41
includes/parser/Parser.php
View file

@ -4225,11 +4225,10 @@ class Parser {
# headline counter
$headlineCount = 0;
$numVisible = 0;
$haveTocEntries = false;
# Ugh .. the TOC should have neat indentation levels which can be
# passed to the skin functions. These are determined here
$toc = '';
$full = '';
$head = [];
$sublevelCount = [];
@ -4237,7 +4236,6 @@ class Parser {
$level = 0;
$prevlevel = 0;
$toclevel = 0;
$prevtoclevel = 0;
$markerRegex = self::MARKER_PREFIX . "-h-(\d+)-" . self::MARKER_SUFFIX;
$baseTitleText = $this->getTitle()->getPrefixedDBkey();
$oldType = $this->mOutputType;
@ -4275,9 +4273,7 @@ class Parser {
$toclevel++;
$sublevelCount[$toclevel] = 0;
if ( $toclevel < $maxTocLevel ) {
$prevtoclevel = $toclevel;
$toc .= Linker::tocIndent();
$numVisible++;
$haveTocEntries = true;
}
} elseif ( $level < $prevlevel && $toclevel > 1 ) {
# Decrease TOC level, find level to jump to
@ -4298,20 +4294,6 @@ class Parser {
if ( $i == 0 ) {
$toclevel = 1;
}
if ( $toclevel < $maxTocLevel ) {
if ( $prevtoclevel < $maxTocLevel ) {
# Unindent only if the previous toc level was shown :p
$toc .= Linker::tocUnindent( $prevtoclevel - $toclevel );
$prevtoclevel = $toclevel;
} else {
$toc .= Linker::tocLineEnd();
}
}
} else {
# No change in level, end TOC line
if ( $toclevel < $maxTocLevel ) {
$toc .= Linker::tocLineEnd();
}
}
$levelCount[$toclevel] = $level;
@ -4422,16 +4404,6 @@ class Parser {
$refers[$fallbackArrayKey] = true;
}
if ( $enoughToc && ( !isset( $maxTocLevel ) || $toclevel < $maxTocLevel ) ) {
$toc .= Linker::tocLine(
$linkAnchor,
$tocline,
$numbering,
$toclevel,
( $isTemplate ? false : $sectionIndex )
);
}
# Add the section to the section tree
# Find the DOM node for this header
$noOffset = ( $isTemplate || $sectionIndex === false );
@ -4454,7 +4426,7 @@ class Parser {
$level,
$tocline,
$numbering,
( $isTemplate ? 'T-' : '' ) . $sectionIndex,
$sectionIndex === false ? '' : ( ( $isTemplate ? 'T-' : '' ) . $sectionIndex ),
$titleText ?: null,
( $noOffset ? null : $byteOffset ),
$anchor,
@ -4508,16 +4480,11 @@ class Parser {
# Never ever show TOC if no headers (or suppressed)
$suppressToc = $this->mOptions->getSuppressTOC();
if ( $numVisible < 1 || $suppressToc ) {
if ( !$haveTocEntries || $suppressToc ) {
$enoughToc = false;
}
if ( $enoughToc ) {
if ( $prevtoclevel > 0 && $prevtoclevel < $maxTocLevel ) {
$toc .= Linker::tocUnindent( $prevtoclevel - 1 );
}
$toc = Linker::tocList( $toc, $this->mOptions->getUserLangObj() );
$this->mOutput->setTOCHTML( $toc );
// Record the fact that the TOC should be shown. T294950
// (We shouldn't be looking at ::getTOCHTML() for this because
// eventually that will be replaced (T293513) and

79
includes/parser/ParserOutput.php
View file

@ -382,6 +382,9 @@ class ParserOutput extends CacheTime implements ContentMetadataCollector {
* - enableSectionEditLinks: (bool) Include section edit links, assuming
* section edit link tokens are present in the HTML. Default is true,
* but might be statefully overridden.
* - userLang: (Language) Language object used for localizing UX messages,
* for example the heading of the table of contents. If omitted, will
* use the language of the main request context.
* - skin: (Skin) Skin object used for transforming section edit links.
* - unwrap: (bool) Return text without a wrapper div. Default is false,
* meaning a wrapper div will be added if getWrapperDivClass() returns
@ -406,6 +409,7 @@ class ParserOutput extends CacheTime implements ContentMetadataCollector {
'injectTOC' => true,
'enableSectionEditLinks' => true,
'skin' => null,
'userLang' => null,
'unwrap' => false,
'deduplicateStyles' => true,
'wrapperDivClass' => $this->getWrapperDivClass(),
@ -466,37 +470,46 @@ class ParserOutput extends CacheTime implements ContentMetadataCollector {
if ( $options['allowTOC'] ) {
if ( $options['injectTOC'] ) {
$toc = $this->getTOCHTML();
// language conversion needs to be done on the TOC fetched
// from parser cache
if ( !$this->getOutputFlag( ParserOutputFlags::NO_TOC_CONVERSION ) ) {
// XXX Use DI to inject this once ::getText() is moved out
// of ParserOutput
if ( count( $this->getSections() ) === 0 ) {
$toc = '';
} else {
$services = MediaWikiServices::getInstance();
$languageFactory =
$services->getLanguageFactory();
$languageConverterFactory =
$services->getLanguageConverterFactory();
// T303329: this should migrate out of extension data
$langCode = $this->getExtensionData( 'core:target-lang' )
// This is a temporary fallback while the ParserCache fills
?? $services->getContentLanguage()->getCode();
$langConv = $languageConverterFactory->getLanguageConverter(
$languageFactory->getLanguage( $langCode )
);
$variant = $this->getExtensionData( 'core:target-lang-variant' )
// This is a temporary fallback while the ParserCache fills
?? $langConv->getPreferredVariant();
$toc = $langConv->convertTo( $toc, $variant );
}
$userLang = $options['userLang'];
$skin = $options['skin'];
if ( ( !$userLang ) && $skin ) {
// TODO: See above comment about replacing the use
// of 'skin' here.
$userLang = $skin->getLanguage();
}
if ( !$userLang ) {
$userLang = RequestContext::getMain()->getLanguage();
}
$config = $services->getMainConfig();
$maxTocLevel = $config->get( MainConfigNames::MaxTocLevel );
$toc = Linker::generateTOC(
$this->getSections(), $userLang, [ "maxtoclevel" => $maxTocLevel ] );
// language conversion needs to be done on the TOC fetched
// from parser cache
if ( !$this->getOutputFlag( ParserOutputFlags::NO_TOC_CONVERSION ) ) {
$languageFactory = $services->getLanguageFactory();
$languageConverterFactory = $services->getLanguageConverterFactory();
// T303329: this should migrate out of extension data
$langCode = $this->getExtensionData( 'core:target-lang' )
// This is a temporary fallback while the ParserCache fills
?? $services->getContentLanguage()->getCode();
$langConv = $languageConverterFactory->getLanguageConverter(
$languageFactory->getLanguage( $langCode )
);
$variant = $this->getExtensionData( 'core:target-lang-variant' )
// This is a temporary fallback while the ParserCache fills
?? $langConv->getPreferredVariant();
$toc = $langConv->convertTo( $toc, $variant );
}
// XXX Use DI to inject this once ::getText() is moved out
// of ParserOutput.
$tidy = MediaWikiServices::getInstance()->getTidy();
$toc = $tidy->tidy(
$toc,
[ Sanitizer::class, 'armorFrenchSpaces' ]
);
// XXX Use DI to inject this once ::getText() is moved out of ParserOutput.
$toc = $services->getTidy()->tidy( $toc, [ Sanitizer::class, 'armorFrenchSpaces' ] );
}
$this->mTOCHTML = $toc;
$text = Parser::replaceTableOfContentsMarker( $text, $toc );
// The line below can be removed once old content has expired
// from the parser cache
@ -946,6 +959,14 @@ class ParserOutput extends CacheTime implements ContentMetadataCollector {
return $old;
}
/**
* @internal
* @deprecated since 1.40
* T293513: We can remove this once we get rid of MW 1.38 and older
* parsercache serialization tests since those serialized
* files have artificial TOC data (which we cannot replicate
* via on-demand TOC generation).
*/
public function setTOCHTML( $tochtml ) {
return wfSetVar( $this->mTOCHTML, $tochtml );
}

89
tests/phpunit/includes/parser/ParserOutputTest.php
View file

@ -4,6 +4,7 @@ use MediaWiki\MainConfigNames;
use MediaWiki\Page\PageReferenceValue;
use MediaWiki\Tests\Parser\ParserCacheSerializationTestCases;
use Wikimedia\Parsoid\Core\SectionMetadata;
use Wikimedia\Parsoid\Core\TOCData;
use Wikimedia\TestingAccessWrapper;
use Wikimedia\Tests\SerializationTestTrait;
@ -256,32 +257,55 @@ class ParserOutputTest extends MediaWikiLangTestCase {
] );
$po = new ParserOutput( $text );
$po->setTOCHTML( self::provideGetTextToC() );
self::initSections( $po );
$actual = $po->getText( $options );
$this->assertSame( $expect, $actual );
}
public static function provideGetTextToC() {
$toc = <<<EOF
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
<ul>
<li class="toclevel-2 tocsection-3"><a href="#Section_2.1"><span class="tocnumber">2.1</span> <span class="toctext">Section 2.1</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-4"><a href="#Section_3"><span class="tocnumber">3</span> <span class="toctext">Section 3</span></a></li>
</ul>
</div>
EOF;
return $toc;
private static function initSections( ParserOutput $po ): void {
$po->setTOCData( new TOCData(
SectionMetadata::fromLegacy( [
'index' => "1",
'level' => 1,
'toclevel' => 1,
'number' => "1",
'line' => "Section 1",
'anchor' => "Section_1"
] ),
SectionMetadata::fromLegacy( [
'index' => "2",
'level' => 1,
'toclevel' => 1,
'number' => "2",
'line' => "Section 2",
'anchor' => "Section_2"
] ),
SectionMetadata::fromLegacy( [
'index' => "3",
'level' => 2,
'toclevel' => 2,
'number' => "2.1",
'line' => "Section 2.1",
'anchor' => "Section_2.1"
] ),
SectionMetadata::fromLegacy( [
'index' => "4",
'level' => 1,
'toclevel' => 1,
'number' => "3",
'line' => "Section 3",
'anchor' => "Section_3"
] ),
) );
}
// REMOVE THIS ONCE Parser::TOC_START IS REMOVED
public static function provideGetTextBackCompat() {
$toc = self::provideGetTextToC();
$dummyPO = new ParserOutput( '' );
self::initSections( $dummyPO );
$dummyPO->getText(); // force TOC generation
$toc = $dummyPO->getTOCHTML();
$text = <<<EOF
<p>Test document.
</p>
@ -305,7 +329,7 @@ EOF;
[], $text, <<<EOF
<p>Test document.
</p>
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
@ -335,7 +359,7 @@ EOF
[ 'enableSectionEditLinks' => false ], $text, <<<EOF
<p>Test document.
</p>
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
@ -387,7 +411,6 @@ EOF
// REMOVE THIS ONCE ParserCache is transitioned to <meta> placeholder
public static function provideGetTextBackCompat2() {
// phpcs:disable Generic.Files.LineLength
$toc = self::provideGetTextToC();
$text = <<<EOF
<p>Test document.
</p>
@ -411,7 +434,7 @@ EOF;
[], $text, <<<EOF
<p>Test document.
</p>
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
@ -441,7 +464,7 @@ EOF
[ 'enableSectionEditLinks' => false ], $text, <<<EOF
<p>Test document.
</p>
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
@ -491,7 +514,6 @@ EOF
}
public static function provideGetText() {
$toc = self::provideGetTextToC();
$text = <<<EOF
<p>Test document.
</p>
@ -528,7 +550,7 @@ EOF;
[], $text, <<<EOF
<p>Test document.
</p>
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
@ -558,7 +580,7 @@ EOF
[ 'enableSectionEditLinks' => false ], $text, <<<EOF
<p>Test document.
</p>
<div id="toc" class="toc"><div class="toctitle"><h2>Contents</h2></div>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="#Section_1"><span class="tocnumber">1</span> <span class="toctext">Section 1</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="#Section_2"><span class="tocnumber">2</span> <span class="toctext">Section 2</span></a>
@ -817,16 +839,21 @@ EOF
] ];
// TOC ------------
$a = new ParserOutput();
$a->setTOCHTML( '<p>TOC A</p>' );
$a = new ParserOutput( '' );
$a->setSections( [ [ 'fromtitle' => 'A1' ], [ 'fromtitle' => 'A2' ] ] );
$a->getText(); // force TOC
$b = new ParserOutput();
$b->setTOCHTML( '<p>TOC B</p>' );
$b = new ParserOutput( '' );
$b->setSections( [ [ 'fromtitle' => 'B1' ], [ 'fromtitle' => 'B2' ] ] );
$b->getText(); // force TOC
$emptyTOC = '<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>' . "\n" .
'<li class="toclevel-0"><a href="#"><span class="tocnumber"></span> <span class="toctext"></span></a></li>' . "\n" .
'<li class="toclevel-0"><a href="#"><span class="tocnumber"></span> <span class="toctext"></span></a>' . "\n" .
"</li></div>\n";
yield 'concat TOC' => [ $a, $b, [
'getTOCHTML' => '<p>TOC A</p><p>TOC B</p>',
'getTOCHTML' => $emptyTOC . $emptyTOC,
'getSections' => [
SectionMetadata::fromLegacy( [ 'fromtitle' => 'A1' ] )->toLegacy(),
SectionMetadata::fromLegacy( [ 'fromtitle' => 'A2' ] )->toLegacy(),