parser: convert Preprocessor to WANCache and inject dependencies
Make the caching logic use getWithSetCallback() and simplify the code given that there is only one Preprocessor subclass. Also, keep the cached values JSON serializable but rely on the serialization in BagOStuff instead for simplicity. Add related class constants for injecting preprocessor flags. Bug: T254608 Change-Id: I72f9f0c0bc352ed5120469090c71294ff0c24999
This commit is contained in:
parent
3ef5ebd504
commit
de6eeead21
6 changed files with 169 additions and 225 deletions
|
|
@ -340,6 +340,7 @@ class PPNode_Hash_Tree implements PPNode {
|
|||
* Like splitTemplate() but for a raw child array. For internal use only.
|
||||
* @param array $children
|
||||
* @return array
|
||||
* @suppress SecurityCheck-XSS
|
||||
*/
|
||||
public static function splitRawTemplate( array $children ) {
|
||||
$parts = [];
|
||||
|
|
|
|||
|
|
@ -112,8 +112,11 @@ class Parser {
|
|||
# Regular expression for a non-newline space
|
||||
private const SPACE_NOT_NL = '(?:\t| |&\#0*160;|&\#[Xx]0*[Aa]0;|\p{Zs})';
|
||||
|
||||
# Flags for preprocessToDom
|
||||
public const PTD_FOR_INCLUSION = 1;
|
||||
/**
|
||||
* @var int Preprocess wikitext in transclusion mode
|
||||
* @deprecated Since 1.36
|
||||
*/
|
||||
public const PTD_FOR_INCLUSION = Preprocessor::DOM_FOR_INCLUSION;
|
||||
|
||||
# Allowed values for $this->mOutputType
|
||||
# Parameter to startExternalParse().
|
||||
|
|
@ -381,6 +384,8 @@ class Parser {
|
|||
'Sitename',
|
||||
'StylePath',
|
||||
'TranscludeCacheExpiry',
|
||||
'PreprocessorCacheThreshold',
|
||||
'DisableLangConversion'
|
||||
];
|
||||
|
||||
/**
|
||||
|
|
@ -957,7 +962,7 @@ class Parser {
|
|||
$this->startParse( $title, $options, self::OT_PLAIN, true );
|
||||
|
||||
$flags = PPFrame::NO_ARGS | PPFrame::NO_TEMPLATES;
|
||||
$dom = $this->preprocessToDom( $text, self::PTD_FOR_INCLUSION );
|
||||
$dom = $this->preprocessToDom( $text, Preprocessor::DOM_FOR_INCLUSION );
|
||||
$text = $this->getPreprocessor()->newFrame()->expand( $dom, $flags );
|
||||
$text = $this->mStripState->unstripBoth( $text );
|
||||
return $text;
|
||||
|
|
@ -1143,8 +1148,16 @@ class Parser {
|
|||
*/
|
||||
public function getPreprocessor() {
|
||||
if ( !isset( $this->mPreprocessor ) ) {
|
||||
$this->mPreprocessor = new Preprocessor_Hash( $this );
|
||||
$this->mPreprocessor = new Preprocessor_Hash(
|
||||
$this,
|
||||
MediaWikiServices::getInstance()->getMainWANObjectCache(),
|
||||
[
|
||||
'cacheThreshold' => $this->svcOptions->get( 'PreprocessorCacheThreshold' ),
|
||||
'disableLangConversion' => $this->svcOptions->get( 'DisableLangConversion' )
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
return $this->mPreprocessor;
|
||||
}
|
||||
|
||||
|
|
@ -1536,7 +1549,7 @@ class Parser {
|
|||
if ( !$frame->depth ) {
|
||||
$flag = 0;
|
||||
} else {
|
||||
$flag = self::PTD_FOR_INCLUSION;
|
||||
$flag = Preprocessor::DOM_FOR_INCLUSION;
|
||||
}
|
||||
$dom = $this->preprocessToDom( $text, $flag );
|
||||
$text = $frame->expand( $dom );
|
||||
|
|
@ -2804,30 +2817,24 @@ class Parser {
|
|||
}
|
||||
|
||||
/**
|
||||
* Preprocess some wikitext and return the document tree.
|
||||
* This is the ghost of replace_variables().
|
||||
* Get the document object model for the given wikitext
|
||||
*
|
||||
* @param string $text The text to parse
|
||||
* @param int $flags Bitwise combination of:
|
||||
* - self::PTD_FOR_INCLUSION: Handle "<noinclude>" and "<includeonly>" as if the text is being
|
||||
* included. Default is to assume a direct page view.
|
||||
* @see Preprocessor::preprocessToObj()
|
||||
*
|
||||
* The generated DOM tree must depend only on the input text and the flags.
|
||||
* The DOM tree must be the same in OT_HTML and OT_WIKI mode, to avoid a regression of T6899.
|
||||
* The DOM tree must be the same in OT_HTML and OT_WIKI mode, to avoid a
|
||||
* regression of T6899.
|
||||
*
|
||||
* Any flag added to the $flags parameter here, or any other parameter liable to cause a
|
||||
* change in the DOM tree for a given text, must be passed through the section identifier
|
||||
* in the section edit link and thus back to extractSections().
|
||||
*
|
||||
* The output of this function is currently only cached in process memory, but a persistent
|
||||
* cache may be implemented at a later date which takes further advantage of these strict
|
||||
* dependency requirements.
|
||||
*
|
||||
* @param string $text Wikitext
|
||||
* @param int $flags Bit field of Preprocessor::DOM_* constants
|
||||
* @return PPNode
|
||||
*/
|
||||
public function preprocessToDom( $text, $flags = 0 ) {
|
||||
$dom = $this->getPreprocessor()->preprocessToObj( $text, $flags );
|
||||
return $dom;
|
||||
return $this->getPreprocessor()->preprocessToObj( $text, $flags );
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -3178,7 +3185,7 @@ class Parser {
|
|||
} else {
|
||||
$text = $this->interwikiTransclude( $title, 'raw' );
|
||||
# Preprocess it like a template
|
||||
$text = $this->preprocessToDom( $text, self::PTD_FOR_INCLUSION );
|
||||
$text = $this->preprocessToDom( $text, Preprocessor::DOM_FOR_INCLUSION );
|
||||
$isChildObj = true;
|
||||
}
|
||||
$found = true;
|
||||
|
|
@ -3410,7 +3417,7 @@ class Parser {
|
|||
return [ false, $title ];
|
||||
}
|
||||
|
||||
$dom = $this->preprocessToDom( $text, self::PTD_FOR_INCLUSION );
|
||||
$dom = $this->preprocessToDom( $text, Preprocessor::DOM_FOR_INCLUSION );
|
||||
$this->mTplDomCache[$titleText] = $dom;
|
||||
|
||||
if ( !$title->equals( $cacheTitle ) ) {
|
||||
|
|
@ -5551,7 +5558,7 @@ class Parser {
|
|||
$sectionIndex = array_pop( $sectionParts );
|
||||
foreach ( $sectionParts as $part ) {
|
||||
if ( $part === 'T' ) {
|
||||
$flags |= self::PTD_FOR_INCLUSION;
|
||||
$flags |= Preprocessor::DOM_FOR_INCLUSION;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,24 +21,27 @@
|
|||
* @ingroup Parser
|
||||
*/
|
||||
|
||||
use MediaWiki\Logger\LoggerFactory;
|
||||
use MediaWiki\MediaWikiServices;
|
||||
|
||||
/**
|
||||
* @ingroup Parser
|
||||
*/
|
||||
abstract class Preprocessor {
|
||||
/** Transclusion mode flag for Preprocessor::preprocessToObj() */
|
||||
public const DOM_FOR_INCLUSION = 1;
|
||||
/** Language conversion construct omission flag for Preprocessor::preprocessToObj() */
|
||||
public const DOM_LANG_CONVERSION_DISABLED = 2;
|
||||
/** Preprocessor cache bypass flag for Preprocessor::preprocessToObj */
|
||||
public const DOM_UNCACHED = 4;
|
||||
|
||||
public const CACHE_VERSION = 1;
|
||||
|
||||
/**
|
||||
* @var Parser
|
||||
*/
|
||||
/** @var Parser */
|
||||
public $parser;
|
||||
|
||||
/**
|
||||
* @var array Brace matching rules.
|
||||
*/
|
||||
/** @var WANObjectCache */
|
||||
protected $wanCache;
|
||||
|
||||
/** @var bool Whether language variant conversion is disabled */
|
||||
protected $disableLangConversion;
|
||||
|
||||
/** @var array Brace matching rules */
|
||||
protected $rules = [
|
||||
'{' => [
|
||||
'end' => '}',
|
||||
|
|
@ -64,76 +67,19 @@ abstract class Preprocessor {
|
|||
];
|
||||
|
||||
/**
|
||||
* Store a document tree in the cache.
|
||||
*
|
||||
* @param string $text
|
||||
* @param int $flags
|
||||
* @param string $tree
|
||||
* @param Parser $parser
|
||||
* @param WANObjectCache|null $wanCache
|
||||
* @param array $options Map of additional options, including:
|
||||
* - disableLangConversion: disable language variant conversion. [Default: false]
|
||||
*/
|
||||
protected function cacheSetTree( $text, $flags, $tree ) {
|
||||
$config = RequestContext::getMain()->getConfig();
|
||||
|
||||
$length = strlen( $text );
|
||||
$threshold = $config->get( 'PreprocessorCacheThreshold' );
|
||||
if ( $threshold === false || $length < $threshold || $length > 1e6 ) {
|
||||
return;
|
||||
}
|
||||
|
||||
$cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
|
||||
$key = $cache->makeKey(
|
||||
// @phan-suppress-next-line PhanUndeclaredConstantOfClass
|
||||
defined( 'static::CACHE_PREFIX' ) ? static::CACHE_PREFIX : static::class,
|
||||
md5( $text ),
|
||||
$flags
|
||||
);
|
||||
$value = sprintf( "%08d", static::CACHE_VERSION ) . $tree;
|
||||
|
||||
$cache->set( $key, $value, 86400 );
|
||||
|
||||
LoggerFactory::getInstance( 'Preprocessor' )
|
||||
->info( "Cached preprocessor output (key: $key)" );
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to load a precomputed document tree for some given wikitext
|
||||
* from the cache.
|
||||
*
|
||||
* @param string $text
|
||||
* @param int $flags
|
||||
* @return PPNode_Hash_Tree|bool
|
||||
*/
|
||||
protected function cacheGetTree( $text, $flags ) {
|
||||
$config = RequestContext::getMain()->getConfig();
|
||||
|
||||
$length = strlen( $text );
|
||||
$threshold = $config->get( 'PreprocessorCacheThreshold' );
|
||||
if ( $threshold === false || $length < $threshold || $length > 1e6 ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
|
||||
|
||||
$key = $cache->makeKey(
|
||||
// @phan-suppress-next-line PhanUndeclaredConstantOfClass
|
||||
defined( 'static::CACHE_PREFIX' ) ? static::CACHE_PREFIX : static::class,
|
||||
md5( $text ),
|
||||
$flags
|
||||
);
|
||||
|
||||
$value = $cache->get( $key );
|
||||
if ( !$value ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$version = intval( substr( $value, 0, 8 ) );
|
||||
if ( $version !== static::CACHE_VERSION ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
LoggerFactory::getInstance( 'Preprocessor' )
|
||||
->info( "Loaded preprocessor output from cache (key: $key)" );
|
||||
|
||||
return substr( $value, 8 );
|
||||
public function __construct(
|
||||
Parser $parser,
|
||||
WANObjectCache $wanCache = null,
|
||||
array $options = []
|
||||
) {
|
||||
$this->parser = $parser;
|
||||
$this->wanCache = $wanCache ?: WANObjectCache::newEmpty();
|
||||
$this->disableLangConversion = !empty( $options['disableLangConversion'] );
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -145,29 +91,41 @@ abstract class Preprocessor {
|
|||
|
||||
/**
|
||||
* Create a new custom frame for programmatic use of parameter replacement
|
||||
* as used in some extensions.
|
||||
*
|
||||
* This is useful for certain types of extensions
|
||||
*
|
||||
* @param array $args
|
||||
*
|
||||
* @return PPFrame
|
||||
*/
|
||||
abstract public function newCustomFrame( $args );
|
||||
|
||||
/**
|
||||
* Create a new custom node for programmatic use of parameter replacement
|
||||
* as used in some extensions.
|
||||
*
|
||||
* This is useful for certain types of extensions
|
||||
*
|
||||
* @param array $values
|
||||
*/
|
||||
abstract public function newPartNodeArray( $values );
|
||||
|
||||
/**
|
||||
* Preprocess text to a PPNode
|
||||
* Get the document object model for the given wikitext
|
||||
*
|
||||
* @param string $text
|
||||
* @param int $flags
|
||||
* Any flag added to the $flags parameter here, or any other parameter liable to cause
|
||||
* a change in the DOM tree for the given wikitext, must be passed through the section
|
||||
* identifier in the section edit link and thus back to extractSections().
|
||||
*
|
||||
* @param string $text Wikitext
|
||||
* @param int $flags Bit field of Preprocessor::DOM_* flags:
|
||||
* - Preprocessor::DOM_FOR_INCLUSION: treat the wikitext as transcluded content from
|
||||
* a page rather than direct content of a page or message. By default, the text is
|
||||
* assumed to be undergoing processing for use by direct page views. The use of this
|
||||
* flag causes text within <noinclude> tags to be ignored, text within <includeonly>
|
||||
* to be included, and text outside of <onlyinclude> to be ignored.
|
||||
* - Preprocessor::DOM_NO_LANG_CONV: do not parse "-{ ... }-" constructs, which are
|
||||
* involved in language variant conversion. (deprecated since 1.36)
|
||||
* - Preprocessor::DOM_UNCACHED: disable use of the preprocessor cache.
|
||||
* @return PPNode
|
||||
*/
|
||||
abstract public function preprocessToObj( $text, $flags = 0 );
|
||||
abstract public function preprocessToObj( $text, $flags = 0 );
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,6 @@
|
|||
* @ingroup Parser
|
||||
*/
|
||||
|
||||
use MediaWiki\MediaWikiServices;
|
||||
|
||||
/**
|
||||
* Differences from DOM schema:
|
||||
* * attribute nodes are children
|
||||
|
|
@ -43,14 +41,27 @@ use MediaWiki\MediaWikiServices;
|
|||
*/
|
||||
// phpcs:ignore Squiz.Classes.ValidClassName.NotCamelCaps
|
||||
class Preprocessor_Hash extends Preprocessor {
|
||||
public const CACHE_PREFIX = 'preprocess-hash';
|
||||
public const CACHE_VERSION = 2;
|
||||
/** Cache format version */
|
||||
protected const CACHE_VERSION = 3;
|
||||
|
||||
/** @var int|bool Min wikitext size for which to cache DOM tree */
|
||||
protected $cacheThreshold;
|
||||
|
||||
/**
|
||||
* @see Preprocessor::__construct()
|
||||
* @param Parser $parser
|
||||
* @param WANObjectCache|null $wanCache
|
||||
* @param array $options Additional options include:
|
||||
* - cacheThreshold: min text size for which to cache DOMs. [Default: false]
|
||||
*/
|
||||
public function __construct( $parser ) {
|
||||
$this->parser = $parser;
|
||||
public function __construct(
|
||||
Parser $parser,
|
||||
WANObjectCache $wanCache = null,
|
||||
array $options = []
|
||||
) {
|
||||
parent::__construct( $parser, $wanCache, $options );
|
||||
|
||||
$this->cacheThreshold = $options['cacheThreshold'] ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -92,41 +103,43 @@ class Preprocessor_Hash extends Preprocessor {
|
|||
$list[] = new PPNode_Hash_Tree( $store, 0 );
|
||||
}
|
||||
|
||||
$node = new PPNode_Hash_Array( $list );
|
||||
return $node;
|
||||
return new PPNode_Hash_Array( $list );
|
||||
}
|
||||
|
||||
public function preprocessToObj( $text, $flags = 0 ) {
|
||||
if ( $this->disableLangConversion ) {
|
||||
// Language conversions are globally disabled; implicitly set flag
|
||||
$flags |= self::DOM_LANG_CONVERSION_DISABLED;
|
||||
}
|
||||
|
||||
if (
|
||||
$this->cacheThreshold !== false &&
|
||||
strlen( $text ) >= $this->cacheThreshold &&
|
||||
( $flags & self::DOM_UNCACHED ) != self::DOM_UNCACHED
|
||||
) {
|
||||
$domTreeArray = $this->wanCache->getWithSetCallback(
|
||||
$this->wanCache->makeKey( 'preprocess-hash', sha1( $text ), $flags ),
|
||||
$this->wanCache::TTL_DAY,
|
||||
function () use ( $text, $flags ) {
|
||||
return $this->buildDomTreeArrayFromText( $text, $flags );
|
||||
},
|
||||
[ 'version' => self::CACHE_VERSION ]
|
||||
);
|
||||
} else {
|
||||
$domTreeArray = $this->buildDomTreeArrayFromText( $text, $flags );
|
||||
}
|
||||
|
||||
return new PPNode_Hash_Tree( $domTreeArray, 0 );
|
||||
}
|
||||
|
||||
/**
|
||||
* Preprocess some wikitext and return the document tree.
|
||||
*
|
||||
* @param string $text The text to parse
|
||||
* @param int $flags Bitwise combination of:
|
||||
* Parser::PTD_FOR_INCLUSION Handle "<noinclude>" and "<includeonly>" as if the text is being
|
||||
* included. Default is to assume a direct page view.
|
||||
*
|
||||
* The generated DOM tree must depend only on the input text and the flags.
|
||||
* The DOM tree must be the same in OT_HTML and OT_WIKI mode, to avoid a regression of T6899.
|
||||
*
|
||||
* Any flag added to the $flags parameter here, or any other parameter liable to cause a
|
||||
* change in the DOM tree for a given text, must be passed through the section identifier
|
||||
* in the section edit link and thus back to extractSections().
|
||||
*
|
||||
* @throws MWException
|
||||
* @return PPNode_Hash_Tree
|
||||
* @param string $text Wikitext
|
||||
* @param int $flags Bit field of Preprocessor::DOM_* flags
|
||||
* @return array JSON-serializable document object model array
|
||||
*/
|
||||
public function preprocessToObj( $text, $flags = 0 ) {
|
||||
$isConversionDisabled = MediaWikiServices::getInstance()->getLanguageConverterFactory()
|
||||
->isConversionDisabled();
|
||||
|
||||
$tree = $this->cacheGetTree( $text, $flags );
|
||||
if ( $tree !== false ) {
|
||||
$store = json_decode( $tree );
|
||||
if ( is_array( $store ) ) {
|
||||
return new PPNode_Hash_Tree( $store, 0 );
|
||||
}
|
||||
}
|
||||
|
||||
$forInclusion = $flags & Parser::PTD_FOR_INCLUSION;
|
||||
private function buildDomTreeArrayFromText( $text, $flags ) {
|
||||
$forInclusion = ( $flags & self::DOM_FOR_INCLUSION );
|
||||
$langConversionDisabled = ( $flags & self::DOM_LANG_CONVERSION_DISABLED );
|
||||
|
||||
$xmlishElements = $this->parser->getStripList();
|
||||
$xmlishAllowMissingEndTag = [ 'includeonly', 'noinclude', 'onlyinclude' ];
|
||||
|
|
@ -153,7 +166,7 @@ class Preprocessor_Hash extends Preprocessor {
|
|||
$stack = new PPDStack_Hash;
|
||||
|
||||
$searchBase = "[{<\n";
|
||||
if ( !$isConversionDisabled ) {
|
||||
if ( !$langConversionDisabled ) {
|
||||
$searchBase .= '-';
|
||||
}
|
||||
|
||||
|
|
@ -182,8 +195,6 @@ class Preprocessor_Hash extends Preprocessor {
|
|||
$fakeLineStart = true;
|
||||
|
||||
while ( true ) {
|
||||
// $this->memCheck();
|
||||
|
||||
if ( $findOnlyinclude ) {
|
||||
// Ignore all input up to the next <onlyinclude>
|
||||
$startPos = strpos( $text, '<onlyinclude>', $i );
|
||||
|
|
@ -777,16 +788,7 @@ class Preprocessor_Hash extends Preprocessor {
|
|||
}
|
||||
}
|
||||
|
||||
$rootStore = [ [ 'root', $stack->rootAccum ] ];
|
||||
$rootNode = new PPNode_Hash_Tree( $rootStore, 0 );
|
||||
|
||||
// Cache
|
||||
$tree = json_encode( $rootStore, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE );
|
||||
if ( $tree !== false ) {
|
||||
$this->cacheSetTree( $text, $flags, $tree );
|
||||
}
|
||||
|
||||
return $rootNode;
|
||||
return [ [ 'root', $stack->rootAccum ] ];
|
||||
}
|
||||
|
||||
private static function addLiteral( array &$accum, $text ) {
|
||||
|
|
|
|||
|
|
@ -36,18 +36,11 @@ require_once __DIR__ . '/dumpIterator.php';
|
|||
* @ingroup Maintenance
|
||||
*/
|
||||
class PreprocessDump extends DumpIterator {
|
||||
/** @var Preprocessor|null */
|
||||
private $preprocessor;
|
||||
|
||||
/* Variables for dressing up as a parser */
|
||||
public $mTitle = 'PreprocessDump';
|
||||
public $mPPNodeCount = 0;
|
||||
/** @var Preprocessor */
|
||||
public $mPreprocessor;
|
||||
|
||||
public function getStripList() {
|
||||
$parser = MediaWikiServices::getInstance()->getParser();
|
||||
|
||||
return $parser->getStripList();
|
||||
}
|
||||
/** @var int Bit field of Preprocessor::DOM_* constants */
|
||||
private $ptoFlags;
|
||||
|
||||
public function __construct() {
|
||||
parent::__construct();
|
||||
|
|
@ -60,15 +53,10 @@ class PreprocessDump extends DumpIterator {
|
|||
}
|
||||
|
||||
public function checkOptions() {
|
||||
global $wgPreprocessorCacheThreshold;
|
||||
|
||||
if ( !$this->hasOption( 'cache' ) ) {
|
||||
$wgPreprocessorCacheThreshold = false;
|
||||
}
|
||||
|
||||
$parser = MediaWikiServices::getInstance()->getParser();
|
||||
$parser->firstCallInit();
|
||||
$this->mPreprocessor = new Preprocessor_Hash( $parser );
|
||||
$parser->firstCallInit(); // make sure strip list is loaded
|
||||
$this->preprocessor = $parser->getPreprocessor();
|
||||
$this->ptoFlags = $this->hasOption( 'cache' ) ? 0 : Preprocessor::DOM_UNCACHED;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -77,7 +65,6 @@ class PreprocessDump extends DumpIterator {
|
|||
*/
|
||||
public function processRevision( WikiRevision $rev ) {
|
||||
$content = $rev->getContent();
|
||||
|
||||
if ( $content->getModel() !== CONTENT_MODEL_WIKITEXT ) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -85,7 +72,7 @@ class PreprocessDump extends DumpIterator {
|
|||
'@phan-var WikitextContent $content';
|
||||
|
||||
try {
|
||||
$this->mPreprocessor->preprocessToObj( strval( $content->getText() ), 0 );
|
||||
$this->preprocessor->preprocessToObj( strval( $content->getText() ), $this->ptoFlags );
|
||||
} catch ( Exception $e ) {
|
||||
$this->error( "Caught exception " . $e->getMessage() . " in "
|
||||
. $rev->getTitle()->getPrefixedText() );
|
||||
|
|
|
|||
|
|
@ -20,48 +20,34 @@ use MediaWiki\MediaWikiServices;
|
|||
class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
||||
protected $mTitle = 'Page title';
|
||||
protected $mPPNodeCount = 0;
|
||||
/**
|
||||
* @var ParserOptions
|
||||
*/
|
||||
/** @var ParserOptions */
|
||||
protected $mOptions;
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
protected $mPreprocessors;
|
||||
|
||||
protected static $classNames = [
|
||||
Preprocessor_Hash::class
|
||||
];
|
||||
/** @var Preprocessor */
|
||||
protected $preprocessor;
|
||||
|
||||
protected function setUp() : void {
|
||||
parent::setUp();
|
||||
$this->mOptions = ParserOptions::newFromUserAndLang( new User,
|
||||
MediaWikiServices::getInstance()->getContentLanguage() );
|
||||
|
||||
$this->mPreprocessors = [];
|
||||
foreach ( self::$classNames as $className ) {
|
||||
$this->mPreprocessors[$className] = new $className( $this );
|
||||
}
|
||||
}
|
||||
$wanCache = new WANObjectCache( [ 'cache' => new HashBagOStuff() ] );
|
||||
$parser = $this->getMockBuilder( Parser::class )
|
||||
->disableOriginalConstructor()
|
||||
->getMock();
|
||||
$parser->method( 'getStripList' )->willReturn( [
|
||||
'gallery', 'display map' /* Used by Maps, see r80025 CR */, '/foo'
|
||||
] );
|
||||
|
||||
public function getStripList() {
|
||||
return [ 'gallery', 'display map' /* Used by Maps, see r80025 CR */, '/foo' ];
|
||||
}
|
||||
|
||||
protected static function addClassArg( $testCases ) {
|
||||
$newTestCases = [];
|
||||
foreach ( self::$classNames as $className ) {
|
||||
foreach ( $testCases as $testCase ) {
|
||||
array_unshift( $testCase, $className );
|
||||
$newTestCases[] = $testCase;
|
||||
}
|
||||
}
|
||||
return $newTestCases;
|
||||
$this->preprocessor = new Preprocessor_Hash(
|
||||
$parser,
|
||||
$wanCache,
|
||||
[ 'cacheThreshold' => 1000 ]
|
||||
);
|
||||
}
|
||||
|
||||
public static function provideCases() {
|
||||
// phpcs:disable Generic.Files.LineLength
|
||||
return self::addClassArg( [
|
||||
return [
|
||||
[ "Foo", "<root>Foo</root>" ],
|
||||
[ "<!-- Foo -->", "<root><comment><!-- Foo --></comment></root>" ],
|
||||
[ "<!-- Foo --><!-- Bar -->", "<root><comment><!-- Foo --></comment><comment><!-- Bar --></comment></root>" ],
|
||||
|
|
@ -147,7 +133,7 @@ class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
|||
[ "{{Foo|} Bar=", "<root>{{Foo|} Bar=</root>" ],
|
||||
[ "{{Foo|} Bar=}}", "<root><template><title>Foo</title><part><name>} Bar</name>=<value></value></part></template></root>" ],
|
||||
/* [ file_get_contents( __DIR__ . '/QuoteQuran.txt' ], file_get_contents( __DIR__ . '/QuoteQuranExpanded.txt' ) ], */
|
||||
] );
|
||||
];
|
||||
// phpcs:enable
|
||||
}
|
||||
|
||||
|
|
@ -155,12 +141,11 @@ class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
|||
* Get XML preprocessor tree from the preprocessor (which may not be the
|
||||
* native XML-based one).
|
||||
*
|
||||
* @param string $className
|
||||
* @param string $wikiText
|
||||
* @return string
|
||||
*/
|
||||
protected function preprocessToXml( $className, $wikiText ) {
|
||||
$preprocessor = $this->mPreprocessors[$className];
|
||||
protected function preprocessToXml( $wikiText ) {
|
||||
$preprocessor = $this->preprocessor;
|
||||
if ( method_exists( $preprocessor, 'preprocessToXml' ) ) {
|
||||
return $this->normalizeXml( $preprocessor->preprocessToXml( $wikiText ) );
|
||||
}
|
||||
|
|
@ -191,9 +176,11 @@ class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
|||
/**
|
||||
* @dataProvider provideCases
|
||||
*/
|
||||
public function testPreprocessorOutput( $className, $wikiText, $expectedXml ) {
|
||||
$this->assertEquals( $this->normalizeXml( $expectedXml ),
|
||||
$this->preprocessToXml( $className, $wikiText ) );
|
||||
public function testPreprocessorOutput( $wikiText, $expectedXml ) {
|
||||
$this->assertEquals(
|
||||
$this->normalizeXml( $expectedXml ),
|
||||
$this->preprocessToXml( $wikiText )
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -201,23 +188,23 @@ class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
|||
*/
|
||||
public static function provideFiles() {
|
||||
// phpcs:disable Generic.Files.LineLength
|
||||
return self::addClassArg( [
|
||||
return [
|
||||
[ "QuoteQuran" ], # https://en.wikipedia.org/w/index.php?title=Template:QuoteQuran/sandbox&oldid=237348988 GFDL + CC BY-SA by Striver
|
||||
[ "Factorial" ], # https://en.wikipedia.org/w/index.php?title=Template:Factorial&oldid=98548758 GFDL + CC BY-SA by Polonium
|
||||
[ "All_system_messages" ], # https://tl.wiktionary.org/w/index.php?title=Suleras:All_system_messages&oldid=2765 GPL text generated by MediaWiki
|
||||
[ "Fundraising" ], # https://tl.wiktionary.org/w/index.php?title=MediaWiki:Sitenotice&oldid=5716 GFDL + CC BY-SA, copied there by Sky Harbor.
|
||||
[ "NestedTemplates" ], # T29936
|
||||
] );
|
||||
];
|
||||
// phpcs:enable
|
||||
}
|
||||
|
||||
/**
|
||||
* @dataProvider provideFiles
|
||||
*/
|
||||
public function testPreprocessorOutputFiles( $className, $filename ) {
|
||||
public function testPreprocessorOutputFiles( $filename ) {
|
||||
$folder = __DIR__ . "/../../../parser/preprocess";
|
||||
$wikiText = file_get_contents( "$folder/$filename.txt" );
|
||||
$output = $this->preprocessToXml( $className, $wikiText );
|
||||
$output = $this->preprocessToXml( $wikiText );
|
||||
|
||||
$expectedFilename = "$folder/$filename.expected";
|
||||
if ( file_exists( $expectedFilename ) ) {
|
||||
|
|
@ -235,7 +222,7 @@ class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
|||
*/
|
||||
public static function provideHeadings() {
|
||||
// phpcs:disable Generic.Files.LineLength
|
||||
return self::addClassArg( [
|
||||
return [
|
||||
/* These should become headings: */
|
||||
[ "== h ==<!--c1-->", "<root><h level=\"2\" i=\"1\">== h ==<comment><!--c1--></comment></h></root>" ],
|
||||
[ "== h == <!--c1-->", "<root><h level=\"2\" i=\"1\">== h == <comment><!--c1--></comment></h></root>" ],
|
||||
|
|
@ -272,15 +259,17 @@ class PreprocessorTest extends MediaWikiIntegrationTestCase {
|
|||
[ "== h == x <!--c1--><!--c2--><!--c3--> ", "<root>== h == x <comment><!--c1--></comment><comment><!--c2--></comment><comment><!--c3--></comment> </root>" ],
|
||||
[ "== h ==<!--c1--> x <!--c2--><!--c3--> ", "<root>== h ==<comment><!--c1--></comment> x <comment><!--c2--></comment><comment><!--c3--></comment> </root>" ],
|
||||
[ "== h ==<!--c1--><!--c2--><!--c3--> x ", "<root>== h ==<comment><!--c1--></comment><comment><!--c2--></comment><comment><!--c3--></comment> x </root>" ],
|
||||
] );
|
||||
];
|
||||
// phpcs:enable
|
||||
}
|
||||
|
||||
/**
|
||||
* @dataProvider provideHeadings
|
||||
*/
|
||||
public function testHeadings( $className, $wikiText, $expectedXml ) {
|
||||
$this->assertEquals( $this->normalizeXml( $expectedXml ),
|
||||
$this->preprocessToXml( $className, $wikiText ) );
|
||||
public function testHeadings( $wikiText, $expectedXml ) {
|
||||
$this->assertEquals(
|
||||
$this->normalizeXml( $expectedXml ),
|
||||
$this->preprocessToXml( $wikiText )
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue