Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ParserCache: always use JSON

Browse source 
When JSON support was introduced into ParserCache in 1.36, it was
controlled by a feature flag, $wgParserCacheUseJson. The feature flag
was "born deprecated" in 1.36. It can now be removed.

This means that ParserCache will always store entries as JSON.
Support for reading old non-JSON entries remains intact.
This is needed when updating wikis from a version older than 1.36
to the current version.

Change-Id: Id04e42bfb458d98414bac50e0d6c505e8878e5c0
This commit is contained in:
daniel 2022-05-09 17:51:35 +02:00
parent 73d1885abe
commit 697f28df32
13 changed files with 50 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
RELEASE-NOTES-1.39
View file

@ -45,6 +45,9 @@ For notes on 1.38.x and older releases, see HISTORY.
Use the MW_WIKI_NAME environment variable to specifiy the name of the site
to load configuration for. Using the WIKI_NAME environment variable for this
purpose is deprecated.
* $wgParserCacheUseJson - the ParserCache now always uses JSON serialization.
Reading old non-JSON cache entries is still supported. The setting had been
deprecated since 1.36.
* $wgAllowJavaUploads - To allow uploads of JAR files, remove application/java
from $wgMimeTypeExclusions.
* $wgMaxRedirects has been removed. This feature never worked as intended,

15
docs/config-schema.yaml
View file

@ -2567,21 +2567,6 @@ config-schema:
```
If set to false, the mtime for each individual JSON file will be checked,
which can be slow if a large number of extensions are being loaded.
ParserCacheUseJson:
default: true
deprecated: 'since 1.36'
description: |-
Enable JSON serialization for ParserCache.
In 1.36 the default serialization format for ParserCache has been changed from PHP
serialization to JSON serialization. The cache is still compatible with old PHP-serialized
entries, so for the most part the change should be unnoticed. However in case some
extensions are installed which write non-JSON-serializable data to
ParserOutput::setExtensionData, the cache will break for some pages. Setting this to 'false'
makes ParserCache use PHP serialization format for writing new cache entries, and all the
cache entries already written in JSON are discarded.
@since 1.36
@deprecated since 1.36
@see https://phabricator.wikimedia.org/T263579
EnableRemoteBagOStuffTests:
default: false
description: |-

7
includes/MainConfigNames.php
View file

@ -1628,13 +1628,6 @@ class MainConfigNames {
*/
public const ExtensionInfoMTime = 'ExtensionInfoMTime';
/**
* Name constant for the ParserCacheUseJson setting, for use with Config::get()
* @see MainConfigSchema::ParserCacheUseJson
* @deprecated since 1.36
*/
public const ParserCacheUseJson = 'ParserCacheUseJson';
/**
* Name constant for the EnableRemoteBagOStuffTests setting, for use with Config::get()
* @see MainConfigSchema::EnableRemoteBagOStuffTests

20
includes/MainConfigSchema.php
View file

@ -4017,26 +4017,6 @@ class MainConfigSchema {
'type' => 'integer|false',
];
/**
* Enable JSON serialization for ParserCache.
*
* In 1.36 the default serialization format for ParserCache has been changed from PHP
* serialization to JSON serialization. The cache is still compatible with old PHP-serialized
* entries, so for the most part the change should be unnoticed. However in case some
* extensions are installed which write non-JSON-serializable data to
* ParserOutput::setExtensionData, the cache will break for some pages. Setting this to 'false'
* makes ParserCache use PHP serialization format for writing new cache entries, and all the
* cache entries already written in JSON are discarded.
*
* @since 1.36
* @deprecated since 1.36
* @see https://phabricator.wikimedia.org/T263579
*/
public const ParserCacheUseJson = [
'default' => true,
'deprecated' => 'since 1.36',
];
/**
* If this is set to true, phpunit will run integration tests against remote
* caches defined in $wgObjectCaches.

4
includes/config-schema.php
View file

@ -538,7 +538,6 @@ return [
'UseGzip' => false,
'InvalidateCacheOnLocalSettingsChange' => true,
'ExtensionInfoMTime' => false,
'ParserCacheUseJson' => true,
'EnableRemoteBagOStuffTests' => false,
'UseCdn' => false,
'VaryOnXFP' => false,
@ -2889,9 +2888,6 @@ return [
'ContentHandlerTextFallback' => [
'deprecated' => 'since 1.37',
],
'ParserCacheUseJson' => [
'deprecated' => 'since 1.36',
],
'SquidPurgeUseHostHeader' => [
'deprecated' => 'since 1.33',
],

7
includes/config-vars.php
View file

@ -1612,13 +1612,6 @@ $wgInvalidateCacheOnLocalSettingsChange = null;
*/
$wgExtensionInfoMTime = null;
/**
* Config variable stub for the ParserCacheUseJson setting, for use by phpdoc and IDEs.
* @see MediaWiki\MainConfigSchema::ParserCacheUseJson
* @deprecated since 1.36
*/
$wgParserCacheUseJson = null;
/**
* Config variable stub for the EnableRemoteBagOStuffTests setting, for use by phpdoc and IDEs.
* @see MediaWiki\MainConfigSchema::EnableRemoteBagOStuffTests

50
includes/parser/ParserCache.php
View file

@ -121,18 +121,6 @@ class ParserCache {
*/
private $metadataProcCache;
/**
* @note Temporary feature flag, remove before 1.36 is released.
* @var bool
*/
private $writeJson = false;
/**
* @note Temporary feature flag, remove before 1.36 is released.
* @var bool
*/
private $readJson = false;
/**
* Setup a cache pathway with a given back-end storage mechanism.
*
@ -148,7 +136,6 @@ class ParserCache {
* @param LoggerInterface $logger
* @param TitleFactory $titleFactory
* @param WikiPageFactory $wikiPageFactory
* @param bool $useJson Temporary feature flag, remove before 1.36 is released.
*/
public function __construct(
string $name,
@ -159,8 +146,7 @@ class ParserCache {
IBufferingStatsdDataFactory $stats,
LoggerInterface $logger,
TitleFactory $titleFactory,
WikiPageFactory $wikiPageFactory,
$useJson = false
WikiPageFactory $wikiPageFactory
) {
$this->name = $name;
$this->cache = $cache;
@ -171,8 +157,6 @@ class ParserCache {
$this->logger = $logger;
$this->titleFactory = $titleFactory;
$this->wikiPageFactory = $wikiPageFactory;
$this->readJson = $useJson;
$this->writeJson = $useJson;
$this->metadataProcCache = new HashBagOStuff( [ 'maxKeys' => 2 ] );
}
@ -252,7 +236,7 @@ class ParserCache {
// NOTE: Support for reading string values from the cache must be
// deployed a while before starting to write JSON to the cache,
// in case we have to revert either change.
if ( is_string( $metadata ) && $this->readJson ) {
if ( is_string( $metadata ) ) {
$metadata = $this->restoreFromJson( $metadata, $pageKey, CacheTime::class );
}
@ -369,7 +353,7 @@ class ParserCache {
// NOTE: Support for reading string values from the cache must be
// deployed a while before starting to write JSON to the cache,
// in case we have to revert either change.
if ( is_string( $value ) && $this->readJson ) {
if ( is_string( $value ) ) {
$value = $this->restoreFromJson( $value, $parserOutputKey, ParserOutput::class );
}
@ -464,23 +448,12 @@ class ParserCache {
$msg = "Saved in parser cache with key $parserOutputKey" .
" and timestamp $cacheTime" .
" and revision id $revId.";
if ( $this->writeJson ) {
$msg .= " Serialized with JSON.";
} else {
$msg .= " Serialized with PHP.";
}
$parserOutput->addCacheMessage( $msg );
$pageKey = $this->makeMetadataKey( $page );
if ( $this->writeJson ) {
$parserOutputData = $this->encodeAsJson( $parserOutput, $parserOutputKey );
$metadataData = $this->encodeAsJson( $metadata, $pageKey );
} else {
// rely on implicit PHP serialization in the cache
$parserOutputData = $parserOutput;
$metadataData = $metadata;
}
$parserOutputData = $this->convertForCache( $parserOutput, $parserOutputKey );
$metadataData = $this->convertForCache( $metadata, $pageKey );
if ( !$parserOutputData || !$metadataData ) {
$this->logger->warning(
@ -585,17 +558,6 @@ class ParserCache {
return false;
}
/**
* @note setter for temporary feature flags, for use in testing.
* @internal
* @param bool $readJson
* @param bool $writeJson
*/
public function setJsonSupport( bool $readJson, bool $writeJson ): void {
$this->readJson = $readJson;
$this->writeJson = $writeJson;
}
/**
* @param string $jsonData
* @param string $key
@ -622,7 +584,7 @@ class ParserCache {
* @param string $key
* @return string|null
*/
private function encodeAsJson( CacheTime $obj, string $key ) {
protected function convertForCache( CacheTime $obj, string $key ) {
try {
return $this->jsonCodec->serialize( $obj );
} catch ( InvalidArgumentException $e ) {

4
includes/parser/ParserCacheFactory.php
View file

@ -80,7 +80,6 @@ class ParserCacheFactory {
* @internal
*/
public const CONSTRUCTOR_OPTIONS = [
MainConfigNames::ParserCacheUseJson, // Temporary feature flag, remove before 1.36 is released.
MainConfigNames::CacheEpoch,
MainConfigNames::OldRevisionParserCacheExpireTime,
];
@ -137,8 +136,7 @@ class ParserCacheFactory {
$this->stats,
$this->logger,
$this->titleFactory,
$this->wikiPageFactory,
$this->options->get( MainConfigNames::ParserCacheUseJson )
$this->wikiPageFactory
);
$this->parserCaches[$name] = $cache;

9
includes/parser/ParserOutput.php
View file

@ -1501,15 +1501,6 @@ class ParserOutput extends CacheTime implements ContentMetadataCollector {
* $output->getExtensionData( 'my_ext_foo' );
* @endcode
*
* In MediaWiki 1.20 and older, you have to use a custom member variable
* within the ParserOutput object, but this will not work in modern
* MediaWiki with $wgParserCacheUseJson set to true (the default):
*
* @par Example:
* @code
* $parser->getOutput()->my_ext_foo = '...';
* @endcode
*
* @param string $name
* @param int|float|string|bool|null $value
* @since 1.38

98
tests/phpunit/includes/parser/ParserCacheTest.php
View file

@ -3,6 +3,7 @@
namespace MediaWiki\Tests\Parser;
use BagOStuff;
use CacheTime;
use EmptyBagOStuff;
use HashBagOStuff;
use InvalidArgumentException;
@ -484,7 +485,7 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
'RejectParserCacheValue' =>
function ( ParserOutput $value, WikiPage $hookPage, ParserOptions $popts )
use ( $wikiPageMock, $parserOutput, $options ) {
$this->assertSame( $parserOutput, $value );
$this->assertEquals( $parserOutput, $value );
$this->assertSame( $wikiPageMock, $hookPage );
$this->assertSame( $options, $popts );
return false;
@ -569,12 +570,11 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
}
public function provideCorruptData() {
yield 'PHP serialization, bad data' => [ false, 'bla bla' ];
yield 'JSON serialization, bad data' => [ true, 'bla bla' ];
yield 'JSON serialization, no _class_' => [ true, '{"test":"test"}' ];
yield 'JSON serialization, non-existing _class_' => [ true, '{"_class_":"NonExistentBogusClass"}' ];
yield 'JSON serialization, bad data' => [ 'bla bla' ];
yield 'JSON serialization, no _class_' => [ '{"test":"test"}' ];
yield 'JSON serialization, non-existing _class_' => [ '{"_class_":"NonExistentBogusClass"}' ];
$wrongInstance = new JsonUnserializableSuperClass( 'test' );
yield 'JSON serialization, wrong class' => [ true, json_encode( $wrongInstance->jsonSerialize() ) ];
yield 'JSON serialization, wrong class' => [ json_encode( $wrongInstance->jsonSerialize() ) ];
}
/**
@ -586,12 +586,10 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
* @dataProvider provideCorruptData
* @covers ParserCache::get
* @covers ParserCache::restoreFromJson
* @param bool $json
* @param string $data
*/
public function testCorruptData( bool $json, string $data ) {
public function testCorruptData( string $data ) {
$cache = $this->createParserCache( null, new HashBagOStuff() );
$cache->setJsonSupport( $json, $json );
$parserOutput = new ParserOutput( 'TEST_TEXT' );
$options1 = ParserOptions::newCanonical( 'canonical' );
@ -639,14 +637,38 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
}
/**
* Test whats happen when we turn on JSON support but there
* are still old entries in the cache.
* Test what happens when upgrading from 1.35 or earlier,
* when old cache entries do not yet use JSON.
*
* @covers ParserCache::get
*/
public function testMigrationToJson() {
$cache = $this->createParserCache();
$cache->setJsonSupport( false, false );
$bagOStuff = new HashBagOStuff();
$cache = $this->getMockBuilder( ParserCache::class )
->setConstructorArgs( [
'test',
$bagOStuff,
'19900220000000',
$this->createHookContainer( [] ),
new JsonCodec(),
new NullStatsdDataFactory(),
new NullLogger(),
$this->getServiceContainer()->getTitleFactory(),
$this->getServiceContainer()->getWikiPageFactory()
] )
->onlyMethods( [ 'convertForCache' ] )
->getMock();
// Emulate pre-1.36 behavior: rely on native PHP serialization.
// Note that backwards compatibility of the actual serialization is covered
// by ParserOutputTest which uses various versions of serialized data
// under tests/phpunit/data/ParserCache.
$cache->method( 'convertForCache' )->willReturnCallback(
static function ( CacheTime $obj, string $key ) {
return $obj;
}
);
$parserOutput1 = new ParserOutput( 'Lorem Ipsum' );
@ -654,7 +676,7 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
$cache->save( $parserOutput1, $this->page, $options, $this->cacheTime );
// emulate migration to JSON
$cache->setJsonSupport( true, true );
$cache = $this->createParserCache( null, $bagOStuff );
// make sure we can load non-json cache data
$cachedOutput = $cache->get( $this->page, $options );
@ -670,50 +692,11 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
}
/**
* Test whats happen when we have to roll back from supporting JSON
* to not supporting JSON.
*
* @covers ParserCache::get
*/
public function testRollbackFromJson() {
$cache = $this->createParserCache();
$cache->setJsonSupport( true, true );
$parserOutput1 = new ParserOutput( 'Lorem Ipsum' );
$options = ParserOptions::newCanonical( 'canonical' );
$cache->save( $parserOutput1, $this->page, $options, $this->cacheTime );
// emulate rolling back to not writing but still reading JSON
$cache->setJsonSupport( true, false );
// make sure we can load json cache data
$cachedOutput = $cache->get( $this->page, $options );
$this->assertEquals( $parserOutput1, $cachedOutput );
// emulate rolling back to not reading JSON
$cache->setJsonSupport( false, false );
// make sure we don't crash and burn
$cachedOutput = $cache->get( $this->page, $options );
$this->assertFalse( $cachedOutput );
// now test that the cache works without JSON
$parserOutput2 = new ParserOutput( 'dolor sit amet' );
$cache->save( $parserOutput2, $this->page, $options, $this->cacheTime );
// make sure we can load non-json cache data
$cachedOutput = $cache->get( $this->page, $options );
$this->assertEquals( $parserOutput2, $cachedOutput );
}
/**
* @covers ParserCache::encodeAsJson
* @covers ParserCache::convertForCache
*/
public function testNonSerializableJsonIsReported() {
$testLogger = new TestLogger( true );
$cache = $this->createParserCache( null, null, $testLogger );
$cache->setJsonSupport( true, true );
$parserOutput = $this->createDummyParserOutput();
$parserOutput->setExtensionData( 'test', new User() );
@ -725,12 +708,11 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
}
/**
* @covers ParserCache::encodeAsJson
* @covers ParserCache::convertForCache
*/
public function testCyclicStructuresDoNotBlowUpInJson() {
$testLogger = new TestLogger( true );
$cache = $this->createParserCache( null, null, $testLogger );
$cache->setJsonSupport( true, true );
$parserOutput = $this->createDummyParserOutput();
$cyclicArray = [ 'a' => 'b' ];
@ -745,12 +727,12 @@ class ParserCacheTest extends MediaWikiIntegrationTestCase {
/**
* Tests that unicode characters are not \u escaped
* @covers ParserCache::encodeAsJson
*
* @covers ParserCache::convertForCache
*/
public function testJsonEncodeUnicode() {
$unicodeCharacter = "Э";
$cache = $this->createParserCache( null, new HashBagOStuff() );
$cache->setJsonSupport( true, true );
$parserOutput = $this->createDummyParserOutput();
$parserOutput->setText( $unicodeCharacter );

1
tests/phpunit/integration/includes/Rest/Handler/PageHTMLHandlerTest.php
View file

@ -74,7 +74,6 @@ class PageHTMLHandlerTest extends MediaWikiIntegrationTestCase {
*/
private function newHandler( Parsoid $parsoid = null ): PageHTMLHandler {
$parserCacheFactoryOptions = new ServiceOptions( ParserCacheFactory::CONSTRUCTOR_OPTIONS, [
'ParserCacheUseJson' => true,
'CacheEpoch' => '20200202112233',
'OldRevisionParserCacheExpireTime' => 60,
] );

1
tests/phpunit/integration/includes/Rest/Handler/RevisionHTMLHandlerTest.php
View file

@ -77,7 +77,6 @@ class RevisionHTMLHandlerTest extends MediaWikiIntegrationTestCase {
*/
private function newHandler( Parsoid $parsoid = null ): RevisionHTMLHandler {
$parserCacheFactoryOptions = new ServiceOptions( ParserCacheFactory::CONSTRUCTOR_OPTIONS, [
'ParserCacheUseJson' => true,
'CacheEpoch' => '20200202112233',
'OldRevisionParserCacheExpireTime' => 60 * 60,
] );

1
tests/phpunit/unit/includes/parser/ParserCacheFactoryTest.php
View file

@ -17,7 +17,6 @@ class ParserCacheFactoryTest extends MediaWikiUnitTestCase {
*/
private function newParserCacheFactory() {
$options = new ServiceOptions( ParserCacheFactory::CONSTRUCTOR_OPTIONS, [
'ParserCacheUseJson' => true,
'CacheEpoch' => '20200202112233',
'OldRevisionParserCacheExpireTime' => 60,
] );