Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

resourceloader: support tracking indirect module dependency paths via BagOStuff

Browse source 
This can be enabled via a configuration flag. Otherwise, SqlModuleDependencyStore
will be used in order to keep using the module_deps table.

Create a dependency store class, wrapping BagOStuff, that stores known module
dependencies. Inject it into ResourceLoader and inject the path lists into
ResourceLoaderModule directly and via callback.

Bug: T113916
Change-Id: I6da55e78d5554e30e5df6b4bc45d84817f5bea15
This commit is contained in:
Aaron Schulz 2019-06-28 21:50:31 -07:00
parent 24f5d0b00e
commit 5282a02961
11 changed files with 643 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
autoload.php
View file

@ -1642,6 +1642,10 @@ $wgAutoloadLocalClasses = [
'WikiRevision' => __DIR__ . '/includes/import/WikiRevision.php',
'WikiStatsOutput' => __DIR__ . '/maintenance/language/StatOutputs.php',
'WikiTextStructure' => __DIR__ . '/includes/content/WikiTextStructure.php',
'Wikimedia\\DependencyStore\\DependencyStore' => __DIR__ . '/includes/resourceloader/dependencystore/DependencyStore.php',
'Wikimedia\\DependencyStore\\DependencyStoreException' => __DIR__ . '/includes/resourceloader/dependencystore/DependencyStoreException.php',
'Wikimedia\\DependencyStore\\KeyValueDependencyStore' => __DIR__ . '/includes/resourceloader/dependencystore/KeyValueDependencyStore.php',
'Wikimedia\\DependencyStore\\SqlModuleDependencyStore' => __DIR__ . '/includes/resourceloader/dependencystore/SqlModuleDependencyStore.php',
'Wikimedia\\Http\\HttpAcceptNegotiator' => __DIR__ . '/includes/libs/http/HttpAcceptNegotiator.php',
'Wikimedia\\Http\\HttpAcceptParser' => __DIR__ . '/includes/libs/http/HttpAcceptParser.php',
'Wikimedia\\Rdbms\\AtomicSectionIdentifier' => __DIR__ . '/includes/libs/rdbms/database/utils/AtomicSectionIdentifier.php',

8
includes/DefaultSettings.php
View file

@ -3752,6 +3752,14 @@ $wgResourceLoaderMaxage = [
'unversioned' => 5 * 60, // 5 minutes
];
/**
* Use the main stash instead of the module_deps table for indirect dependency tracking
*
* @since 1.35
* @warning EXPERIMENTAL
*/
$wgResourceLoaderUseObjectCacheForDeps = false;
/**
* The default debug mode (on/off) for of ResourceLoader requests.
*

7
includes/ServiceWiring.php
View file

@ -87,6 +87,8 @@ use MediaWiki\Storage\BlobStoreFactory;
use MediaWiki\Storage\NameTableStoreFactory;
use MediaWiki\Storage\PageEditStash;
use MediaWiki\Storage\SqlBlobStore;
use Wikimedia\DependencyStore\KeyValueDependencyStore;
use Wikimedia\DependencyStore\SqlModuleDependencyStore;
use Wikimedia\Message\IMessageFormatterFactory;
use Wikimedia\ObjectFactory;
@ -777,7 +779,10 @@ return [
$rl = new ResourceLoader(
$config,
LoggerFactory::getInstance( 'resourceloader' )
LoggerFactory::getInstance( 'resourceloader' ),
$config->get( 'ResourceLoaderUseObjectCacheForDeps' )
? new KeyValueDependencyStore( $services->getMainObjectStash() )
: new SqlModuleDependencyStore( $services->getDBLoadBalancer() )
);
$rl->addSource( $config->get( 'ResourceLoaderSources' ) );

180
includes/resourceloader/ResourceLoader.php
View file

@ -20,10 +20,13 @@
* @author Trevor Parscal
*/
use MediaWiki\HeaderCallback;
use MediaWiki\MediaWikiServices;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Wikimedia\DependencyStore\DependencyStore;
use Wikimedia\DependencyStore\KeyValueDependencyStore;
use Wikimedia\Rdbms\DBConnectionError;
use Wikimedia\Timestamp\ConvertibleTimestamp;
use Wikimedia\WrappedString;
@ -47,6 +50,8 @@ class ResourceLoader implements LoggerAwareInterface {
protected $config;
/** @var MessageBlobStore */
protected $blobStore;
/** @var DependencyStore */
protected $depStore;
/** @var LoggerInterface */
private $logger;
@ -63,7 +68,6 @@ class ResourceLoader implements LoggerAwareInterface {
protected $testModuleNames = [];
/** @var string[] List of module names that contain QUnit test suites */
protected $testSuiteModuleNames = [];
/** @var array Map of (source => path); E.g. [ 'source-id' => 'http://.../load.php' ] */
protected $sources = [];
/** @var array Errors accumulated during current respond() call */
@ -71,79 +75,69 @@ class ResourceLoader implements LoggerAwareInterface {
/** @var string[] Extra HTTP response headers from modules loaded in makeModuleResponse() */
protected $extraHeaders = [];
/** @var array Map of (module-variant => buffered DependencyStore updates) */
private $depStoreUpdateBuffer = [];
/** @var bool */
protected static $debugMode = null;
/** @var int */
const CACHE_VERSION = 8;
/** @var string */
private const RL_DEP_STORE_PREFIX = 'ResourceLoaderModule';
/** @var int Expiry (in seconds) of indirect dependency information for modules */
private const RL_MODULE_DEP_TTL = BagOStuff::TTL_WEEK;
/** @var string JavaScript / CSS pragma to disable minification. * */
const FILTER_NOMIN = '/*@nomin*/';
/**
* Load information stored in the database about modules.
* Load information stored in the database and dependency tracking store about modules
*
* This method grabs modules dependencies from the database and updates modules
* objects.
*
* This is not inside the module code because it is much faster to
* request all of the information at once than it is to have each module
* requests its own information. This sacrifice of modularity yields a substantial
* performance improvement.
*
* @param array $moduleNames List of module names to preload information for
* @param ResourceLoaderContext $context Context to load the information within
* @param string[] $moduleNames Module names
* @param ResourceLoaderContext $context ResourceLoader-specific context of the request
*/
public function preloadModuleInfo( array $moduleNames, ResourceLoaderContext $context ) {
if ( !$moduleNames ) {
// Or else Database*::select() will explode, plus it's cheaper!
return;
}
$dbr = wfGetDB( DB_REPLICA );
$lang = $context->getLanguage();
// Batched version of ResourceLoaderModule::getFileDependencies
// Load all tracked indirect file dependencies for the modules
$vary = ResourceLoaderModule::getVary( $context );
$res = $dbr->select( 'module_deps', [ 'md_module', 'md_deps' ], [
'md_module' => $moduleNames,
'md_skin' => $vary,
], __METHOD__
);
// Prime in-object cache for file dependencies
$modulesWithDeps = [];
foreach ( $res as $row ) {
$module = $this->getModule( $row->md_module );
if ( $module ) {
$module->setFileDependencies( $context, ResourceLoaderModule::expandRelativePaths(
json_decode( $row->md_deps, true )
) );
$modulesWithDeps[] = $row->md_module;
}
$entitiesByModule = [];
foreach ( $moduleNames as $moduleName ) {
$entitiesByModule[$moduleName] = "$moduleName|$vary";
}
// Register the absence of a dependency row too
foreach ( array_diff( $moduleNames, $modulesWithDeps ) as $name ) {
$module = $this->getModule( $name );
$depsByEntity = $this->depStore->retrieveMulti(
self::RL_DEP_STORE_PREFIX,
$entitiesByModule
);
// Inject the indirect file dependencies for all the modules
foreach ( $moduleNames as $moduleName ) {
$module = $this->getModule( $moduleName );
if ( $module ) {
$module->setFileDependencies( $context, [] );
$entity = $entitiesByModule[$moduleName];
$deps = $depsByEntity[$entity];
$paths = ResourceLoaderModule::expandRelativePaths( $deps['paths'] );
$module->setFileDependencies( $context, $paths );
}
}
// Batched version of ResourceLoaderWikiModule::getTitleInfo
$dbr = wfGetDB( DB_REPLICA );
ResourceLoaderWikiModule::preloadTitleInfo( $context, $dbr, $moduleNames );
// Prime in-object cache for message blobs for modules with messages
$modules = [];
foreach ( $moduleNames as $name ) {
$module = $this->getModule( $name );
$modulesWithMessages = [];
foreach ( $moduleNames as $moduleName ) {
$module = $this->getModule( $moduleName );
if ( $module && $module->getMessages() ) {
$modules[$name] = $module;
$modulesWithMessages[$moduleName] = $module;
}
}
// Prime in-object cache for message blobs for modules with messages
$lang = $context->getLanguage();
$store = $this->getMessageBlobStore();
$blobs = $store->getBlobs( $modules, $lang );
foreach ( $blobs as $name => $blob ) {
$modules[$name]->setMessageBlob( $blob, $lang );
$blobs = $store->getBlobs( $modulesWithMessages, $lang );
foreach ( $blobs as $moduleName => $blob ) {
$modulesWithMessages[$moduleName]->setMessageBlob( $blob, $lang );
}
}
@ -218,8 +212,13 @@ class ResourceLoader implements LoggerAwareInterface {
* Register core modules and runs registration hooks.
* @param Config|null $config
* @param LoggerInterface|null $logger [optional]
* @param DependencyStore|null $tracker [optional]
*/
public function __construct( Config $config = null, LoggerInterface $logger = null ) {
public function __construct(
Config $config = null,
LoggerInterface $logger = null,
DependencyStore $tracker = null
) {
$this->logger = $logger ?: new NullLogger();
$services = MediaWikiServices::getInstance();
@ -238,6 +237,9 @@ class ResourceLoader implements LoggerAwareInterface {
$this->setMessageBlobStore(
new MessageBlobStore( $this, $this->logger, $services->getMainWANObjectCache() )
);
$tracker = $tracker ?: new KeyValueDependencyStore( new HashBagOStuff() );
$this->setDependencyStore( $tracker );
}
/**
@ -279,6 +281,14 @@ class ResourceLoader implements LoggerAwareInterface {
$this->blobStore = $blobStore;
}
/**
* @since 1.35
* @param DependencyStore $tracker
*/
public function setDependencyStore( DependencyStore $tracker ) {
$this->depStore = $tracker;
}
/**
* Register a module with the ResourceLoader system.
*
@ -506,12 +516,84 @@ class ResourceLoader implements LoggerAwareInterface {
$object->setConfig( $this->getConfig() );
$object->setLogger( $this->logger );
$object->setName( $name );
$object->setDependencyAccessCallbacks(
[ $this, 'loadModuleDependenciesInternal' ],
[ $this, 'saveModuleDependenciesInternal' ]
);
$this->modules[$name] = $object;
}
return $this->modules[$name];
}
/**
* @param string $moduleName Module name
* @param string $variant Language/skin variant
* @return string[] List of absolute file paths
* @private
*/
public function loadModuleDependenciesInternal( $moduleName, $variant ) {
$deps = $this->depStore->retrieve( self::RL_DEP_STORE_PREFIX, "$moduleName|$variant" );
return ResourceLoaderModule::expandRelativePaths( $deps['paths'] );
}
/**
* @param string $moduleName Module name
* @param string $variant Language/skin variant
* @param string[] $paths List of relative paths referenced during computation
* @param string[] $priorPaths List of relative paths tracked in the dependency store
* @private
*/
public function saveModuleDependenciesInternal( $moduleName, $variant, $paths, $priorPaths ) {
$hasPendingUpdate = (bool)$this->depStoreUpdateBuffer;
$entity = "$moduleName|$variant";
if ( array_diff( $paths, $priorPaths ) || array_diff( $priorPaths, $paths ) ) {
// Dependency store needs to be updated with the new path list
if ( $paths ) {
$deps = $this->depStore->newEntityDependencies( $paths, time() );
$this->depStoreUpdateBuffer[$entity] = $deps;
} else {
$this->depStoreUpdateBuffer[$entity] = null;
}
} elseif ( $priorPaths ) {
// Dependency store needs to store the existing path list for longer
$this->depStoreUpdateBuffer[$entity] = '*';
}
// Use a DeferrableUpdate to flush the buffered dependency updates...
if ( !$hasPendingUpdate ) {
DeferredUpdates::addCallableUpdate( function () {
$updatesByEntity = $this->depStoreUpdateBuffer;
$this->depStoreUpdateBuffer = []; // consume
$cache = ObjectCache::getLocalClusterInstance();
$scopeLocks = [];
$depsByEntity = [];
$entitiesUnreg = [];
$entitiesRenew = [];
foreach ( $updatesByEntity as $entity => $update ) {
$scopeLocks[$entity] = $cache->getScopedLock( "rl-deps:$entity", 0 );
if ( !$scopeLocks[$entity] ) {
continue; // avoid duplicate write request slams (T124649)
} elseif ( $update === null ) {
$entitiesUnreg[] = $entity;
} elseif ( $update === '*' ) {
$entitiesRenew[] = $entity;
} else {
$depsByEntity[$entity] = $update;
}
}
$ttl = self::RL_MODULE_DEP_TTL;
$this->depStore->storeMulti( self::RL_DEP_STORE_PREFIX, $depsByEntity, $ttl );
$this->depStore->remove( self::RL_DEP_STORE_PREFIX, $entitiesUnreg );
$this->depStore->renew( self::RL_DEP_STORE_PREFIX, $entitiesRenew, $ttl );
} );
}
}
/**
* Whether the module is a ResourceLoaderFileModule (including subclasses).
*
@ -865,7 +947,7 @@ class ResourceLoader implements LoggerAwareInterface {
protected function sendResponseHeaders(
ResourceLoaderContext $context, $etag, $errors, array $extra = []
) {
\MediaWiki\HeaderCallback::warnIfHeadersSent();
HeaderCallback::warnIfHeadersSent();
$rlMaxage = $this->config->get( 'ResourceLoaderMaxage' );
// Use a short cache expiry so that updates propagate to clients quickly, if:
// - No version specified (shared resources, e.g. stylesheets)

5
includes/resourceloader/ResourceLoaderFileModule.php
View file

@ -436,7 +436,9 @@ class ResourceLoaderFileModule extends ResourceLoaderModule {
$this->getStyleFiles( $context ),
$context
);
// Collect referenced files
// Track indirect file dependencies so that ResourceLoaderStartUpModule can check for
// on-disk file changes to any of this files without having to recompute the file list
$this->saveFileDependencies( $context, $this->localFileRefs );
return $styles;
@ -527,7 +529,6 @@ class ResourceLoaderFileModule extends ResourceLoaderModule {
/**
* Helper method for getDefinitionSummary.
*
* @see ResourceLoaderModule::getFileDependencies
* @param ResourceLoaderContext $context
* @return string
*/

156
includes/resourceloader/ResourceLoaderModule.php
View file

@ -26,7 +26,6 @@ use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Wikimedia\AtEase\AtEase;
use Wikimedia\RelPath;
use Wikimedia\ScopedCallback;
/**
* Abstraction for ResourceLoader modules, with name registration and maxage functionality.
@ -63,6 +62,11 @@ abstract class ResourceLoaderModule implements LoggerAwareInterface {
/** @var array Map of (context hash => cached module content) */
protected $contents = [];
/** @var callback Function of (module name, variant) to get indirect file dependencies */
private $depLoadCallback;
/** @var callback Function of (module name, variant) to get indirect file dependencies */
private $depSaveCallback;
/** @var string|bool Deprecation string or true if deprecated; false otherwise */
protected $deprecated = false;
@ -113,6 +117,18 @@ abstract class ResourceLoaderModule implements LoggerAwareInterface {
$this->name = $name;
}
/**
* Inject the functions that load/save the indirect file path dependency list from storage
*
* @param callable $loadCallback Function of (module name, variant)
* @param callable $saveCallback Function of (module name, variant, current paths, stored paths)
* @since 1.35
*/
public function setDependencyAccessCallbacks( callable $loadCallback, callable $saveCallback ) {
$this->depLoadCallback = $loadCallback;
$this->depSaveCallback = $saveCallback;
}
/**
* Get this module's origin. This is set when the module is registered
* with ResourceLoader::register()
@ -390,125 +406,89 @@ abstract class ResourceLoaderModule implements LoggerAwareInterface {
}
/**
* Get the files this module depends on indirectly for a given skin.
* Get the indirect dependencies for this module persuant to the skin/language context
*
* These are only image files referenced by the module's stylesheet.
* These are only image files referenced by the module's stylesheet
*
* If niether setFileDependencies() nor setDependencyLoadCallback() was called, this
* will simply return a placeholder with an empty file list
*
* @see ResourceLoader::setFileDependencies()
* @see ResourceLoader::saveFileDependencies()
*
* @param ResourceLoaderContext $context
* @return array List of files
* @return string[] List of absolute file paths
* @throws RuntimeException When setFileDependencies() has not yet been called
*/
protected function getFileDependencies( ResourceLoaderContext $context ) {
$vary = self::getVary( $context );
$variant = self::getVary( $context );
// Try in-object cache first
if ( !isset( $this->fileDeps[$vary] ) ) {
$dbr = wfGetDB( DB_REPLICA );
$deps = $dbr->selectField( 'module_deps',
'md_deps',
[
'md_module' => $this->getName(),
'md_skin' => $vary,
],
__METHOD__
);
if ( $deps !== null ) {
$this->fileDeps[$vary] = self::expandRelativePaths(
(array)json_decode( $deps, true )
);
if ( !isset( $this->fileDeps[$variant] ) ) {
if ( $this->depLoadCallback ) {
$this->fileDeps[$variant] =
call_user_func( $this->depLoadCallback, $this->getName(), $variant );
} else {
$this->fileDeps[$vary] = [];
$this->getLogger()->info( __METHOD__ . ": no callback registered" );
$this->fileDeps[$variant] = [];
}
}
return $this->fileDeps[$vary];
return $this->fileDeps[$variant];
}
/**
* Set in-object cache for file dependencies.
* Set the indirect dependencies for this module persuant to the skin/language context
*
* This is used to retrieve data in batches. See ResourceLoader::preloadModuleInfo().
* To save the data, use saveFileDependencies().
* These are only image files referenced by the module's stylesheet
*
* @see ResourceLoader::getFileDependencies()
* @see ResourceLoader::saveFileDependencies()
*
* @param ResourceLoaderContext $context
* @param string[] $files Array of file names
* @param string[] $paths List of absolute file paths
*/
public function setFileDependencies( ResourceLoaderContext $context, $files ) {
$vary = self::getVary( $context );
$this->fileDeps[$vary] = $files;
public function setFileDependencies( ResourceLoaderContext $context, array $paths ) {
$variant = self::getVary( $context );
$this->fileDeps[$variant] = $paths;
}
/**
* Set the files this module depends on indirectly for a given skin.
* Save the indirect dependencies for this module persuant to the skin/language context
*
* @param ResourceLoaderContext $context
* @param string[] $curFileRefs List of newly computed indirect file dependencies
* @since 1.27
* @param ResourceLoaderContext $context
* @param array $localFileRefs List of files
*/
protected function saveFileDependencies( ResourceLoaderContext $context, array $localFileRefs ) {
protected function saveFileDependencies( ResourceLoaderContext $context, array $curFileRefs ) {
if ( !$this->depSaveCallback ) {
$this->getLogger()->info( __METHOD__ . ": no callback registered" );
return;
}
try {
// Related bugs and performance considerations:
// 1. Don't needlessly change the database value with the same list in a
// different order or with duplicates.
// Pitfalls and performance considerations:
// 1. Don't keep updating the tracked paths due to duplicates or sorting.
// 2. Use relative paths to avoid ghost entries when $IP changes. (T111481)
// 3. Don't needlessly replace the database with the same value
// 3. Don't needlessly replace tracked paths with the same value
// just because $IP changed (e.g. when upgrading a wiki).
// 4. Don't create an endless replace loop on every request for this
// module when '../' is used anywhere. Even though both are expanded
// (one expanded by getFileDependencies from the DB, the other is
// still raw as originally read by RL), the latter has not
// been normalized yet.
// Normalise
$localFileRefs = array_values( array_unique( $localFileRefs ) );
sort( $localFileRefs );
$localPaths = self::getRelativePaths( $localFileRefs );
$storedPaths = self::getRelativePaths( $this->getFileDependencies( $context ) );
if ( $localPaths === $storedPaths ) {
// Unchanged. Avoid needless database query (especially master conn!).
return;
}
// The file deps list has changed, we want to update it.
$vary = self::getVary( $context );
$cache = ObjectCache::getLocalClusterInstance();
$key = $cache->makeKey( __METHOD__, $this->getName(), $vary );
$scopeLock = $cache->getScopedLock( $key, 0 );
if ( !$scopeLock ) {
// Another request appears to be doing this update already.
// Avoid write slams (T124649).
return;
}
// No needless escaping as this isn't HTML output.
// Only stored in the database and parsed in PHP.
$deps = json_encode( $localPaths, JSON_UNESCAPED_SLASHES );
$dbw = wfGetDB( DB_MASTER );
$dbw->upsert( 'module_deps',
[
'md_module' => $this->getName(),
'md_skin' => $vary,
'md_deps' => $deps,
],
[ [ 'md_module', 'md_skin' ] ],
[
'md_deps' => $deps,
],
__METHOD__
call_user_func(
$this->depSaveCallback,
$this->getName(),
self::getVary( $context ),
self::getRelativePaths( $curFileRefs ),
self::getRelativePaths( $this->getFileDependencies( $context ) )
);
if ( $dbw->trxLevel() ) {
$dbw->onTransactionResolution(
function () use ( &$scopeLock ) {
ScopedCallback::consume( $scopeLock ); // release after commit
},
__METHOD__
);
}
} catch ( Exception $e ) {
// Probably a DB failure. Either the read query from getFileDependencies(),
// or the write query above.
$this->getLogger()->error( "Failed to update DB: $e", [ 'exception' => $e ] );
$this->getLogger()->warning(
__METHOD__ . ": failed to update dependencies: {$e->getMessage()}",
[ 'exception' => $e ]
);
}
}

128
includes/resourceloader/dependencystore/DependencyStore.php Normal file
View file

@ -0,0 +1,128 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
namespace Wikimedia\DependencyStore;
/**
* Class for tracking per-entity dependency path lists that are expensive to mass compute
*
* @internal This should not be used outside of ResourceLoader and ResourceLoaderModule
*
* @since 1.35
*/
abstract class DependencyStore {
/** @var string */
const KEY_PATHS = 'paths';
/** @var string */
const KEY_AS_OF = 'asOf';
/**
* @param string[] $paths List of dependency paths
* @param int|null $asOf UNIX timestamp or null
* @return array
*/
public function newEntityDependencies( array $paths = [], $asOf = null ) {
return [ self::KEY_PATHS => $paths, self::KEY_AS_OF => $asOf ];
}
/**
* Get the currently tracked dependencies for an entity
*
* The "paths" field contains a sorted list of unique paths
*
* The "asOf" field reflects the last-modified timestamp of the dependency data itself.
* It will be null if there is no tracking data available. Note that if empty path lists
* are never stored (as an optimisation) then it will not be possible to discern whether
* the result is up-to-date.
*
* @param string $type Entity type
* @param string $entity Entity name
* @return array Map of (paths: paths, asOf: UNIX timestamp or null)
* @throws DependencyStoreException
*/
final public function retrieve( $type, $entity ) {
return $this->retrieveMulti( $type, [ $entity ] )[$entity];
}
/**
* Get the currently tracked dependencies for a set of entities
*
* @see KeyValueDependencyStore::retrieve()
*
* @param string $type Entity type
* @param string[] $entities Entity names
* @return array[] Map of (entity => (paths: paths, asOf: UNIX timestamp or null))
* @throws DependencyStoreException
*/
abstract public function retrieveMulti( $type, array $entities );
/**
* Set the currently tracked dependencies for an entity
*
* Dependency data should be set to persist as long as anything might rely on it existing
* in order to check the validity of some previously computed work. This can be achieved
* while minimizing storage space under the following scheme:
* - a) computed work has a TTL (time-to-live)
* - b) when work is computed, the dependency data is updated
* - c) the dependency data has a TTL higher enough to accounts for skew/latency
* - d) the TTL of tracked dependency data is renewed upon access
*
* @param string $type Entity type
* @param string $entity Entity name
* @param array $data Map of (paths: paths, asOf: UNIX timestamp or null)
* @param int $ttl New time-to-live in seconds
* @throws DependencyStoreException
*/
final public function store( $type, $entity, array $data, $ttl ) {
$this->storeMulti( $type, [ $entity => $data ], $ttl );
}
/**
* Set the currently tracked dependencies for a set of entities
*
* @see KeyValueDependencyStore::store()
*
* @param string $type Entity type
* @param array[] $dataByEntity Map of (entity => (paths: paths, asOf: UNIX timestamp or null))
* @param int $ttl New time-to-live in seconds
* @throws DependencyStoreException
*
*/
abstract public function storeMulti( $type, array $dataByEntity, $ttl );
/**
* Delete the currently tracked dependencies for an entity or set of entities
*
* @param string $type Entity type
* @param string|string[] $entities Entity name(s)
* @throws DependencyStoreException
*/
abstract public function remove( $type, $entities );
/**
* Set the expiry for the currently tracked dependencies for an entity or set of entities
*
* @param string $type Entity type
* @param string|string[] $entities Entity name(s)
* @param int $ttl New time-to-live in seconds
* @throws DependencyStoreException
*/
abstract public function renew( $type, $entities, $ttl );
}

12
includes/resourceloader/dependencystore/DependencyStoreException.php Normal file
View file

@ -0,0 +1,12 @@
<?php
namespace Wikimedia\DependencyStore;
use RuntimeException;
/**
* @since 1.35
*/
class DependencyStoreException extends RuntimeException {
}

118
includes/resourceloader/dependencystore/KeyValueDependencyStore.php Normal file
View file

@ -0,0 +1,118 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
namespace Wikimedia\DependencyStore;
use BagOStuff;
use InvalidArgumentException;
/**
* Lightweight class for tracking path dependencies lists via an object cache instance
*
* This does not throw DependencyStoreException due to I/O errors since it is optimized for
* speed and availability. Read methods return empty placeholders on failure. Write methods
* might issue I/O in the background and return immediately. However, reads methods will at
* least block on the resolution (success/failure) of any such pending writes.
*
* @since 1.35
*/
class KeyValueDependencyStore extends DependencyStore {
/** @var BagOStuff */
private $stash;
/**
* @param BagOStuff $stash Storage backend
*/
public function __construct( BagOStuff $stash ) {
$this->stash = $stash;
}
public function retrieveMulti( $type, array $entities ) {
$entitiesByKey = [];
foreach ( $entities as $entity ) {
$entitiesByKey[$this->getStoreKey( $type, $entity )] = $entity;
}
$blobsByKey = $this->stash->getMulti( array_keys( $entitiesByKey ) );
$results = [];
foreach ( $entitiesByKey as $key => $entity ) {
$blob = $blobsByKey[$key] ?? null;
$data = is_string( $blob ) ? json_decode( $blob, true ) : null;
$results[$entity] = $this->newEntityDependencies(
$data[self::KEY_PATHS] ?? [],
$data[self::KEY_AS_OF] ?? null
);
}
return $results;
}
public function storeMulti( $type, array $dataByEntity, $ttl ) {
$blobsByKey = [];
foreach ( $dataByEntity as $entity => $data ) {
if ( !is_array( $data[self::KEY_PATHS] ) || !is_int( $data[self::KEY_AS_OF] ) ) {
throw new InvalidArgumentException( "Invalid entry for '$entity'" );
}
// Normalize the list by removing duplicates and sorting
$data[self::KEY_PATHS] = array_values( array_unique( $data[self::KEY_PATHS] ) );
sort( $data[self::KEY_PATHS], SORT_STRING );
$blob = json_encode( $data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE );
$blobsByKey[$this->getStoreKey( $type, $entity )] = $blob;
}
if ( $blobsByKey ) {
$this->stash->setMulti( $blobsByKey, $ttl, BagOStuff::WRITE_BACKGROUND );
}
}
public function remove( $type, $entities ) {
$keys = [];
foreach ( (array)$entities as $entity ) {
$keys[] = $this->getStoreKey( $type, $entity );
}
if ( $keys ) {
$this->stash->deleteMulti( $keys, BagOStuff::WRITE_BACKGROUND );
}
}
public function renew( $type, $entities, $ttl ) {
$keys = [];
foreach ( (array)$entities as $entity ) {
$keys[] = $this->getStoreKey( $type, $entity );
}
if ( $keys ) {
$this->stash->changeTTLMulti( $keys, $ttl, BagOStuff::WRITE_BACKGROUND );
}
}
/**
* @param string $type
* @param string $entity
* @return string
*/
private function getStoreKey( $type, $entity ) {
return $this->stash->makeKey( "{$type}-dependencies", $entity );
}
}

163
includes/resourceloader/dependencystore/SqlModuleDependencyStore.php Normal file
View file

@ -0,0 +1,163 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
namespace Wikimedia\DependencyStore;
use InvalidArgumentException;
use Wikimedia\Rdbms\DBError;
use Wikimedia\Rdbms\ILoadBalancer;
/**
* Class for tracking per-entity dependency path lists in the module_deps table
*
* This should not be used outside of ResourceLoader and ResourceLoaderModule
*
* @internal For use with ResourceLoader/ResourceLoaderModule only
* @since 1.35
*/
class SqlModuleDependencyStore extends DependencyStore {
/** @var ILoadBalancer */
private $lb;
/**
* @param ILoadBalancer $lb Storage backend
*/
public function __construct( ILoadBalancer $lb ) {
$this->lb = $lb;
}
public function retrieveMulti( $type, array $entities ) {
try {
$dbr = $this->lb->getConnectionRef( DB_REPLICA );
$modulesByVariant = [];
foreach ( $entities as $entity ) {
list( $module, $variant ) = $this->getEntityNameComponents( $entity );
$modulesByVariant[$variant][] = $module;
}
$condsByVariant = [];
foreach ( $modulesByVariant as $variant => $modules ) {
$condsByVariant[] = $dbr->makeList(
[ 'md_module' => $modules, 'md_skin' => $variant ],
$dbr::LIST_AND
);
}
if ( !$condsByVariant ) {
return [];
}
$conds = $dbr->makeList( $condsByVariant, $dbr::LIST_OR );
$res = $dbr->select(
'module_deps',
[ 'md_module', 'md_skin', 'md_deps' ],
$conds,
__METHOD__
);
$pathsByEntity = [];
foreach ( $res as $row ) {
$entity = "{$row->md_module}|{$row->md_skin}";
$pathsByEntity[$entity] = json_decode( $row->md_deps, true );
}
$results = [];
foreach ( $entities as $entity ) {
$paths = $pathsByEntity[$entity] ?? [];
$results[$entity] = $this->newEntityDependencies( $paths, null );
}
return $results;
} catch ( DBError $e ) {
throw new DependencyStoreException( $e->getMessage() );
}
}
public function storeMulti( $type, array $dataByEntity, $ttl ) {
try {
$dbw = $this->lb->getConnectionRef( DB_MASTER );
$rows = [];
foreach ( $dataByEntity as $entity => $data ) {
list( $module, $variant ) = $this->getEntityNameComponents( $entity );
if ( !is_array( $data[self::KEY_PATHS] ) ) {
throw new InvalidArgumentException( "Invalid entry for '$entity'" );
}
// Normalize the list by removing duplicates and sortings
$paths = array_values( array_unique( $data[self::KEY_PATHS] ) );
sort( $paths, SORT_STRING );
$blob = json_encode( $paths, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE );
$rows[] = [
'md_module' => $module,
'md_skin' => $variant,
'md_deps' => $blob
];
}
if ( $rows ) {
$dbw->insert( 'module_deps', $rows, __METHOD__ );
}
} catch ( DBError $e ) {
throw new DependencyStoreException( $e->getMessage() );
}
}
public function remove( $type, $entities ) {
try {
$dbw = $this->lb->getConnectionRef( DB_MASTER );
$condsPerRow = [];
foreach ( (array)$entities as $entity ) {
list( $module, $variant ) = $this->getEntityNameComponents( $entity );
$condsPerRow[] = $dbw->makeList(
[ 'md_module' => $module, 'md_skin' => $variant ],
$dbw::LIST_AND
);
}
if ( $condsPerRow ) {
$conds = $dbw->makeList( $condsPerRow, $dbw::LIST_OR );
$dbw->delete( 'module_deps', $conds, __METHOD__ );
}
} catch ( DBError $e ) {
throw new DependencyStoreException( $e->getMessage() );
}
}
public function renew( $type, $entities, $ttl ) {
// no-op
}
/**
* @param string $entity
* @return string[]
*/
private function getEntityNameComponents( $entity ) {
$parts = explode( '|', $entity, 2 );
if ( count( $parts ) !== 2 ) {
throw new InvalidArgumentException( "Invalid module entity '$entity'" );
}
return $parts;
}
}

2
tests/phpunit/includes/OutputPageTest.php
View file

@ -1,6 +1,7 @@
<?php
use MediaWiki\MediaWikiServices;
use Wikimedia\DependencyStore\KeyValueDependencyStore;
use Wikimedia\TestingAccessWrapper;
/**
@ -2462,6 +2463,7 @@ class OutputPageTest extends MediaWikiTestCase {
$nonce->setValue( $out->getCSP(), 'secret' );
$rl = $out->getResourceLoader();
$rl->setMessageBlobStore( $this->createMock( MessageBlobStore::class ) );
$rl->setDependencyStore( $this->createMock( KeyValueDependencyStore::class ) );
$rl->register( [
'test.foo' => [
'class' => ResourceLoaderTestModule::class,