Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add small HtmlCacheUpdater service class to normalize purging code (2)

Browse source 
This is a re-submit of 35da1bbd7c, which was accidentally merged before
CR (and reverted with aa4da3c2e8).

The purge() method handles purging of both file cache and CDN, using
a PRESEND deferred update. This avoids code duplication and missing
file cache purge calls.

Also:
* Migrate HTMLCacheUpdate callers to just directly using HTMLCacheUpdateJob
* Add HtmlFileCacheUpdate class and defer such updates just like with CDN
* Simplify HTMLCacheUpdate constructor parameters
* Remove BacklinkCache::clear() calls which do nothing since the backlink
  query does not actually happen until the job runs

Bug: T230025
Change-Id: Ic1005e70e2c22d5bd1ca36dcdb618108ebe290f3
This commit is contained in:
Aaron Schulz 2019-03-14 17:23:26 -07:00 committed by Krinkle
parent 9ff7146049
commit 3c7f29a6b9
20 changed files with 390 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
RELEASE-NOTES-1.35
View file

@ -176,6 +176,8 @@ For notes on 1.34.x and older releases, see HISTORY.
* ParserOutput now has methods addExtraCSPStyleSrc, addExtraCSPDefaultSrc
addExtraCSPScriptSrc for parser tags/functions to be able to add sources
to the Content Security Policy.
* The HtmlCacheUpdater service was added to unify the logic of purging CDN cache
and HTML file cache to simplify callers and make them more consistent.
* …
=== External library changes in 1.35 ===
@ -873,6 +875,7 @@ because of Phabricator reports.
wiki editing if $wgWatchlistPurgeRate is > 0. This maintenance script only
has effect if $wgWatchlistExpiry is true. It is recommended that a cronjob or
similar be set up to run it at least daily.
* Title::purgeSquid is deprecated. Use MediaWikiServices::getHtmlCacheUpdater.
* …
== Compatibility ==

2
autoload.php
View file

@ -638,6 +638,8 @@ $wgAutoloadLocalClasses = [
'Hooks' => __DIR__ . '/includes/Hooks.php',
'Html' => __DIR__ . '/includes/Html.php',
'HtmlArmor' => __DIR__ . '/includes/libs/HtmlArmor.php',
'HtmlCacheUpdater' => __DIR__ . '/includes/cache/HtmlCacheUpdater.php',
'HtmlFileCacheUpdate' => __DIR__ . '/includes/deferred/HtmlFileCacheUpdate.php',
'Http' => __DIR__ . '/includes/http/Http.php',
'HttpError' => __DIR__ . '/includes/exception/HttpError.php',
'HttpStatus' => __DIR__ . '/includes/libs/HttpStatus.php',

9
includes/MediaWikiServices.php
View file

@ -17,6 +17,7 @@ use FileBackendGroup;
use GenderCache;
use GlobalVarConfig;
use Hooks;
use HtmlCacheUpdater;
use IBufferingStatsdDataFactory;
use JobRunner;
use Language;
@ -690,6 +691,14 @@ class MediaWikiServices extends ServiceContainer {
return $this->getService( 'GlobalIdGenerator' );
}
/**
* @since 1.35
* @return HtmlCacheUpdater
*/
public function getHtmlCacheUpdater() {
return $this->getService( 'HtmlCacheUpdater' );
}
/**
* @since 1.31
* @return HttpRequestFactory

9
includes/ServiceWiring.php
View file

@ -340,6 +340,15 @@ return [
);
},
'HtmlCacheUpdater' => function ( MediaWikiServices $services ) : HtmlCacheUpdater {
$config = $services->getMainConfig();
return new HtmlCacheUpdater(
$config->get( 'CdnReboundPurgeDelay' ),
$config->get( 'UseFileCache' )
);
},
'HttpRequestFactory' =>
function ( MediaWikiServices $services ) : HttpRequestFactory {
return new HttpRequestFactory();

7
includes/Title.php
View file

@ -3577,12 +3577,11 @@ class Title implements LinkTarget, IDBAccessObject {
/**
* Purge all applicable CDN URLs
* @deprecated 1.35 Use HtmlCacheUpdater
*/
public function purgeSquid() {
DeferredUpdates::addUpdate(
new CdnCacheUpdate( [ $this ] ),
DeferredUpdates::PRESEND
);
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $this, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
}
/**

14
includes/cache/HTMLFileCache.php vendored
View file

@ -38,15 +38,15 @@ class HTMLFileCache extends FileCacheBase {
/**
* @param Title|string $title Title object or prefixed DB key string
* @param string $action
* @throws MWException
* @throws InvalidArgumentException
*/
public function __construct( $title, $action ) {
parent::__construct();
$allowedTypes = self::cacheablePageActions();
if ( !in_array( $action, $allowedTypes ) ) {
throw new MWException( 'Invalid file cache type given.' );
if ( !in_array( $action, self::cacheablePageActions() ) ) {
throw new InvalidArgumentException( 'Invalid file cache type given.' );
}
$this->mKey = ( $title instanceof Title )
? $title->getPrefixedDBkey()
: (string)$title;
@ -217,12 +217,12 @@ class HTMLFileCache extends FileCacheBase {
/**
* Clear the file caches for a page for all actions
* @param Title $title
*
* @param Title|string $title Title or prefixed DB key
* @return bool Whether $wgUseFileCache is enabled
*/
public static function clearFileCache( Title $title ) {
public static function clearFileCache( $title ) {
$config = MediaWikiServices::getInstance()->getMainConfig();
if ( !$config->get( 'UseFileCache' ) ) {
return false;
}

118
includes/cache/HtmlCacheUpdater.php vendored Normal file
View file

@ -0,0 +1,118 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
/**
* Class to invalidate the CDN and HTMLFileCache entries associated with URLs/titles
*
* @ingroup Cache
* @since 1.35
*/
class HtmlCacheUpdater {
/** @var int Seconds between initial and rebound purges; 0 if disabled */
private $reboundDelay;
/** @var int Whether filesystem-based HTML output caching is enabled */
private $useFileCache;
/** @var int Issue purge immediately and do not schedule a rebound purge */
public const PURGE_NAIVE = 0;
/**
* @var int Defer purge via PRESEND deferred updates. The pending DeferrableUpdate instances
* will combined/de-duplicated into a single DeferrableUpdate instance; this lowers overhead
* and improves HTTP PURGE request pipelining.
*/
public const PURGE_PRESEND = 1;
/**
* @var int Upon purge, schedule a delayed CDN purge if rebound purges are enabled
* ($wgCdnReboundPurgeDelay). Rebound purges are schedule via the job queue.
*/
public const PURGE_REBOUND = 2;
/**
* @var int Defer purge until no LBFactory transaction round is pending and then schedule
* a delayed rebound purge if rebound purges are enabled ($wgCdnReboundPurgeDelay). This is
* useful for CDN purges triggered by data changes in the current or last transaction round.
* Even if the origin server uses lagged replicas, the use of rebound purges corrects the
* cache in cases where lag is less than the rebound delay. If the lag is anywhere near the
* rebound delay, then auxiliary mechanisms should lower the cache TTL ($wgCdnMaxageLagged).
*/
public const PURGE_INTENT_TXROUND_REFLECTED = self::PURGE_PRESEND | self::PURGE_REBOUND;
/**
* @param int $reboundDelay $wgCdnReboundPurgeDelay
* @param bool $useFileCache $wgUseFileCache
* @internal For use with MediaWikiServices->getHtmlCacheUpdater()
*/
public function __construct( $reboundDelay, $useFileCache ) {
$this->reboundDelay = $reboundDelay;
$this->useFileCache = $useFileCache;
}
/**
* Purge the CDN for a URL or list of URLs
*
* @param string[]|string $urls URL or list of URLs
* @param int $flags Bit field of class PURGE_* constants
* [Default: HtmlCacheUpdater::PURGE_PRESEND]
*/
public function purgeUrls( $urls, $flags = self::PURGE_PRESEND ) {
$urls = is_string( $urls ) ? [ $urls ] : $urls;
$reboundDelay = ( ( $flags & self::PURGE_REBOUND ) == self::PURGE_REBOUND )
? $this->reboundDelay
: 0; // no second purge
$update = new CdnCacheUpdate( $urls, [ 'reboundDelay' => $reboundDelay ] );
if ( ( $flags & self::PURGE_PRESEND ) == self::PURGE_PRESEND ) {
DeferredUpdates::addUpdate( $update, DeferredUpdates::PRESEND );
} else {
$update->doUpdate();
}
}
/**
* Purge the CDN/HTMLFileCache for a title or the titles yielded by an iterator
*
* All cacheable canonical URLs associated with the titles will be purged from CDN.
* All cacheable actions associated with the titles will be purged from HTMLFileCache.
*
* @param Traversable|Title[]|Title $titles Title or iterator yielding Title instances
* @param int $flags Bit field of class PURGE_* constants
* [Default: HtmlCacheUpdater::PURGE_PRESEND]
*/
public function purgeTitleUrls( $titles, $flags = self::PURGE_PRESEND ) {
$titles = $titles instanceof Title ? [ $titles ] : $titles;
if ( $this->useFileCache ) {
$update = HtmlFileCacheUpdate::newFromTitles( $titles );
if ( ( $flags & self::PURGE_PRESEND ) == self::PURGE_PRESEND ) {
DeferredUpdates::addUpdate( $update, DeferredUpdates::PRESEND );
} else {
$update->doUpdate();
}
}
$urls = [];
foreach ( $titles as $title ) {
/** @var Title $title */
$urls = array_merge( $urls, $title->getCdnUrls() );
}
$this->purgeUrls( $urls, $flags );
}
}

123
includes/deferred/CdnCacheUpdate.php
View file

@ -1,7 +1,5 @@
<?php
/**
* CDN cache purging.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@ -28,14 +26,33 @@ use Wikimedia\Assert\Assert;
* @ingroup Cache
*/
class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
/** @var (string|Title)[] Collection of URLs or Titles to purge */
private $urls = [];
/** @var array[] List of (URL, rebound purge delay) tuples */
private $urlTuples = [];
/** @var array[] List of (Title, rebound purge delay) tuples */
private $titleTuples = [];
/** @var int Maximum seconds of rebound purge delay (sanity) */
const MAX_REBOUND_DELAY = 300;
/**
* @param (string|Title)[] $urlArr Collection of URLs or Titles to purge
* @param string[]|Title[] $targets Collection of URLs/titles to be purged from CDN
* @param array $options Options map. Supports:
* - reboundDelay: how many seconds after the first purge to send a rebound purge.
* No rebound purge will be sent if this is not positive. [Default: 0]
*/
public function __construct( array $urlArr ) {
$this->urls = $urlArr;
public function __construct( array $targets, array $options = [] ) {
$delay = min(
(int)max( $options['reboundDelay'] ?? 0, 0 ),
self::MAX_REBOUND_DELAY
);
foreach ( $targets as $target ) {
if ( $target instanceof Title ) {
$this->titleTuples[] = [ $target, $delay ];
} else {
$this->urlTuples[] = [ $target, $delay ];
}
}
}
public function merge( MergeableUpdate $update ) {
@ -43,52 +60,46 @@ class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
Assert::parameterType( __CLASS__, $update, '$update' );
'@phan-var self $update';
$this->urls = array_merge( $this->urls, $update->urls );
$this->urlTuples = array_merge( $this->urlTuples, $update->urlTuples );
$this->titleTuples = array_merge( $this->titleTuples, $update->titleTuples );
}
/**
* Create an update object from an array of Title objects, or a TitleArray object
*
* @param Traversable|Title[] $titles
* @param string[] $urlArr
* @param string[] $urls
* @return CdnCacheUpdate
* @deprecated Since 1.35 Use HtmlCacheUpdater instead
*/
public static function newFromTitles( $titles, $urlArr = [] ) {
return new CdnCacheUpdate( array_merge( $titles, $urlArr ) );
public static function newFromTitles( $titles, $urls = [] ) {
return new CdnCacheUpdate( array_merge( $titles, $urls ) );
}
/**
* Purges the list of URLs passed to the constructor.
*/
public function doUpdate() {
global $wgCdnReboundPurgeDelay;
// Resolve the final list of URLs just before purging them (T240083)
$reboundDelayByUrl = $this->resolveReboundDelayByUrl();
$urls = [];
$titles = [];
foreach ( $this->urls as $u ) {
if ( $u instanceof Title ) {
$titles[] = $u;
} else {
$urls[] = $u;
// Send the immediate purges to CDN
self::purge( array_keys( $reboundDelayByUrl ) );
$immediatePurgeTimestamp = time();
// Get the URLs that need rebound purges, grouped by seconds of purge delay
$urlsWithReboundByDelay = [];
foreach ( $reboundDelayByUrl as $url => $delay ) {
if ( $delay > 0 ) {
$urlsWithReboundByDelay[$delay][] = $url;
}
}
if ( $titles ) {
( new LinkBatch( $titles ) )->execute();
/** @var Title $title */
foreach ( $titles as $title ) {
$urls = array_merge( $urls, $title->getCdnUrls() );
}
}
self::purge( $urls );
if ( $wgCdnReboundPurgeDelay > 0 ) {
JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob( [
// Enqueue delayed purge jobs for these URLs (usually only one job)
$jobs = [];
foreach ( $urlsWithReboundByDelay as $delay => $urls ) {
$jobs[] = new CdnPurgeJob( [
'urls' => $urls,
'jobReleaseTimestamp' => time() + $wgCdnReboundPurgeDelay
] ) );
'jobReleaseTimestamp' => $immediatePurgeTimestamp + $delay
] );
}
JobQueueGroup::singleton()->lazyPush( $jobs );
}
/**
@ -159,6 +170,44 @@ class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
}
}
/**
* @return string[] List of URLs
*/
public function getUrls() {
return array_keys( $this->resolveReboundDelayByUrl() );
}
/**
* @return int[] Map of (URL => rebound purge delay)
*/
private function resolveReboundDelayByUrl() {
/** @var Title $title */
// Avoid multiple queries for getCdnUrls() call
$lb = MediaWikiServices::getInstance()->getLinkBatchFactory()->newLinkBatch();
foreach ( $this->titleTuples as list( $title, $delay ) ) {
$lb->addObj( $title );
}
$lb->execute();
$reboundDelayByUrl = [];
// Resolve the titles into CDN URLs
foreach ( $this->titleTuples as list( $title, $delay ) ) {
foreach ( $title->getCdnUrls() as $url ) {
// Use the highest rebound for duplicate URLs in order to handle the most lag
$reboundDelayByUrl[$url] = max( $reboundDelayByUrl[$url] ?? 0, $delay );
}
}
foreach ( $this->urlTuples as list( $url, $delay ) ) {
// Use the highest rebound for duplicate URLs in order to handle the most lag
$reboundDelayByUrl[$url] = max( $reboundDelayByUrl[$url] ?? 0, $delay );
}
return $reboundDelayByUrl;
}
/**
* Send Hyper Text Caching Protocol (HTCP) CLR requests.
*

7
includes/deferred/HTMLCacheUpdate.php
View file

@ -1,7 +1,5 @@
<?php
/**
* HTML cache invalidation of all pages linking to a given title.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@ -18,13 +16,13 @@
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup Cache
*/
/**
* Class to invalidate the HTML/file cache of all the pages linking to a given title
* HTML file cache invalidation all the pages linking to a given title
*
* @ingroup Cache
* @deprecated Since 1.34; Enqueue jobs from HTMLCacheUpdateJob::newForBacklinks instead
*/
class HTMLCacheUpdate extends DataUpdate {
/** @var Title */
@ -53,7 +51,6 @@ class HTMLCacheUpdate extends DataUpdate {
$this->table,
[ 'causeAction' => $this->getCauseAction(), 'causeAgent' => $this->getCauseAgent() ]
);
JobQueueGroup::singleton()->lazyPush( $job );
}
}

66
includes/deferred/HtmlFileCacheUpdate.php Normal file
View file

@ -0,0 +1,66 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
use Wikimedia\Assert\Assert;
/**
* HTMLFileCache purge update for a set of titles
*
* @ingroup Cache
* @since 1.35
*/
class HtmlFileCacheUpdate implements DeferrableUpdate, MergeableUpdate {
/** @var string[] List of page prefixed DB keys */
private $prefixedDbKeys = [];
/**
* @param string[] $prefixedDbKeys List of page prefixed DB keys
*/
public function __construct( array $prefixedDbKeys ) {
$this->prefixedDbKeys = $prefixedDbKeys;
}
public function merge( MergeableUpdate $update ) {
/** @var self $update */
Assert::parameterType( __CLASS__, $update, '$update' );
'@phan-var self $update';
$this->prefixedDbKeys = array_merge( $this->prefixedDbKeys, $update->prefixedDbKeys );
}
/**
* @param Traversable|Title[] $titles Array or iterator of Title instances
* @return HtmlFileCacheUpdate
*/
public static function newFromTitles( $titles ) {
$prefixedDbKeys = [];
foreach ( $titles as $title ) {
$prefixedDbKeys[] = $title->getPrefixedDBkey();
}
return new self( $prefixedDbKeys );
}
public function doUpdate() {
foreach ( array_unique( $this->prefixedDbKeys ) as $prefixedDbKey ) {
HTMLFileCache::clearFileCache( $prefixedDbKey );
}
}
}

3
includes/filerepo/file/File.php
View file

@ -1495,7 +1495,8 @@ abstract class File implements IDBAccessObject {
$title = $this->getTitle();
if ( $title ) {
$title->invalidateCache();
$title->purgeSquid();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $title, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
}
}

51
includes/filerepo/file/LocalFile.php
View file

@ -1038,9 +1038,12 @@ class LocalFile extends File {
$this->purgeThumbnails( $options );
// Purge CDN cache for this file
DeferredUpdates::addUpdate(
new CdnCacheUpdate( [ $this->getUrl() ] ),
DeferredUpdates::PRESEND
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls(
$this->getUrl(),
!empty( $options['forThumbRefresh'] )
? $hcu::PURGE_PRESEND // just a manual purge
: $hcu::PURGE_INTENT_TXROUND_REFLECTED
);
}
@ -1064,7 +1067,9 @@ class LocalFile extends File {
foreach ( $files as $file ) {
$urls[] = $this->getArchiveThumbUrl( $archiveName, $file );
}
DeferredUpdates::addUpdate( new CdnCacheUpdate( $urls ), DeferredUpdates::PRESEND );
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls( $urls, $hcu::PURGE_PRESEND );
}
/**
@ -1097,7 +1102,13 @@ class LocalFile extends File {
$this->purgeThumbList( $dir, $files );
// Purge the CDN
DeferredUpdates::addUpdate( new CdnCacheUpdate( $urls ), DeferredUpdates::PRESEND );
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls(
$urls,
!empty( $options['forThumbRefresh'] )
? $hcu::PURGE_PRESEND // just a manual purge
: $hcu::PURGE_INTENT_TXROUND_REFLECTED
);
}
/**
@ -1694,8 +1705,10 @@ class LocalFile extends File {
}
} else {
# Existing file page: invalidate description page cache
$wikiPage->getTitle()->invalidateCache();
$wikiPage->getTitle()->purgeSquid();
$title = $wikiPage->getTitle();
$title->invalidateCache();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $title, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
# Allow the new file version to be patrolled from the page footer
Article::purgePatrolFooterCache( $descId );
}
@ -1743,10 +1756,8 @@ class LocalFile extends File {
# Delete old thumbnails
$this->purgeThumbnails();
# Remove the old file from the CDN cache
DeferredUpdates::addUpdate(
new CdnCacheUpdate( [ $this->getUrl() ] ),
DeferredUpdates::PRESEND
);
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls( $this->getUrl(), $hcu::PURGE_INTENT_TXROUND_REFLECTED );
} else {
# Update backlink pages pointing to this title if created
LinksUpdate::queueRecursiveJobsForTable(
@ -1769,9 +1780,12 @@ class LocalFile extends File {
}
# Invalidate cache for all pages using this file
DeferredUpdates::addUpdate(
new HTMLCacheUpdate( $this->getTitle(), 'imagelinks', 'file-upload' )
$job = HTMLCacheUpdateJob::newForBacklinks(
$this->getTitle(),
'imagelinks',
[ 'causeAction' => 'file-upload', 'causeAgent' => $user->getName() ]
);
JobQueueGroup::singleton()->lazyPush( $job );
return Status::newGood();
}
@ -2001,7 +2015,9 @@ class LocalFile extends File {
foreach ( $archiveNames as $archiveName ) {
$purgeUrls[] = $this->getArchiveUrl( $archiveName );
}
DeferredUpdates::addUpdate( new CdnCacheUpdate( $purgeUrls ), DeferredUpdates::PRESEND );
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls( $purgeUrls, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
return $status;
}
@ -2066,10 +2082,9 @@ class LocalFile extends File {
$this->purgeDescription();
}
DeferredUpdates::addUpdate(
new CdnCacheUpdate( [ $this->getArchiveUrl( $archiveName ) ] ),
DeferredUpdates::PRESEND
);
$url = $this->getArchiveUrl( $archiveName );
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls( $url, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
return $status;
}

26
includes/jobqueue/jobs/HTMLCacheUpdateJob.php
View file

@ -1,7 +1,5 @@
<?php
/**
* HTML cache invalidation of all pages linking to a given title.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@ -18,14 +16,12 @@
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup JobQueue
* @ingroup Cache
*/
use MediaWiki\MediaWikiServices;
/**
* Job to purge the cache for all pages that link to or use another page or file
* Job to purge the HTML/file cache for all pages that link to or use another page or file
*
* This job comes in a few variants:
* - a) Recursive jobs to purge caches for backlink pages for a given title.
@ -34,6 +30,7 @@ use MediaWiki\MediaWikiServices;
* These jobs have (pages:(<page ID>:(<namespace>,<title>),...) set.
*
* @ingroup JobQueue
* @ingroup Cache
*/
class HTMLCacheUpdateJob extends Job {
public function __construct( Title $title, array $params ) {
@ -110,7 +107,7 @@ class HTMLCacheUpdateJob extends Job {
* @param array $pages Map of (page ID => (namespace, DB key)) entries
*/
protected function invalidateTitles( array $pages ) {
global $wgUpdateRowsPerQuery, $wgUseFileCache, $wgPageLanguageUseDB;
global $wgUpdateRowsPerQuery, $wgPageLanguageUseDB;
// Get all page IDs in this query into an array
$pageIds = array_keys( $pages );
@ -160,20 +157,9 @@ class HTMLCacheUpdateJob extends Job {
__METHOD__
) );
// Update CDN; call purge() directly so as to not bother with secondary purges
$urls = [];
foreach ( $titleArray as $title ) {
/** @var Title $title */
$urls = array_merge( $urls, $title->getCdnUrls() );
}
CdnCacheUpdate::purge( $urls );
// Update file cache
if ( $wgUseFileCache ) {
foreach ( $titleArray as $title ) {
HTMLFileCache::clearFileCache( $title );
}
}
// Update CDN and file caches (avoiding secondary purge overhead)
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $titleArray, $hcu::PURGE_NAIVE );
}
public function getDeduplicationInfo() {

29
includes/page/WikiPage.php
View file

@ -1306,13 +1306,9 @@ class WikiPage implements Page, IDBAccessObject {
$this->mTitle->invalidateCache();
// Clear file cache
HTMLFileCache::clearFileCache( $this->getTitle() );
// Send purge after above page_touched update was committed
DeferredUpdates::addUpdate(
new CdnCacheUpdate( [ $this->mTitle ] ),
DeferredUpdates::PRESEND
);
// Clear file cache and send purge after above page_touched update was committed
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $this->mTitle, $hcu::PURGE_PRESEND );
if ( $this->mTitle->getNamespace() == NS_MEDIAWIKI ) {
MediaWikiServices::getInstance()->getMessageCache()
@ -3407,10 +3403,10 @@ class WikiPage implements Page, IDBAccessObject {
// Update existence markers on article/talk tabs...
$other = $title->getOtherPage();
$other->purgeSquid();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( [ $title, $other ], $hcu::PURGE_INTENT_TXROUND_REFLECTED );
$title->touchLinks();
$title->purgeSquid();
$title->deleteTitleProtection();
MediaWikiServices::getInstance()->getLinkCache()->invalidateTitle( $title );
@ -3441,20 +3437,16 @@ class WikiPage implements Page, IDBAccessObject {
// TODO: move this into a PageEventEmitter service
// Update existence markers on article/talk tabs...
// Clear Backlink cache first so that purge jobs use more up-to-date backlink information
BacklinkCache::get( $title )->clear();
$other = $title->getOtherPage();
$other->purgeSquid();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( [ $title, $other ], $hcu::PURGE_INTENT_TXROUND_REFLECTED );
$title->touchLinks();
$title->purgeSquid();
$services = MediaWikiServices::getInstance();
$services->getLinkCache()->invalidateTitle( $title );
// File cache
HTMLFileCache::clearFileCache( $title );
InfoAction::invalidateCache( $title );
// Messages
@ -3501,6 +3493,7 @@ class WikiPage implements Page, IDBAccessObject {
$slotsChanged = null
) {
// TODO: move this into a PageEventEmitter service
$jobs = [];
if ( $slotsChanged === null || in_array( SlotRecord::MAIN, $slotsChanged ) ) {
// Invalidate caches of articles which include this page.
@ -3522,10 +3515,8 @@ class WikiPage implements Page, IDBAccessObject {
MediaWikiServices::getInstance()->getLinkCache()->invalidateTitle( $title );
// Purge CDN for this page only
$title->purgeSquid();
// Clear file cache for this page only
HTMLFileCache::clearFileCache( $title );
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $title, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
// Purge ?action=info cache
$revid = $revision ? $revision->getId() : null;

7
includes/revisiondelete/RevDelFileList.php
View file

@ -122,10 +122,9 @@ class RevDelFileList extends RevDelList {
$file->purgeOldThumbnails( $archiveName );
$purgeUrls[] = $file->getArchiveUrl( $archiveName );
}
DeferredUpdates::addUpdate(
new CdnCacheUpdate( $purgeUrls ),
DeferredUpdates::PRESEND
);
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls( $purgeUrls, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
return Status::newGood();
}

4
includes/revisiondelete/RevDelRevisionList.php
View file

@ -179,12 +179,14 @@ class RevDelRevisionList extends RevDelList {
}
public function doPostCommitUpdates( array $visibilityChangeMap ) {
$this->title->purgeSquid();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $this->title, $hcu::PURGE_INTENT_TXROUND_REFLECTED );
// Extensions that require referencing previous revisions may need this
Hooks::run( 'ArticleRevisionVisibilitySet', [ $this->title, $this->ids, $visibilityChangeMap ] );
MediaWikiServices::getInstance()
->getMainWANObjectCache()
->touchCheckKey( "RevDelRevisionList:page:{$this->title->getArticleID()}}" );
return Status::newGood();
}
}

3
includes/user/User.php
View file

@ -3994,7 +3994,8 @@ class User implements IDBAccessObject, UserIdentity {
Hooks::run( 'UserSaveSettings', [ $this ] );
$this->clearSharedCache( 'changed' );
$this->getUserPage()->purgeSquid();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeTitleUrls( $this->getUserPage(), $hcu::PURGE_INTENT_TXROUND_REFLECTED );
}
/**

5
maintenance/purgeChangedPages.php
View file

@ -23,6 +23,7 @@
require_once __DIR__ . '/Maintenance.php';
use MediaWiki\MediaWikiServices;
use Wikimedia\Rdbms\IResultWrapper;
/**
@ -137,8 +138,8 @@ class PurgeChangedPages extends Maintenance {
}
// Send batch of purge requests out to CDN servers
$cdn = new CdnCacheUpdate( $urls );
$cdn->doUpdate();
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
$hcu->purgeUrls( $urls, $hcu::PURGE_NAIVE );
if ( $this->hasOption( 'sleep-per-batch' ) ) {
// sleep-per-batch is milliseconds, usleep wants micro seconds.

9
maintenance/purgeList.php
View file

@ -21,6 +21,8 @@
* @ingroup Maintenance
*/
use MediaWiki\MediaWikiServices;
require_once __DIR__ . '/Maintenance.php';
/**
@ -152,21 +154,20 @@ class PurgeList extends Maintenance {
* @param array $urls List of URLS to purge from CDNs
*/
private function sendPurgeRequest( $urls ) {
$hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
if ( $this->delay > 0 ) {
foreach ( $urls as $url ) {
if ( $this->hasOption( 'verbose' ) ) {
$this->output( $url . "\n" );
}
$u = new CdnCacheUpdate( [ $url ] );
$u->doUpdate();
$hcu->purgeUrls( $url, $hcu::PURGE_NAIVE );
usleep( $this->delay * 1e6 );
}
} else {
if ( $this->hasOption( 'verbose' ) ) {
$this->output( implode( "\n", $urls ) . "\n" );
}
$u = new CdnCacheUpdate( $urls );
$u->doUpdate();
$hcu->purgeUrls( $urls, $hcu::PURGE_NAIVE );
}
}
}

27
tests/phpunit/includes/deferred/CdnCacheUpdateTest.php
View file

@ -1,7 +1,5 @@
<?php
use Wikimedia\TestingAccessWrapper;
class CdnCacheUpdateTest extends MediaWikiTestCase {
/**
@ -10,13 +8,13 @@ class CdnCacheUpdateTest extends MediaWikiTestCase {
public function testPurgeMergeWeb() {
$this->setMwGlobals( 'wgCommandLineMode', false );
$urls1 = [];
$title = Title::newMainPage();
$urls1 = [];
$urls1[] = $title->getCanonicalURL( '?x=1' );
$urls1[] = $title->getCanonicalURL( '?x=2' );
$urls1[] = $title->getCanonicalURL( '?x=3' );
$update1 = $this->newCdnCacheUpdate( $urls1 );
DeferredUpdates::addUpdate( $update1 );
$urls2 = [];
$urls2[] = $title->getCanonicalURL( '?x=2' );
@ -24,11 +22,21 @@ class CdnCacheUpdateTest extends MediaWikiTestCase {
$urls2[] = $title->getCanonicalURL( '?x=4' );
$urls2[] = $title;
$update2 = $this->newCdnCacheUpdate( $urls2 );
$expected = [
$title->getInternalURL(),
$title->getInternalURL( 'action=history' ),
$title->getCanonicalURL( '?x=1' ),
$title->getCanonicalURL( '?x=2' ),
$title->getCanonicalURL( '?x=3' ),
$title->getCanonicalURL( '?x=4' ),
];
DeferredUpdates::addUpdate( $update1 );
DeferredUpdates::addUpdate( $update2 );
$wrapper = TestingAccessWrapper::newFromObject( $update1 );
$this->assertEquals( array_merge( $urls1, $urls2 ), $wrapper->urls );
$this->assertEquals( $expected, $update1->getUrls() );
/** @var CdnCacheUpdate $update */
$update = null;
DeferredUpdates::clearPendingUpdates();
DeferredUpdates::addCallableUpdate( function () use ( $urls1, $urls2, &$update ) {
@ -36,12 +44,13 @@ class CdnCacheUpdateTest extends MediaWikiTestCase {
DeferredUpdates::addUpdate( $update );
DeferredUpdates::addUpdate( $this->newCdnCacheUpdate( $urls2 ) );
DeferredUpdates::addUpdate(
$this->newCdnCacheUpdate( $urls2 ), DeferredUpdates::PRESEND );
$this->newCdnCacheUpdate( $urls2 ),
DeferredUpdates::PRESEND
);
} );
DeferredUpdates::doUpdates();
$wrapper = TestingAccessWrapper::newFromObject( $update );
$this->assertEquals( array_merge( $urls1, $urls2 ), $wrapper->urls );
$this->assertEquals( $expected, $update->getUrls() );
$this->assertEquals( DeferredUpdates::pendingUpdatesCount(), 0, 'PRESEND update run' );
}