Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

objectcache: add "coalesceKeys" option to WANObjectCache for key grouping

Browse source 
This is useful for grouping related keys on the same servers to reduce
the need for cache server connections and availability. A cache key that
uses "lockTSE" can already involve accessing several keys during the
read/write cache-aside paths:
a) The value key itself
b) The check key (named after the main key, a common pattern)
c) The mutex key (used if the value looks stale)
d) The cool-off key (used if regeneration took a while)

Any problems accessing the first two could cause extra value regenerations.
Problems with the mutex key could lead to stampedes due to threads assuming
another thread was regerating a soon-to-expire value when, in fact, none was.
A similar problem could happen with cool-off keys, with threads assuming
that another saved the newly regenerated value when, in fact, none did.

The use of hash stops puts the tiny related keys on the same server as the
main cache key that they serve. This is only for hash-based routing, and not
route prefix routing (e.g. All*Route still sends the key to multiple child
routes, but the PoolRoute/HashRoute function will hash differently).

The option is not enabled by default yet.

Change-Id: I37e92a88f356ef1e2a2b7728263040e2f6f09a13
This commit is contained in:
Aaron Schulz 2019-08-22 20:57:11 -07:00 committed by Krinkle
parent bfac0ffc4f
commit 85bc62c5a8
2 changed files with 357 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

193
includes/libs/objectcache/wancache/WANObjectCache.php
View file

@ -137,6 +137,8 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
protected $epoch; protected $epoch;
/** @var string Stable secret used for hasing long strings into key components */ /** @var string Stable secret used for hasing long strings into key components */
protected $secret; protected $secret;
/** @var bool Whether to use Hash Tags and Hash Stops in key names */
protected $coalesceKeys;
/** @var int Callback stack depth for getWithSetCallback() */ /** @var int Callback stack depth for getWithSetCallback() */
private $callbackDepth = 0; private $callbackDepth = 0;
@ -239,12 +241,18 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
/** @var int Key to how long it took to generate the value */ /** @var int Key to how long it took to generate the value */
private static $FLD_GENERATION_TIME = 6; private static $FLD_GENERATION_TIME = 6;
private static $VALUE_KEY_PREFIX = 'WANCache:v:'; /** @var string Single character value mutex key component */
private static $INTERIM_KEY_PREFIX = 'WANCache:i:'; private static $TYPE_VALUE = 'v';
private static $TIME_KEY_PREFIX = 'WANCache:t:'; /** @var string Single character timestamp key component */
private static $MUTEX_KEY_PREFIX = 'WANCache:m:'; private static $TYPE_TIMESTAMP = 't';
private static $COOLOFF_KEY_PREFIX = 'WANCache:c:'; /** @var string Single character mutex key component */
private static $TYPE_MUTEX = 'm';
/** @var string Single character interium key component */
private static $TYPE_INTERIM = 'i';
/** @var string Single character cool-off key component */
private static $TYPE_COOLOFF = 'c';
/** @var string Prefix for tombstone key values */
private static $PURGE_VAL_PREFIX = 'PURGED:'; private static $PURGE_VAL_PREFIX = 'PURGED:';
/** /**
@ -271,6 +279,9 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
* requires that "region" and "cluster" are both set above. [optional] * requires that "region" and "cluster" are both set above. [optional]
* - epoch: lowest UNIX timestamp a value/tombstone must have to be valid. [optional] * - epoch: lowest UNIX timestamp a value/tombstone must have to be valid. [optional]
* - secret: stable secret used for hashing long strings into key components. [optional] * - secret: stable secret used for hashing long strings into key components. [optional]
* - coalesceKeys: whether to use Hash Tags and Hash Stops in key names so that related
* keys use the same cache server. This can reduce network overhead and reduce the
* chance the a single down cache server causes disruption. [default: false]
*/ */
public function __construct( array $params ) { public function __construct( array $params ) {
$this->cache = $params['cache']; $this->cache = $params['cache'];
@ -279,6 +290,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
$this->mcrouterAware = !empty( $params['mcrouterAware'] ); $this->mcrouterAware = !empty( $params['mcrouterAware'] );
$this->epoch = $params['epoch'] ?? 0; $this->epoch = $params['epoch'] ?? 0;
$this->secret = $params['secret'] ?? (string)$this->epoch; $this->secret = $params['secret'] ?? (string)$this->epoch;
$this->coalesceKeys = $params['coalesce'] ?? false;
$this->setLogger( $params['logger'] ?? new NullLogger() ); $this->setLogger( $params['logger'] ?? new NullLogger() );
$this->stats = $params['stats'] ?? new NullStatsdDataFactory(); $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
@ -405,14 +417,13 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
$curTTLs = []; $curTTLs = [];
$infoByKey = []; $infoByKey = [];
$vPrefixLen = strlen( self::$VALUE_KEY_PREFIX ); $valueKeys = $this->makeSisterKeys( $keys, self::$TYPE_VALUE );
$valueKeys = self::prefixCacheKeys( $keys, self::$VALUE_KEY_PREFIX );
$checkKeysForAll = []; $checkKeysForAll = [];
$checkKeysByKey = []; $checkKeysByKey = [];
$checkKeysFlat = []; $checkKeysFlat = [];
foreach ( $checkKeys as $i => $checkKeyGroup ) { foreach ( $checkKeys as $i => $checkKeyGroup ) {
$prefixed = self::prefixCacheKeys( (array)$checkKeyGroup, self::$TIME_KEY_PREFIX ); $prefixed = $this->makeSisterKeys( (array)$checkKeyGroup, self::$TYPE_TIMESTAMP );
$checkKeysFlat = array_merge( $checkKeysFlat, $prefixed ); $checkKeysFlat = array_merge( $checkKeysFlat, $prefixed );
// Are these check keys for a specific cache key, or for all keys being fetched? // Are these check keys for a specific cache key, or for all keys being fetched?
if ( is_int( $i ) ) { if ( is_int( $i ) ) {
@ -447,7 +458,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
// Get the main cache value for each key and validate them // Get the main cache value for each key and validate them
foreach ( $valueKeys as $vKey ) { foreach ( $valueKeys as $vKey ) {
$key = substr( $vKey, $vPrefixLen ); // unprefix $key = $this->extractBaseKey( $vKey ); // unprefix
list( $value, $keyInfo ) = $this->unwrap( list( $value, $keyInfo ) = $this->unwrap(
array_key_exists( $vKey, $wrappedValues ) ? $wrappedValues[$vKey] : false, array_key_exists( $vKey, $wrappedValues ) ? $wrappedValues[$vKey] : false,
$now $now
@ -669,10 +680,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
$storeTTL = $ttl + $staleTTL; $storeTTL = $ttl + $staleTTL;
if ( $creating ) { if ( $creating ) {
$ok = $this->cache->add( self::$VALUE_KEY_PREFIX . $key, $wrapped, $storeTTL ); $ok = $this->cache->add(
$this->makeSisterKey( $key, self::$TYPE_VALUE ),
$wrapped,
$storeTTL
);
} else { } else {
$ok = $this->cache->merge( $ok = $this->cache->merge(
self::$VALUE_KEY_PREFIX . $key, $this->makeSisterKey( $key, self::$TYPE_VALUE ),
function ( $cache, $key, $cWrapped ) use ( $wrapped ) { function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
// A string value means that it is a tombstone; do nothing in that case // A string value means that it is a tombstone; do nothing in that case
return ( is_string( $cWrapped ) ) ? false : $wrapped; return ( is_string( $cWrapped ) ) ? false : $wrapped;
@ -749,10 +764,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
final public function delete( $key, $ttl = self::HOLDOFF_TTL ) { final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
if ( $ttl <= 0 ) { if ( $ttl <= 0 ) {
// Publish the purge to all datacenters // Publish the purge to all datacenters
$ok = $this->relayDelete( self::$VALUE_KEY_PREFIX . $key ); $ok = $this->relayDelete( $this->makeSisterKey( $key, self::$TYPE_VALUE ) );
} else { } else {
// Publish the purge to all datacenters // Publish the purge to all datacenters
$ok = $this->relayPurge( self::$VALUE_KEY_PREFIX . $key, $ttl, self::HOLDOFF_TTL_NONE ); $ok = $this->relayPurge(
$this->makeSisterKey( $key, self::$TYPE_VALUE ),
$ttl,
self::HOLDOFF_TTL_NONE
);
} }
$kClass = $this->determineKeyClassForStats( $key ); $kClass = $this->determineKeyClassForStats( $key );
@ -848,7 +867,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
final public function getMultiCheckKeyTime( array $keys ) { final public function getMultiCheckKeyTime( array $keys ) {
$rawKeys = []; $rawKeys = [];
foreach ( $keys as $key ) { foreach ( $keys as $key ) {
$rawKeys[$key] = self::$TIME_KEY_PREFIX . $key; $rawKeys[$key] = $this->makeSisterKey( $key, self::$TYPE_TIMESTAMP );
} }
$rawValues = $this->cache->getMulti( $rawKeys ); $rawValues = $this->cache->getMulti( $rawKeys );
@ -912,7 +931,11 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
*/ */
final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) { final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
// Publish the purge to all datacenters // Publish the purge to all datacenters
$ok = $this->relayPurge( self::$TIME_KEY_PREFIX . $key, self::$CHECK_KEY_TTL, $holdoff ); $ok = $this->relayPurge(
$this->makeSisterKey( $key, self::$TYPE_TIMESTAMP ),
self::$CHECK_KEY_TTL,
$holdoff
);
$kClass = $this->determineKeyClassForStats( $key ); $kClass = $this->determineKeyClassForStats( $key );
$this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) ); $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
@ -949,7 +972,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
*/ */
final public function resetCheckKey( $key ) { final public function resetCheckKey( $key ) {
// Publish the purge to all datacenters // Publish the purge to all datacenters
$ok = $this->relayDelete( self::$TIME_KEY_PREFIX . $key ); $ok = $this->relayDelete( $this->makeSisterKey( $key, self::$TYPE_TIMESTAMP ) );
$kClass = $this->determineKeyClassForStats( $key ); $kClass = $this->determineKeyClassForStats( $key );
$this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) ); $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
@ -1495,7 +1518,11 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
*/ */
private function claimStampedeLock( $key ) { private function claimStampedeLock( $key ) {
// Note that locking is not bypassed due to I/O errors; this avoids stampedes // Note that locking is not bypassed due to I/O errors; this avoids stampedes
return $this->cache->add( self::$MUTEX_KEY_PREFIX . $key, 1, self::$LOCK_TTL ); return $this->cache->add(
$this->makeSisterKey( $key, self::$TYPE_MUTEX ),
1,
self::$LOCK_TTL
);
} }
/** /**
@ -1507,10 +1534,75 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
// The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local // The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local
// datacenter cache servers via OperationSelectorRoute (for increased consistency). // datacenter cache servers via OperationSelectorRoute (for increased consistency).
// Since that would be excessive for these locks, use TOUCH to expire the key. // Since that would be excessive for these locks, use TOUCH to expire the key.
$this->cache->changeTTL( self::$MUTEX_KEY_PREFIX . $key, $this->getCurrentTime() - 60 ); $this->cache->changeTTL(
$this->makeSisterKey( $key, self::$TYPE_MUTEX ),
$this->getCurrentTime() - 60
);
} }
} }
/**
* Get cache keys that should be collocated with their corresponding base keys
*
* @param string[] $baseKeys Cache keys made from makeKey()/makeGlobalKey()
* @param string $type Consistent hashing agnostic suffix character matching [a-zA-Z]
* @return string[] List of cache keys
*/
private function makeSisterKeys( array $baseKeys, $type ) {
$keys = [];
foreach ( $baseKeys as $baseKey ) {
$keys[] = $this->makeSisterKey( $baseKey, $type );
}
return $keys;
}
/**
* Get a cache key that should be collocated with a base key
*
* @param string $baseKey Cache key made from makeKey()/makeGlobalKey()
* @param string $type Consistent hashing agnostic suffix character matching [a-zA-Z]
* @return string Cache key
*/
private function makeSisterKey( $baseKey, $type ) {
if ( !$this->coalesceKeys ) {
// Old key style: "WANCache:<character>:<base key>"
return 'WANCache:' . $type . ':' . $baseKey;
}
return $this->mcrouterAware
// https://github.com/facebook/mcrouter/wiki/Key-syntax
// Note that the prefix prevents callers from making route keys.
? 'WANCache:' . $baseKey . '|#|' . $type
// https://redis.io/topics/cluster-spec
// https://github.com/twitter/twemproxy/blob/master/notes/recommendation.md
// https://github.com/Netflix/dynomite/blob/master/notes/recommendation.md
: 'WANCache:{' . $baseKey . '}:' . $type;
}
/**
* Get a cache key that should be collocated with a base key
*
* @param string $sisterKey Cache key made from makeSisterKey()
* @return string Original cache key made from makeKey()/makeGlobalKey()
*/
private function extractBaseKey( $sisterKey ) {
if ( !$this->coalesceKeys ) {
// Old key style: "WANCache:<character>:<base key>"
return substr( $sisterKey, 11 );
}
return $this->mcrouterAware
// Key style: "WANCache:<base key>|#|<character>"
// https://github.com/facebook/mcrouter/wiki/Key-syntax
? substr( $sisterKey, 9, -4 )
// Key style: "WANCache:{<base key>}:<character>"
// https://redis.io/topics/cluster-spec
// https://github.com/twitter/twemproxy/blob/master/notes/recommendation.md
// https://github.com/Netflix/dynomite/blob/master/notes/recommendation.md
: substr( $sisterKey, 10, -3 );
}
/** /**
* @param float $age Age of volatile/interim key in seconds * @param float $age Age of volatile/interim key in seconds
* @return bool Whether the age of a volatile value is negligible * @return bool Whether the age of a volatile value is negligible
@ -1544,7 +1636,11 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
$this->cache->clearLastError(); $this->cache->clearLastError();
if ( if (
!$this->cache->add( self::$COOLOFF_KEY_PREFIX . $key, 1, self::$COOLOFF_TTL ) && !$this->cache->add(
$this->makeSisterKey( $key, self::$TYPE_COOLOFF ),
1,
self::$COOLOFF_TTL
) &&
// Don't treat failures due to I/O errors as the key being in cooloff // Don't treat failures due to I/O errors as the key being in cooloff
$this->cache->getLastError() === BagOStuff::ERR_NONE $this->cache->getLastError() === BagOStuff::ERR_NONE
) { ) {
@ -1599,7 +1695,9 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
$now = $this->getCurrentTime(); $now = $this->getCurrentTime();
if ( $this->useInterimHoldOffCaching ) { if ( $this->useInterimHoldOffCaching ) {
$wrapped = $this->cache->get( self::$INTERIM_KEY_PREFIX . $key ); $wrapped = $this->cache->get(
$this->makeSisterKey( $key, self::$TYPE_INTERIM )
);
list( $value, $keyInfo ) = $this->unwrap( $wrapped, $now ); list( $value, $keyInfo ) = $this->unwrap( $wrapped, $now );
if ( $this->isValid( $value, $keyInfo['asOf'], $minAsOf ) ) { if ( $this->isValid( $value, $keyInfo['asOf'], $minAsOf ) ) {
@ -1622,7 +1720,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
$wrapped = $this->wrap( $value, $ttl, $version, $this->getCurrentTime(), $walltime ); $wrapped = $this->wrap( $value, $ttl, $version, $this->getCurrentTime(), $walltime );
$this->cache->merge( $this->cache->merge(
self::$INTERIM_KEY_PREFIX . $key, $this->makeSisterKey( $key, self::$TYPE_INTERIM ),
function () use ( $wrapped ) { function () use ( $wrapped ) {
return $wrapped; return $wrapped;
}, },
@ -1888,12 +1986,15 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
*/ */
final public function reap( $key, $purgeTimestamp, &$isStale = false ) { final public function reap( $key, $purgeTimestamp, &$isStale = false ) {
$minAsOf = $purgeTimestamp + self::HOLDOFF_TTL; $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL;
$wrapped = $this->cache->get( self::$VALUE_KEY_PREFIX . $key ); $wrapped = $this->cache->get( $this->makeSisterKey( $key, self::$TYPE_VALUE ) );
if ( is_array( $wrapped ) && $wrapped[self::$FLD_TIME] < $minAsOf ) { if ( is_array( $wrapped ) && $wrapped[self::$FLD_TIME] < $minAsOf ) {
$isStale = true; $isStale = true;
$this->logger->warning( "Reaping stale value key '$key'." ); $this->logger->warning( "Reaping stale value key '$key'." );
$ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation
$ok = $this->cache->changeTTL( self::$VALUE_KEY_PREFIX . $key, $ttlReap ); $ok = $this->cache->changeTTL(
$this->makeSisterKey( $key, self::$TYPE_VALUE ),
$ttlReap
);
if ( !$ok ) { if ( !$ok ) {
$this->logger->error( "Could not complete reap of key '$key'." ); $this->logger->error( "Could not complete reap of key '$key'." );
} }
@ -1916,11 +2017,16 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
* @since 1.28 * @since 1.28
*/ */
final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) { final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) {
$purge = $this->parsePurgeValue( $this->cache->get( self::$TIME_KEY_PREFIX . $key ) ); $purge = $this->parsePurgeValue(
$this->cache->get( $this->makeSisterKey( $key, self::$TYPE_TIMESTAMP ) )
);
if ( $purge && $purge[self::$PURGE_TIME] < $purgeTimestamp ) { if ( $purge && $purge[self::$PURGE_TIME] < $purgeTimestamp ) {
$isStale = true; $isStale = true;
$this->logger->warning( "Reaping stale check key '$key'." ); $this->logger->warning( "Reaping stale check key '$key'." );
$ok = $this->cache->changeTTL( self::$TIME_KEY_PREFIX . $key, self::TTL_SECOND ); $ok = $this->cache->changeTTL(
$this->makeSisterKey( $key, self::$TYPE_TIMESTAMP ),
self::TTL_SECOND
);
if ( !$ok ) { if ( !$ok ) {
$this->logger->error( "Could not complete reap of check key '$key'." ); $this->logger->error( "Could not complete reap of check key '$key'." );
} }
@ -2242,7 +2348,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
* *
* This must set the key to "PURGED:<UNIX timestamp>:<holdoff>" * This must set the key to "PURGED:<UNIX timestamp>:<holdoff>"
* *
* @param string $key Cache key * @param string $key Sister cache key
* @param int $ttl Seconds to keep the tombstone around * @param int $ttl Seconds to keep the tombstone around
* @param int $holdoff HOLDOFF_* constant controlling how long to ignore sets for this key * @param int $holdoff HOLDOFF_* constant controlling how long to ignore sets for this key
* @return bool Success * @return bool Success
@ -2271,7 +2377,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
/** /**
* Do the actual async bus delete of a key * Do the actual async bus delete of a key
* *
* @param string $key Cache key * @param string $key Sister cache key
* @return bool Success * @return bool Success
*/ */
protected function relayDelete( $key ) { protected function relayDelete( $key ) {
@ -2525,20 +2631,6 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
return [ $value, $info ]; return [ $value, $info ];
} }
/**
* @param string[] $keys
* @param string $prefix
* @return string[] Prefix keys; the order of $keys is preserved
*/
protected static function prefixCacheKeys( array $keys, $prefix ) {
$res = [];
foreach ( $keys as $key ) {
$res[] = $prefix . $key;
}
return $res;
}
/** /**
* @param string $key String of the format <scope>:<class>[:<class or variable>]... * @param string $key String of the format <scope>:<class>[:<class or variable>]...
* @return string A collection name to describe this class of key * @return string A collection name to describe this class of key
@ -2651,21 +2743,18 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
return []; return [];
} }
$keysWarmUp = [];
// Get all the value keys to fetch... // Get all the value keys to fetch...
foreach ( $keys as $key ) { $keysWarmUp = $this->makeSisterKeys( $keys, self::$TYPE_VALUE );
$keysWarmUp[] = self::$VALUE_KEY_PREFIX . $key;
}
// Get all the check keys to fetch... // Get all the check keys to fetch...
foreach ( $checkKeys as $i => $checkKeyOrKeys ) { foreach ( $checkKeys as $i => $checkKeyOrKeys ) {
if ( is_int( $i ) ) { if ( is_int( $i ) ) {
// Single check key that applies to all value keys // Single check key that applies to all value keys
$keysWarmUp[] = self::$TIME_KEY_PREFIX . $checkKeyOrKeys; $keysWarmUp[] = $this->makeSisterKey( $checkKeyOrKeys, self::$TYPE_TIMESTAMP );
} else { } else {
// List of check keys that apply to value key $i // List of check keys that apply to value key $i
$keysWarmUp = array_merge( $keysWarmUp = array_merge(
$keysWarmUp, $keysWarmUp,
self::prefixCacheKeys( $checkKeyOrKeys, self::$TIME_KEY_PREFIX ) $this->makeSisterKeys( $checkKeyOrKeys, self::$TYPE_TIMESTAMP )
); );
} }
} }
@ -2676,6 +2765,16 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
return $warmupCache; return $warmupCache;
} }
/**
* Set/clear the mcrouter support flag for testing
*
* @param bool $enabled
* @since 1.35
*/
public function setMcRouterAware( $enabled ) {
$this->mcrouterAware = $enabled;
}
/** /**
* @return float UNIX timestamp * @return float UNIX timestamp
* @codeCoverageIgnore * @codeCoverageIgnore

344
tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
View file

@ -1,4 +1,4 @@
<?php <?php /** @noinspection PhpStaticAsDynamicMethodCallInspection */
use Wikimedia\TestingAccessWrapper; use Wikimedia\TestingAccessWrapper;
@ -9,7 +9,9 @@ use Wikimedia\TestingAccessWrapper;
* @covers WANObjectCache::worthRefreshPopular * @covers WANObjectCache::worthRefreshPopular
* @covers WANObjectCache::isValid * @covers WANObjectCache::isValid
* @covers WANObjectCache::getWarmupKeyMisses * @covers WANObjectCache::getWarmupKeyMisses
* @covers WANObjectCache::prefixCacheKeys * @covers WANObjectCache::makeSisterKey
* @covers WANObjectCache::makeSisterKeys
* @covers WANObjectCache::extractBaseKey
* @covers WANObjectCache::getProcessCache * @covers WANObjectCache::getProcessCache
* @covers WANObjectCache::getNonProcessCachedMultiKeys * @covers WANObjectCache::getNonProcessCachedMultiKeys
* @covers WANObjectCache::getRawKeysForWarmup * @covers WANObjectCache::getRawKeysForWarmup
@ -20,22 +22,21 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
use MediaWikiCoversValidator; use MediaWikiCoversValidator;
/** @var WANObjectCache */ /**
private $cache; * @param array $params [optional]
/** @var BagOStuff */ * @return WANObjectCache[]|BagOStuff[] (WANObjectCache, BagOStuff)
private $internalCache; */
private function newWanCache( array $params = [] ) {
if ( !empty( $params['mcrouterAware'] ) ) {
// Convert mcrouter broadcast keys to regular keys in HashBagOStuff::delete() calls
$bag = new McrouterHashBagOStuff();
} else {
$bag = new HashBagOStuff();
}
protected function setUp() : void { $cache = new WANObjectCache( [ 'cache' => $bag ] + $params );
parent::setUp();
$this->cache = new WANObjectCache( [ return [ $cache, $bag ];
'cache' => new HashBagOStuff(),
'logger' => \MediaWiki\Logger\LoggerFactory::getInstance( 'objectcache' )
] );
$wanCache = TestingAccessWrapper::newFromObject( $this->cache );
/** @noinspection PhpUndefinedFieldInspection */
$this->internalCache = $wanCache->cache;
} }
/** /**
@ -47,7 +48,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @param int $ttl * @param int $ttl
*/ */
public function testSetAndGet( $value, $ttl ) { public function testSetAndGet( $value, $ttl ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$curTTL = null; $curTTL = null;
$asOf = null; $asOf = null;
@ -97,9 +98,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::makeGlobalKey() * @covers WANObjectCache::makeGlobalKey()
*/ */
public function testGetNotExists() { public function testGetNotExists() {
$key = $this->cache->makeGlobalKey( 'y', wfRandomString(), 'p' ); list( $cache ) = $this->newWanCache();
$key = $cache->makeGlobalKey( 'y', wfRandomString(), 'p' );
$curTTL = null; $curTTL = null;
$value = $this->cache->get( $key, $curTTL ); $value = $cache->get( $key, $curTTL );
$this->assertSame( false, $value, "Return value" ); $this->assertSame( false, $value, "Return value" );
$this->assertSame( null, $curTTL, "current TTL" ); $this->assertSame( null, $curTTL, "current TTL" );
@ -109,12 +112,14 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::set() * @covers WANObjectCache::set()
*/ */
public function testSetOver() { public function testSetOver() {
list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
for ( $i = 0; $i < 3; ++$i ) { for ( $i = 0; $i < 3; ++$i ) {
$value = wfRandomString(); $value = wfRandomString();
$this->cache->set( $key, $value, 3 ); $cache->set( $key, $value, 3 );
$this->assertSame( $this->cache->get( $key ), $value ); $this->assertSame( $cache->get( $key ), $value );
} }
} }
@ -122,18 +127,20 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::set() * @covers WANObjectCache::set()
*/ */
public function testStaleSet() { public function testStaleSet() {
list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$value = wfRandomString(); $value = wfRandomString();
$this->cache->set( $key, $value, 3, [ 'since' => microtime( true ) - 30 ] ); $cache->set( $key, $value, 3, [ 'since' => microtime( true ) - 30 ] );
$this->assertSame( false, $this->cache->get( $key ), "Stale set() value ignored" ); $this->assertSame( false, $cache->get( $key ), "Stale set() value ignored" );
} }
/** /**
* @covers WANObjectCache::getWithSetCallback * @covers WANObjectCache::getWithSetCallback
*/ */
public function testProcessCacheTTL() { public function testProcessCacheTTL() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
@ -159,7 +166,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getWithSetCallback * @covers WANObjectCache::getWithSetCallback
*/ */
public function testProcessCacheLruAndDelete() { public function testProcessCacheLruAndDelete() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
@ -204,7 +211,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getWithSetCallback * @covers WANObjectCache::getWithSetCallback
*/ */
public function testProcessCacheInterimKeys() { public function testProcessCacheInterimKeys() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
@ -241,7 +248,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getWithSetCallback * @covers WANObjectCache::getWithSetCallback
*/ */
public function testProcessCacheNesting() { public function testProcessCacheNesting() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
@ -303,10 +310,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::getWithSetCallback()
* @covers WANObjectCache::fetchOrRegenerate() * @covers WANObjectCache::fetchOrRegenerate()
* @param array $extOpts * @param array $extOpts
* @param bool $versioned
*/ */
public function testGetWithSetCallback( array $extOpts, $versioned ) { public function testGetWithSetCallback( array $extOpts ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$value = wfRandomString(); $value = wfRandomString();
@ -493,10 +499,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::getWithSetCallback()
* @covers WANObjectCache::fetchOrRegenerate() * @covers WANObjectCache::fetchOrRegenerate()
* @param array $extOpts * @param array $extOpts
* @param bool $versioned
*/ */
public function testGetWithSetcallback_touched( array $extOpts, $versioned ) { public function testGetWithSetCallback_touched( array $extOpts ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
@ -663,10 +668,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::makeMultiKeys * @covers WANObjectCache::makeMultiKeys
* @covers WANObjectCache::getMulti * @covers WANObjectCache::getMulti
* @param array $extOpts * @param array $extOpts
* @param bool $versioned
*/ */
public function testGetMultiWithSetCallback( array $extOpts, $versioned ) { public function testGetMultiWithSetCallback( array $extOpts ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$keyA = wfRandomString(); $keyA = wfRandomString();
$keyB = wfRandomString(); $keyB = wfRandomString();
@ -789,7 +793,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
"Correct values in correct order" "Correct values in correct order"
); );
$this->assertSame( $this->assertSame(
array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ), array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $cache ) ),
array_keys( $values ), array_keys( $values ),
"Correct keys in correct order" "Correct keys in correct order"
); );
@ -919,10 +923,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getMultiWithUnionSetCallback() * @covers WANObjectCache::getMultiWithUnionSetCallback()
* @covers WANObjectCache::makeMultiKeys() * @covers WANObjectCache::makeMultiKeys()
* @param array $extOpts * @param array $extOpts
* @param bool $versioned
*/ */
public function testGetMultiWithUnionSetCallback( array $extOpts, $versioned ) { public function testGetMultiWithUnionSetCallback( array $extOpts ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$keyA = wfRandomString(); $keyA = wfRandomString();
$keyB = wfRandomString(); $keyB = wfRandomString();
@ -1046,7 +1049,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
"Correct values in correct order" "Correct values in correct order"
); );
$this->assertSame( $this->assertSame(
array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ), array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $cache ) ),
array_keys( $values ), array_keys( $values ),
"Correct keys in correct order" "Correct keys in correct order"
); );
@ -1063,6 +1066,15 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
]; ];
} }
public static function provideCoalesceAndMcrouterSettings() {
return [
[ [ 'mcrouterAware' => false, 'coalesceKeys' => false ] ],
[ [ 'mcrouterAware' => false, 'coalesceKeys' => true ] ],
[ [ 'mcrouterAware' => true, 'cluster' => 'test', 'coalesceKeys' => false ] ],
[ [ 'mcrouterAware' => true, 'cluster' => 'test', 'coalesceKeys' => true ] ]
];
}
/** /**
* @dataProvider getMultiWithUnionSetCallbackRefresh_provider * @dataProvider getMultiWithUnionSetCallbackRefresh_provider
* @param bool $expiring * @param bool $expiring
@ -1156,9 +1168,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
/** /**
* @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::getWithSetCallback()
* @covers WANObjectCache::fetchOrRegenerate() * @covers WANObjectCache::fetchOrRegenerate()
* @dataProvider provideCoalesceAndMcrouterSettings
* @param array $params
*/ */
public function testLockTSE() { public function testLockTSE( array $params ) {
$cache = $this->cache; list( $cache, $bag ) = $this->newWanCache( $params );
$key = wfRandomString(); $key = wfRandomString();
$value = wfRandomString(); $value = wfRandomString();
@ -1176,7 +1190,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertSame( 1, $calls, 'Value was populated' ); $this->assertSame( 1, $calls, 'Value was populated' );
// Acquire the mutex to verify that getWithSetCallback uses lockTSE properly // Acquire the mutex to verify that getWithSetCallback uses lockTSE properly
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$checkKeys = [ wfRandomString() ]; // new check keys => force misses $checkKeys = [ wfRandomString() ]; // new check keys => force misses
$ret = $cache->getWithSetCallback( $key, 30, $func, $ret = $cache->getWithSetCallback( $key, 30, $func,
@ -1184,7 +1198,10 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertSame( $value, $ret, 'Old value used' ); $this->assertSame( $value, $ret, 'Old value used' );
$this->assertSame( 1, $calls, 'Callback was not used' ); $this->assertSame( 1, $calls, 'Callback was not used' );
$cache->delete( $key ); $cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->delete( $key ); // no value at all anymore and still locked
$cache->setMcRouterAware( $params['mcrouterAware'] );
$mockWallClock += 0.001; // cached values will be newer than tombstone $mockWallClock += 0.001; // cached values will be newer than tombstone
$ret = $cache->getWithSetCallback( $key, 30, $func, $ret = $cache->getWithSetCallback( $key, 30, $func,
[ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] );
@ -1197,13 +1214,27 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertSame( 2, $calls, 'Callback was not used; used interim (mutex failed)' ); $this->assertSame( 2, $calls, 'Callback was not used; used interim (mutex failed)' );
} }
private function setMutexKey( BagOStuff $bag, $key ) {
$bag->add( "{$key}:m", 1, 0 );
$bag->add( "{$key}|#|m", 1, 0 );
$bag->add( "WANCache:m:$key", 1, 0 );
}
private function clearMutexKey( BagOStuff $bag, $key ) {
$bag->delete( "{$key}:m" );
$bag->delete( "{$key}|#|m" );
$bag->delete( "WANCache:m:$key" );
}
/** /**
* @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::getWithSetCallback()
* @covers WANObjectCache::fetchOrRegenerate() * @covers WANObjectCache::fetchOrRegenerate()
* @covers WANObjectCache::set() * @covers WANObjectCache::set()
* @dataProvider provideCoalesceAndMcrouterSettings
* @param array $params
*/ */
public function testLockTSESlow() { public function testLockTSESlow( array $params ) {
$cache = $this->cache; list( $cache, $bag ) = $this->newWanCache( $params );
$key = wfRandomString(); $key = wfRandomString();
$key2 = wfRandomString(); $key2 = wfRandomString();
$value = wfRandomString(); $value = wfRandomString();
@ -1223,7 +1254,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
$this->assertSame( $value, $ret ); $this->assertSame( $value, $ret );
$this->assertSame( $value, $cache->get( $key, $curTTL ), 'Value was populated' ); $this->assertSame( $value, $cache->get( $key, $curTTL ), 'Value was populated' );
$this->assertSame( 1.0, $curTTL, 'Value has reduced logical TTL', 0.01 ); $this->assertEqualsWithDelta( 1.0, $curTTL, 0.01, 'Value has reduced logical TTL' );
$this->assertSame( 1, $calls, 'Value was generated' ); $this->assertSame( 1, $calls, 'Value was generated' );
$mockWallClock += 2; // low logical TTL expired $mockWallClock += 2; // low logical TTL expired
@ -1238,7 +1269,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$mockWallClock += 2; // low logical TTL expired $mockWallClock += 2; // low logical TTL expired
// Acquire a lock to verify that getWithSetCallback uses lockTSE properly // Acquire a lock to verify that getWithSetCallback uses lockTSE properly
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
$this->assertSame( $value, $ret ); $this->assertSame( $value, $ret );
@ -1246,7 +1277,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$mockWallClock += 301; // physical TTL expired $mockWallClock += 301; // physical TTL expired
// Acquire a lock to verify that getWithSetCallback uses lockTSE properly // Acquire a lock to verify that getWithSetCallback uses lockTSE properly
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
$this->assertSame( $value, $ret ); $this->assertSame( $value, $ret );
@ -1281,9 +1312,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
/** /**
* @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::getWithSetCallback()
* @covers WANObjectCache::fetchOrRegenerate() * @covers WANObjectCache::fetchOrRegenerate()
* @dataProvider provideCoalesceAndMcrouterSettings
* @param array $params
*/ */
public function testBusyValueBasic() { public function testBusyValueBasic( array $params ) {
$cache = $this->cache; list( $cache, $bag ) = $this->newWanCache( $params );
$key = wfRandomString(); $key = wfRandomString();
$value = wfRandomString(); $value = wfRandomString();
$busyValue = wfRandomString(); $busyValue = wfRandomString();
@ -1304,7 +1337,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$mockWallClock += 0.2; // interim keys not brand new $mockWallClock += 0.2; // interim keys not brand new
// Acquire a lock to verify that getWithSetCallback uses busyValue properly // Acquire a lock to verify that getWithSetCallback uses busyValue properly
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$checkKeys = [ wfRandomString() ]; // new check keys => force misses $checkKeys = [ wfRandomString() ]; // new check keys => force misses
$ret = $cache->getWithSetCallback( $key, 30, $func, $ret = $cache->getWithSetCallback( $key, 30, $func,
@ -1317,20 +1350,23 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertSame( $value, $ret, 'Old value used' ); $this->assertSame( $value, $ret, 'Old value used' );
$this->assertSame( 2, $calls, 'Callback was not used' ); $this->assertSame( 2, $calls, 'Callback was not used' );
$cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->delete( $key ); // no value at all anymore and still locked $cache->delete( $key ); // no value at all anymore and still locked
$cache->setMcRouterAware( $params['mcrouterAware'] );
$ret = $cache->getWithSetCallback( $key, 30, $func, $ret = $cache->getWithSetCallback( $key, 30, $func,
[ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
$this->assertSame( $busyValue, $ret, 'Callback was not used; used busy value' ); $this->assertSame( $busyValue, $ret, 'Callback was not used; used busy value' );
$this->assertSame( 2, $calls, 'Callback was not used; used busy value' ); $this->assertSame( 2, $calls, 'Callback was not used; used busy value' );
$this->internalCache->delete( 'WANCache:m:' . $key ); $this->clearMutexKey( $bag, $key );
$mockWallClock += 0.001; // cached values will be newer than tombstone $mockWallClock += 0.001; // cached values will be newer than tombstone
$ret = $cache->getWithSetCallback( $key, 30, $func, $ret = $cache->getWithSetCallback( $key, 30, $func,
[ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
$this->assertSame( $value, $ret, 'Callback was used; saved interim' ); $this->assertSame( $value, $ret, 'Callback was used; saved interim' );
$this->assertSame( 3, $calls, 'Callback was used; saved interim' ); $this->assertSame( 3, $calls, 'Callback was used; saved interim' );
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$ret = $cache->getWithSetCallback( $key, 30, $func, $ret = $cache->getWithSetCallback( $key, 30, $func,
[ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
$this->assertSame( $value, $ret, 'Callback was not used; used interim' ); $this->assertSame( $value, $ret, 'Callback was not used; used interim' );
@ -1363,7 +1399,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @param mixed $expected * @param mixed $expected
*/ */
public function testBusyValueTypes( $busyValue, $expected ) { public function testBusyValueTypes( $busyValue, $expected ) {
$cache = $this->cache; list( $cache, $bag ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
@ -1376,7 +1412,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
}; };
// Acquire a lock to verify that getWithSetCallback uses busyValue properly // Acquire a lock to verify that getWithSetCallback uses busyValue properly
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] ); $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] );
$this->assertSame( $expected, $ret, 'busyValue used as expected' ); $this->assertSame( $expected, $ret, 'busyValue used as expected' );
@ -1387,7 +1423,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::getMulti() * @covers WANObjectCache::getMulti()
*/ */
public function testGetMulti() { public function testGetMulti() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$value1 = [ 'this' => 'is', 'a' => 'test' ]; $value1 = [ 'this' => 'is', 'a' => 'test' ];
$value2 = [ 'this' => 'is', 'another' => 'test' ]; $value2 = [ 'this' => 'is', 'another' => 'test' ];
@ -1449,9 +1485,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
/** /**
* @covers WANObjectCache::getMulti() * @covers WANObjectCache::getMulti()
* @covers WANObjectCache::processCheckKeys() * @covers WANObjectCache::processCheckKeys()
* @param array $params
* @dataProvider provideCoalesceAndMcrouterSettings
*/ */
public function testGetMultiCheckKeys() { public function testGetMultiCheckKeys( array $params ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache( $params );
$checkAll = wfRandomString(); $checkAll = wfRandomString();
$check1 = wfRandomString(); $check1 = wfRandomString();
@ -1466,7 +1504,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
// Fake initial check key to be set in the past. Otherwise we'd have to sleep for // Fake initial check key to be set in the past. Otherwise we'd have to sleep for
// several seconds during the test to assert the behaviour. // several seconds during the test to assert the behaviour.
foreach ( [ $checkAll, $check1, $check2 ] as $checkKey ) { foreach ( [ $checkAll, $check1, $check2 ] as $checkKey ) {
$cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->touchCheckKey( $checkKey, WANObjectCache::HOLDOFF_TTL_NONE ); $cache->touchCheckKey( $checkKey, WANObjectCache::HOLDOFF_TTL_NONE );
$cache->setMcRouterAware( $params['mcrouterAware'] );
} }
$mockWallClock += 0.100; $mockWallClock += 0.100;
@ -1492,7 +1532,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertLessThanOrEqual( 10.5, $curTTLs['key2'], 'Initial ttls' ); $this->assertLessThanOrEqual( 10.5, $curTTLs['key2'], 'Initial ttls' );
$mockWallClock += 0.100; $mockWallClock += 0.100;
$cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->touchCheckKey( $check1 ); $cache->touchCheckKey( $check1 );
$cache->setMcRouterAware( $params['mcrouterAware'] );
$curTTLs = []; $curTTLs = [];
$result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [
@ -1509,7 +1551,9 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertLessThan( 0, $curTTLs['key1'], 'key1 TTL expired' ); $this->assertLessThan( 0, $curTTLs['key1'], 'key1 TTL expired' );
$this->assertGreaterThan( 0, $curTTLs['key2'], 'key2 still valid' ); $this->assertGreaterThan( 0, $curTTLs['key2'], 'key2 still valid' );
$cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->touchCheckKey( $checkAll ); $cache->touchCheckKey( $checkAll );
$cache->setMcRouterAware( $params['mcrouterAware'] );
$curTTLs = []; $curTTLs = [];
$result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [
@ -1532,7 +1576,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::processCheckKeys() * @covers WANObjectCache::processCheckKeys()
*/ */
public function testCheckKeyInitHoldoff() { public function testCheckKeyInitHoldoff() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
for ( $i = 0; $i < 500; ++$i ) { for ( $i = 0; $i < 500; ++$i ) {
$key = wfRandomString(); $key = wfRandomString();
@ -1565,7 +1609,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::processCheckKeys() * @covers WANObjectCache::processCheckKeys()
*/ */
public function testCheckKeyHoldoff() { public function testCheckKeyHoldoff() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$checkKey = wfRandomString(); $checkKey = wfRandomString();
@ -1595,37 +1639,38 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::relayPurge * @covers WANObjectCache::relayPurge
*/ */
public function testDelete() { public function testDelete() {
list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$value = wfRandomString(); $value = wfRandomString();
$this->cache->set( $key, $value ); $cache->set( $key, $value );
$curTTL = null; $curTTL = null;
$v = $this->cache->get( $key, $curTTL ); $v = $cache->get( $key, $curTTL );
$this->assertSame( $value, $v, "Key was created with value" ); $this->assertSame( $value, $v, "Key was created with value" );
$this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" ); $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" );
$this->cache->delete( $key ); $cache->delete( $key );
$curTTL = null; $curTTL = null;
$v = $this->cache->get( $key, $curTTL ); $v = $cache->get( $key, $curTTL );
$this->assertSame( false, $v, "Deleted key has false value" ); $this->assertSame( false, $v, "Deleted key has false value" );
$this->assertLessThan( 0, $curTTL, "Deleted key has current TTL < 0" ); $this->assertLessThan( 0, $curTTL, "Deleted key has current TTL < 0" );
$this->cache->set( $key, $value . 'more' ); $cache->set( $key, $value . 'more' );
$v = $this->cache->get( $key, $curTTL ); $v = $cache->get( $key, $curTTL );
$this->assertSame( false, $v, "Deleted key is tombstoned and has false value" ); $this->assertSame( false, $v, "Deleted key is tombstoned and has false value" );
$this->assertLessThan( 0, $curTTL, "Deleted key is tombstoned and has current TTL < 0" ); $this->assertLessThan( 0, $curTTL, "Deleted key is tombstoned and has current TTL < 0" );
$this->cache->set( $key, $value ); $cache->set( $key, $value );
$this->cache->delete( $key, WANObjectCache::HOLDOFF_TTL_NONE ); $cache->delete( $key, WANObjectCache::HOLDOFF_TTL_NONE );
$curTTL = null; $curTTL = null;
$v = $this->cache->get( $key, $curTTL ); $v = $cache->get( $key, $curTTL );
$this->assertSame( false, $v, "Deleted key has false value" ); $this->assertSame( false, $v, "Deleted key has false value" );
$this->assertSame( null, $curTTL, "Deleted key has null current TTL" ); $this->assertSame( null, $curTTL, "Deleted key has null current TTL" );
$this->cache->set( $key, $value ); $cache->set( $key, $value );
$v = $this->cache->get( $key, $curTTL ); $v = $cache->get( $key, $curTTL );
$this->assertSame( $value, $v, "Key was created with value" ); $this->assertSame( $value, $v, "Key was created with value" );
$this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" ); $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" );
} }
@ -1638,7 +1683,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @param bool $versioned * @param bool $versioned
*/ */
public function testGetWithSetCallback_versions( array $extOpts, $versioned ) { public function testGetWithSetCallback_versions( array $extOpts, $versioned ) {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$valueV1 = wfRandomString(); $valueV1 = wfRandomString();
@ -1716,9 +1761,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
/** /**
* @covers WANObjectCache::useInterimHoldOffCaching * @covers WANObjectCache::useInterimHoldOffCaching
* @covers WANObjectCache::getInterimValue * @covers WANObjectCache::getInterimValue
* @dataProvider provideCoalesceAndMcrouterSettings
* @param array $params
*/ */
public function testInterimHoldOffCaching() { public function testInterimHoldOffCaching( array $params ) {
$cache = $this->cache; list( $cache, $bag ) = $this->newWanCache( $params );
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
@ -1734,43 +1781,50 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$cache->useInterimHoldOffCaching( true ); $cache->useInterimHoldOffCaching( true );
$key = wfRandomString( 32 ); $key = wfRandomString( 32 );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 1, $wasCalled, 'Value cached' ); $this->assertSame( 1, $wasCalled, 'Value cached' );
$cache->delete( $key ); $cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->delete( $key ); // no value at all anymore and still locked
$cache->setMcRouterAware( $params['mcrouterAware'] );
$mockWallClock += 0.001; // cached values will be newer than tombstone $mockWallClock += 0.001; // cached values will be newer than tombstone
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim $this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 2, $wasCalled, 'Value interim cached' ); // reuses interim $this->assertSame( 2, $wasCalled, 'Value interim cached' ); // reuses interim
$mockWallClock += 0.2; // interim key not brand new $mockWallClock += 0.2; // interim key not brand new
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 3, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim $this->assertSame( 3, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim
// Lock up the mutex so interim cache is used // Lock up the mutex so interim cache is used
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 3, $wasCalled, 'Value interim cached (failed mutex)' ); $this->assertSame( 3, $wasCalled, 'Value interim cached (failed mutex)' );
$this->internalCache->delete( 'WANCache:m:' . $key ); $this->clearMutexKey( $bag, $key );
$cache->useInterimHoldOffCaching( false ); $cache->useInterimHoldOffCaching( false );
$wasCalled = 0; $wasCalled = 0;
$key = wfRandomString( 32 ); $key = wfRandomString( 32 );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 1, $wasCalled, 'Value cached' ); $this->assertSame( 1, $wasCalled, 'Value cached' );
$cache->delete( $key );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->setMcRouterAware( false ); // broadcast keys don't work with HashBagOStuff
$cache->delete( $key ); // no value at all anymore and still locked
$cache->setMcRouterAware( $params['mcrouterAware'] );
$cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); $this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 3, $wasCalled, 'Value still regenerated (got mutex)' ); $this->assertSame( 3, $wasCalled, 'Value still regenerated (got mutex)' );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 4, $wasCalled, 'Value still regenerated (got mutex)' ); $this->assertSame( 4, $wasCalled, 'Value still regenerated (got mutex)' );
// Lock up the mutex so interim cache is used // Lock up the mutex so interim cache is used
$this->internalCache->add( 'WANCache:m:' . $key, 1, 0 ); $this->setMutexKey( $bag, $key );
$v = $cache->getWithSetCallback( $key, 60, $func ); $cache->getWithSetCallback( $key, 60, $func );
$this->assertSame( 5, $wasCalled, 'Value still regenerated (failed mutex)' ); $this->assertSame( 5, $wasCalled, 'Value still regenerated (failed mutex)' );
} }
@ -1783,7 +1837,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::parsePurgeValue * @covers WANObjectCache::parsePurgeValue
*/ */
public function testTouchKeys() { public function testTouchKeys() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$key = wfRandomString(); $key = wfRandomString();
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
@ -1822,8 +1876,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
/** /**
* @covers WANObjectCache::getMulti() * @covers WANObjectCache::getMulti()
* @param array $params
* @dataProvider provideCoalesceAndMcrouterSettings
*/ */
public function testGetWithSeveralCheckKeys() { public function testGetWithSeveralCheckKeys( array $params ) {
list( $cache, $bag ) = $this->newWanCache( $params );
$key = wfRandomString(); $key = wfRandomString();
$tKey1 = wfRandomString(); $tKey1 = wfRandomString();
$tKey2 = wfRandomString(); $tKey2 = wfRandomString();
@ -1831,25 +1888,25 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$mockWallClock = 1549343530.2053; $mockWallClock = 1549343530.2053;
$priorTime = $mockWallClock; // reference time $priorTime = $mockWallClock; // reference time
$this->cache->setMockTime( $mockWallClock ); $cache->setMockTime( $mockWallClock );
// Two check keys are newer (given hold-off) than $key, another is older // Two check keys are newer (given hold-off) than $key, another is older
$this->internalCache->set( $bag->set(
'WANCache:t:' . $tKey2, 'WANCache:t:' . $tKey2,
'PURGED:' . ( $priorTime - 3 ) 'PURGED:' . ( $priorTime - 3 )
); );
$this->internalCache->set( $bag->set(
'WANCache:t:' . $tKey2, 'WANCache:t:' . $tKey2,
'PURGED:' . ( $priorTime - 5 ) 'PURGED:' . ( $priorTime - 5 )
); );
$this->internalCache->set( $bag->set(
'WANCache:t:' . $tKey1, 'WANCache:t:' . $tKey1,
'PURGED:' . ( $priorTime - 30 ) 'PURGED:' . ( $priorTime - 30 )
); );
$this->cache->set( $key, $value, 30 ); $cache->set( $key, $value, 30 );
$curTTL = null; $curTTL = null;
$v = $this->cache->get( $key, $curTTL, [ $tKey1, $tKey2 ] ); $v = $cache->get( $key, $curTTL, [ $tKey1, $tKey2 ] );
$this->assertSame( $value, $v, "Value matches" ); $this->assertSame( $value, $v, "Value matches" );
$this->assertLessThan( -4.9, $curTTL, "Correct CTL" ); $this->assertLessThan( -4.9, $curTTL, "Correct CTL" );
$this->assertGreaterThan( -5.1, $curTTL, "Correct CTL" ); $this->assertGreaterThan( -5.1, $curTTL, "Correct CTL" );
@ -1860,6 +1917,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::reapCheckKey() * @covers WANObjectCache::reapCheckKey()
*/ */
public function testReap() { public function testReap() {
list( $cache, $bag ) = $this->newWanCache();
$vKey1 = wfRandomString(); $vKey1 = wfRandomString();
$vKey2 = wfRandomString(); $vKey2 = wfRandomString();
$tKey1 = wfRandomString(); $tKey1 = wfRandomString();
@ -1870,7 +1928,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$goodTime = microtime( true ) - 5; $goodTime = microtime( true ) - 5;
$badTime = microtime( true ) - 300; $badTime = microtime( true ) - 300;
$this->internalCache->set( $bag->set(
'WANCache:v:' . $vKey1, 'WANCache:v:' . $vKey1,
[ [
0 => 1, 0 => 1,
@ -1879,7 +1937,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
3 => $goodTime 3 => $goodTime
] ]
); );
$this->internalCache->set( $bag->set(
'WANCache:v:' . $vKey2, 'WANCache:v:' . $vKey2,
[ [
0 => 1, 0 => 1,
@ -1888,25 +1946,25 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
3 => $badTime 3 => $badTime
] ]
); );
$this->internalCache->set( $bag->set(
'WANCache:t:' . $tKey1, 'WANCache:t:' . $tKey1,
'PURGED:' . $goodTime 'PURGED:' . $goodTime
); );
$this->internalCache->set( $bag->set(
'WANCache:t:' . $tKey2, 'WANCache:t:' . $tKey2,
'PURGED:' . $badTime 'PURGED:' . $badTime
); );
$this->assertSame( $value, $this->cache->get( $vKey1 ) ); $this->assertSame( $value, $cache->get( $vKey1 ) );
$this->assertSame( $value, $this->cache->get( $vKey2 ) ); $this->assertSame( $value, $cache->get( $vKey2 ) );
$this->cache->reap( $vKey1, $knownPurge, $bad1 ); $cache->reap( $vKey1, $knownPurge, $bad1 );
$this->cache->reap( $vKey2, $knownPurge, $bad2 ); $cache->reap( $vKey2, $knownPurge, $bad2 );
$this->assertSame( false, $bad1 ); $this->assertSame( false, $bad1 );
$this->assertTrue( $bad2 ); $this->assertTrue( $bad2 );
$this->cache->reapCheckKey( $tKey1, $knownPurge, $tBad1 ); $cache->reapCheckKey( $tKey1, $knownPurge, $tBad1 );
$this->cache->reapCheckKey( $tKey2, $knownPurge, $tBad2 ); $cache->reapCheckKey( $tKey2, $knownPurge, $tBad2 );
$this->assertSame( false, $tBad1 ); $this->assertSame( false, $tBad1 );
$this->assertTrue( $tBad2 ); $this->assertTrue( $tBad2 );
} }
@ -1941,34 +1999,36 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::set() * @covers WANObjectCache::set()
*/ */
public function testSetWithLag() { public function testSetWithLag() {
list( $cache ) = $this->newWanCache();
$value = 1; $value = 1;
$key = wfRandomString(); $key = wfRandomString();
$opts = [ 'lag' => 300, 'since' => microtime( true ) ]; $opts = [ 'lag' => 300, 'since' => microtime( true ) ];
$this->cache->set( $key, $value, 30, $opts ); $cache->set( $key, $value, 30, $opts );
$this->assertSame( $value, $this->cache->get( $key ), "Rep-lagged value written." ); $this->assertSame( $value, $cache->get( $key ), "Rep-lagged value written." );
$key = wfRandomString(); $key = wfRandomString();
$opts = [ 'lag' => 0, 'since' => microtime( true ) - 300 ]; $opts = [ 'lag' => 0, 'since' => microtime( true ) - 300 ];
$this->cache->set( $key, $value, 30, $opts ); $cache->set( $key, $value, 30, $opts );
$this->assertSame( false, $this->cache->get( $key ), "Trx-lagged value not written." ); $this->assertSame( false, $cache->get( $key ), "Trx-lagged value not written." );
$key = wfRandomString(); $key = wfRandomString();
$opts = [ 'lag' => 5, 'since' => microtime( true ) - 5 ]; $opts = [ 'lag' => 5, 'since' => microtime( true ) - 5 ];
$this->cache->set( $key, $value, 30, $opts ); $cache->set( $key, $value, 30, $opts );
$this->assertSame( false, $this->cache->get( $key ), "Lagged value not written." ); $this->assertSame( false, $cache->get( $key ), "Lagged value not written." );
} }
/** /**
* @covers WANObjectCache::set() * @covers WANObjectCache::set()
*/ */
public function testWritePending() { public function testWritePending() {
list( $cache ) = $this->newWanCache();
$value = 1; $value = 1;
$key = wfRandomString(); $key = wfRandomString();
$opts = [ 'pending' => true ]; $opts = [ 'pending' => true ];
$this->cache->set( $key, $value, 30, $opts ); $cache->set( $key, $value, 30, $opts );
$this->assertSame( false, $this->cache->get( $key ), "Pending value not written." ); $this->assertSame( false, $cache->get( $key ), "Pending value not written." );
} }
public function testMcRouterSupport() { public function testMcRouterSupport() {
@ -2091,14 +2151,15 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @param int $adaptiveTTL * @param int $adaptiveTTL
*/ */
public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) { public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) {
list( $cache ) = $this->newWanCache();
$mtime = $ago ? time() - $ago : $ago; $mtime = $ago ? time() - $ago : $ago;
$margin = 5; $margin = 5;
$ttl = $this->cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor ); $ttl = $cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor );
$this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl ); $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl );
$this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl ); $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl );
$ttl = $this->cache->adaptiveTTL( (string)$mtime, $maxTTL, $minTTL, $factor ); $ttl = $cache->adaptiveTTL( (string)$mtime, $maxTTL, $minTTL, $factor );
$this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl ); $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl );
$this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl ); $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl );
@ -2129,7 +2190,8 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::setLogger * @covers WANObjectCache::setLogger
*/ */
public function testSetLogger() { public function testSetLogger() {
$this->assertSame( null, $this->cache->setLogger( new Psr\Log\NullLogger ) ); list( $cache ) = $this->newWanCache();
$this->assertSame( null, $cache->setLogger( new Psr\Log\NullLogger ) );
} }
/** /**
@ -2194,8 +2256,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
/** /**
* @dataProvider statsKeyProvider * @dataProvider statsKeyProvider
* @covers WANObjectCache::determineKeyClassForStats * @covers WANObjectCache::determineKeyClassForStats
* @param string $key
* @param string $class
*/ */
public function testStatsKeyClass( $key, $class ) { public function testStatsKeyClass( $key, $class ) {
/** @var WANObjectCache $wanCache */
$wanCache = TestingAccessWrapper::newFromObject( new WANObjectCache( [ $wanCache = TestingAccessWrapper::newFromObject( new WANObjectCache( [
'cache' => new HashBagOStuff 'cache' => new HashBagOStuff
] ) ); ] ) );
@ -2207,7 +2272,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::makeMultiKeys * @covers WANObjectCache::makeMultiKeys
*/ */
public function testMakeMultiKeys() { public function testMakeMultiKeys() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$ids = [ 1, 2, 3, 4, 4, 5, 6, 6, 7, 7 ]; $ids = [ 1, 2, 3, 4, 4, 5, 6, 6, 7, 7 ];
$keyCallback = function ( $id, WANObjectCache $cache ) { $keyCallback = function ( $id, WANObjectCache $cache ) {
@ -2248,7 +2313,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::makeMultiKeys * @covers WANObjectCache::makeMultiKeys
*/ */
public function testMakeMultiKeysIntString() { public function testMakeMultiKeysIntString() {
$cache = $this->cache; list( $cache ) = $this->newWanCache();
$ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7, '7' ]; $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7, '7' ];
$keyCallback = function ( $id, WANObjectCache $cache ) { $keyCallback = function ( $id, WANObjectCache $cache ) {
return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' ); return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' );
@ -2272,10 +2337,11 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::makeMultiKeys * @covers WANObjectCache::makeMultiKeys
*/ */
public function testMakeMultiKeysCollision() { public function testMakeMultiKeysCollision() {
list( $cache ) = $this->newWanCache();
$ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7 ]; $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7 ];
$this->expectException( UnexpectedValueException::class ); $this->expectException( UnexpectedValueException::class );
$this->cache->makeMultiKeys( $cache->makeMultiKeys(
$ids, $ids,
function ( $id ) { function ( $id ) {
return "keymod:" . $id % 3; return "keymod:" . $id % 3;
@ -2287,12 +2353,13 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::multiRemap * @covers WANObjectCache::multiRemap
*/ */
public function testMultiRemap() { public function testMultiRemap() {
list( $cache ) = $this->newWanCache();
$a = [ 'a', 'b', 'c' ]; $a = [ 'a', 'b', 'c' ];
$res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3 ]; $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3 ];
$this->assertSame( $this->assertSame(
[ 'a' => 1, 'b' => 2, 'c' => 3 ], [ 'a' => 1, 'b' => 2, 'c' => 3 ],
$this->cache->multiRemap( $a, $res ) $cache->multiRemap( $a, $res )
); );
$a = [ 'a', 'b', 'c', 'c', 'd' ]; $a = [ 'a', 'b', 'c', 'c', 'd' ];
@ -2300,7 +2367,7 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
$this->assertSame( $this->assertSame(
[ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ], [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ],
$this->cache->multiRemap( $a, $res ) $cache->multiRemap( $a, $res )
); );
} }
@ -2308,26 +2375,25 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
* @covers WANObjectCache::hash256 * @covers WANObjectCache::hash256
*/ */
public function testHash256() { public function testHash256() {
$bag = new HashBagOStuff(); list( $cache ) = $this->newWanCache( [ 'epoch' => 5 ] );
$cache = new WANObjectCache( [ 'cache' => $bag, 'epoch' => 5 ] ); $this->assertEquals(
$this->assertSame(
'f402bce76bfa1136adc705d8d5719911ce1fe61f0ad82ddf79a15f3c4de6ec4c', 'f402bce76bfa1136adc705d8d5719911ce1fe61f0ad82ddf79a15f3c4de6ec4c',
$cache->hash256( 'x' ) $cache->hash256( 'x' )
); );
$cache = new WANObjectCache( [ 'cache' => $bag, 'epoch' => 50 ] ); list( $cache ) = $this->newWanCache( [ 'epoch' => 50 ] );
$this->assertSame( $this->assertSame(
'f79a126722f0a682c4c500509f1b61e836e56c4803f92edc89fc281da5caa54e', 'f79a126722f0a682c4c500509f1b61e836e56c4803f92edc89fc281da5caa54e',
$cache->hash256( 'x' ) $cache->hash256( 'x' )
); );
$cache = new WANObjectCache( [ 'cache' => $bag, 'secret' => 'garden' ] ); list( $cache ) = $this->newWanCache( [ 'secret' => 'garden' ] );
$this->assertSame( $this->assertSame(
'48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093', '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093',
$cache->hash256( 'x' ) $cache->hash256( 'x' )
); );
$cache = new WANObjectCache( [ 'cache' => $bag, 'secret' => 'garden', 'epoch' => 3 ] ); list( $cache ) = $this->newWanCache( [ 'secret' => 'garden', 'epoch' => 3 ] );
$this->assertSame( $this->assertSame(
'48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093', '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093',
$cache->hash256( 'x' ) $cache->hash256( 'x' )
@ -2335,6 +2401,18 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
} }
} }
class McrouterHashBagOStuff extends HashBagOStuff {
public function delete( $key, $flags = 0 ) {
// Convert mcrouter broadcast keys to regular keys in HashBagOStuff::delete() calls
// https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
if ( preg_match( '#^/\*/[^/]+/(.*)$#', $key, $m ) ) {
$key = $m[1];
}
return parent::delete( $key, $flags );
}
}
class NearExpiringWANObjectCache extends WANObjectCache { class NearExpiringWANObjectCache extends WANObjectCache {
const CLOCK_SKEW = 1; const CLOCK_SKEW = 1;