Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

objectcache: add WANObjectCache::getMultiWithSetCallback()

Browse source 
This does what it says on the tin, e.g. a way to fetch
multiple keys at once. Callbacks are still called in a
serial manner when needed.

Change-Id: I8e24a6de7f46499a53ec41636c5a4f106b9b3d09
This commit is contained in:
Aaron Schulz 2016-09-13 00:00:11 -07:00 committed by Ori.livneh
parent 0829be4de2
commit 24200e88d2
2 changed files with 274 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
includes/libs/objectcache/WANObjectCache.php
View file

@ -88,6 +88,9 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
/** @var int ERR_* constant for the "last error" registry */
protected $lastRelayError = self::ERR_NONE;
/** @var mixed[] Temporary warm-up cache */
private $warmupCache = [];
/** Max time expected to pass between delete() and DB commit finishing */
const MAX_COMMIT_DELAY = 3;
/** Max replication+snapshot lag before applying TTL_LAGGED or disallowing set() */
@ -284,7 +287,14 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
}
// Fetch all of the raw values
$wrappedValues = $this->cache->getMulti( array_merge( $valueKeys, $checkKeysFlat ) );
$keysGet = array_merge( $valueKeys, $checkKeysFlat );
if ( $this->warmupCache ) {
$wrappedValues = array_intersect_key( $this->warmupCache, array_flip( $keysGet ) );
$keysGet = array_diff( $keysGet, array_keys( $wrappedValues ) ); // keys left to fetch
} else {
$wrappedValues = [];
}
$wrappedValues += $this->cache->getMulti( $keysGet );
// Time used to compare/init "check" keys (derived after getMulti() to be pessimistic)
$now = microtime( true );
@ -1016,6 +1026,95 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
return $value;
}
/**
* Method to fetch/regenerate multiple cache keys at once
*
* This works the same as getWithSetCallback() except:
* - a) The $keys argument expects the result of WANObjectCache::makeMultiKeys()
* - b) The $callback argument expects a callback taking the following arguments:
* - $id: ID of an entity to query
* - $oldValue : the prior cache value or false if none was present
* - &$ttl : a reference to the new value TTL in seconds
* - &$setOpts : a reference to options for set() which can be altered
* - $oldAsOf : generation UNIX timestamp of $oldValue or null if not present
* Aside from the additional $id argument, the other arguments function the same
* way they do in getWithSetCallback().
* - c) The return value is a map of (cache key => value) in the order of $keyedIds
*
* @see WANObjectCache::getWithSetCallback()
*
* Example usage:
* @code
* $rows = $cache->getMultiWithSetCallback(
* // Map of cache keys to entitiy IDs
* $cache->makeMultiKeys(
* $this->fileVersionIds(),
* function ( $id, WANObjectCache $cache ) {
* return $cache->makeKey( 'file-version', $id );
* }
* ),
* // Time-to-live (in seconds)
* $cache::TTL_DAY,
* // Function that derives the new key value
* return function ( $id, $oldValue, &$ttl, array &$setOpts ) {
* $dbr = wfGetDB( DB_REPLICA );
* // Account for any snapshot/replica DB lag
* $setOpts += Database::getCacheSetOptions( $dbr );
*
* // Load the row for this file
* $row = $dbr->selectRow( 'file', '*', [ 'id' => $id ], __METHOD__ );
*
* return $row ? (array)$row : false;
* },
* [
* // Process cache for 30 seconds
* 'pcTTL' => 30,
* // Use a dedicated 500 item cache (initialized on-the-fly)
* 'pcGroup' => 'file-versions:500'
* ]
* );
* $files = array_map( [ __CLASS__, 'newFromRow' ], $rows );
* @endcode
*
* @param ArrayIterator $keyedIds Result of WANObjectCache::makeMultiKeys()
* @param integer $ttl Seconds to live for key updates
* @param callable $callback Callback the yields entity regeneration callbacks
* @param array $opts Options map
* @return array Map of (cache key => value) in the same order as $keyedIds
* @since 1.28
*/
final public function getMultiWithSetCallback(
ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
) {
$keysWarmUp = iterator_to_array( $keyedIds, true );
$checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : [];
foreach ( $checkKeys as $i => $checkKeyOrKeys ) {
if ( is_int( $i ) ) {
$keysWarmUp[] = $checkKeyOrKeys;
} else {
$keysWarmUp = array_merge( $keysWarmUp, $checkKeyOrKeys );
}
}
$this->warmupCache = $this->cache->getMulti( $keysWarmUp );
$this->warmupCache += array_fill_keys( $keysWarmUp, false );
// Wrap $callback to match the getWithSetCallback() format while passing $id to $callback
$id = null;
$func = function ( $oldValue, &$ttl, array $setOpts, $oldAsOf ) use ( $callback, &$id ) {
return $callback( $id, $oldValue, $ttl, $setOpts, $oldAsOf );
};
$values = [];
foreach ( $keyedIds as $key => $id ) {
$values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
}
$this->warmupCache = [];
return $values;
}
/**
* @see BagOStuff::makeKey()
* @param string ... Key component
@ -1036,6 +1135,21 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
return call_user_func_array( [ $this->cache, __FUNCTION__ ], func_get_args() );
}
/**
* @param array $entities List of entity IDs
* @param callable $keyFunc Callback yielding a key from (entity ID, this WANObjectCache)
* @return ArrayIterator Iterator yielding (cache key => entity ID) in $entities order
* @since 1.28
*/
public function makeMultiKeys( array $entities, callable $keyFunc ) {
$map = [];
foreach ( $entities as $entity ) {
$map[$keyFunc( $entity, $this )] = $entity;
}
return new ArrayIterator( $map );
}
/**
* Get the "last error" registered; clearLastError() should be called manually
* @return int ERR_* class constant for the "last error" registry

166
tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
View file

@ -22,6 +22,7 @@ class WANObjectCacheTest extends MediaWikiTestCase {
}
$wanCache = TestingAccessWrapper::newFromObject( $this->cache );
/** @noinspection PhpUndefinedFieldInspection */
$this->internalCache = $wanCache->cache;
}
@ -29,13 +30,14 @@ class WANObjectCacheTest extends MediaWikiTestCase {
* @dataProvider provideSetAndGet
* @covers WANObjectCache::set()
* @covers WANObjectCache::get()
* @covers WANObjectCache::makeKey()
* @param mixed $value
* @param integer $ttl
*/
public function testSetAndGet( $value, $ttl ) {
$curTTL = null;
$asOf = null;
$key = wfRandomString();
$key = $this->cache->makeKey( 'x', wfRandomString() );
$this->cache->get( $key, $curTTL, [], $asOf );
$this->assertNull( $curTTL, "Current TTL is null" );
@ -71,9 +73,10 @@ class WANObjectCacheTest extends MediaWikiTestCase {
/**
* @covers WANObjectCache::get()
* @covers WANObjectCache::makeGlobalKey()
*/
public function testGetNotExists() {
$key = wfRandomString();
$key = $this->cache->makeGlobalKey( 'y', wfRandomString(), 'p' );
$curTTL = null;
$value = $this->cache->get( $key, $curTTL );
@ -165,7 +168,7 @@ class WANObjectCacheTest extends MediaWikiTestCase {
$priorAsOf = null;
$wasSet = 0;
$func = function( $old, &$ttl, &$opts, $asOf )
use ( &$wasSet, &$priorValue, &$priorAsOf, $value )
use ( &$wasSet, &$priorValue, &$priorAsOf, $value )
{
++$wasSet;
$priorValue = $old;
@ -188,9 +191,9 @@ class WANObjectCacheTest extends MediaWikiTestCase {
$wasSet = 0;
$v = $cache->getWithSetCallback( $key, 30, $func, [
'lowTTL' => 0,
'lockTSE' => 5,
] + $extOpts );
'lowTTL' => 0,
'lockTSE' => 5,
] + $extOpts );
$this->assertEquals( $value, $v, "Value returned" );
$this->assertEquals( 0, $wasSet, "Value not regenerated" );
@ -247,6 +250,150 @@ class WANObjectCacheTest extends MediaWikiTestCase {
];
}
/**
* @dataProvider getMultiWithSetCallback_provider
* @covers WANObjectCache::geMultitWithSetCallback()
* @covers WANObjectCache::makeMultiKeys()
* @param array $extOpts
* @param bool $versioned
*/
public function testGetMultiWithSetCallback( array $extOpts, $versioned ) {
$cache = $this->cache;
$keyA = wfRandomString();
$keyB = wfRandomString();
$keyC = wfRandomString();
$cKey1 = wfRandomString();
$cKey2 = wfRandomString();
$priorValue = null;
$priorAsOf = null;
$wasSet = 0;
$genFunc = function ( $id, $old, &$ttl, &$opts, $asOf ) use (
&$wasSet, &$priorValue, &$priorAsOf
) {
++$wasSet;
$priorValue = $old;
$priorAsOf = $asOf;
$ttl = 20; // override with another value
return "@$id$";
};
$wasSet = 0;
$keyedIds = new ArrayIterator( [ $keyA => 3353 ] );
$value = "@3353$";
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'lockTSE' => 5 ] + $extOpts );
$this->assertEquals( $value, $v[$keyA], "Value returned" );
$this->assertEquals( 1, $wasSet, "Value regenerated" );
$this->assertFalse( $priorValue, "No prior value" );
$this->assertNull( $priorAsOf, "No prior value" );
$curTTL = null;
$cache->get( $keyA, $curTTL );
$this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' );
$this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' );
$wasSet = 0;
$value = "@efef$";
$keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5, ] + $extOpts );
$this->assertEquals( $value, $v[$keyB], "Value returned" );
$this->assertEquals( 1, $wasSet, "Value regenerated" );
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5, ] + $extOpts );
$this->assertEquals( $value, $v[$keyB], "Value returned" );
$this->assertEquals( 1, $wasSet, "Value not regenerated" );
$priorTime = microtime( true );
usleep( 1 );
$wasSet = 0;
$keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
);
$this->assertEquals( $value, $v[$keyB], "Value returned" );
$this->assertEquals( 1, $wasSet, "Value regenerated due to check keys" );
$this->assertEquals( $value, $priorValue, "Has prior value" );
$this->assertType( 'float', $priorAsOf, "Has prior value" );
$t1 = $cache->getCheckKeyTime( $cKey1 );
$this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' );
$t2 = $cache->getCheckKeyTime( $cKey2 );
$this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' );
$priorTime = microtime( true );
$value = "@43636$";
$wasSet = 0;
$keyedIds = new ArrayIterator( [ $keyC => 43636 ] );
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
);
$this->assertEquals( $value, $v[$keyC], "Value returned" );
$this->assertEquals( 1, $wasSet, "Value regenerated due to still-recent check keys" );
$t1 = $cache->getCheckKeyTime( $cKey1 );
$this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' );
$t2 = $cache->getCheckKeyTime( $cKey2 );
$this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' );
$curTTL = null;
$v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] );
if ( $versioned ) {
$this->assertEquals( $value, $v[$cache::VFLD_DATA], "Value returned" );
} else {
$this->assertEquals( $value, $v, "Value returned" );
}
$this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" );
$wasSet = 0;
$key = wfRandomString();
$keyedIds = new ArrayIterator( [ $key => 242424 ] );
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
$this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value returned" );
$cache->delete( $key );
$keyedIds = new ArrayIterator( [ $key => 242424 ] );
$v = $cache->getMultiWithSetCallback(
$keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
$this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" );
$this->assertEquals( 1, $wasSet, "Value process cached while deleted" );
$calls = 0;
$ids = [ 1, 2, 3, 4, 5, 6 ];
$keyFunc = function ( $id, WANObjectCache $wanCache ) {
return $wanCache->makeKey( 'test', $id );
};
$keyedIds = $cache->makeMultiKeys( $ids, $keyFunc );
$genFunc = function ( $id, $oldValue, &$ttl, array &$setops ) use ( &$calls ) {
++$calls;
return "val-{$id}";
};
$values = $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc );
$this->assertEquals(
[ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ],
array_values( $values ),
"Correct values in correct order"
);
$this->assertEquals(
array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ),
array_keys( $values ),
"Correct keys in correct order"
);
$this->assertEquals( count( $ids ), $calls );
$cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc );
$this->assertEquals( count( $ids ), $calls, "Values cached" );
}
public static function getMultiWithSetCallback_provider() {
return [
[ [], false ],
[ [ 'version' => 1 ], true ]
];
}
/**
* @covers WANObjectCache::getWithSetCallback()
* @covers WANObjectCache::doGetWithSetCallback()
@ -777,9 +924,14 @@ class WANObjectCacheTest extends MediaWikiTestCase {
/**
* @dataProvider provideAdaptiveTTL
* @covers WANObjectCache::adaptiveTTL()
* @param float|int $ago
* @param int $maxTTL
* @param int $minTTL
* @param float $factor
* @param int $adaptiveTTL
*/
public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) {
$mtime = is_int( $ago ) ? time() - $ago : $ago;
$mtime = $ago ? time() - $ago : $ago;
$margin = 5;
$ttl = $this->cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor );