Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

objectcache: add statsd key metrics to BagOStuff classes

Browse source 
Update SQL, REST, and redis subclasses to emit call count and
payload size metrics for cache key operations. These metrics
are bucketed by cache key collection (similar to WANCache).

Bug: T235705
Change-Id: Icaa3fa1ae9c8b0f664c26ce70b7e1c4fc5f92767
This commit is contained in:
Aaron Schulz 2020-12-28 14:34:29 -08:00
parent 9834fd052c
commit 57325ba3bd
13 changed files with 392 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
includes/ServiceWiring.php
View file

@ -708,6 +708,7 @@ return [
}
$params = $mainConfig->get( 'ObjectCaches' )[$id];
$params['stats'] = $services->getStatsdDataFactory();
$store = ObjectCache::newFromParams( $params, $mainConfig );
$store->getLogger()->debug( 'MainObjectStash using store {class}', [
@ -739,6 +740,7 @@ return [
"wgObjectCaches must have \"$cacheId\" set (via wgWANObjectCaches)"
);
}
$storeParams['stats'] = $services->getStatsdDataFactory();
$store = ObjectCache::newFromParams( $storeParams, $mainConfig );
$logger = $store->getLogger();
$logger->debug( 'MainWANObjectCache using store {class}', [

95
includes/libs/objectcache/BagOStuff.php
View file

@ -26,6 +26,7 @@
* @defgroup Cache Cache
*/
use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
@ -69,10 +70,17 @@ abstract class BagOStuff implements
IStoreKeyEncoder,
LoggerAwareInterface
{
/** @var StatsdDataFactoryInterface */
protected $stats;
/** @var LoggerInterface */
protected $logger;
/** @var callable|null */
protected $asyncHandler;
/**
* @var array<string,array> Cache key processing callbacks and info for metrics
* @phan-var array<string,array{0:string,1:callable}>
*/
protected $wrapperInfoByPrefix = [];
/** @var int[] Map of (ATTR_* class constant => QOS_* class constant) */
protected $attrMap = [];
@ -117,11 +125,17 @@ abstract class BagOStuff implements
/** @var int Item does not involve any keys */
protected const RES_NONKEY = 1;
/** Key to the metric group to use for the relevant cache wrapper */
private const WRAPPER_STATS_GROUP = 0;
/** Key to the callback that extracts collection names from cache wrapper keys */
private const WRAPPER_COLLECTION_CALLBACK = 1;
/**
* Parameters include:
* - keyspace: Keyspace to use for keys in makeKey(). [Default: "local"]
* - asyncHandler: Callable to use for scheduling tasks after the web request ends.
* In CLI mode, it should run the task immediately. [Default: null]
* - stats: IStatsdDataFactory instance. [optional]
* - logger: Psr\Log\LoggerInterface instance. [optional]
* @param array $params
* @codingStandardsIgnoreStart
@ -130,8 +144,9 @@ abstract class BagOStuff implements
*/
public function __construct( array $params = [] ) {
$this->keyspace = $params['keyspace'] ?? 'local';
$this->setLogger( $params['logger'] ?? new NullLogger() );
$this->asyncHandler = $params['asyncHandler'] ?? null;
$this->stats = $params['stats'] ?? new NullStatsdDataFactory();
$this->setLogger( $params['logger'] ?? new NullLogger() );
}
/**
@ -494,22 +509,26 @@ abstract class BagOStuff implements
/**
* Make a cache key for the default keyspace and given components
*
* @param string $class Key collection name component
* @param string|int ...$components Key components for entity IDs
* @return string Keyspace-prepended list of encoded components as a colon-separated value
* @see IStoreKeyEncoder::makeGlobalKey()
*
* @param string $collection Key collection name component
* @param string|int ...$components Additional, ordered, key components for entity IDs
* @return string Colon-separated, keyspace-prepended, ordered list of encoded components
* @since 1.27
*/
abstract public function makeGlobalKey( $class, ...$components );
abstract public function makeGlobalKey( $collection, ...$components );
/**
* Make a cache key for the global keyspace and given components
*
* @param string $class Key collection name component
* @param string|int ...$components Key components for entity IDs
* @return string Keyspace-prepended list of encoded components as a colon-separated value
* @see IStoreKeyEncoder::makeKey()
*
* @param string $collection Key collection name component
* @param string|int ...$components Additional, ordered, key components for entity IDs
* @return string Colon-separated, keyspace-prepended, ordered list of encoded components
* @since 1.27
*/
abstract public function makeKey( $class, ...$components );
abstract public function makeKey( $collection, ...$components );
/**
* Check whether a cache key is in the global keyspace
@ -605,6 +624,39 @@ abstract class BagOStuff implements
*/
abstract public function setNewPreparedValues( array $valueByKey );
/**
* Register info about a caching layer class that uses BagOStuff as a backing store
*
* Object cache wrappers are classes that implement generic caching/storage functionality,
* use a BagOStuff instance as the backing store, and implement IStoreKeyEncoder with the
* same "generic" style key encoding as BagOStuff. Such wrappers transform keys before
* passing them to BagOStuff methods; a wrapper-specific prefix component will be prepended
* along with other possible additions. Transformed keys still use the "generic" BagOStuff
* encoding.
*
* The provided callback takes a transformed key, having the specified prefix component,
* and extracts the key collection name. For sanity, the callback must be able to handle
* keys that bear the prefix (by coincidence) but do not originate from the wrapper class.
*
* Calls to this method should be idempotent.
*
* @param string $prefixComponent Key prefix component used by the wrapper
* @param string $statsGroup Stats group to use for metrics from this wrapper
* @param callable $collectionCallback Static callback that gets the key collection name
* @internal For use with BagOStuff and WANObjectCache only
* @since 1.36
*/
public function registerWrapperInfoForStats(
string $prefixComponent,
string $statsGroup,
callable $collectionCallback
) {
$this->wrapperInfoByPrefix[$prefixComponent] = [
self::WRAPPER_STATS_GROUP => $statsGroup,
self::WRAPPER_COLLECTION_CALLBACK => $collectionCallback
];
}
/**
* At a minimum, there must be a keyspace and collection name component
*
@ -701,6 +753,31 @@ abstract class BagOStuff implements
return $genericRes;
}
/**
* @param string $key Key generated by an IStoreKeyEncoder instance
* @return string A stats prefix to describe this class of key (e.g. "objectcache.file")
*/
protected function determineKeyPrefixForStats( $key ) {
$firstComponent = substr( $key, 0, strcspn( $key, ':' ) );
$wrapperInfo = $this->wrapperInfoByPrefix[$firstComponent] ?? null;
if ( $wrapperInfo ) {
// Key has the prefix of a cache wrapper class that wraps BagOStuff
$collection = $wrapperInfo[self::WRAPPER_COLLECTION_CALLBACK]( $key );
$statsGroup = $wrapperInfo[self::WRAPPER_STATS_GROUP];
} else {
// Key came directly from BagOStuff::makeKey() or BagOStuff::makeGlobalKey()
// and thus has the format of "<scope>:<collection>[:<constant or variable>]..."
$components = explode( ':', $key, 3 );
// Handle legacy callers that fail to use the key building methods
$collection = $components[1] ?? $components[0];
$statsGroup = 'objectcache';
}
// Replace dots because they are special in StatsD (T232907)
return $statsGroup . '.' . strtr( $collection, '.', '_' );
}
/**
* @internal For testing only
* @return float UNIX timestamp

8
includes/libs/objectcache/CachedBagOStuff.php
View file

@ -165,12 +165,12 @@ class CachedBagOStuff extends BagOStuff {
return $this->genericKeyFromComponents( $keyspace, ...$components );
}
public function makeKey( $class, ...$components ) {
return $this->genericKeyFromComponents( $this->keyspace, $class, ...$components );
public function makeKey( $collection, ...$components ) {
return $this->genericKeyFromComponents( $this->keyspace, $collection, ...$components );
}
public function makeGlobalKey( $class, ...$components ) {
return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $class, ...$components );
public function makeGlobalKey( $collection, ...$components ) {
return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $collection, ...$components );
}
protected function convertGenericKey( $key ) {

22
includes/libs/objectcache/IStoreKeyEncoder.php
View file

@ -10,18 +10,24 @@ interface IStoreKeyEncoder {
/**
* Make a cache key using the "global" keyspace for the given components
*
* @param string $class Key collection name component
* @param string|int ...$components Key components for entity IDs
* @return string Keyspace-prepended list of encoded components as a colon-separated value
* Encoding is limited to the escaping of delimiter (":") and escape ("%") characters.
* Any backend-specific encoding should be delegated to methods that use the network.
*
* @param string $collection Key collection name component
* @param string|int ...$components Additional, ordered, key components for entity IDs
* @return string Colon-separated, keyspace-prepended, ordered list of encoded components
*/
public function makeGlobalKey( $class, ...$components );
public function makeGlobalKey( $collection, ...$components );
/**
* Make a cache key using the default keyspace for the given components
*
* @param string $class Key collection name component
* @param string|int ...$components Key components for entity IDs
* @return string Keyspace-prepended list of encoded components as a colon-separated value
* Encoding is limited to the escaping of delimiter (":") and escape ("%") characters.
* Any backend-specific encoding should be delegated to methods that use the network.
*
* @param string $collection Key collection name component
* @param string|int ...$components Additional, ordered, key components for entity IDs
* @return string Colon-separated, keyspace-prepended, ordered list of encoded components
*/
public function makeKey( $class, ...$components );
public function makeKey( $collection, ...$components );
}

76
includes/libs/objectcache/MediumSpecificBagOStuff.php
View file

@ -62,6 +62,15 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
/** @var int Idiom for doGet() to return extra information by reference */
protected const PASS_BY_REF = -1;
protected const METRIC_OP_GET = 'get';
protected const METRIC_OP_SET = 'set';
protected const METRIC_OP_DELETE = 'delete';
protected const METRIC_OP_CHANGE_TTL = 'change_ttl';
protected const METRIC_OP_ADD = 'add';
protected const METRIC_OP_INCR = 'incr';
protected const METRIC_OP_DECR = 'decr';
protected const METRIC_OP_CAS = 'cas';
/**
* @see BagOStuff::__construct()
* Additional $params options include:
@ -357,7 +366,7 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
* @return bool Success
*/
protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
// @TODO: the lock() call assumes that all other relavent sets() use one
// @TODO: the use of lock() assumes that all other relevant sets() use a lock
if ( !$this->lock( $key, 0 ) ) {
return false; // non-blocking
}
@ -419,6 +428,7 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
* @return bool
*/
protected function doChangeTTL( $key, $exptime, $flags ) {
// @TODO: the use of lock() assumes that all other relevant sets() use a lock
if ( !$this->lock( $key, 0 ) ) {
return false;
}
@ -574,7 +584,7 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
* Get an associative array containing the item for each of the keys that have items.
* @param string[] $keys List of keys
* @param int $flags Bitfield; supports READ_LATEST [optional]
* @return array Map of (key => value) for existing keys
* @return array Map of (key => value) for existing keys; preserves the order of $keys
*/
protected function doGetMulti( array $keys, $flags = 0 ) {
$res = [];
@ -660,11 +670,20 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
* @param int $exptime TTL or UNIX timestamp
* @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33)
* @return bool Success
* @see BagOStuff::changeTTL()
*
* @since 1.34
*/
public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
return $this->doChangeTTLMulti( $keys, $exptime, $flags );
}
/**
* @param string[] $keys List of keys
* @param int $exptime TTL or UNIX timestamp
* @param int $flags Bitfield of BagOStuff::WRITE_* constants
* @return bool Success
*/
protected function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
$res = true;
foreach ( $keys as $key ) {
$res = $this->doChangeTTL( $key, $exptime, $flags ) && $res;
@ -883,11 +902,11 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
return ( $value === (string)$integer );
}
public function makeGlobalKey( $class, ...$components ) {
public function makeGlobalKey( $collection, ...$components ) {
return $this->makeKeyInternal( self::GLOBAL_KEYSPACE, func_get_args() );
}
public function makeKey( $class, ...$components ) {
public function makeKey( $collection, ...$components ) {
return $this->makeKeyInternal( $this->keyspace, func_get_args() );
}
@ -1042,4 +1061,51 @@ abstract class MediumSpecificBagOStuff extends BagOStuff {
$this->logger->debug( "{class} debug: $text", [ 'class' => static::class ] );
}
}
/**
* @param string $op Operation name as a MediumSpecificBagOStuff::METRIC_OP_* constant
* @param string[] $keys List of cache keys referenced by this operation
* @param int[]|null $sSizes List of corresponding send payload sizes; null if not applicable
* @param int[]|false[]|null $rSizes List of corresponding receive payload sizes,
* with "false" entries indicating that the key was not found; null if not applicable
*/
protected function updateOpStats(
string $op,
array $keys,
?array $sSizes = null,
?array $rSizes = null
) {
$deltasByMetric = [];
foreach ( $keys as $i => $key ) {
// Metric prefix for the cache wrapper and key collection name
$prefix = $this->determineKeyPrefixForStats( $key );
if ( $op === self::METRIC_OP_GET && $rSizes ) {
// This operation was either a "hit" or "miss" for this key
$name = ( $rSizes[$i] === false )
? "{$prefix}.{$op}_miss_rate"
: "{$prefix}.{$op}_hit_rate";
} else {
// There is no concept of "hit" or "miss" for this operation
$name = "{$prefix}.{$op}_rate";
}
$deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + 1;
$bytesSent = ( $sSizes ? $sSizes[$i] : 0 );
if ( $bytesSent > 0 ) {
$name = "{$prefix}.{$op}_bytes_sent";
$deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + $bytesSent;
}
$bytesRead = ( $rSizes ? $rSizes[$i] : 0 );
if ( $bytesRead > 0 ) {
$name = "{$prefix}.{$op}_bytes_read";
$deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + $bytesRead;
}
}
foreach ( $deltasByMetric as $name => $delta ) {
$this->stats->updateCount( $name, $delta );
}
}
}

8
includes/libs/objectcache/MultiWriteBagOStuff.php
View file

@ -373,12 +373,12 @@ class MultiWriteBagOStuff extends BagOStuff {
return $this->genericKeyFromComponents( $keyspace, ...$components );
}
public function makeKey( $class, ...$components ) {
return $this->genericKeyFromComponents( $this->keyspace, $class, ...$components );
public function makeKey( $collection, ...$components ) {
return $this->genericKeyFromComponents( $this->keyspace, $collection, ...$components );
}
public function makeGlobalKey( $class, ...$components ) {
return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $class, ...$components );
public function makeGlobalKey( $collection, ...$components ) {
return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $collection, ...$components );
}
protected function convertGenericKey( $key ) {

50
includes/libs/objectcache/RESTBagOStuff.php
View file

@ -146,23 +146,23 @@ class RESTBagOStuff extends MediumSpecificBagOStuff {
'headers' => $this->httpParams['readHeaders'],
];
$value = false;
$valueSize = false;
list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->client->run( $req );
if ( $rcode === 200 ) {
if ( is_string( $rbody ) ) {
$value = $this->decodeBody( $rbody );
/// @FIXME: use some kind of hash or UUID header as CAS token
if ( $getToken && $value !== false ) {
$casToken = $rbody;
}
return $value;
if ( $rcode === 200 && is_string( $rbody ) ) {
$value = $this->decodeBody( $rbody );
$valueSize = strlen( $rbody );
/// @FIXME: use some kind of hash or UUID header as CAS token
if ( $getToken && $value !== false ) {
$casToken = $rbody;
}
return false;
} elseif ( $rcode === 0 || ( $rcode >= 400 && $rcode != 404 ) ) {
$this->handleError( "Failed to fetch $key", $rcode, $rerr, $rhdrs, $rbody );
}
if ( $rcode === 0 || ( $rcode >= 400 && $rcode != 404 ) ) {
return $this->handleError( "Failed to fetch $key", $rcode, $rerr, $rhdrs, $rbody );
}
return false;
$this->updateOpStats( self::METRIC_OP_GET, [ $key ], null, [ $valueSize ] );
return $value;
}
protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
@ -176,10 +176,14 @@ class RESTBagOStuff extends MediumSpecificBagOStuff {
];
list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->client->run( $req );
if ( $rcode === 200 || $rcode === 201 || $rcode === 204 ) {
return true;
$res = ( $rcode === 200 || $rcode === 201 || $rcode === 204 );
if ( !$res ) {
$this->handleError( "Failed to store $key", $rcode, $rerr, $rhdrs, $rbody );
}
return $this->handleError( "Failed to store $key", $rcode, $rerr, $rhdrs, $rbody );
$this->updateOpStats( self::METRIC_OP_SET, [ $key ], [ strlen( $rbody ) ] );
return $res;
}
protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) {
@ -200,10 +204,14 @@ class RESTBagOStuff extends MediumSpecificBagOStuff {
];
list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->client->run( $req );
if ( in_array( $rcode, [ 200, 204, 205, 404, 410 ] ) ) {
return true;
$res = in_array( $rcode, [ 200, 204, 205, 404, 410 ] );
if ( !$res ) {
$this->handleError( "Failed to delete $key", $rcode, $rerr, $rhdrs, $rbody );
}
return $this->handleError( "Failed to delete $key", $rcode, $rerr, $rhdrs, $rbody );
$this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] );
return $res;
}
public function incr( $key, $value = 1, $flags = 0 ) {
@ -307,7 +315,6 @@ class RESTBagOStuff extends MediumSpecificBagOStuff {
* @param string $rerr Error message from client
* @param array $rhdrs Response headers
* @param string $rbody Error body from client (if any)
* @return false
*/
protected function handleError( $msg, $rcode, $rerr, $rhdrs, $rbody ) {
$message = "$msg : ({code}) {error}";
@ -334,6 +341,5 @@ class RESTBagOStuff extends MediumSpecificBagOStuff {
$this->logger->error( $message, $context );
$this->setLastError( $rcode === 0 ? self::ERR_UNREACHABLE : self::ERR_UNEXPECTED );
return false;
}
}

73
includes/libs/objectcache/RedisBagOStuff.php
View file

@ -99,18 +99,22 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$e = null;
try {
$value = $conn->get( $key );
if ( $getToken && $value !== false ) {
$casToken = $value;
$blob = $conn->get( $key );
if ( $getToken && $blob !== false ) {
$casToken = $blob;
}
$result = $this->unserialize( $value );
$result = $this->unserialize( $blob );
$valueSize = strlen( $blob );
} catch ( RedisException $e ) {
$result = false;
$valueSize = false;
$this->handleException( $conn, $e );
}
$this->logRequest( 'get', $key, $conn->getServer(), $e );
$this->updateOpStats( self::METRIC_OP_GET, [ $key ], null, [ $valueSize ] );
return $result;
}
@ -121,13 +125,14 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
}
$ttl = $this->getExpirationAsTTL( $exptime );
$serialized = $this->getSerialized( $value, $key );
$e = null;
try {
if ( $ttl ) {
$result = $conn->setex( $key, $ttl, $this->getSerialized( $value, $key ) );
$result = $conn->setex( $key, $ttl, $serialized );
} else {
$result = $conn->set( $key, $this->getSerialized( $value, $key ) );
$result = $conn->set( $key, $serialized );
}
} catch ( RedisException $e ) {
$result = false;
@ -136,6 +141,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'set', $key, $conn->getServer(), $e );
$this->updateOpStats( self::METRIC_OP_SET, [ $key ], [ strlen( $serialized ) ] );
return $result;
}
@ -156,6 +163,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'delete', $key, $conn->getServer(), $e );
$this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] );
return $result;
}
@ -172,7 +181,7 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
}
}
$result = [];
$blobsFound = [];
foreach ( $batches as $server => $batchKeys ) {
$conn = $conns[$server];
@ -189,9 +198,9 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
continue;
}
foreach ( $batchResult as $i => $value ) {
if ( $value !== false ) {
$result[$batchKeys[$i]] = $this->unserialize( $value );
foreach ( $batchResult as $i => $blob ) {
if ( $blob !== false ) {
$blobsFound[$batchKeys[$i]] = $blob;
}
}
} catch ( RedisException $e ) {
@ -201,6 +210,24 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
}
// Preserve the order of $keys
$result = [];
$valueSizes = [];
foreach ( $keys as $key ) {
if ( array_key_exists( $key, $blobsFound ) ) {
$blob = $blobsFound[$key];
$value = $this->unserialize( $blob );
if ( $value !== false ) {
$result[$key] = $value;
}
$valueSizes[] = strlen( $blob );
} else {
$valueSizes[] = false;
}
}
$this->updateOpStats( self::METRIC_OP_GET, $keys, [], $valueSizes );
return $result;
}
@ -221,18 +248,21 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$op = $ttl ? 'setex' : 'set';
$result = true;
$valueSizes = [];
foreach ( $batches as $server => $batchKeys ) {
$conn = $conns[$server];
$e = null;
$serialized = $this->getSerialized( $data[$key], $key );
$valueSizes[] = strlen( $serialized );
try {
// Avoid mset() to reduce CPU hogging from a single request
$conn->multi( Redis::PIPELINE );
foreach ( $batchKeys as $key ) {
if ( $ttl ) {
$conn->setex( $key, $ttl, $this->getSerialized( $data[$key], $key ) );
$conn->setex( $key, $ttl, $serialized );
} else {
$conn->set( $key, $this->getSerialized( $data[$key], $key ) );
$conn->set( $key, $serialized );
}
}
$batchResult = $conn->exec();
@ -249,6 +279,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
}
$this->updateOpStats( self::METRIC_OP_SET, array_keys( $data ), $valueSizes );
return $result;
}
@ -291,10 +323,12 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
}
$this->updateOpStats( self::METRIC_OP_DELETE, $keys );
return $result;
}
public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
public function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
/** @var RedisConnRef[]|Redis[] $conns */
$conns = [];
$batches = [];
@ -342,6 +376,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
}
$this->updateOpStats( self::METRIC_OP_CHANGE_TTL, $keys );
return $result;
}
@ -352,10 +388,11 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
}
$ttl = $this->getExpirationAsTTL( $expiry );
$serialized = $this->getSerialized( $value, $key );
try {
$result = $conn->set(
$key,
$this->getSerialized( $value, $key ),
$serialized,
$ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
);
} catch ( RedisException $e ) {
@ -365,6 +402,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'add', $key, $conn->getServer(), $result );
$this->updateOpStats( self::METRIC_OP_ADD, [ $key ], [ strlen( $serialized ) ] );
return $result;
}
@ -387,6 +426,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'incr', $key, $conn->getServer(), $result );
$this->updateOpStats( self::METRIC_OP_INCR, [ $key ] );
return $result;
}
@ -409,6 +450,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->logRequest( 'decr', $key, $conn->getServer(), $result );
$this->updateOpStats( self::METRIC_OP_DECR, [ $key ] );
return $result;
}
@ -435,6 +478,8 @@ class RedisBagOStuff extends MediumSpecificBagOStuff {
$this->handleException( $conn, $e );
}
$this->updateOpStats( self::METRIC_OP_CHANGE_TTL, [ $key ] );
return $result;
}

8
includes/libs/objectcache/ReplicatedBagOStuff.php
View file

@ -277,12 +277,12 @@ class ReplicatedBagOStuff extends BagOStuff {
return $this->genericKeyFromComponents( $keyspace, ...$components );
}
public function makeKey( $class, ...$components ) {
return $this->genericKeyFromComponents( $this->keyspace, $class, ...$components );
public function makeKey( $collection, ...$components ) {
return $this->genericKeyFromComponents( $this->keyspace, $collection, ...$components );
}
public function makeGlobalKey( $class, ...$components ) {
return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $class, ...$components );
public function makeGlobalKey( $collection, ...$components ) {
return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $collection, ...$components );
}
protected function convertGenericKey( $key ) {

61
includes/libs/objectcache/wancache/WANObjectCache.php
View file

@ -344,6 +344,12 @@ class WANObjectCache implements
$this->setLogger( $params['logger'] ?? new NullLogger() );
$this->stats = $params['stats'] ?? new NullStatsdDataFactory();
$this->asyncHandler = $params['asyncHandler'] ?? null;
$this->cache->registerWrapperInfoForStats(
'WANCache',
'wanobjectcache',
[ __CLASS__, 'getCollectionFromKey' ]
);
}
/**
@ -1680,6 +1686,29 @@ class WANObjectCache implements
return $fullKey;
}
/**
* @param string $sisterKey Sister key from makeSisterKey()
* @return string Key collection name
* @internal For use by WANObjectCache/BagOStuff only
* @since 1.36
*/
public static function getCollectionFromKey( string $sisterKey ) {
if ( substr( $sisterKey, -4 ) === '|#|v' ) {
// Key style: "WANCache:<base key>|#|<character>"
$collection = substr( $sisterKey, 9, strcspn( $sisterKey, ':|', 9 ) );
} elseif ( substr( $sisterKey, -3 ) === '}:v' ) {
// Key style: "WANCache:{<base key>}:<character>"
$collection = substr( $sisterKey, 10, strcspn( $sisterKey, ':}', 10 ) );
} elseif ( substr( $sisterKey, 9, 2 ) === 'v:' ) {
// Old key style: "WANCache:<character>:<base key>"
$collection = substr( $sisterKey, 11, strcspn( $sisterKey, ':', 11 ) );
} else {
$collection = 'internal';
}
return $collection;
}
/**
* @param float $age Age of volatile/interim key in seconds
* @return bool Whether the age of a volatile value is negligible
@ -2157,25 +2186,31 @@ class WANObjectCache implements
}
/**
* @see BagOStuff::makeKey()
* @param string $class Key collection name
* @param string|int ...$components Key components (starting with a key collection name)
* @return string Colon-delimited list of $keyspace followed by escaped components
* Make a cache key for the global keyspace and given components
*
* @see IStoreKeyEncoder::makeGlobalKey()
*
* @param string $collection Key collection name component
* @param string|int ...$components Additional, ordered, key components for entity IDs
* @return string Colon-separated, keyspace-prepended, ordered list of encoded components
* @since 1.27
*/
public function makeKey( $class, ...$components ) {
return $this->cache->makeKey( ...func_get_args() );
public function makeGlobalKey( $collection, ...$components ) {
return $this->cache->makeGlobalKey( ...func_get_args() );
}
/**
* @see BagOStuff::makeGlobalKey()
* @param string $class Key collection name
* @param string|int ...$components Key components (starting with a key collection name)
* @return string Colon-delimited list of $keyspace followed by escaped components
* Make a cache key using the "global" keyspace for the given components
*
* @see IStoreKeyEncoder::makeKey()
*
* @param string $collection Key collection name component
* @param string|int ...$components Additional, ordered, key components for entity IDs
* @return string Colon-separated, keyspace-prepended, ordered list of encoded components
* @since 1.27
*/
public function makeGlobalKey( $class, ...$components ) {
return $this->cache->makeGlobalKey( ...func_get_args() );
public function makeKey( $collection, ...$components ) {
return $this->cache->makeKey( ...func_get_args() );
}
/**
@ -2774,7 +2809,7 @@ class WANObjectCache implements
}
/**
* @param string $key String of the format <scope>:<class>[:<class or variable>]...
* @param string $key String of the format <scope>:<collection>[:<constant or variable>]...
* @return string A collection name to describe this class of key
*/
private function determineKeyClassForStats( $key ) {

4
includes/objectcache/ObjectCache.php
View file

@ -149,6 +149,10 @@ class ObjectCache {
'reportDupes' => true,
];
if ( !isset( $params['stats'] ) ) {
$params['stats'] = MediaWikiServices::getInstance()->getStatsdDataFactory();
}
if ( isset( $params['factory'] ) ) {
return call_user_func( $params['factory'], $params );
}

58
includes/objectcache/SqlBagOStuff.php
View file

@ -265,25 +265,41 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$blobs = $this->fetchBlobMulti( [ $key ] );
if ( array_key_exists( $key, $blobs ) ) {
$blob = $blobs[$key];
$value = $this->unserialize( $blob );
if ( $getToken && $value !== false ) {
$result = $this->unserialize( $blob );
if ( $getToken && $blob !== false ) {
$casToken = $blob;
}
return $value;
$valueSize = strlen( $blob );
} else {
$result = false;
$valueSize = false;
}
return false;
$this->updateOpStats( self::METRIC_OP_GET, [ $key ], null, [ $valueSize ] );
return $result;
}
protected function doGetMulti( array $keys, $flags = 0 ) {
$values = [];
$valueSizes = [];
$blobs = $this->fetchBlobMulti( $keys );
foreach ( $blobs as $key => $blob ) {
$values[$key] = $this->unserialize( $blob );
$blobsByKey = $this->fetchBlobMulti( $keys );
foreach ( $keys as $key ) {
if ( array_key_exists( $key, $blobsByKey ) ) {
$blob = $blobsByKey[$key];
$value = $this->unserialize( $blob );
if ( $value !== false ) {
$values[$key] = $value;
}
$valueSizes[] = strlen( $blob );
} else {
$valueSizes[] = false;
}
}
$this->updateOpStats( self::METRIC_OP_GET, $keys, null, $valueSizes );
return $values;
}
@ -424,14 +440,17 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
private function updateTable( $op, $db, $table, $tableKeys, $data, $dbExpiry ) {
$success = true;
$valueSizes = [];
if ( $op === self::$OP_ADD ) {
$rows = [];
foreach ( $tableKeys as $key ) {
$serialized = $this->serialize( $data[$key] );
$rows[] = [
'keyname' => $key,
'value' => $db->encodeBlob( $this->serialize( $data[$key] ) ),
'value' => $db->encodeBlob( $serialized ),
'exptime' => $dbExpiry
];
$valueSizes[] = strlen( $serialized );
}
$db->delete(
$table,
@ -444,18 +463,26 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$db->insert( $table, $rows, __METHOD__, [ 'IGNORE' ] );
$success = ( $db->affectedRows() == count( $rows ) );
$this->updateOpStats( self::METRIC_OP_ADD, $tableKeys, $valueSizes );
} elseif ( $op === self::$OP_SET ) {
$rows = [];
foreach ( $tableKeys as $key ) {
$serialized = $this->serialize( $data[$key] );
$rows[] = [
'keyname' => $key,
'value' => $db->encodeBlob( $this->serialize( $data[$key] ) ),
'value' => $db->encodeBlob( $serialized ),
'exptime' => $dbExpiry
];
$valueSizes[] = strlen( $serialized );
}
$db->replace( $table, 'keyname', $rows, __METHOD__ );
$this->updateOpStats( self::METRIC_OP_SET, $tableKeys, $valueSizes );
} elseif ( $op === self::$OP_DELETE ) {
$db->delete( $table, [ 'keyname' => $tableKeys ], __METHOD__ );
$this->updateOpStats( self::METRIC_OP_DELETE, $tableKeys );
} elseif ( $op === self::$OP_TOUCH ) {
$db->update(
$table,
@ -468,6 +495,8 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
);
$success = ( $db->affectedRows() == count( $tableKeys ) );
$this->updateOpStats( self::METRIC_OP_CHANGE_TTL, $tableKeys );
} else {
throw new InvalidArgumentException( "Invalid operation '$op'" );
}
@ -486,6 +515,7 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
list( $shardIndex, $tableName ) = $this->getKeyLocation( $key );
$exptime = $this->getExpirationAsTimestamp( $exptime );
$serialized = $this->serialize( $value );
/** @noinspection PhpUnusedLocalVariableInspection */
$silenceScope = $this->silenceTransactionProfiler();
@ -498,7 +528,7 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$tableName,
[
'keyname' => $key,
'value' => $db->encodeBlob( $this->serialize( $value ) ),
'value' => $db->encodeBlob( $serialized ),
'exptime' => $exptime
? $db->timestamp( $exptime )
: $this->getMaxDateTime( $db )
@ -521,6 +551,8 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$success = $this->waitForReplication( $shardIndex ) && $success;
}
$this->updateOpStats( self::METRIC_OP_CAS, [ $key ], [ strlen( $serialized ) ] );
return $success;
}
@ -568,6 +600,8 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$this->handleWriteError( $e, $db, $shardIndex );
}
$this->updateOpStats( $step >= 0 ? self::METRIC_OP_INCR : self::METRIC_OP_DECR, [ $key ] );
return $newCount;
}
@ -575,7 +609,7 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
return $this->incr( $key, -$value, $flags );
}
public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
public function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
return $this->modifyMulti(
array_fill_keys( $keys, null ),
$exptime,

22
tests/phpunit/unit/includes/libs/objectcache/WANObjectCacheTest.php
View file

@ -2467,6 +2467,28 @@ class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
}
}
}
/**
* @param string $key
* @param string $expectedCollection
* @covers WANObjectCache::getCollectionFromKey()
* @dataProvider provideCollectionKeys
*/
public function testGetCollectionFromKey( $key, $expectedCollection ) {
$this->assertSame( $expectedCollection, WANObjectCache::getCollectionFromKey( $key ) );
}
public static function provideCollectionKeys() {
return [
[ 'WANCache:collection:a:b|#|v', 'collection' ],
[ 'WANCache:{collection:a:b}:v', 'collection' ],
[ 'WANCache:v:collection:a:b', 'collection' ],
[ 'WANCache:collection:a:b|#|t', 'internal' ],
[ 'WANCache:{collection:a:b}:t', 'internal' ],
[ 'WANCache:t:collection:a:b', 'internal' ],
[ 'WANCache:improper-key', 'internal' ],
];
}
}
class McrouterHashBagOStuff extends HashBagOStuff {