Move LBFactorySimple to /libs/rdbms

* Refactored LBFactory a bit to make this possible.
* Move newChronologyProtector() up to LBFactory and
  make a lazy-loading method instead.
* Move appendPreShutdownTimeAsQuery() up to LBFactory.
* Inject the web request values for LBFactory from Setup.php.
* Remove unused laggedSlaveUsed() method.

Change-Id: Ie8a38a6f4d6359680eb6a5be24a34e30b9816479
This commit is contained in:
Aaron Schulz 2016-09-15 11:52:55 -07:00
parent 1afef61fb0
commit d175b391ae
9 changed files with 212 additions and 179 deletions

View file

@ -660,7 +660,7 @@ $wgAutoloadLocalClasses = [
'LBFactory' => __DIR__ . '/includes/libs/rdbms/lbfactory/LBFactory.php',
'LBFactoryMW' => __DIR__ . '/includes/db/loadbalancer/LBFactoryMW.php',
'LBFactoryMulti' => __DIR__ . '/includes/db/loadbalancer/LBFactoryMulti.php',
'LBFactorySimple' => __DIR__ . '/includes/db/loadbalancer/LBFactorySimple.php',
'LBFactorySimple' => __DIR__ . '/includes/libs/rdbms/lbfactory/LBFactorySimple.php',
'LBFactorySingle' => __DIR__ . '/includes/db/loadbalancer/LBFactorySingle.php',
'LCStore' => __DIR__ . '/includes/cache/localisation/LCStore.php',
'LCStoreCDB' => __DIR__ . '/includes/cache/localisation/LCStoreCDB.php',

View file

@ -43,15 +43,53 @@ use MediaWiki\MediaWikiServices;
return [
'DBLoadBalancerFactory' => function( MediaWikiServices $services ) {
$config = $services->getMainConfig()->get( 'LBFactoryConf' );
$mainConfig = $services->getMainConfig();
$class = LBFactoryMW::getLBFactoryClass( $config );
if ( !isset( $config['readOnlyReason'] ) ) {
$lbConf = $mainConfig->get( 'LBFactoryConf' );
$lbConf += [
'localDomain' => new DatabaseDomain(
$mainConfig->get( 'DBname' ), null, $mainConfig->get( 'DBprefix' ) ),
// TODO: replace the global wfConfiguredReadOnlyReason() with a service.
$config['readOnlyReason'] = wfConfiguredReadOnlyReason();
'readOnlyReason' => wfConfiguredReadOnlyReason(),
];
$class = LBFactoryMW::getLBFactoryClass( $lbConf );
if ( $class === 'LBFactorySimple' ) {
if ( is_array( $mainConfig->get( 'DBservers' ) ) ) {
foreach ( $mainConfig->get( 'DBservers' ) as $i => $server ) {
$lbConf['servers'][$i] = $server + [
'schema' => $mainConfig->get( 'DBmwschema' ),
'tablePrefix' => $mainConfig->get( 'DBprefix' ),
'flags' => DBO_DEFAULT,
'sqlMode' => $mainConfig->get( 'SQLMode' ),
'utf8Mode' => $mainConfig->get( 'DBmysql5' )
];
}
} else {
$flags = DBO_DEFAULT;
$flags |= $mainConfig->get( 'DebugDumpSql' ) ? DBO_DEBUG : 0;
$flags |= $mainConfig->get( 'DBssl' ) ? DBO_SSL : 0;
$flags |= $mainConfig->get( 'DBcompress' ) ? DBO_COMPRESS : 0;
$lbConf['servers'] = [
[
'host' => $mainConfig->get( 'DBserver' ),
'user' => $mainConfig->get( 'DBuser' ),
'password' => $mainConfig->get( 'DBpassword' ),
'dbname' => $mainConfig->get( 'DBname' ),
'schema' => $mainConfig->get( 'DBmwschema' ),
'tablePrefix' => $mainConfig->get( 'DBprefix' ),
'type' => $mainConfig->get( 'DBtype' ),
'load' => 1,
'flags' => $flags,
'sqlMode' => $mainConfig->get( 'SQLMode' ),
'utf8Mode' => $mainConfig->get( 'DBmysql5' )
]
];
}
$lbConf['externalServers'] = $mainConfig->get( 'ExternalServers' );
}
return new $class( $config );
return new $class( LBFactoryMW::applyDefaultConfig( $lbConf ) );
},
'DBLoadBalancer' => function( MediaWikiServices $services ) {

View file

@ -504,8 +504,9 @@ if ( !class_exists( 'AutoLoader' ) ) {
// Reset the global service locator, so any services that have already been created will be
// re-created while taking into account any custom settings and extensions.
MediaWikiServices::resetGlobalInstance( new GlobalVarConfig(), 'quick' );
// Apply $wgSharedDB table aliases for the local LB (all non-foreign DB connections)
if ( $wgSharedDB && $wgSharedTables ) {
// Apply $wgSharedDB table aliases for the local LB (all non-foreign DB connections)
MediaWikiServices::getInstance()->getDBLoadBalancer()->setTableAliases(
array_fill_keys(
$wgSharedTables,
@ -661,6 +662,12 @@ if ( !$wgDBerrorLogTZ ) {
// initialize the request object in $wgRequest
$wgRequest = RequestContext::getMain()->getRequest(); // BackCompat
// Set user IP/agent information for causal consistency purposes
MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->setRequestInfo( [
'IPAddress' => $wgRequest->getIP(),
'UserAgent' => $wgRequest->getHeader( 'User-Agent' ),
'ChronologyProtection' => $wgRequest->getHeader( 'ChronologyProtection' )
] );
// Useful debug output
if ( $wgCommandLineMode ) {

View file

@ -21,15 +21,13 @@
* @ingroup Database
*/
use MediaWiki\MediaWikiServices;
use MediaWiki\Services\DestructibleService;
use MediaWiki\Logger\LoggerFactory;
/**
* Legacy MediaWiki-specific class for generating database load balancers
* @ingroup Database
*/
abstract class LBFactoryMW extends LBFactory implements DestructibleService {
abstract class LBFactoryMW extends LBFactory {
/** @noinspection PhpMissingParentConstructorInspection */
/**
* Construct a factory based on a configuration array (typically from $wgLBFactoryConf)
@ -37,6 +35,15 @@ abstract class LBFactoryMW extends LBFactory implements DestructibleService {
* @TODO: inject objects via dependency framework
*/
public function __construct( array $conf ) {
parent::__construct( self::applyDefaultConfig( $conf ) );
}
/**
* @param array $conf
* @return array
* @TODO: inject objects via dependency framework
*/
public static function applyDefaultConfig( array $conf ) {
global $wgCommandLineMode, $wgSQLMode, $wgDBmysql5, $wgDBname, $wgDBprefix;
$defaults = [
@ -48,7 +55,9 @@ abstract class LBFactoryMW extends LBFactory implements DestructibleService {
'queryLogger' => LoggerFactory::getInstance( 'wfLogDBError' ),
'connLogger' => LoggerFactory::getInstance( 'wfLogDBError' ),
'perfLogger' => LoggerFactory::getInstance( 'DBPerformance' ),
'errorLogger' => [ MWExceptionHandler::class, 'logException' ]
'errorLogger' => [ MWExceptionHandler::class, 'logException' ],
'cliMode' => $wgCommandLineMode,
'agent' => ''
];
// Use APC/memcached style caching, but avoids loops with CACHE_DB (T141804)
$sCache = ObjectCache::getLocalServerInstance();
@ -64,15 +73,12 @@ abstract class LBFactoryMW extends LBFactory implements DestructibleService {
$defaults['wanCache'] = $wCache;
}
$this->agent = isset( $params['agent'] ) ? $params['agent'] : '';
$this->cliMode = isset( $params['cliMode'] ) ? $params['cliMode'] : $wgCommandLineMode;
if ( isset( $conf['serverTemplate'] ) ) { // LBFactoryMulti
$conf['serverTemplate']['sqlMode'] = $wgSQLMode;
$conf['serverTemplate']['utf8Mode'] = $wgDBmysql5;
}
parent::__construct( $conf + $defaults );
return $conf + $defaults;
}
/**
@ -104,57 +110,4 @@ abstract class LBFactoryMW extends LBFactory implements DestructibleService {
return $class;
}
/**
* @return bool
* @since 1.27
* @deprecated Since 1.28; use laggedReplicaUsed()
*/
public function laggedSlaveUsed() {
return $this->laggedReplicaUsed();
}
protected function newChronologyProtector() {
$request = RequestContext::getMain()->getRequest();
$chronProt = new ChronologyProtector(
ObjectCache::getMainStashInstance(),
[
'ip' => $request->getIP(),
'agent' => $request->getHeader( 'User-Agent' ),
],
$request->getFloat( 'cpPosTime', $request->getCookie( 'cpPosTime', '' ) )
);
if ( PHP_SAPI === 'cli' ) {
$chronProt->setEnabled( false );
} elseif ( $request->getHeader( 'ChronologyProtection' ) === 'false' ) {
// Request opted out of using position wait logic. This is useful for requests
// done by the job queue or background ETL that do not have a meaningful session.
$chronProt->setWaitEnabled( false );
}
return $chronProt;
}
/**
* Append ?cpPosTime parameter to a URL for ChronologyProtector purposes if needed
*
* Note that unlike cookies, this works accross domains
*
* @param string $url
* @param float $time UNIX timestamp just before shutdown() was called
* @return string
* @since 1.28
*/
public function appendPreShutdownTimeAsQuery( $url, $time ) {
$usedCluster = 0;
$this->forEachLB( function ( LoadBalancer $lb ) use ( &$usedCluster ) {
$usedCluster |= ( $lb->getServerCount() > 1 );
} );
if ( !$usedCluster ) {
return $url; // no master/replica clusters touched
}
return wfAppendQuery( $url, [ 'cpPosTime' => $time ] );
}
}

View file

@ -254,7 +254,7 @@ class LBFactoryMulti extends LBFactoryMW {
$section = $this->getSectionForWiki( $wiki );
if ( !isset( $this->mainLBs[$section] ) ) {
$lb = $this->newMainLB( $wiki );
$this->chronProt->initLB( $lb );
$this->getChronologyProtector()->initLB( $lb );
$this->mainLBs[$section] = $lb;
}
@ -295,7 +295,7 @@ class LBFactoryMulti extends LBFactoryMW {
public function getExternalLB( $cluster, $wiki = false ) {
if ( !isset( $this->extLBs[$cluster] ) ) {
$this->extLBs[$cluster] = $this->newExternalLB( $cluster, $wiki );
$this->chronProt->initLB( $this->extLBs[$cluster] );
$this->getChronologyProtector()->initLB( $this->extLBs[$cluster] );
}
return $this->extLBs[$cluster];

View file

@ -55,6 +55,9 @@ abstract class LBFactory {
protected $localDomain;
/** @var string Local hostname of the app server */
protected $hostname;
/** @var array Web request information about the client */
protected $requestInfo;
/** @var mixed */
protected $ticket;
/** @var string|bool String if a requested DBO_TRX transaction round is active */
@ -103,22 +106,25 @@ abstract class LBFactory {
: function ( Exception $e ) {
trigger_error( E_WARNING, get_class( $e ) . ': ' . $e->getMessage() );
};
$this->hostname = isset( $conf['hostname'] )
? $conf['hostname']
: gethostname();
$this->chronProt = isset( $conf['chronProt'] )
? $conf['chronProt']
: $this->newChronologyProtector();
$this->chronProt = isset( $conf['chronProt'] ) ? $conf['chronProt'] : null;
$this->profiler = isset( $params['profiler'] ) ? $params['profiler'] : null;
$this->trxProfiler = isset( $conf['trxProfiler'] )
? $conf['trxProfiler']
: new TransactionProfiler();
$this->ticket = mt_rand();
$this->requestInfo = [
'IPAddress' => isset( $_SERVER[ 'REMOTE_ADDR' ] ) ? $_SERVER[ 'REMOTE_ADDR' ] : '',
'UserAgent' => isset( $_SERVER['HTTP_USER_AGENT'] ) ? $_SERVER['HTTP_USER_AGENT'] : '',
'ChronologyProtection' => 'true'
];
$this->cliMode = isset( $params['cliMode'] ) ? $params['cliMode'] : PHP_SAPI === 'cli';
$this->hostname = isset( $conf['hostname'] ) ? $conf['hostname'] : gethostname();
$this->agent = isset( $params['agent'] ) ? $params['agent'] : '';
$this->ticket = mt_rand();
}
/**
@ -186,10 +192,11 @@ abstract class LBFactory {
public function shutdown(
$mode = self::SHUTDOWN_CHRONPROT_SYNC, callable $workCallback = null
) {
$chronProt = $this->getChronologyProtector();
if ( $mode === self::SHUTDOWN_CHRONPROT_SYNC ) {
$this->shutdownChronologyProtector( $this->chronProt, $workCallback, 'sync' );
$this->shutdownChronologyProtector( $chronProt, $workCallback, 'sync' );
} elseif ( $mode === self::SHUTDOWN_CHRONPROT_ASYNC ) {
$this->shutdownChronologyProtector( $this->chronProt, null, 'async' );
$this->shutdownChronologyProtector( $chronProt, null, 'async' );
}
$this->commitMasterChanges( __METHOD__ ); // sanity
@ -540,7 +547,7 @@ abstract class LBFactory {
* @since 1.28
*/
public function getChronologyProtectorTouched( $dbName ) {
return $this->chronProt->getTouched( $dbName );
return $this->getChronologyProtector()->getTouched( $dbName );
}
/**
@ -551,27 +558,39 @@ abstract class LBFactory {
* @since 1.27
*/
public function disableChronologyProtection() {
$this->chronProt->setEnabled( false );
$this->getChronologyProtector()->setEnabled( false );
}
/**
* @return ChronologyProtector
*/
protected function newChronologyProtector() {
$chronProt = new ChronologyProtector(
protected function getChronologyProtector() {
if ( $this->chronProt ) {
return $this->chronProt;
}
$this->chronProt = new ChronologyProtector(
$this->memCache,
[
'ip' => isset( $_SERVER[ 'REMOTE_ADDR' ] ) ? $_SERVER[ 'REMOTE_ADDR' ] : '',
'agent' => isset( $_SERVER['HTTP_USER_AGENT'] ) ? $_SERVER['HTTP_USER_AGENT'] : ''
'ip' => $this->requestInfo['IPAddress'],
'agent' => $this->requestInfo['UserAgent'],
],
isset( $_GET['cpPosTime'] ) ? $_GET['cpPosTime'] : null
);
$chronProt->setLogger( $this->replLogger );
$this->chronProt->setLogger( $this->replLogger );
if ( $this->cliMode ) {
$chronProt->setEnabled( false );
$this->chronProt->setEnabled( false );
} elseif ( $this->requestInfo['ChronologyProtection'] === 'false' ) {
// Request opted out of using position wait logic. This is useful for requests
// done by the job queue or background ETL that do not have a meaningful session.
$this->chronProt->setWaitEnabled( false );
}
return $chronProt;
$this->replLogger->debug( __METHOD__ . ': using request info ' .
json_encode( $this->requestInfo, JSON_PRETTY_PRINT ) );
return $this->chronProt;
}
/**
@ -671,4 +690,42 @@ abstract class LBFactory {
public function setAgentName( $agent ) {
$this->agent = $agent;
}
/**
* Append ?cpPosTime parameter to a URL for ChronologyProtector purposes if needed
*
* Note that unlike cookies, this works accross domains
*
* @param string $url
* @param float $time UNIX timestamp just before shutdown() was called
* @return string
* @since 1.28
*/
public function appendPreShutdownTimeAsQuery( $url, $time ) {
$usedCluster = 0;
$this->forEachLB( function ( ILoadBalancer $lb ) use ( &$usedCluster ) {
$usedCluster |= ( $lb->getServerCount() > 1 );
} );
if ( !$usedCluster ) {
return $url; // no master/replica clusters touched
}
return strpos( $url, '?' ) === false ? "$url?cpPosTime=$time" : "$url&cpPosTime=$time";
}
/**
* @param array $info Map of fields, including:
* - IPAddress : IP address
* - UserAgent : User-Agent HTTP header
* - ChronologyProtection : cookie/header value specifying ChronologyProtector usage
* @since 1.28
*/
public function setRequestInfo( array $info ) {
$this->requestInfo = $info + $this->requestInfo;
}
function __destruct() {
$this->destroy();
}
}

View file

@ -24,88 +24,56 @@
/**
* A simple single-master LBFactory that gets its configuration from the b/c globals
*/
class LBFactorySimple extends LBFactoryMW {
class LBFactorySimple extends LBFactory {
/** @var LoadBalancer */
private $mainLB;
/** @var LoadBalancer[] */
private $extLBs = [];
/** @var array[] Map of (server index => server config) */
private $servers = [];
/** @var array[] Map of (cluster => (server index => server config)) */
private $externalClusters = [];
/** @var string */
private $loadMonitorClass;
public function __construct( array $conf ) {
parent::__construct( $conf );
$this->servers = isset( $conf['servers'] ) ? $conf['servers'] : [];
foreach ( $this->servers as $i => $server ) {
if ( $i == 0 ) {
$this->servers[$i]['master'] = true;
} else {
$this->servers[$i]['replica'] = true;
}
}
$this->externalClusters = isset( $conf['externalClusters'] )
? $conf['externalClusters']
: [];
$this->loadMonitorClass = isset( $conf['loadMonitorClass'] )
? $conf['loadMonitorClass']
: null;
}
/**
* @param bool|string $wiki
* @param bool|string $domain
* @return LoadBalancer
*/
public function newMainLB( $wiki = false ) {
global $wgDBservers, $wgDBprefix, $wgDBmwschema, $wgSQLMode, $wgDBmysql5;
if ( is_array( $wgDBservers ) ) {
$servers = $wgDBservers;
foreach ( $servers as $i => &$server ) {
if ( $i == 0 ) {
$server['master'] = true;
} else {
$server['replica'] = true;
}
$server += [
'schema' => $wgDBmwschema,
'tablePrefix' => $wgDBprefix,
'flags' => DBO_DEFAULT,
'sqlMode' => $wgSQLMode,
'utf8Mode' => $wgDBmysql5
];
}
} else {
global $wgDBserver, $wgDBuser, $wgDBpassword, $wgDBname, $wgDBtype, $wgDebugDumpSql;
global $wgDBssl, $wgDBcompress;
$flags = DBO_DEFAULT;
if ( $wgDebugDumpSql ) {
$flags |= DBO_DEBUG;
}
if ( $wgDBssl ) {
$flags |= DBO_SSL;
}
if ( $wgDBcompress ) {
$flags |= DBO_COMPRESS;
}
$servers = [ [
'host' => $wgDBserver,
'user' => $wgDBuser,
'password' => $wgDBpassword,
'dbname' => $wgDBname,
'schema' => $wgDBmwschema,
'tablePrefix' => $wgDBprefix,
'type' => $wgDBtype,
'load' => 1,
'flags' => $flags,
'master' => true,
'sqlMode' => $wgSQLMode,
'utf8Mode' => $wgDBmysql5
] ];
}
return $this->newLoadBalancer( $servers );
public function newMainLB( $domain = false ) {
return $this->newLoadBalancer( $this->servers );
}
/**
* @param bool|string $wiki
* @param bool|string $domain
* @return LoadBalancer
*/
public function getMainLB( $wiki = false ) {
public function getMainLB( $domain = false ) {
if ( !isset( $this->mainLB ) ) {
$this->mainLB = $this->newMainLB( $wiki );
$this->chronProt->initLB( $this->mainLB );
$this->mainLB = $this->newMainLB( $domain );
$this->getChronologyProtector()->initLB( $this->mainLB );
}
return $this->mainLB;
@ -113,28 +81,27 @@ class LBFactorySimple extends LBFactoryMW {
/**
* @param string $cluster
* @param bool|string $wiki
* @param bool|string $domain
* @return LoadBalancer
* @throws InvalidArgumentException
*/
protected function newExternalLB( $cluster, $wiki = false ) {
global $wgExternalServers;
if ( !isset( $wgExternalServers[$cluster] ) ) {
protected function newExternalLB( $cluster, $domain = false ) {
if ( !isset( $this->externalClusters[$cluster] ) ) {
throw new InvalidArgumentException( __METHOD__ . ": Unknown cluster \"$cluster\"" );
}
return $this->newLoadBalancer( $wgExternalServers[$cluster] );
return $this->newLoadBalancer( $this->externalClusters[$cluster] );
}
/**
* @param string $cluster
* @param bool|string $wiki
* @param bool|string $domain
* @return array
*/
public function getExternalLB( $cluster, $wiki = false ) {
public function getExternalLB( $cluster, $domain = false ) {
if ( !isset( $this->extLBs[$cluster] ) ) {
$this->extLBs[$cluster] = $this->newExternalLB( $cluster, $wiki );
$this->chronProt->initLB( $this->extLBs[$cluster] );
$this->extLBs[$cluster] = $this->newExternalLB( $cluster, $domain );
$this->getChronologyProtector()->initLB( $this->extLBs[$cluster] );
}
return $this->extLBs[$cluster];

View file

@ -147,9 +147,6 @@ class MediaWikiServicesTest extends MediaWikiTestCase {
->disableOriginalConstructor()
->getMock();
$lbFactory->expects( $this->once() )
->method( 'destroy' );
$newServices->redefineService(
'DBLoadBalancerFactory',
function() use ( $lbFactory ) {
@ -164,12 +161,11 @@ class MediaWikiServicesTest extends MediaWikiTestCase {
try {
MediaWikiServices::getInstance()->getService( 'DBLoadBalancerFactory' );
$this->fail( 'DBLoadBalancerFactory shoudl have been disabled' );
$this->fail( 'DBLoadBalancerFactory should have been disabled' );
}
catch ( ServiceDisabledException $ex ) {
// ok, as expected
}
catch ( Throwable $ex ) {
} catch ( Throwable $ex ) {
$this->fail( 'ServiceDisabledException expected, caught ' . get_class( $ex ) );
}

View file

@ -58,9 +58,21 @@ class LBFactoryTest extends MediaWikiTestCase {
}
public function testLBFactorySimpleServer() {
$this->setMwGlobals( 'wgDBservers', false );
global $wgDBserver, $wgDBname, $wgDBuser, $wgDBpassword, $wgDBtype;
$factory = new LBFactorySimple( [] );
$servers = [
[
'host' => $wgDBserver,
'dbname' => $wgDBname,
'user' => $wgDBuser,
'password' => $wgDBpassword,
'type' => $wgDBtype,
'load' => 0,
'flags' => DBO_TRX // REPEATABLE-READ for consistency
],
];
$factory = new LBFactorySimple( [ 'servers' => $servers ] );
$lb = $factory->getMainLB();
$dbw = $lb->getConnection( DB_MASTER );
@ -76,28 +88,31 @@ class LBFactoryTest extends MediaWikiTestCase {
public function testLBFactorySimpleServers() {
global $wgDBserver, $wgDBname, $wgDBuser, $wgDBpassword, $wgDBtype;
$this->setMwGlobals( 'wgDBservers', [
$servers = [
[ // master
'host' => $wgDBserver,
'dbname' => $wgDBname,
'user' => $wgDBuser,
'password' => $wgDBpassword,
'type' => $wgDBtype,
'load' => 0,
'flags' => DBO_TRX // REPEATABLE-READ for consistency
'host' => $wgDBserver,
'dbname' => $wgDBname,
'user' => $wgDBuser,
'password' => $wgDBpassword,
'type' => $wgDBtype,
'load' => 0,
'flags' => DBO_TRX // REPEATABLE-READ for consistency
],
[ // emulated slave
'host' => $wgDBserver,
'dbname' => $wgDBname,
'user' => $wgDBuser,
'password' => $wgDBpassword,
'type' => $wgDBtype,
'load' => 100,
'flags' => DBO_TRX // REPEATABLE-READ for consistency
'host' => $wgDBserver,
'dbname' => $wgDBname,
'user' => $wgDBuser,
'password' => $wgDBpassword,
'type' => $wgDBtype,
'load' => 100,
'flags' => DBO_TRX // REPEATABLE-READ for consistency
]
] );
];
$factory = new LBFactorySimple( [ 'loadMonitorClass' => 'LoadMonitorNull' ] );
$factory = new LBFactorySimple( [
'servers' => $servers,
'loadMonitorClass' => 'LoadMonitorNull'
] );
$lb = $factory->getMainLB();
$dbw = $lb->getConnection( DB_MASTER );