rdbms: Introduce concept of virtual domains and mapping to ext cluster

This would simplify any caller that's trying to use extension1 cluster

Bug: T330590
Change-Id: Icccde8e10070686870601cae74b21ca9bed71ece

RELEASE-NOTES-1.41

@@ -33,6 +33,8 @@ For notes on 1.40.x and older releases, see HISTORY.
   external link queries.
 * $wgResourceLoaderEnableSourceMapLinks - Add a SourceMap header to
   ResourceLoader responses for JavaScript modules (T47514).
+* $wgVirtualDomainsMapping - Mapping of virtual domains to other dbs. It's
+  useful to get connection to the external cluster.
 * …
 
 ==== Changed configuration ====

docs/config-schema.yaml

@@ -1974,6 +1974,19 @@ config-schema:
             Max execution time for queries of several expensive special pages such as RecentChanges
             in milliseconds.
             @since 1.38
+    VirtualDomainsMapping:
+        default: {  }
+        type: object
+        description: |-
+            Mapping of virtual domain to external cluster db.
+            If no entry is set, the code assumes local database.
+            For example, for routing queries of virtual domain 'vdomain'
+            to 'wikishared' database in 'extension1' cluster. The config should be like this:
+             [ 'vdomain' => [ 'cluster' => 'extension1', 'db' => 'wikishared' ] ]
+            If the database needs to be the local domain, just set the 'db' to false.
+            If you want to get another db in the main cluster, just omit 'cluster'. For example:
+             [ 'centralauth' => [ 'db' => 'centralauth' ] ]
+            @since 1.41
     TemplateLinksSchemaMigrationStage:
         default: 768
         type: integer

docs/config-vars.php

@@ -1223,6 +1223,12 @@ $wgDatabaseReplicaLagCritical = null;
  */
 $wgMaxExecutionTimeForExpensiveQueries = null;
 
+/**
+ * Config variable stub for the VirtualDomainsMapping setting, for use by phpdoc and IDEs.
+ * @see MediaWiki\MainConfigSchema::VirtualDomainsMapping
+ */
+$wgVirtualDomainsMapping = null;
+
 /**
  * Config variable stub for the TemplateLinksSchemaMigrationStage setting, for use by phpdoc and IDEs.
  * @see MediaWiki\MainConfigSchema::TemplateLinksSchemaMigrationStage

docs/extension.schema.v1.json

@@ -1127,6 +1127,13 @@
 				"type": "string"
 			}
 		},
+		"DatabaseVirtualDomains": {
+			"type": "array",
+			"description": "Names of virtual domains that are used to get a db connection.",
+			"items": {
+				"type": "string"
+			}
+		},
 		"ReauthenticateTime": {
 			"type": "object",
 			"patternProperties": {

docs/extension.schema.v2.json

@@ -1302,6 +1302,13 @@
 				"type": "string"
 			}
 		},
+		"DatabaseVirtualDomains": {
+			"type": "array",
+			"description": "Names of virtual domains that are used to get a db connection.",
+			"items": {
+				"type": "string"
+			}
+		},
 		"ReauthenticateTime": {
 			"type": "object",
 			"patternProperties": {

includes/MainConfigNames.php

@@ -1238,6 +1238,12 @@ class MainConfigNames {
 	 */
 	public const MaxExecutionTimeForExpensiveQueries = 'MaxExecutionTimeForExpensiveQueries';
 
+	/**
+	 * Name constant for the VirtualDomainsMapping setting, for use with Config::get()
+	 * @see MainConfigSchema::VirtualDomainsMapping
+	 */
+	public const VirtualDomainsMapping = 'VirtualDomainsMapping';
+
 	/**
 	 * Name constant for the TemplateLinksSchemaMigrationStage setting, for use with Config::get()
 	 * @see MainConfigSchema::TemplateLinksSchemaMigrationStage

includes/MainConfigSchema.php

@@ -3229,6 +3229,26 @@ class MainConfigSchema {
 		'default' => 0,
 	];
 
+	/**
+	 * Mapping of virtual domain to external cluster db.
+	 *
+	 * If no entry is set, the code assumes local database.
+	 * For example, for routing queries of virtual domain 'vdomain'
+	 * to 'wikishared' database in 'extension1' cluster. The config should be like this:
+	 *  [ 'vdomain' => [ 'cluster' => 'extension1', 'db' => 'wikishared' ] ]
+	 *
+	 * If the database needs to be the local domain, just set the 'db' to false.
+	 *
+	 * If you want to get another db in the main cluster, just omit 'cluster'. For example:
+	 *  [ 'centralauth' => [ 'db' => 'centralauth' ] ]
+	 *
+	 * @since 1.41
+	 */
+	public const VirtualDomainsMapping = [
+		'default' => [],
+		'type' => 'map',
+	];
+
 	/**
 	 * Templatelinks table schema migration stage, for normalizing tl_namespace and tl_title fields.
 	 *

includes/ServiceWiring.php

@@ -709,7 +709,8 @@ return [
 			$srvCache,
 			$wanCache,
 			$services->getCriticalSectionProvider(),
-			$services->getStatsdDataFactory()
+			$services->getStatsdDataFactory(),
+			ExtensionRegistry::getInstance()->getAttribute( 'DatabaseVirtualDomains' )
 		);
 	},
 

includes/config-schema.php

@@ -387,6 +387,8 @@ return [
 			'DatabaseReplicaLagWarning' => 10,
 			'DatabaseReplicaLagCritical' => 30,
 			'MaxExecutionTimeForExpensiveQueries' => 0,
+			'VirtualDomainsMapping' => [
+			],
 			'TemplateLinksSchemaMigrationStage' => 768,
 			'PageLinksSchemaMigrationStage' => 3,
 			'ExternalLinksDomainGaps' => [
@@ -2625,6 +2627,7 @@ return [
 			],
 			'LBFactoryConf' => 'object',
 			'LocalDatabases' => 'array',
+			'VirtualDomainsMapping' => 'object',
 			'TemplateLinksSchemaMigrationStage' => 'integer',
 			'PageLinksSchemaMigrationStage' => 'integer',
 			'ExternalLinksDomainGaps' => 'object',

includes/db/MWLBFactory.php

@@ -67,6 +67,7 @@ class MWLBFactory {
 		MainConfigNames::ExternalServers,
 		MainConfigNames::SQLiteDataDir,
 		MainConfigNames::SQLMode,
+		MainConfigNames::VirtualDomainsMapping,
 	];
 	/**
 	 * @var ServiceOptions
@@ -96,6 +97,7 @@ class MWLBFactory {
 	 * @var StatsdDataFactoryInterface
 	 */
 	private $statsdDataFactory;
+	private array $virtualDomains = [];
 
 	/**
 	 * @param ServiceOptions $options
@@ -113,7 +115,8 @@ class MWLBFactory {
 		BagOStuff $srvCache,
 		WANObjectCache $wanCache,
 		CriticalSectionProvider $csProvider,
-		StatsdDataFactoryInterface $statsdDataFactory
+		StatsdDataFactoryInterface $statsdDataFactory,
+		array $virtualDomains
 	) {
 		$this->options = $options;
 		$this->readOnlyMode = $readOnlyMode;
@@ -122,6 +125,7 @@ class MWLBFactory {
 		$this->wanCache = $wanCache;
 		$this->csProvider = $csProvider;
 		$this->statsdDataFactory = $statsdDataFactory;
+		$this->virtualDomains = $virtualDomains;
 	}
 
 	/**
@@ -215,6 +219,8 @@ class MWLBFactory {
 		$lbConf['chronologyProtector'] = $this->chronologyProtector;
 		$lbConf['srvCache'] = $this->srvCache;
 		$lbConf['wanCache'] = $this->wanCache;
+		$lbConf['virtualDomains'] = $this->virtualDomains;
+		$lbConf['virtualDomainsMapping'] = $this->options->get( MainConfigNames::VirtualDomainsMapping );
 
 		return $lbConf;
 	}

includes/libs/rdbms/lbfactory/LBFactory.php

@@ -76,6 +76,10 @@ abstract class LBFactory implements ILBFactory {
 	private $indexAliases = [];
 	/** @var DatabaseDomain[]|string[] Map of (domain alias => DB domain) */
 	protected $domainAliases = [];
+	/** @var array[] Map of virtual domain to array of cluster and domain */
+	protected array $virtualDomainsMapping = [];
+	/** @var string[] List of registered virtual domains */
+	protected array $virtualDomains = [];
 	/** @var callable[] */
 	private $replicationWaitCallbacks = [];
 
@@ -150,6 +154,8 @@ abstract class LBFactory implements ILBFactory {
 		$this->agent = $conf['agent'] ?? '';
 		$this->defaultGroup = $conf['defaultGroup'] ?? null;
 		$this->replicationWaitTimeout = $this->cliMode ? 60 : 1;
+		$this->virtualDomainsMapping = $conf['virtualDomainsMapping'] ?? [];
+		$this->virtualDomains = $conf['virtualDomains'] ?? [];
 
 		static $nextTicket;
 		$this->ticket = $nextTicket = ( is_int( $nextTicket ) ? $nextTicket++ : mt_rand() );
@@ -515,6 +521,20 @@ abstract class LBFactory implements ILBFactory {
 	}
 
 	public function getPrimaryDatabase( $domain = false ): IDatabase {
+		if ( $domain !== false && in_array( $domain, $this->virtualDomains ) ) {
+			if ( isset( $this->virtualDomainsMapping[$domain] ) ) {
+				$config = $this->virtualDomainsMapping[$domain];
+				if ( isset( $config['cluster'] ) ) {
+					return $this
+						->getExternalLB( $config['cluster'] )
+						->getConnection( DB_PRIMARY, [], $config['db'] );
+				}
+				$domain = $config['db'];
+			} else {
+				// It's not configured, assume local db.
+				$domain = false;
+			}
+		}
 		return $this->getMainLB( $domain )->getConnection( DB_PRIMARY, [], $domain );
 	}
 
@@ -524,6 +544,20 @@ abstract class LBFactory implements ILBFactory {
 		} else {
 			$groups = [ $group ];
 		}
+		if ( $domain !== false && in_array( $domain, $this->virtualDomains ) ) {
+			if ( isset( $this->virtualDomainsMapping[$domain] ) ) {
+				$config = $this->virtualDomainsMapping[$domain];
+				if ( isset( $config['cluster'] ) ) {
+					return $this
+						->getExternalLB( $config['cluster'] )
+						->getConnection( DB_REPLICA, $groups, $config['db'] );
+				}
+				$domain = $config['db'];
+			} else {
+				// It's not configured, assume local db.
+				$domain = false;
+			}
+		}
 		return $this->getMainLB( $domain )->getConnection( DB_REPLICA, $groups, $domain );
 	}
 

includes/registration/ExtensionProcessor.php

@@ -82,6 +82,7 @@ class ExtensionProcessor implements Processor {
 		'LateJSConfigVarNames',
 		'TempUserSerialProviders',
 		'TempUserSerialMappings',
+		'DatabaseVirtualDomains',
 	];
 
 	/**

tests/phpunit/includes/db/LBFactoryTest.php

@@ -642,6 +642,55 @@ class LBFactoryTest extends MediaWikiIntegrationTestCase {
 		$factory->destroy();
 	}
 
+	public function testVirtualDomains() {
+		$baseOverrides = [
+			'localDomain' => ( new DatabaseDomain( 'localdomain', null, '' ) )->getId(),
+			'sectionLoads' => [
+				'DEFAULT' => [
+					'test-db1' => 1,
+				],
+				'shareddb' => [
+					'test-db1' => 1,
+				],
+			],
+			'externalLoads' => [
+				'extension1' => [
+					'test-db1' => 1,
+				],
+			],
+			'virtualDomains' => [ 'virtualdomain1', 'virtualdomain2', 'virtualdomain3', 'virtualdomain4' ],
+			'virtualDomainsMapping' => [
+				'virtualdomain1' => [ 'db' => 'extdomain', 'cluster' => 'extension1' ],
+				'virtualdomain2' => [ 'db' => false, 'cluster' => 'extension1' ],
+				'virtualdomain3' => [ 'db' => 'shareddb' ],
+			]
+		];
+		$factory = $this->newLBFactoryMulti( $baseOverrides );
+		$db1 = $factory->getPrimaryDatabase( 'virtualdomain1' );
+		$this->assertEquals(
+			'extdomain',
+			$db1->getDomainID()
+		);
+
+		$db2 = $factory->getPrimaryDatabase( 'virtualdomain2' );
+		$this->assertEquals(
+			'localdomain',
+			$db2->getDomainID()
+		);
+
+		$db3 = $factory->getPrimaryDatabase( 'virtualdomain3' );
+		$this->assertEquals(
+			'shareddb',
+			$db3->getDomainID()
+		);
+
+		$db3 = $factory->getPrimaryDatabase( 'virtualdomain4' );
+		$this->assertEquals(
+			'localdomain',
+			$db3->getDomainID()
+		);
+	}
+
 	private function quoteTable( IReadableDatabase $db, $table ) {
 		if ( $db->getType() === 'sqlite' ) {
 			return $table;

tests/phpunit/unit/includes/db/MWLBFactoryTest.php

@@ -41,7 +41,8 @@ class MWLBFactoryTest extends MediaWikiUnitTestCase {
 			new EmptyBagOStuff(),
 			new WANObjectCache( [ 'cache' => new EmptyBagOStuff() ] ),
 			new CriticalSectionProvider( RequestTimeout::singleton(), 1, null, null ),
-			new NullStatsdDataFactory()
+			new NullStatsdDataFactory(),
+			[]
 		);
 	}