Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add support for write new for templatelinks migration

Browse source 
- schema change to allow tl_namespace and tl_title being empty
   This is done by removing them from primary key. They don't need to be
   nullable as they have default value.
 - Make sure with WRITE_NEW, updater avoids writing to the old columns

Bug: T306674
Change-Id: I2b8a29043e952060e7a79b6a7a3d647d48cd16fb
This commit is contained in:
Amir Sarabadani 2022-05-30 14:23:30 +02:00
parent 0b804b56f5
commit 692dde00df
19 changed files with 237 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/config-schema.yaml
View file

@ -1907,17 +1907,18 @@ config-schema:
in milliseconds.
@since 1.38
TemplateLinksSchemaMigrationStage:
default: 3
default: 259
type: integer
description: |-
Templatelinks table schema migration stage, for normalizing tl_namespace and tl_title fields.
Use the SCHEMA_COMPAT_XXX flags. Supported values:
- SCHEMA_COMPAT_OLD
- SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_OLD
- SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_NEW
- SCHEMA_COMPAT_NEW
History:
- 1.38: Added
- 1.39: Default has changed to SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_OLD
and support for SCHEMA_COMPAT_OLD is dropped.
ContentHandlers:
default:
wikitext: WikitextContentHandler

5
includes/MainConfigSchema.php
View file

@ -3007,16 +3007,17 @@ class MainConfigSchema {
*
* Use the SCHEMA_COMPAT_XXX flags. Supported values:
*
* - SCHEMA_COMPAT_OLD
* - SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_OLD
* - SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_NEW
* - SCHEMA_COMPAT_NEW
*
* History:
* - 1.38: Added
* - 1.39: Default has changed to SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_OLD
* and support for SCHEMA_COMPAT_OLD is dropped.
*/
public const TemplateLinksSchemaMigrationStage = [
'default' => SCHEMA_COMPAT_OLD,
'default' => SCHEMA_COMPAT_WRITE_BOTH | SCHEMA_COMPAT_READ_OLD,
'type' => 'integer',
];

2
includes/config-schema.php
View file

@ -380,7 +380,7 @@ return [
'DatabaseReplicaLagWarning' => 10,
'DatabaseReplicaLagCritical' => 30,
'MaxExecutionTimeForExpensiveQueries' => 0,
'TemplateLinksSchemaMigrationStage' => 3,
'TemplateLinksSchemaMigrationStage' => 259,
'ContentHandlers' => [
'wikitext' => 'WikitextContentHandler',
'javascript' => 'JavaScriptContentHandler',

6
includes/deferred/LinksUpdate/GenericPageLinksTable.php
View file

@ -111,9 +111,11 @@ abstract class GenericPageLinksTable extends TitleLinksTable {
protected function insertLink( $linkId ) {
$row = [
$this->getFromNamespaceField() => $this->getSourcePage()->getNamespace(),
$this->getNamespaceField() => $linkId[0],
$this->getTitleField() => $linkId[1]
];
if ( $this->linksTargetNormalizationStage() & SCHEMA_COMPAT_WRITE_OLD ) {
$row[$this->getNamespaceField()] = $linkId[0];
$row[$this->getTitleField()] = $linkId[1];
}
if ( $this->linksTargetNormalizationStage() & SCHEMA_COMPAT_WRITE_NEW ) {
$row[$this->getTargetIdField()] = $this->linkTargetLookup->acquireLinkTargetId(
$this->makeTitle( $linkId ),

21
includes/installer/DatabaseUpdater.php
View file

@ -1205,6 +1205,27 @@ abstract class DatabaseUpdater {
$this->output( "done.\n" );
}
protected function migrateTemplatelinks() {
if ( $this->updateRowExists( MigrateLinksTable::class . 'templatelinks' ) ) {
$this->output( "Templatelinks table have been already migrated...\n" );
return;
}
/**
* @var MigrateLinksTable $task
*/
$task = $this->maintenance->runChild(
MigrateLinksTable::class, 'migrateLinksTable.php'
);
'@phan-var MigrateLinksTable $task';
$task->loadParamsAndArgs( MigrateLinksTable::class, [
'force' => true,
'table' => 'templatelinks'
] );
$this->output( "Running migrate templatelinks...\n" );
$task->execute();
$this->output( "done.\n" );
}
/**
* Migrate comments to the new 'comment' table
* @since 1.30

2
includes/installer/MysqlUpdater.php
View file

@ -220,6 +220,8 @@ class MysqlUpdater extends DatabaseUpdater {
[ 'dropTable', 'revision_actor_temp' ],
[ 'runMaintenance', UpdateRestrictions::class, 'maintenance/updateRestrictions.php' ],
[ 'dropField', 'page', 'page_restrictions', 'patch-page-drop-page_restrictions.sql' ],
[ 'migrateTemplatelinks' ],
[ 'modifyField', 'templatelinks', 'tl_namespace', 'patch-templatelinks-tl_title-nullable.sql' ],
];
}

3
includes/installer/PostgresUpdater.php
View file

@ -600,6 +600,9 @@ class PostgresUpdater extends DatabaseUpdater {
[ 'dropTable', 'revision_actor_temp' ],
[ 'runMaintenance', UpdateRestrictions::class, 'maintenance/updateRestrictions.php' ],
[ 'dropPgField', 'page', 'page_restrictions' ],
[ 'migrateTemplatelinks' ],
[ 'changeNullableField', 'templatelinks', 'tl_target_id', 'NOT NULL', true ],
[ 'changePrimaryKey', 'templatelinks', [ 'tl_from', 'tl_target_id' ], 'templatelinks_pk' ],
];
}

2
includes/installer/SqliteUpdater.php
View file

@ -192,6 +192,8 @@ class SqliteUpdater extends DatabaseUpdater {
[ 'dropTable', 'revision_actor_temp' ],
[ 'runMaintenance', UpdateRestrictions::class, 'maintenance/updateRestrictions.php' ],
[ 'dropField', 'page', 'page_restrictions', 'patch-page-drop-page_restrictions.sql' ],
[ 'migrateTemplatelinks' ],
[ 'modifyField', 'templatelinks', 'tl_namespace', 'patch-templatelinks-tl_title-nullable.sql' ],
];
}

8
includes/linker/LinksMigration.php
View file

@ -44,6 +44,7 @@ class LinksMigration {
'ns' => 'tl_namespace',
'title' => 'tl_title',
'target_id' => 'tl_target_id',
'deprecated_configs' => [ SCHEMA_COMPAT_OLD ],
],
];
@ -139,5 +140,12 @@ class LinksMigration {
"LinksMigration doesn't support the $table table yet"
);
}
$config = $this->config->get( self::$mapping[$table]['config'] );
if ( in_array( $config, self::$mapping[$table]['deprecated_configs'] ) ) {
throw new InvalidArgumentException(
"LinksMigration config $config on $table table is not supported anymore"
);
}
}
}

125
maintenance/abstractSchemaChanges/patch-templatelinks-tl_title-nullable.json Normal file
View file

@ -0,0 +1,125 @@
{
"comment": "Making tl_title and tl_namespace nullable as part of normalizing templatelinks (T306674)",
"before": {
"name": "templatelinks",
"comment": "Track template inclusions. The target page may or may not exist, and due to renames and deletions may refer to different page records as time goes by.",
"columns": [
{
"name": "tl_from",
"comment": "Key to the page_id of the page containing the link.",
"type": "integer",
"options": { "notnull": true, "unsigned": true, "default": 0 }
},
{
"name": "tl_from_namespace",
"type": "integer",
"comment": "Namespace for this page",
"options": { "notnull": true, "default": 0 }
},
{
"name": "tl_namespace",
"type": "integer",
"options": { "notnull": true, "default": 0 }
},
{
"name": "tl_title",
"type": "binary",
"options": { "notnull": true, "length": 255, "default": "" }
},
{
"name": "tl_target_id",
"type": "bigint",
"comment": "Foreign key to linktarget.lt_id",
"options": { "notnull": false, "unsigned": true, "default": null }
}
],
"indexes": [
{
"name": "tl_namespace",
"columns": [ "tl_namespace", "tl_title", "tl_from" ],
"comment": "Reverse index, for Special:Whatlinkshere",
"unique": false
},
{
"name": "tl_backlinks_namespace",
"columns": [ "tl_from_namespace", "tl_namespace", "tl_title", "tl_from" ],
"comment": "Index for Special:Whatlinkshere with namespace filter",
"unique": false
},
{
"name": "tl_target_id",
"columns": [ "tl_target_id", "tl_from" ],
"comment": "Reverse index, for Special:Whatlinkshere",
"unique": false
},
{
"name": "tl_backlinks_namespace_target_id",
"columns": [ "tl_from_namespace", "tl_target_id", "tl_from" ],
"comment": "Index for Special:Whatlinkshere with namespace filter",
"unique": false
}
],
"pk": [ "tl_from", "tl_namespace", "tl_title" ]
},
"after": {
"name": "templatelinks",
"comment": "Track template inclusions. The target page may or may not exist, and due to renames and deletions may refer to different page records as time goes by.",
"columns": [
{
"name": "tl_from",
"comment": "Key to the page_id of the page containing the link.",
"type": "integer",
"options": { "notnull": true, "unsigned": true, "default": 0 }
},
{
"name": "tl_from_namespace",
"type": "integer",
"comment": "Namespace for this page",
"options": { "notnull": true, "default": 0 }
},
{
"name": "tl_namespace",
"type": "integer",
"options": { "default": 0 }
},
{
"name": "tl_title",
"type": "binary",
"options": { "length": 255, "default": "" }
},
{
"name": "tl_target_id",
"type": "bigint",
"comment": "Foreign key to linktarget.lt_id",
"options": { "notnull": true, "unsigned": true }
}
],
"indexes": [
{
"name": "tl_namespace",
"columns": [ "tl_namespace", "tl_title", "tl_from" ],
"comment": "Reverse index, for Special:Whatlinkshere",
"unique": false
},
{
"name": "tl_backlinks_namespace",
"columns": [ "tl_from_namespace", "tl_namespace", "tl_title", "tl_from" ],
"comment": "Index for Special:Whatlinkshere with namespace filter",
"unique": false
},
{
"name": "tl_target_id",
"columns": [ "tl_target_id", "tl_from" ],
"comment": "Reverse index, for Special:Whatlinkshere",
"unique": false
},
{
"name": "tl_backlinks_namespace_target_id",
"columns": [ "tl_from_namespace", "tl_target_id", "tl_from" ],
"comment": "Index for Special:Whatlinkshere with namespace filter",
"unique": false
}
],
"pk": [ "tl_from", "tl_target_id" ]
}
}

10
maintenance/archives/patch-templatelinks-tl_title-nullable.sql Normal file
View file

@ -0,0 +1,10 @@
-- This file is automatically generated using maintenance/generateSchemaChangeSql.php.
-- Source: maintenance/abstractSchemaChanges/patch-templatelinks-tl_title-nullable.json
-- Do not modify this file directly.
-- See https://www.mediawiki.org/wiki/Manual:Schema_changes
ALTER TABLE /*_*/templatelinks
DROP PRIMARY KEY;
ALTER TABLE /*_*/templatelinks
CHANGE tl_target_id tl_target_id BIGINT UNSIGNED NOT NULL;
ALTER TABLE /*_*/templatelinks
ADD PRIMARY KEY (tl_from, tl_target_id);

8
maintenance/migrateLinksTable.php
View file

@ -55,13 +55,19 @@ class MigrateLinksTable extends LoggedUpdateMaintenance {
$this->output( "Starting the populating $targetColumn column\n" );
$updated = 0;
$highestPageId = $dbw->newSelectQueryBuilder()
->select( 'page_id' )
->from( 'page' )
->limit( 1 )
->caller( __METHOD__ )
->orderBy( 'page_id', 'DESC' )
->fetchResultSet()->fetchRow()[0];
->fetchResultSet()->fetchRow();
if ( !$highestPageId ) {
$this->output( "Page table is empty.\n" );
return true;
}
$highestPageId = $highestPageId[0];
$pageId = 0;
while ( $pageId < $highestPageId ) {
// Given the indexes and the structure of links tables,

6
maintenance/postgres/tables-generated.sql
View file

@ -184,11 +184,11 @@ CREATE INDEX pl_backlinks_namespace ON pagelinks (
CREATE TABLE templatelinks (
tl_from INT DEFAULT 0 NOT NULL,
tl_target_id BIGINT NOT NULL,
tl_from_namespace INT DEFAULT 0 NOT NULL,
tl_namespace INT DEFAULT 0 NOT NULL,
tl_title TEXT DEFAULT '' NOT NULL,
tl_from_namespace INT DEFAULT 0 NOT NULL,
tl_target_id BIGINT DEFAULT NULL,
PRIMARY KEY(tl_from, tl_namespace, tl_title)
PRIMARY KEY(tl_from, tl_target_id)
);
CREATE INDEX tl_namespace ON templatelinks (tl_namespace, tl_title, tl_from);

21
maintenance/sqlite/archives/patch-templatelinks-tl_title-nullable.sql Normal file
View file

@ -0,0 +1,21 @@
-- This file is automatically generated using maintenance/generateSchemaChangeSql.php.
-- Source: maintenance/abstractSchemaChanges/patch-templatelinks-tl_title-nullable.json
-- Do not modify this file directly.
-- See https://www.mediawiki.org/wiki/Manual:Schema_changes
DROP INDEX tl_namespace;
DROP INDEX tl_backlinks_namespace;
DROP INDEX tl_target_id;
DROP INDEX tl_backlinks_namespace_target_id;
CREATE TEMPORARY TABLE /*_*/__temp__templatelinks AS
SELECT tl_from, tl_namespace, tl_title, tl_from_namespace, tl_target_id
FROM /*_*/templatelinks;
DROP TABLE /*_*/templatelinks;
CREATE TABLE /*_*/templatelinks ( tl_from INTEGER UNSIGNED DEFAULT 0 NOT NULL, tl_target_id BIGINT UNSIGNED NOT NULL, tl_namespace INTEGER DEFAULT 0 NOT NULL, tl_title BLOB DEFAULT '' NOT NULL, tl_from_namespace INTEGER DEFAULT 0 NOT NULL, PRIMARY KEY(tl_from, tl_target_id) );
INSERT INTO /*_*/templatelinks ( tl_from, tl_namespace, tl_title, tl_from_namespace, tl_target_id )
SELECT tl_from, tl_namespace, tl_title, tl_from_namespace, tl_target_id
FROM /*_*/__temp__templatelinks;
DROP TABLE /*_*/__temp__templatelinks;
CREATE INDEX tl_namespace ON /*_*/templatelinks (tl_namespace, tl_title, tl_from);
CREATE INDEX tl_backlinks_namespace ON /*_*/templatelinks ( tl_from_namespace, tl_namespace, tl_title, tl_from );
CREATE INDEX tl_target_id ON /*_*/templatelinks (tl_target_id, tl_from);
CREATE INDEX tl_backlinks_namespace_target_id ON /*_*/templatelinks ( tl_from_namespace, tl_target_id, tl_from );

6
maintenance/sqlite/tables-generated.sql
View file

@ -177,11 +177,11 @@ CREATE INDEX pl_backlinks_namespace ON /*_*/pagelinks (
CREATE TABLE /*_*/templatelinks (
tl_from INTEGER UNSIGNED DEFAULT 0 NOT NULL,
tl_target_id BIGINT UNSIGNED NOT NULL,
tl_from_namespace INTEGER DEFAULT 0 NOT NULL,
tl_namespace INTEGER DEFAULT 0 NOT NULL,
tl_title BLOB DEFAULT '' NOT NULL,
tl_from_namespace INTEGER DEFAULT 0 NOT NULL,
tl_target_id BIGINT UNSIGNED DEFAULT NULL,
PRIMARY KEY(tl_from, tl_namespace, tl_title)
PRIMARY KEY(tl_from, tl_target_id)
);
CREATE INDEX tl_namespace ON /*_*/templatelinks (tl_namespace, tl_title, tl_from);

6
maintenance/tables-generated.sql
View file

@ -169,10 +169,10 @@ CREATE TABLE /*_*/pagelinks (
CREATE TABLE /*_*/templatelinks (
tl_from INT UNSIGNED DEFAULT 0 NOT NULL,
tl_target_id BIGINT UNSIGNED NOT NULL,
tl_from_namespace INT DEFAULT 0 NOT NULL,
tl_namespace INT DEFAULT 0 NOT NULL,
tl_title VARBINARY(255) DEFAULT '' NOT NULL,
tl_from_namespace INT DEFAULT 0 NOT NULL,
tl_target_id BIGINT UNSIGNED DEFAULT NULL,
INDEX tl_namespace (tl_namespace, tl_title, tl_from),
INDEX tl_backlinks_namespace (
tl_from_namespace, tl_namespace,
@ -183,7 +183,7 @@ CREATE TABLE /*_*/templatelinks (
tl_from_namespace, tl_target_id,
tl_from
),
PRIMARY KEY(tl_from, tl_namespace, tl_title)
PRIMARY KEY(tl_from, tl_target_id)
) /*$wgDBTableOptions*/;

8
maintenance/tables.json
View file

@ -557,18 +557,18 @@
{
"name": "tl_namespace",
"type": "integer",
"options": { "notnull": true, "default": 0 }
"options": { "default": 0 }
},
{
"name": "tl_title",
"type": "binary",
"options": { "notnull": true, "length": 255, "default": "" }
"options": { "length": 255, "default": "" }
},
{
"name": "tl_target_id",
"type": "bigint",
"comment": "Foreign key to linktarget.lt_id",
"options": { "notnull": false, "unsigned": true, "default": null }
"options": { "notnull": true, "unsigned": true }
}
],
"indexes": [
@ -597,7 +597,7 @@
"unique": false
}
],
"pk": [ "tl_from", "tl_namespace", "tl_title" ]
"pk": [ "tl_from", "tl_target_id" ]
},
{
"name": "imagelinks",

19
tests/phpunit/includes/cache/BacklinkCacheTest.php vendored
View file

@ -140,12 +140,21 @@ class BacklinkCacheTest extends MediaWikiIntegrationTestCase {
* @covers BacklinkCache::partition
*/
public function testPartition() {
$targetId = $this->getServiceContainer()->getLinkTargetLookup()->acquireLinkTargetId(
Title::newFromText( 'BLCTest1234' ),
$this->db
);
$targetRow = [
'tl_target_id' => $targetId,
'tl_namespace' => 0,
'tl_title' => 'BLCTest1234',
];
$this->db->insert( 'templatelinks', [
[ 'tl_from' => 56890, 'tl_from_namespace' => 0, 'tl_namespace' => 0, 'tl_title' => 'BLCTest1234' ],
[ 'tl_from' => 56891, 'tl_from_namespace' => 0, 'tl_namespace' => 0, 'tl_title' => 'BLCTest1234' ],
[ 'tl_from' => 56892, 'tl_from_namespace' => 0, 'tl_namespace' => 0, 'tl_title' => 'BLCTest1234' ],
[ 'tl_from' => 56893, 'tl_from_namespace' => 0, 'tl_namespace' => 0, 'tl_title' => 'BLCTest1234' ],
[ 'tl_from' => 56894, 'tl_from_namespace' => 0, 'tl_namespace' => 0, 'tl_title' => 'BLCTest1234' ],
[ 'tl_from' => 56890, 'tl_from_namespace' => 0 ] + $targetRow,
[ 'tl_from' => 56891, 'tl_from_namespace' => 0 ] + $targetRow,
[ 'tl_from' => 56892, 'tl_from_namespace' => 0 ] + $targetRow,
[ 'tl_from' => 56893, 'tl_from_namespace' => 0 ] + $targetRow,
[ 'tl_from' => 56894, 'tl_from_namespace' => 0 ] + $targetRow,
] );
$blcFactory = $this->getServiceContainer()->getBacklinkCacheFactory();
$backlinkCache = $blcFactory->getBacklinkCache( Title::newFromText( 'BLCTest1234' ) );

1
tests/phpunit/unit/linker/LinksMigrationTest.php
View file

@ -41,7 +41,6 @@ class LinksMigrationTest extends MediaWikiUnitTestCase {
public function provideReadOld() {
yield [ SCHEMA_COMPAT_READ_OLD ];
yield [ SCHEMA_COMPAT_OLD ];
}
/**