Populate externallinks.el_index_60 and drop default

Adds a maintenance script to populate the field, has that be
automatically run during update.php, and drops the no-longer-needed
default value on the column (where possible: mssql has some sort of
constraint thing going on that I have no idea how it works).

Bug: T59176
Change-Id: I971edf013a1a39466aca3b6e34c915cb24fd3aa7

autoload.php

@@ -1161,6 +1161,7 @@ $wgAutoloadLocalClasses = [
 	'PopulateBacklinkNamespace' => __DIR__ . '/maintenance/populateBacklinkNamespace.php',
 	'PopulateCategory' => __DIR__ . '/maintenance/populateCategory.php',
 	'PopulateContentModel' => __DIR__ . '/maintenance/populateContentModel.php',
+	'PopulateExternallinksIndex60' => __DIR__ . '/maintenance/populateExternallinksIndex60.php',
 	'PopulateFilearchiveSha1' => __DIR__ . '/maintenance/populateFilearchiveSha1.php',
 	'PopulateImageSha1' => __DIR__ . '/maintenance/populateImageSha1.php',
 	'PopulateInterwiki' => __DIR__ . '/maintenance/populateInterwiki.php',

includes/deferred/LinksUpdate.php

@@ -569,6 +569,7 @@ class LinksUpdate extends DataUpdate implements EnqueueableDataUpdate {
 					'el_from' => $this->mId,
 					'el_to' => $url,
 					'el_index' => $index,
+					'el_index_60' => substr( $index, 0, 60 ),
 				];
 			}
 		}

includes/installer/DatabaseUpdater.php

@@ -1287,4 +1287,21 @@ abstract class DatabaseUpdater {
 		 }
 	 }
 
+	/**
+	 * Populates the externallinks.el_index_60 field
+	 * @since 1.32
+	 */
+	protected function populateExternallinksIndex60() {
+		if ( !$this->updateRowExists( 'populate externallinks.el_index_60' ) ) {
+			$this->output(
+				"Populating el_index_60 field, printing progress markers. For large\n" .
+				"databases, you may want to hit Ctrl-C and do this manually with\n" .
+				"maintenance/populateExternallinksIndex60.php.\n"
+			);
+			$task = $this->maintenance->runChild( 'PopulateExternallinksIndex60',
+				'populateExternallinksIndex60.php' );
+			$task->execute();
+			$this->output( "done.\n" );
+		}
+	}
 }

includes/installer/MssqlUpdater.php

@@ -131,6 +131,9 @@ class MssqlUpdater extends DatabaseUpdater {
 
 			// 1.32
 			[ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+			[ 'populateExternallinksIndex60' ],
+			[ 'modifyfield', 'externallinks', 'el_index_60',
+				'patch-externallinks-el_index_60-drop-default.sql' ],
 		];
 	}
 

includes/installer/MysqlUpdater.php

@@ -351,6 +351,9 @@ class MysqlUpdater extends DatabaseUpdater {
 
 			// 1.32
 			[ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+			[ 'populateExternallinksIndex60' ],
+			[ 'modifyfield', 'externallinks', 'el_index_60',
+				'patch-externallinks-el_index_60-drop-default.sql' ],
 		];
 	}
 

includes/installer/OracleUpdater.php

@@ -150,6 +150,7 @@ class OracleUpdater extends DatabaseUpdater {
 
 			// 1.32
 			[ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+			[ 'populateExternallinksIndex60' ],
 
 			// KEEP THIS AT THE BOTTOM!!
 			[ 'doRebuildDuplicateFunction' ],

includes/installer/PostgresUpdater.php

@@ -572,6 +572,8 @@ class PostgresUpdater extends DatabaseUpdater {
 
 			// 1.32
 			[ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+			[ 'populateExternallinksIndex60' ],
+			[ 'dropDefault', 'externallinks', 'el_index_60' ],
 		];
 	}
 
@@ -907,6 +909,20 @@ END;
 		}
 	}
 
+	/**
+	 * Drop a default value from a field
+	 * @since 1.32
+	 * @param string $table
+	 * @param string $field
+	 */
+	protected function dropDefault( $table, $field ) {
+		$info = $this->db->fieldInfo( $table, $field );
+		if ( $info->defaultValue() !== false ) {
+			$this->output( "Removing '$table.$field' default value\n" );
+			$this->db->query( "ALTER TABLE $table ALTER $field DROP DEFAULT" );
+		}
+	}
+
 	protected function changeNullableField( $table, $field, $null, $update = false ) {
 		$fi = $this->db->fieldInfo( $table, $field );
 		if ( is_null( $fi ) ) {

includes/installer/SqliteUpdater.php

@@ -215,6 +215,9 @@ class SqliteUpdater extends DatabaseUpdater {
 
 			// 1.32
 			[ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ],
+			[ 'populateExternallinksIndex60' ],
+			[ 'modifyfield', 'externallinks', 'el_index_60',
+				'patch-externallinks-el_index_60-drop-default.sql' ],
 		];
 	}
 

maintenance/archives/patch-externallinks-el_index_60-drop-default.sql

@@ -0,0 +1,2 @@
+-- @since 1.32
+ALTER TABLE /*$wgDBprefix*/externallinks ALTER COLUMN el_index_60 DROP DEFAULT;

maintenance/fixExtLinksProtocolRelative.php

@@ -71,11 +71,13 @@ class FixExtLinksProtocolRelative extends LoggedUpdateMaintenance {
 						'el_from' => $row->el_from,
 						'el_to' => $row->el_to,
 						'el_index' => "http:{$row->el_index}",
+						'el_index_60' => substr( "http:{$row->el_index}", 0, 60 ),
 					],
 					[
 						'el_from' => $row->el_from,
 						'el_to' => $row->el_to,
 						'el_index' => "https:{$row->el_index}",
+						'el_index_60' => substr( "https:{$row->el_index}", 0, 60 ),
 					]
 				], __METHOD__, [ 'IGNORE' ]
 			);

maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql

@@ -0,0 +1,13 @@
+DECLARE @sql nvarchar(max)
+SET @sql=''
+
+SELECT @sql= @sql + 'ALTER TABLE /*_*/externallinks DROP CONSTRAINT ' + df.name + '; '
+FROM sys.default_constraints df
+JOIN sys.columns c
+	ON c.object_id = df.parent_object_id
+	AND c.column_id = df.parent_column_id
+WHERE
+	df.parent_object_id =  OBJECT_ID('/*_*/externallinks')
+	AND c.name = 'el_index_60';--
+
+EXEC sp_executesql @sql;

maintenance/mssql/tables.sql

@@ -549,8 +549,7 @@ CREATE TABLE /*_*/externallinks (
 
   -- This is el_index truncated to 60 bytes to allow for sortable queries that
   -- aren't supported by a partial index.
-  -- @todo Drop the default once this is deployed everywhere and code is populating it.
-  el_index_60 varbinary(60) NOT NULL default ''
+  el_index_60 varbinary(60) NOT NULL
 );
 
 CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from);

maintenance/populateExternallinksIndex60.php

@@ -0,0 +1,89 @@
+<?php
+/**
+ * Populates the el_index_60 field in the externallinks table.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+
+require_once __DIR__ . '/Maintenance.php';
+
+/**
+ * Maintenance script that populates the el_index_60 field in the externallinks
+ * table.
+ *
+ * @ingroup Maintenance
+ * @since 1.32
+ */
+class PopulateExternallinksIndex60 extends LoggedUpdateMaintenance {
+	public function __construct() {
+		parent::__construct();
+		$this->addDescription(
+			'Populates the el_index_60 field in the externallinks table' );
+		$this->setBatchSize( 200 );
+	}
+
+	protected function getUpdateKey() {
+		return 'populate externallinks.el_index_60';
+	}
+
+	protected function updateSkippedMessage() {
+		return 'externallinks.el_index_60 already populated.';
+	}
+
+	protected function doDBUpdates() {
+		$dbw = $this->getDB( DB_MASTER );
+		$this->output( "Populating externallinks.el_index_60...\n" );
+
+		$count = 0;
+		$start = 0;
+		$last = $dbw->selectField( 'externallinks', 'MAX(el_id)', '', __METHOD__ );
+		while ( $start <= $last ) {
+			$end = $start + $this->mBatchSize;
+			$this->output( "el_id $start - $end of $last\n" );
+			$res = $dbw->select( 'externallinks', [ 'el_id', 'el_index' ],
+				[
+					"el_id > $start",
+					"el_id <= $end",
+					'el_index_60' => '',
+				],
+				__METHOD__,
+				[ 'ORDER BY' => 'el_id' ]
+			);
+			foreach ( $res as $row ) {
+				$count++;
+				$dbw->update( 'externallinks',
+					[
+						'el_index_60' => substr( $row->el_index, 0, 60 ),
+					],
+					[
+						'el_id' => $row->el_id,
+					], __METHOD__, [ 'IGNORE' ]
+				);
+			}
+			wfWaitForSlaves();
+			$start = $end;
+		}
+		$this->output( "Done, $count rows updated.\n" );
+
+		return true;
+	}
+}
+
+$maintClass = "PopulateExternallinksIndex60";
+require_once RUN_MAINTENANCE_IF_MAIN;

maintenance/postgres/tables.sql

@@ -373,7 +373,7 @@ CREATE TABLE externallinks (
   el_from     INTEGER     NOT NULL  REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   el_to       TEXT        NOT NULL,
   el_index    TEXT        NOT NULL,
-  el_index_60 BYTEA       NOT NULL  DEFAULT ''
+  el_index_60 BYTEA       NOT NULL
 );
 ALTER SEQUENCE externallinks_el_id_seq OWNED BY externallinks.el_id;
 CREATE INDEX externallinks_from_to ON externallinks (el_from,el_to);

maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql

@@ -0,0 +1,21 @@
+-- To change the default on one column, sqlite requires we copy the whole table
+
+CREATE TABLE /*_*/externallinks_tmp (
+  el_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  el_from int unsigned NOT NULL default 0,
+  el_to blob NOT NULL,
+  el_index blob NOT NULL,
+  el_index_60 varbinary(60) NOT NULL
+) /*$wgDBTableOptions*/;
+
+INSERT INTO /*_*/externallinks_tmp
+	SELECT el_id, el_from, el_to, el_index, el_index_60 FROM /*_*/externallinks;
+
+DROP TABLE /*_*/externallinks;
+ALTER TABLE /*_*/externallinks_tmp RENAME TO /*_*/externallinks;
+
+CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from, el_to(40));
+CREATE INDEX /*i*/el_to ON /*_*/externallinks (el_to(60), el_from);
+CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index(60));
+CREATE INDEX /*i*/el_index_60 ON /*_*/externallinks (el_index_60, el_id);
+CREATE INDEX /*i*/el_from_index_60 ON /*_*/externallinks (el_from, el_index_60, el_id);

maintenance/tables.sql

@@ -932,8 +932,7 @@ CREATE TABLE /*_*/externallinks (
 
   -- This is el_index truncated to 60 bytes to allow for sortable queries that
   -- aren't supported by a partial index.
-  -- @todo Drop the default once this is deployed everywhere and code is populating it.
-  el_index_60 varbinary(60) NOT NULL default ''
+  el_index_60 varbinary(60) NOT NULL
 ) /*$wgDBTableOptions*/;
 
 -- Forward index, for page edit, save