Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Migrate Database::upsert() calls to InsertQueryBuilder

Browse source 
Bug: T335377
Change-Id: I0e0c3f3a9150c7a62d8fff95fe8867bdce356071
This commit is contained in:
Amir Sarabadani 2023-09-05 21:21:03 +02:00 committed by James D. Forrester
parent 9f3c7996ee
commit f783b02ff2
16 changed files with 192 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
includes/Category/Category.php
View file

@ -444,22 +444,22 @@ class Category {
} elseif ( $shouldExist ) {
# The category row doesn't exist but should, so create it. Use
# upsert in case of races.
$dbw->upsert(
'category',
[
$dbw->newInsertQueryBuilder()
->insert( 'category' )
->row( [
'cat_title' => $this->mName,
'cat_pages' => $result->pages,
'cat_subcats' => $result->subcats,
'cat_files' => $result->files
],
'cat_title',
[
] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'cat_title' ] )
->set( [
'cat_pages' => $result->pages,
'cat_subcats' => $result->subcats,
'cat_files' => $result->files
],
__METHOD__
);
'cat_files' => $result->files
] )
->caller( __METHOD__ )->execute();
// @todo: Should we update $this->mID here? Or not since Category
// objects tend to be short lived enough to not matter?
}

16
includes/ResourceLoader/dependencystore/SqlModuleDependencyStore.php
View file

@ -104,15 +104,13 @@ class SqlModuleDependencyStore extends DependencyStore {
// @TODO: use a single query with VALUES()/aliases support in DB wrapper
// See https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html
foreach ( $rows as $row ) {
$dbw->upsert(
'module_deps',
$row,
[ [ 'md_module', 'md_skin' ] ],
[
'md_deps' => $row['md_deps'],
],
__METHOD__
);
$dbw->newInsertQueryBuilder()
->insert( 'module_deps' )
->row( $row )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'md_module', 'md_skin' ] )
->set( [ 'md_deps' => $row['md_deps'] ] )
->caller( __METHOD__ )->execute();
}
}

29
includes/SiteStats/SiteStatsInit.php
View file

@ -198,14 +198,13 @@ class SiteStatsInit {
'ss_images' => $this->getShardedValue( $this->files ?? $this->files(), $shardCnt, $i ),
];
$row = [ 'ss_row_id' => $i ] + $set;
self::getDB( DB_PRIMARY )->upsert(
'site_stats',
$row,
'ss_row_id',
$set,
__METHOD__
);
self::getDB( DB_PRIMARY )->newInsertQueryBuilder()
->insert( 'site_stats' )
->row( $row )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'ss_row_id' ] )
->set( $set )
->caller( __METHOD__ )->execute();
}
} else {
$set = [
@ -217,13 +216,13 @@ class SiteStatsInit {
];
$row = [ 'ss_row_id' => 1 ] + $set;
self::getDB( DB_PRIMARY )->upsert(
'site_stats',
$row,
'ss_row_id',
$set,
__METHOD__
);
self::getDB( DB_PRIMARY )->newInsertQueryBuilder()
->insert( 'site_stats' )
->row( $row )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'ss_row_id' ] )
->set( $set )
->caller( __METHOD__ )->execute();
}
}

23
includes/changetags/ChangeTagsStore.php
View file

@ -227,18 +227,17 @@ class ChangeTagsStore {
*/
public function defineTag( $tag ) {
$dbw = $this->dbProvider->getPrimaryDatabase();
$tagDef = [
'ctd_name' => $tag,
'ctd_user_defined' => 1,
'ctd_count' => 0
];
$dbw->upsert(
self::CHANGE_TAG_DEF,
$tagDef,
'ctd_name',
[ 'ctd_user_defined' => 1 ],
__METHOD__
);
$dbw->newInsertQueryBuilder()
->insert( self::CHANGE_TAG_DEF )
->row( [
'ctd_name' => $tag,
'ctd_user_defined' => 1,
'ctd_count' => 0
] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'ctd_name' ] )
->set( [ 'ctd_user_defined' => 1 ] )
->caller( __METHOD__ )->execute();
// clear the memcache of defined tags
$this->purgeTagCacheAll();

14
includes/deferred/SiteStatsUpdate.php
View file

@ -152,13 +152,13 @@ class SiteStatsUpdate implements DeferrableUpdate, MergeableUpdate {
->where( [ 'ss_row_id' => $shard ] )
->caller( $fname )->execute();
} else {
$dbw->upsert(
'site_stats',
array_merge( [ 'ss_row_id' => $shard ], $initValues ),
'ss_row_id',
$set,
$fname
);
$dbw->newInsertQueryBuilder()
->insert( 'site_stats' )
->row( array_merge( [ 'ss_row_id' => $shard ], $initValues ) )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'ss_row_id' ] )
->set( $set )
->caller( $fname )->execute();
}
}
}

14
includes/installer/Pingback.php
View file

@ -116,13 +116,13 @@ class Pingback {
// so we don't submit data multiple times.
$dbw = $this->dbProvider->getPrimaryDatabase();
$timestamp = ConvertibleTimestamp::time();
$dbw->upsert(
'updatelog',
[ 'ul_key' => $this->key, 'ul_value' => $timestamp ],
'ul_key',
[ 'ul_value' => $timestamp ],
__METHOD__
);
$dbw->newInsertQueryBuilder()
->insert( 'updatelog' )
->row( [ 'ul_key' => $this->key, 'ul_value' => $timestamp ] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'ul_key' ] )
->set( [ 'ul_value' => $timestamp ] )
->caller( __METHOD__ )->execute();
$this->logger->debug( __METHOD__ . ": pingback sent OK ({$this->key})" );
}

121
includes/objectcache/SqlBagOStuff.php
View file

@ -579,13 +579,13 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
}
if ( $this->multiPrimaryMode ) {
$db->upsert(
$ptable,
$rows,
[ [ 'keyname' ] ],
$this->buildMultiUpsertSetForOverwrite( $db, $mt ),
__METHOD__
);
$db->newInsertQueryBuilder()
->insert( $ptable )
->rows( $rows )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildMultiUpsertSetForOverwrite( $db, $mt ) )
->caller( __METHOD__ )->execute();
} else {
// T288998: use REPLACE, if possible, to avoid cluttering the binlogs
$db->replace( $ptable, 'keyname', $rows, __METHOD__ );
@ -625,17 +625,15 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
// Tombstone keys in order to respect eventual consistency
$mt = $this->makeTimestampedModificationToken( $mtime, $db );
$expiry = $this->makeNewKeyExpiry( self::TOMB_EXPTIME, (int)$mtime );
$rows = [];
$queryBuilder = $db->newInsertQueryBuilder()
->insert( $ptable )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildMultiUpsertSetForOverwrite( $db, $mt ) );
foreach ( $argsByKey as $key => $arg ) {
$rows[] = $this->buildUpsertRow( $db, $key, self::TOMB_SERIAL, $expiry, $mt );
$queryBuilder->row( $this->buildUpsertRow( $db, $key, self::TOMB_SERIAL, $expiry, $mt ) );
}
$db->upsert(
$ptable,
$rows,
[ [ 'keyname' ] ],
$this->buildMultiUpsertSetForOverwrite( $db, $mt ),
__METHOD__
);
$queryBuilder->caller( __METHOD__ )->execute();
} else {
// Just purge the keys since there is only one primary (e.g. "source of truth")
$db->newDeleteQueryBuilder()
@ -700,18 +698,19 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$serialValue = $this->getSerialized( $value, $key );
$expiry = $this->makeNewKeyExpiry( $exptime, (int)$mtime );
$rows[] = $this->buildUpsertRow( $db, $key, $serialValue, $expiry, $mt );
$valueSizesByKey[$key] = [ strlen( $serialValue ), 0 ];
$rows[] = $this->buildUpsertRow( $db, $key, $serialValue, $expiry, $mt );
}
$db->upsert(
$ptable,
$rows,
[ [ 'keyname' ] ],
$this->buildMultiUpsertSetForOverwrite( $db, $mt ),
__METHOD__
);
if ( !$rows ) {
return;
}
$db->newInsertQueryBuilder()
->insert( $ptable )
->rows( $rows )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildMultiUpsertSetForOverwrite( $db, $mt ) )
->caller( __METHOD__ )->execute();
foreach ( $argsByKey as $key => $unused ) {
$resByKey[$key] = !isset( $existingByKey[$key] );
@ -764,8 +763,8 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$curTokensByKey[$row->keyname] = $this->getCasTokenFromRow( $db, $row );
}
$rows = [];
$nonMatchingByKey = [];
$rows = [];
foreach ( $argsByKey as $key => [ $value, $exptime, $casToken ] ) {
$curToken = $curTokensByKey[$key] ?? null;
if ( $curToken === null ) {
@ -782,18 +781,20 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$serialValue = $this->getSerialized( $value, $key );
$expiry = $this->makeNewKeyExpiry( $exptime, (int)$mtime );
$rows[] = $this->buildUpsertRow( $db, $key, $serialValue, $expiry, $mt );
$valueSizesByKey[$key] = [ strlen( $serialValue ), 0 ];
}
$db->upsert(
$ptable,
$rows,
[ [ 'keyname' ] ],
$this->buildMultiUpsertSetForOverwrite( $db, $mt ),
__METHOD__
);
$rows[] = $this->buildUpsertRow( $db, $key, $serialValue, $expiry, $mt );
}
if ( !$rows ) {
return;
}
$db->newInsertQueryBuilder()
->insert( $ptable )
->rows( $rows )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildMultiUpsertSetForOverwrite( $db, $mt ) )
->caller( __METHOD__ )->execute();
foreach ( $argsByKey as $key => $unused ) {
$resByKey[$key] = !isset( $nonMatchingByKey[$key] );
@ -848,14 +849,16 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
$expiry = $this->makeNewKeyExpiry( $exptime, (int)$mtime );
$rows[] = $this->buildUpsertRow( $db, $key, $serialValue, $expiry, $mt );
}
$db->upsert(
$ptable,
$rows,
[ [ 'keyname' ] ],
$this->buildMultiUpsertSetForOverwrite( $db, $mt ),
__METHOD__
);
if ( !$rows ) {
return;
}
$db->newInsertQueryBuilder()
->insert( $ptable )
->rows( $rows )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildMultiUpsertSetForOverwrite( $db, $mt ) )
->caller( __METHOD__ )->execute();
foreach ( $argsByKey as $key => $unused ) {
$resByKey[$key] = isset( $existingKeys[$key] );
@ -922,13 +925,13 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
// replication since the TTL for such keys is either indefinite or very short.
$atomic = $db->startAtomic( __METHOD__, IDatabase::ATOMIC_CANCELABLE );
try {
$db->upsert(
$ptable,
$this->buildUpsertRow( $db, $key, $init, $expiry, $mt ),
[ [ 'keyname' ] ],
$this->buildIncrUpsertSet( $db, $step, $init, $expiry, $mt, (int)$mtime ),
__METHOD__
);
$db->newInsertQueryBuilder()
->insert( $ptable )
->rows( $this->buildUpsertRow( $db, $key, $init, $expiry, $mt ) )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildIncrUpsertSet( $db, $step, $init, $expiry, $mt, (int)$mtime ) )
->caller( __METHOD__ )->execute();
$affectedCount = $db->affectedRows();
$row = $db->newSelectQueryBuilder()
->select( 'value' )
@ -980,13 +983,13 @@ class SqlBagOStuff extends MediumSpecificBagOStuff {
foreach ( $argsByKey as $key => [ $step, $init, $exptime ] ) {
$mt = $this->makeTimestampedModificationToken( $mtime, $db );
$expiry = $this->makeNewKeyExpiry( $exptime, (int)$mtime );
$db->upsert(
$ptable,
$this->buildUpsertRow( $db, $key, $init, $expiry, $mt ),
[ [ 'keyname' ] ],
$this->buildIncrUpsertSet( $db, $step, $init, $expiry, $mt, (int)$mtime ),
__METHOD__
);
$db->newInsertQueryBuilder()
->insert( $ptable )
->rows( $this->buildUpsertRow( $db, $key, $init, $expiry, $mt ) )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'keyname' ] )
->set( $this->buildIncrUpsertSet( $db, $step, $init, $expiry, $mt, (int)$mtime ) )
->caller( __METHOD__ )->execute();
if ( !$db->affectedRows() ) {
$this->logger->warning( __METHOD__ . ": failed to set new $key value" );
} else {

36
includes/page/WikiPage.php
View file

@ -1075,24 +1075,24 @@ class WikiPage implements Page, IDBAccessObject, PageRecord {
if ( !$oldLatest || $oldLatest == $this->lockAndGetLatest() ) {
$truncatedFragment = mb_strcut( $rt->getFragment(), 0, 255 );
$dbw->upsert(
'redirect',
[
$dbw->newInsertQueryBuilder()
->insert( 'redirect' )
->row( [
'rd_from' => $this->getId(),
'rd_namespace' => $rt->getNamespace(),
'rd_title' => $rt->getDBkey(),
'rd_fragment' => $truncatedFragment,
'rd_interwiki' => $rt->getInterwiki(),
],
'rd_from',
[
] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'rd_from' ] )
->set( [
'rd_namespace' => $rt->getNamespace(),
'rd_title' => $rt->getDBkey(),
'rd_fragment' => $truncatedFragment,
'rd_interwiki' => $rt->getInterwiki(),
],
__METHOD__
);
] )
->caller( __METHOD__ )->execute();
$success = true;
} else {
$success = false;
@ -3024,22 +3024,20 @@ class WikiPage implements Page, IDBAccessObject, PageRecord {
}
if ( $missingAdded ) {
$insertRows = [];
$queryBuilder = $dbw->newInsertQueryBuilder()
->insert( 'category' )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'cat_title' ] )
->set( $addFields );
foreach ( $missingAdded as $cat ) {
$insertRows[] = [
$queryBuilder->row( [
'cat_title' => $cat,
'cat_pages' => 1,
'cat_subcats' => ( $type === 'subcat' ) ? 1 : 0,
'cat_files' => ( $type === 'file' ) ? 1 : 0,
];
] );
}
$dbw->upsert(
'category',
$insertRows,
'cat_title',
$addFields,
__METHOD__
);
$queryBuilder->caller( __METHOD__ )->execute();
}
if ( $existingDeleted ) {

19
includes/specialpage/QueryPage.php
View file

@ -447,18 +447,13 @@ abstract class QueryPage extends SpecialPage {
->where( [ 'qc_type' => $this->getName() ] )
->caller( $fname )->execute();
// Update the querycache_info record for the page
$dbw->upsert(
'querycache_info',
[
'qci_type' => $this->getName(),
'qci_timestamp' => $dbw->timestamp(),
],
'qci_type',
[
'qci_timestamp' => $dbw->timestamp(),
],
$fname
);
$dbw->newInsertQueryBuilder()
->insert( 'querycache_info' )
->row( [ 'qci_type' => $this->getName(), 'qci_timestamp' => $dbw->timestamp() ] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'qci_type' ] )
->set( [ 'qci_timestamp' => $dbw->timestamp() ] )
->caller( $fname )->execute();
}
);
// Save results into the querycache table on the primary DB

14
includes/user/ActorMigrationBase.php
View file

@ -380,13 +380,13 @@ class ActorMigrationBase {
}
$set[$to] = $extra[$from];
}
$dbw->upsert(
$tempTableInfo['table'],
[ $tempTableInfo['pk'] => $pk ] + $set,
[ [ $tempTableInfo['pk'] ] ],
$set,
$func
);
$dbw->newInsertQueryBuilder()
->insert( $tempTableInfo['table'] )
->row( [ $tempTableInfo['pk'] => $pk ] + $set )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ $tempTableInfo['pk'] ] )
->set( $set )
->caller( $func )->execute();
};
}
if ( $this->writeStage & SCHEMA_COMPAT_WRITE_NEW ) {

15
includes/user/ActorStore.php
View file

@ -518,14 +518,13 @@ class ActorStore implements UserIdentityLookup, ActorNormalization {
);
}
}
$dbw->upsert(
'actor',
[ 'actor_name' => $userName, 'actor_user' => $userId ],
[ [ 'actor_name' ] ],
[ 'actor_user' => $userId ],
__METHOD__
);
$dbw->newInsertQueryBuilder()
->insert( 'actor' )
->row( [ 'actor_name' => $userName, 'actor_user' => $userId ] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'actor_name' ] )
->set( [ 'actor_user' => $userId ] )
->caller( __METHOD__ )->execute();
if ( !$dbw->affectedRows() ) {
throw new CannotCreateActorException(
'Failed to replace user for actor: ' .

17
includes/user/TempUser/DBSerialProvider.php
View file

@ -33,16 +33,13 @@ abstract class DBSerialProvider implements SerialProvider {
$dbw = $this->getDB();
$table = $this->getTableName();
$dbw->startAtomic( __METHOD__ );
$dbw->upsert(
$table,
[
'uas_shard' => $shard,
'uas_value' => 1,
],
[ [ 'uas_shard' ] ],
[ 'uas_value=uas_value+1' ],
__METHOD__
);
$dbw->newInsertQueryBuilder()
->insert( $table )
->row( [ 'uas_shard' => $shard, 'uas_value' => 1 ] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'uas_shard' ] )
->set( [ 'uas_value=uas_value+1' ] )
->caller( __METHOD__ )->execute();
$value = $dbw->newSelectQueryBuilder()
->select( 'uas_value' )
->from( $table )

14
includes/watcheditem/WatchedItemStore.php
View file

@ -1135,13 +1135,13 @@ class WatchedItemStore implements WatchedItemStoreInterface, StatsdAwareInterfac
}, $wlIds );
// Insert into watchlist_expiry, updating the expiry for duplicate rows.
$dbw->upsert(
'watchlist_expiry',
$weRows,
'we_item',
[ 'we_expiry' => $expiry ],
__METHOD__
);
$dbw->newInsertQueryBuilder()
->insert( 'watchlist_expiry' )
->rows( $weRows )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'we_item' ] )
->set( [ 'we_expiry' => $expiry ] )
->caller( __METHOD__ )->execute();
return $dbw->affectedRows();
}

18
maintenance/initUserPreference.php
View file

@ -59,19 +59,17 @@ class InitUserPreference extends Maintenance {
$processed = 0;
foreach ( $iterator as $batch ) {
foreach ( $batch as $row ) {
$dbw->upsert(
'user_properties',
[
$dbw->newInsertQueryBuilder()
->insert( 'user_properties' )
->row( [
'up_user' => $row->up_user,
'up_property' => $target,
'up_value' => $row->up_value,
],
[ [ 'up_user', 'up_property' ] ],
[
'up_value' => $row->up_value,
],
__METHOD__
);
] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'up_user', 'up_property' ] )
->set( [ 'up_value' => $row->up_value ] )
->caller( __METHOD__ )->execute();
$processed += $dbw->affectedRows();
}

17
maintenance/populateChangeTagDef.php
View file

@ -161,18 +161,17 @@ class PopulateChangeTagDef extends LoggedUpdateMaintenance {
$this->output( 'This row will be updated: ' . $row->ct_tag . $row->hitcount . "\n" );
continue;
}
$dbw->upsert(
'change_tag_def',
[
$dbw->newInsertQueryBuilder()
->insert( 'change_tag_def' )
->row( [
'ctd_name' => $row->ct_tag,
'ctd_user_defined' => 0,
'ctd_count' => $row->hitcount
],
'ctd_name',
[ 'ctd_count' => $row->hitcount ],
__METHOD__
);
] )
->onDuplicateKeyUpdate()
->uniqueIndexFields( [ 'ctd_name' ] )
->set( [ 'ctd_count' => $row->hitcount ] )
->caller( __METHOD__ )->execute();
}
$this->lbFactory->waitForReplication();
}

7
tests/phpunit/unit/includes/PingbackTest.php
View file

@ -6,6 +6,7 @@ use MediaWiki\MainConfigNames;
use Psr\Log\NullLogger;
use Wikimedia\Rdbms\DBConnRef;
use Wikimedia\Rdbms\IConnectionProvider;
use Wikimedia\Rdbms\InsertQueryBuilder;
use Wikimedia\Rdbms\SelectQueryBuilder;
use Wikimedia\Timestamp\ConvertibleTimestamp;
@ -115,7 +116,10 @@ class PingbackTest extends MediaWikiUnitTestCase {
// - empty ($priorPing is false)
// - has a ping from over a month ago ($piorPing)
// - cache lock and db lock are available
$database = $this->createNoOpMock( DBConnRef::class, [ 'selectField', 'lock', 'upsert', 'newSelectQueryBuilder' ] );
$database = $this->createNoOpMock(
DBConnRef::class,
[ 'selectField', 'lock', 'upsert', 'newSelectQueryBuilder', 'newInsertQueryBuilder' ]
);
$httpRequestFactory = $this->createNoOpMock( HttpRequestFactory::class, [ 'post' ] );
$database->expects( $this->once() )->method( 'selectField' )->willReturn( $priorPing );
@ -126,6 +130,7 @@ class PingbackTest extends MediaWikiUnitTestCase {
->willReturn( true );
$database->expects( $this->once() )->method( 'upsert' );
$database->method( 'newSelectQueryBuilder' )->willReturnCallback( static fn() => new SelectQueryBuilder( $database ) );
$database->method( 'newInsertQueryBuilder' )->willReturnCallback( static fn() => new InsertQueryBuilder( $database ) );
$pingback = $this->makePingback(
$database,