Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[JobQueue] Throttle refreshLinks2 jobs based on finishing the refreshLinks jobs.

Browse source 
* This should lower the rate of queue activity when major templates change.
* Also fixed a broken array_diff() call in nextJobDB.php.
* Additionally removed checkJob from nextJobDB.php. This is redundant to
  runJobs.php de-listing queues via JobQueueGroup::pop() when the queue
  is empty.

Change-Id: I1518a0de9e7ada22350d9993dd7ffe5f2ce23745
This commit is contained in:
Aaron Schulz 2013-03-01 14:37:51 -08:00 committed by Gerrit Code Review
parent 9ace77f03c
commit 04c610200f
3 changed files with 22 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
includes/DefaultSettings.php
View file

@ -5546,6 +5546,8 @@ $wgJobTypesExcludedFromDefaultQueue = array( 'AssembleUploadChunks', 'PublishSta
/** /**
* Map of job types to configuration arrays. * Map of job types to configuration arrays.
* This determines which queue class and storage system is used for each job type.
* Job types that do not have explicit configuration will use the 'default' config.
* These settings should be global to all wikis. * These settings should be global to all wikis.
*/ */
$wgJobTypeConf = array( $wgJobTypeConf = array(

17
includes/job/JobQueueGroup.php
View file

@ -236,6 +236,23 @@ class JobQueueGroup {
return $types; return $types;
} }
/**
* Check if jobs should not be popped of a queue right now.
* This is only used for performance, such as to avoid spamming
* the queue with many sub-jobs before they actually get run.
*
* @param $type string
* @return bool
*/
public function isQueueDeprioritized( $type ) {
if ( $type === 'refreshLinks2' ) {
// Don't keep converting refreshLinks2 => refreshLinks jobs if the
// later jobs have not been done yet. This helps throttle queue spam.
return !$this->get( 'refreshLinks' )->isEmpty();
}
return false;
}
/** /**
* Execute any due periodic queue maintenance tasks for all queues. * Execute any due periodic queue maintenance tasks for all queues.
* *

18
maintenance/nextJobDB.php
View file

@ -77,10 +77,9 @@ class nextJobDB extends Maintenance {
return; // no jobs for this type return; // no jobs for this type
} }
list( $type, $db ) = $candidates[ mt_rand( 0, count( $candidates ) - 1 ) ]; list( $type, $db ) = $candidates[mt_rand( 0, count( $candidates ) - 1 )];
if ( !$this->checkJob( $type, $db ) ) { // queue is actually empty? if ( JobQueueGroup::singleton( $db )->isQueueDeprioritized( $type ) ) {
$pendingDBs[$type] = array_diff( $pendingDBs[$type], $db ); $pendingDBs[$type] = array_diff( $pendingDBs[$type], array( $db ) );
JobQueueAggregator::singleton()->notifyQueueEmpty( $db, $type );
$again = true; $again = true;
} }
} while ( $again ); } while ( $again );
@ -92,17 +91,6 @@ class nextJobDB extends Maintenance {
} }
} }
/**
* Check if the specified database has a job of the specified type in it.
* The type may be false to indicate "all".
* @param $type string
* @param $dbName string
* @return bool
*/
private function checkJob( $type, $dbName ) {
return !JobQueueGroup::singleton( $dbName )->get( $type )->isEmpty();
}
/** /**
* Do all ready periodic jobs for all databases every 5 minutes (and .1% of the time) * Do all ready periodic jobs for all databases every 5 minutes (and .1% of the time)
* @return integer * @return integer