2021-03-17 12:37:06 +00:00
|
|
|
<?php
|
|
|
|
|
/**
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
|
|
|
* (at your option) any later version.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License along
|
|
|
|
|
* with this program; if not, write to the Free Software Foundation, Inc.,
|
|
|
|
|
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
|
* http://www.gnu.org/copyleft/gpl.html
|
|
|
|
|
*
|
|
|
|
|
* @file
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
namespace MediaWiki\JobQueue;
|
|
|
|
|
|
|
|
|
|
use IBufferingStatsdDataFactory;
|
|
|
|
|
use JobQueueGroup;
|
2023-07-11 09:02:03 +00:00
|
|
|
use LogicException;
|
2021-03-17 12:37:06 +00:00
|
|
|
use MediaWiki\Config\ServiceOptions;
|
2022-04-25 15:19:41 +00:00
|
|
|
use MediaWiki\MainConfigNames;
|
2023-02-23 20:44:38 +00:00
|
|
|
use MediaWiki\WikiMap\WikiMap;
|
2024-09-27 18:13:02 +00:00
|
|
|
use Wikimedia\ObjectCache\WANObjectCache;
|
2023-09-11 18:39:11 +00:00
|
|
|
use Wikimedia\Rdbms\ReadOnlyMode;
|
2021-03-17 12:37:06 +00:00
|
|
|
use Wikimedia\UUID\GlobalIdGenerator;
|
|
|
|
|
|
|
|
|
|
/**
|
2024-03-25 22:00:38 +00:00
|
|
|
* Factory for JobQueueGroup objects.
|
2021-03-17 12:37:06 +00:00
|
|
|
*
|
|
|
|
|
* @since 1.37
|
2024-03-25 22:00:38 +00:00
|
|
|
* @ingroup JobQueue
|
2021-03-17 12:37:06 +00:00
|
|
|
*/
|
|
|
|
|
class JobQueueGroupFactory {
|
|
|
|
|
/**
|
|
|
|
|
* @internal For use by ServiceWiring
|
|
|
|
|
*/
|
|
|
|
|
public const CONSTRUCTOR_OPTIONS = [
|
2022-04-25 15:19:41 +00:00
|
|
|
MainConfigNames::JobClasses,
|
|
|
|
|
MainConfigNames::JobTypeConf,
|
|
|
|
|
MainConfigNames::JobTypesExcludedFromDefaultQueue,
|
|
|
|
|
MainConfigNames::LocalDatabases,
|
2021-03-17 12:37:06 +00:00
|
|
|
];
|
|
|
|
|
|
|
|
|
|
/** @var JobQueueGroup[] */
|
|
|
|
|
private $instances;
|
|
|
|
|
|
|
|
|
|
/** @var ServiceOptions */
|
|
|
|
|
private $options;
|
|
|
|
|
|
2023-09-11 18:39:11 +00:00
|
|
|
/** @var ReadOnlyMode */
|
2021-03-17 12:37:06 +00:00
|
|
|
private $readOnlyMode;
|
|
|
|
|
|
|
|
|
|
/** @var IBufferingStatsdDataFactory */
|
|
|
|
|
private $statsdDataFactory;
|
|
|
|
|
|
|
|
|
|
/** @var WANObjectCache */
|
|
|
|
|
private $wanCache;
|
|
|
|
|
|
|
|
|
|
/** @var GlobalIdGenerator */
|
|
|
|
|
private $globalIdGenerator;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @param ServiceOptions $options
|
2023-09-11 18:39:11 +00:00
|
|
|
* @param ReadOnlyMode $readOnlyMode
|
2021-03-17 12:37:06 +00:00
|
|
|
* @param IBufferingStatsdDataFactory $statsdDataFactory
|
|
|
|
|
* @param WANObjectCache $wanCache
|
|
|
|
|
* @param GlobalIdGenerator $globalIdGenerator
|
|
|
|
|
*/
|
|
|
|
|
public function __construct(
|
|
|
|
|
ServiceOptions $options,
|
2023-09-11 18:39:11 +00:00
|
|
|
ReadOnlyMode $readOnlyMode,
|
2021-03-17 12:37:06 +00:00
|
|
|
IBufferingStatsdDataFactory $statsdDataFactory,
|
|
|
|
|
WANObjectCache $wanCache,
|
|
|
|
|
GlobalIdGenerator $globalIdGenerator
|
|
|
|
|
) {
|
|
|
|
|
$options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );
|
|
|
|
|
$this->instances = [];
|
|
|
|
|
$this->options = $options;
|
|
|
|
|
$this->readOnlyMode = $readOnlyMode;
|
|
|
|
|
$this->statsdDataFactory = $statsdDataFactory;
|
|
|
|
|
$this->wanCache = $wanCache;
|
|
|
|
|
$this->globalIdGenerator = $globalIdGenerator;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @since 1.37
|
|
|
|
|
*
|
|
|
|
|
* @param false|string $domain Wiki domain ID. False uses the current wiki domain ID
|
|
|
|
|
* @return JobQueueGroup
|
|
|
|
|
*/
|
2021-07-22 03:11:47 +00:00
|
|
|
public function makeJobQueueGroup( $domain = false ): JobQueueGroup {
|
2021-03-17 12:37:06 +00:00
|
|
|
if ( $domain === false ) {
|
|
|
|
|
$domain = WikiMap::getCurrentWikiDbDomain()->getId();
|
|
|
|
|
}
|
|
|
|
|
|
jobqueue: Disallow cross-wiki JobQueueGroup calls that require JobClasses
=== Change ===
Follows-up Ia61e00d6dc98c20650 which moved injection of $wgJobClasses
from happening lazily in methods that are never called in any
production code, to happen unconditionally when any JobQueueGroup
is constructed and injected, which happens a lot nowadays given that
most code is now dependency-injected.
For example, when logging in, CentralAuth can call into services
for getUserGroupManager( $wikiID )->getUserGroupMemberships() which
merely performs a simple database select query. But, because this
service class does contain other methods that can queue a job, this
config now needs to be computed across wikis via a potentially slow
shell execution even in simple GET requests.
One way to solve this, which brings back the previous state, is to
keep this logically dependency-injected but deferred by wrapping it
in a closure. We could inject `callable $getJobClassesFn` as param
from JobQueueGroupFactory to JobQueueGroup.
However, given that $wgConf->getConfig() was broken in production
for two weeks and there is not 1 log entry in Logstash during this
time, I think that means these methods are actually never called.
Hence, I'm instead going in the opposite direction of extending the
restriction of pop() to these other methods as well.
Thus, we reduce JobQueueGroup support for cross-wiki method calls
to get(), push() and lazyPush().
=== History ===
The natural question is, why was this added, and do we know that
this reason no longer applies? The getConfig() call was introduced
in 2013 with commit 04e0d75f86b (I7e6904ead1). This mentions as
reasoning to support maintenance/nextJobDB.php, which was was removed
in commit ce2ae144e62 (Ia74386c650) a year later.
It looks like this script supported only JobQueueDB, as used by
a WMF-specific jobrunner known as "jobs-loop.sh", which predates
WMF's JobQueue migration from MySQL to Redis, and later to Kafka,
as per
<https://wikitech.wikimedia.org/wiki/History_of_job_queue_runners_at_WMF>
Bug: T344223
Bug: T343291
Change-Id: Ic2293c4b4be10a698a2f891eaa63c1de7383f982
2023-08-15 17:50:49 +00:00
|
|
|
// Make sure jobs are not getting pushed to bogus wikis. This can confuse
|
|
|
|
|
// the job runner system into spawning endless RPC requests that fail (T171371).
|
|
|
|
|
$wikiId = WikiMap::getWikiIdFromDbDomain( $domain );
|
|
|
|
|
if (
|
|
|
|
|
!WikiMap::isCurrentWikiDbDomain( $domain ) &&
|
|
|
|
|
!in_array( $wikiId, $this->options->get( MainConfigNames::LocalDatabases ) )
|
|
|
|
|
) {
|
|
|
|
|
// Do not enqueue job that cannot be run (T171371)
|
|
|
|
|
throw new LogicException( "Domain '{$domain}' is not recognized." );
|
|
|
|
|
}
|
2021-03-17 12:37:06 +00:00
|
|
|
|
jobqueue: Disallow cross-wiki JobQueueGroup calls that require JobClasses
=== Change ===
Follows-up Ia61e00d6dc98c20650 which moved injection of $wgJobClasses
from happening lazily in methods that are never called in any
production code, to happen unconditionally when any JobQueueGroup
is constructed and injected, which happens a lot nowadays given that
most code is now dependency-injected.
For example, when logging in, CentralAuth can call into services
for getUserGroupManager( $wikiID )->getUserGroupMemberships() which
merely performs a simple database select query. But, because this
service class does contain other methods that can queue a job, this
config now needs to be computed across wikis via a potentially slow
shell execution even in simple GET requests.
One way to solve this, which brings back the previous state, is to
keep this logically dependency-injected but deferred by wrapping it
in a closure. We could inject `callable $getJobClassesFn` as param
from JobQueueGroupFactory to JobQueueGroup.
However, given that $wgConf->getConfig() was broken in production
for two weeks and there is not 1 log entry in Logstash during this
time, I think that means these methods are actually never called.
Hence, I'm instead going in the opposite direction of extending the
restriction of pop() to these other methods as well.
Thus, we reduce JobQueueGroup support for cross-wiki method calls
to get(), push() and lazyPush().
=== History ===
The natural question is, why was this added, and do we know that
this reason no longer applies? The getConfig() call was introduced
in 2013 with commit 04e0d75f86b (I7e6904ead1). This mentions as
reasoning to support maintenance/nextJobDB.php, which was was removed
in commit ce2ae144e62 (Ia74386c650) a year later.
It looks like this script supported only JobQueueDB, as used by
a WMF-specific jobrunner known as "jobs-loop.sh", which predates
WMF's JobQueue migration from MySQL to Redis, and later to Kafka,
as per
<https://wikitech.wikimedia.org/wiki/History_of_job_queue_runners_at_WMF>
Bug: T344223
Bug: T343291
Change-Id: Ic2293c4b4be10a698a2f891eaa63c1de7383f982
2023-08-15 17:50:49 +00:00
|
|
|
$localJobClasses = WikiMap::isCurrentWikiDbDomain( $domain )
|
|
|
|
|
? $this->options->get( MainConfigNames::JobClasses )
|
|
|
|
|
: null;
|
|
|
|
|
|
|
|
|
|
if ( !isset( $this->instances[$domain] ) ) {
|
2021-03-17 12:37:06 +00:00
|
|
|
$this->instances[$domain] = new JobQueueGroup(
|
|
|
|
|
$domain,
|
|
|
|
|
$this->readOnlyMode,
|
jobqueue: Disallow cross-wiki JobQueueGroup calls that require JobClasses
=== Change ===
Follows-up Ia61e00d6dc98c20650 which moved injection of $wgJobClasses
from happening lazily in methods that are never called in any
production code, to happen unconditionally when any JobQueueGroup
is constructed and injected, which happens a lot nowadays given that
most code is now dependency-injected.
For example, when logging in, CentralAuth can call into services
for getUserGroupManager( $wikiID )->getUserGroupMemberships() which
merely performs a simple database select query. But, because this
service class does contain other methods that can queue a job, this
config now needs to be computed across wikis via a potentially slow
shell execution even in simple GET requests.
One way to solve this, which brings back the previous state, is to
keep this logically dependency-injected but deferred by wrapping it
in a closure. We could inject `callable $getJobClassesFn` as param
from JobQueueGroupFactory to JobQueueGroup.
However, given that $wgConf->getConfig() was broken in production
for two weeks and there is not 1 log entry in Logstash during this
time, I think that means these methods are actually never called.
Hence, I'm instead going in the opposite direction of extending the
restriction of pop() to these other methods as well.
Thus, we reduce JobQueueGroup support for cross-wiki method calls
to get(), push() and lazyPush().
=== History ===
The natural question is, why was this added, and do we know that
this reason no longer applies? The getConfig() call was introduced
in 2013 with commit 04e0d75f86b (I7e6904ead1). This mentions as
reasoning to support maintenance/nextJobDB.php, which was was removed
in commit ce2ae144e62 (Ia74386c650) a year later.
It looks like this script supported only JobQueueDB, as used by
a WMF-specific jobrunner known as "jobs-loop.sh", which predates
WMF's JobQueue migration from MySQL to Redis, and later to Kafka,
as per
<https://wikitech.wikimedia.org/wiki/History_of_job_queue_runners_at_WMF>
Bug: T344223
Bug: T343291
Change-Id: Ic2293c4b4be10a698a2f891eaa63c1de7383f982
2023-08-15 17:50:49 +00:00
|
|
|
$localJobClasses,
|
2022-04-25 15:19:41 +00:00
|
|
|
$this->options->get( MainConfigNames::JobTypeConf ),
|
|
|
|
|
$this->options->get( MainConfigNames::JobTypesExcludedFromDefaultQueue ),
|
2021-03-17 12:37:06 +00:00
|
|
|
$this->statsdDataFactory,
|
|
|
|
|
$this->wanCache,
|
|
|
|
|
$this->globalIdGenerator
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $this->instances[$domain];
|
|
|
|
|
}
|
|
|
|
|
}
|