Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

poolcounter: Merge Client and ConnectionManager from extension repo

Browse source 
Code moved as-is from the extension repo with minor changes:

* Adopt PSR-4 namespace.
* Keep backward-compatibility with "PoolCounter_Client"
  in LocalSettings, from before the extension was namespaced recently.
* Document how `connect_timeout` actually works, and that it
  was introduced in MW 1.28 (via extension).
* Add stable interface annotations.

Bug: T201223
Change-Id: Iadec5b4b5d2fc7e76509c9be0a8fa605d95c64a7
This commit is contained in:
Timo Tijhof 2022-09-23 00:47:42 +01:00
parent baed2a618c
commit adb9c0cc1b
12 changed files with 407 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
autoload.php
View file

@ -1601,6 +1601,8 @@ $wgAutoloadLocalClasses = [
'MediaWiki\\Permissions\\SimpleAuthority' => __DIR__ . '/includes/Permissions/SimpleAuthority.php',
'MediaWiki\\Permissions\\UltimateAuthority' => __DIR__ . '/includes/Permissions/UltimateAuthority.php',
'MediaWiki\\Permissions\\UserAuthority' => __DIR__ . '/includes/Permissions/UserAuthority.php',
'MediaWiki\\PoolCounter\\PoolCounterClient' => __DIR__ . '/includes/poolcounter/PoolCounterClient.php',
'MediaWiki\\PoolCounter\\PoolCounterConnectionManager' => __DIR__ . '/includes/poolcounter/PoolCounterConnectionManager.php',
'MediaWiki\\Preferences\\DefaultPreferencesFactory' => __DIR__ . '/includes/preferences/DefaultPreferencesFactory.php',
'MediaWiki\\Preferences\\Filter' => __DIR__ . '/includes/preferences/Filter.php',
'MediaWiki\\Preferences\\Hook\\GetPreferencesHook' => __DIR__ . '/includes/preferences/Hook/GetPreferencesHook.php',

24
docs/config-schema.yaml
View file

@ -2206,16 +2206,36 @@ config-schema:
'redisConfig' => []
] ];
```
**Example using C daemon from https://www.mediawiki.org/wiki/Extension:PoolCounter:**
**Example using C daemon from <https://gerrit.wikimedia.org/g/mediawiki/services/poolcounter>**
```
$wgPoolCountClientConf = [
'servers' => [ '127.0.0.1' ],
'timeout' => 0.5,
'connect_timeout' => 0.01,
];
$wgPoolCounterConf = [ 'ArticleView' => [
'class' => MediaWiki\Extension\PoolCounter\Client::class,
'class' => MediaWiki\PoolCounter\PoolCounterClient::class,
'timeout' => 15, // wait timeout in seconds
'workers' => 5, // maximum number of active threads in each pool
'maxqueue' => 50, // maximum number of total threads in each pool
... any extension-specific options...
] ];
```
@since 1.16
PoolCountClientConf:
default:
servers: [127.0.0.1]
timeout: 0.1
type: object
description: |-
Configuration array for the PoolCounter client.
- servers: Array of hostnames, or hostname:port. The default port is 7531.
- timeout: Connection timeout.
- connect_timeout: [Since 1.28] Alternative connection timeout. If set, it is used
instead of `timeout` and will be retried once if a connection fails
to be established. Background: https://phabricator.wikimedia.org/T105378.
@see MediaWiki\PoolCounter\PoolCounterClient
@since 1.16
MaxUserDBWriteDuration:
default: false
type:

1
includes/AutoLoader.php
View file

@ -66,6 +66,7 @@ class AutoLoader {
'MediaWiki\\Mail\\' => __DIR__ . '/mail/',
'MediaWiki\\Page\\' => __DIR__ . '/page/',
'MediaWiki\\Parser\\' => __DIR__ . '/parser/',
'MediaWiki\\PoolCounter\\' => __DIR__ . '/poolcounter/',
'MediaWiki\\Preferences\\' => __DIR__ . '/preferences/',
'MediaWiki\\Search\\' => __DIR__ . '/search/',
'MediaWiki\\Search\\SearchWidgets\\' => __DIR__ . '/search/searchwidgets/',

6
includes/MainConfigNames.php
View file

@ -1393,6 +1393,12 @@ class MainConfigNames {
*/
public const PoolCounterConf = 'PoolCounterConf';
/**
* Name constant for the PoolCountClientConf setting, for use with Config::get()
* @see MainConfigSchema::PoolCountClientConf
*/
public const PoolCountClientConf = 'PoolCountClientConf';
/**
* Name constant for the MaxUserDBWriteDuration setting, for use with Config::get()
* @see MainConfigSchema::MaxUserDBWriteDuration

34
includes/MainConfigSchema.php
View file

@ -3584,23 +3584,53 @@ class MainConfigSchema {
* ] ];
* ```
*
* **Example using C daemon from https://www.mediawiki.org/wiki/Extension:PoolCounter:**
* **Example using C daemon from <https://gerrit.wikimedia.org/g/mediawiki/services/poolcounter>**
*
* ```
* $wgPoolCountClientConf = [
* 'servers' => [ '127.0.0.1' ],
* 'timeout' => 0.5,
* 'connect_timeout' => 0.01,
* ];
*
* $wgPoolCounterConf = [ 'ArticleView' => [
* 'class' => MediaWiki\Extension\PoolCounter\Client::class,
* 'class' => MediaWiki\PoolCounter\PoolCounterClient::class,
* 'timeout' => 15, // wait timeout in seconds
* 'workers' => 5, // maximum number of active threads in each pool
* 'maxqueue' => 50, // maximum number of total threads in each pool
* ... any extension-specific options...
* ] ];
* ```
*
* @since 1.16
*/
public const PoolCounterConf = [
'default' => null,
'type' => '?map',
];
/**
* Configuration array for the PoolCounter client.
*
* - servers: Array of hostnames, or hostname:port. The default port is 7531.
* - timeout: Connection timeout.
* - connect_timeout: [Since 1.28] Alternative connection timeout. If set, it is used
* instead of `timeout` and will be retried once if a connection fails
* to be established. Background: https://phabricator.wikimedia.org/T105378.
*
* @see MediaWiki\PoolCounter\PoolCounterClient
* @since 1.16
*/
public const PoolCountClientConf = [
'default' => [
'servers' => [
'127.0.0.1'
],
'timeout' => 0.1,
],
'type' => 'map',
];
/**
* Max time (in seconds) a user-generated transaction can spend in writes.
*

7
includes/config-schema.php
View file

@ -423,6 +423,12 @@ return [
'MaxArticleSize' => 2048,
'MemoryLimit' => '50M',
'PoolCounterConf' => null,
'PoolCountClientConf' => [
'servers' => [
0 => '127.0.0.1',
],
'timeout' => 0.1,
],
'MaxUserDBWriteDuration' => false,
'MaxJobDBWriteDuration' => false,
'LinkHolderBatchSize' => 1000,
@ -2575,6 +2581,7 @@ return [
0 => 'object',
1 => 'null',
],
'PoolCountClientConf' => 'object',
'MaxUserDBWriteDuration' => [
0 => 'integer',
1 => 'boolean',

6
includes/config-vars.php
View file

@ -1377,6 +1377,12 @@ $wgMemoryLimit = null;
*/
$wgPoolCounterConf = null;
/**
* Config variable stub for the PoolCountClientConf setting, for use by phpdoc and IDEs.
* @see MediaWiki\MainConfigSchema::PoolCountClientConf
*/
$wgPoolCountClientConf = null;
/**
* Config variable stub for the MaxUserDBWriteDuration setting, for use by phpdoc and IDEs.
* @see MediaWiki\MainConfigSchema::MaxUserDBWriteDuration

11
includes/poolcounter/PoolCounter.php
View file

@ -46,6 +46,9 @@ use Wikimedia\ObjectFactory\ObjectFactory;
* Install the poolcounterd service from
* <https://gerrit.wikimedia.org/g/mediawiki/services/poolcounter> to
* enable this feature.
*
* @since 1.16
* @stable to extend
*/
abstract class PoolCounter {
/* Return codes */
@ -127,7 +130,15 @@ abstract class PoolCounter {
}
$conf = $poolCounterConf[$type];
if ( ( $conf['class'] ?? null ) === 'PoolCounter_Client' ) {
// Since 1.16: Introduced, as an extension.
// Since 1.36: Namespaced extension, with alias.
// Since 1.40: Moved to core.
$conf['class'] = MediaWiki\PoolCounter\PoolCounterClient::class;
}
/** @var PoolCounter $poolCounter */
// @phan-suppress-next-line PhanTypeInvalidCallableArraySize https://github.com/phan/phan/issues/1648
$poolCounter = ObjectFactory::getObjectFromSpec(
$conf,
[

170
includes/poolcounter/PoolCounterClient.php Normal file
View file

@ -0,0 +1,170 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
namespace MediaWiki\PoolCounter;
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use PoolCounter;
use Status;
/**
* @since 1.16
*/
class PoolCounterClient extends PoolCounter {
/**
* @var ?resource the socket connection to the poolcounter. Closing this
* releases all locks acquired.
*/
private $conn;
/**
* @var string The server host name
*/
private $hostName;
/**
* @var PoolCounterConnectionManager
*/
private static $manager;
/**
* @param array $conf
* @param string $type
* @param string $key
*/
public function __construct( $conf, $type, $key ) {
parent::__construct( $conf, $type, $key );
if ( !self::$manager ) {
// TODO: Inject from PoolCounter::factory
$config = MediaWikiServices::getInstance()->getMainConfig();
self::$manager = new PoolCounterConnectionManager(
$config->get( MainConfigNames::PoolCountClientConf )
);
}
}
/**
* @return Status
*/
public function getConn() {
if ( !isset( $this->conn ) ) {
$status = self::$manager->get( $this->key );
if ( !$status->isOK() ) {
return $status;
}
// @phan-suppress-next-line PhanTypeArraySuspiciousNullable
$this->conn = $status->value['conn'];
// @phan-suppress-next-line PhanTypeArraySuspiciousNullable
$this->hostName = $status->value['hostName'];
// Set the read timeout to be 1.5 times the pool timeout.
// This allows the server to time out gracefully before we give up on it.
stream_set_timeout( $this->conn, 0, (int)( $this->timeout * 1e6 * 1.5 ) );
}
return Status::newGood( $this->conn );
}
/**
* @param string|int|float ...$args
* @return Status
*/
public function sendCommand( ...$args ) {
$args = str_replace( ' ', '%20', $args );
$cmd = implode( ' ', $args );
$status = $this->getConn();
if ( !$status->isOK() ) {
return $status;
}
$conn = $status->value;
wfDebug( "Sending pool counter command: $cmd\n" );
if ( fwrite( $conn, "$cmd\n" ) === false ) {
return Status::newFatal( 'poolcounter-write-error', $this->hostName );
}
$response = fgets( $conn );
if ( $response === false ) {
return Status::newFatal( 'poolcounter-read-error', $this->hostName );
}
$response = rtrim( $response, "\r\n" );
wfDebug( "Got pool counter response: $response\n" );
$parts = explode( ' ', $response, 2 );
$responseType = $parts[0];
switch ( $responseType ) {
case 'LOCKED':
$this->onAcquire();
break;
case 'RELEASED':
$this->onRelease();
break;
case 'DONE':
case 'NOT_LOCKED':
case 'QUEUE_FULL':
case 'TIMEOUT':
case 'LOCK_HELD':
break;
case 'ERROR':
default:
$parts = explode( ' ', $parts[1], 2 );
$errorMsg = $parts[1] ?? '(no message given)';
return Status::newFatal( 'poolcounter-remote-error', $errorMsg, $this->hostName );
}
return Status::newGood( constant( "PoolCounter::$responseType" ) );
}
/**
* @param int|null $timeout
* @return Status
*/
public function acquireForMe( $timeout = null ) {
$status = $this->precheckAcquire();
if ( !$status->isGood() ) {
return $status;
}
return $this->sendCommand( 'ACQ4ME', $this->key, $this->workers, $this->maxqueue,
$timeout ?? $this->timeout );
}
/**
* @param int|null $timeout
* @return Status
*/
public function acquireForAnyone( $timeout = null ) {
$status = $this->precheckAcquire();
if ( !$status->isGood() ) {
return $status;
}
return $this->sendCommand( 'ACQ4ANY', $this->key, $this->workers, $this->maxqueue,
$timeout ?? $this->timeout );
}
/**
* @return Status
*/
public function release() {
$status = $this->sendCommand( 'RELEASE' );
if ( $this->conn ) {
self::$manager->close( $this->conn );
$this->conn = null;
}
return $status;
}
}

147
includes/poolcounter/PoolCounterConnectionManager.php Normal file
View file

@ -0,0 +1,147 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
namespace MediaWiki\PoolCounter;
use MWException;
use Status;
use Wikimedia\AtEase\AtEase;
/**
* Helper for \MediaWiki\PoolCounter\PoolCounterClient.
*
* @internal
* @since 1.16
*/
class PoolCounterConnectionManager {
/** @var string[] */
public $hostNames;
/** @var array */
public $conns = [];
/** @var array */
public $refCounts = [];
/** @var float */
public $timeout;
/** @var int */
public $connect_timeout;
/**
* @param array $conf
* @throws MWException
*/
public function __construct( $conf ) {
$this->hostNames = $conf['servers'];
$this->timeout = $conf['timeout'] ?? 0.1;
$this->connect_timeout = $conf['connect_timeout'] ?? 0;
if ( !count( $this->hostNames ) ) {
throw new MWException( __METHOD__ . ': no servers configured' );
}
}
/**
* @param string $key
* @return Status
*/
public function get( $key ) {
$hashes = [];
foreach ( $this->hostNames as $hostName ) {
$hashes[$hostName] = md5( $hostName . $key );
}
asort( $hashes );
$errno = 0;
$errstr = '';
$hostName = '';
$conn = null;
foreach ( $hashes as $hostName => $hash ) {
if ( isset( $this->conns[$hostName] ) ) {
$this->refCounts[$hostName]++;
return Status::newGood(
[ 'conn' => $this->conns[$hostName], 'hostName' => $hostName ] );
}
$parts = explode( ':', $hostName, 2 );
if ( count( $parts ) < 2 ) {
$parts[] = 7531;
}
AtEase::suppressWarnings();
$conn = $this->open( $parts[0], $parts[1], $errno, $errstr );
AtEase::restoreWarnings();
if ( $conn ) {
break;
}
}
if ( !$conn ) {
return Status::newFatal( 'poolcounter-connection-error', $errstr, $hostName );
}
wfDebug( "Connected to pool counter server: $hostName\n" );
$this->conns[$hostName] = $conn;
$this->refCounts[$hostName] = 1;
return Status::newGood( [ 'conn' => $conn, 'hostName' => $hostName ] );
}
/**
* Open a socket. Just a wrapper for fsockopen()
* @param string $host
* @param int $port
* @param int &$errno
* @param string &$errstr
* @return null|resource
*/
private function open( $host, $port, &$errno, &$errstr ) {
// If connect_timeout is set, we try to open the socket twice.
// You usually want to set the connection timeout to a very
// small value so that in case of failure of a server the
// connection to poolcounter is not a SPOF.
if ( $this->connect_timeout > 0 ) {
$tries = 2;
$timeout = $this->connect_timeout;
} else {
$tries = 1;
$timeout = $this->timeout;
}
$fp = null;
while ( true ) {
$fp = fsockopen( $host, $port, $errno, $errstr, $timeout );
if ( $fp !== false || --$tries < 1 ) {
break;
}
usleep( 1000 );
}
return $fp;
}
/**
* @param resource $conn
*/
public function close( $conn ) {
foreach ( $this->conns as $hostName => $otherConn ) {
if ( $conn === $otherConn ) {
if ( $this->refCounts[$hostName] ) {
$this->refCounts[$hostName]--;
}
if ( !$this->refCounts[$hostName] ) {
fclose( $conn );
unset( $this->conns[$hostName] );
}
}
}
}
}

3
includes/poolcounter/PoolCounterNull.php
View file

@ -20,6 +20,9 @@
/**
* A default PoolCounter, which provides no locking.
*
* @internal
* @since 1.33
*/
class PoolCounterNull extends PoolCounter {

1
includes/poolcounter/PoolCounterRedis.php
View file

@ -48,7 +48,6 @@ use Psr\Log\LoggerInterface;
* pools to appear as full when they are not. Using volatile-ttl and bumping memory-samples
* in redis.conf can be helpful otherwise.
*
* @ingroup Redis
* @since 1.23
*/
class PoolCounterRedis extends PoolCounter {