Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add StatsD metric logging

Browse source 
This patch adds a metric data service object to the IContextSource interface,
with full support for StatsD meters, gauges, counters and timing metrics, via
the liuggio/statsd-php-client, which this patch also introduces.

Usage example:

    $stats = $context->getStats();
    $stats->increment( 'resourceloader.cache.hits' );
    $stats->timing( 'resourceloader.cache.rtt', $rtt );

The metrics are flushed to a StatsD server, which may be specified via the
'StatsdServer' configuration key. If no such configuration key exists, the
metrics are discarded.

The StatsD client supplants MediaWiki's StatCounter class. wfIncrStats()
will continue to work, but it will delegate to the StatsD data object.

Change-Id: Ie10db1c154d225971398e189737de7c560bf0f90
This commit is contained in:
Ori Livneh 2015-02-20 00:23:18 -08:00
parent e83bb46bf4
commit 87dfc20b1e
11 changed files with 134 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
RELEASE-NOTES-1.25
View file

@ -111,6 +111,8 @@ production.
used for conditional registration of API modules.
* New hook 'EnhancedChangesList::getLogText' to alter, remove or add to the
links of a group of changes in EnhancedChangesList.
* A full interface for StatsD metric reporting has been added to the context
interface, reachable via IContextSource::getStats().
==== External libraries ====
* MediaWiki now requires certain external libraries to be installed. In the past
@ -137,6 +139,8 @@ production.
This library was formerly a part of MediaWiki core, and has been moved into a separate library.
It provides CDB functions which are used in the Interwiki and Localization caches.
More information about the library can be found at https://www.mediawiki.org/wiki/CDB.
** liuggio/statsd-php-client
This library provides a StatsD client API for logging application metrics to a remote server.
=== Bug fixes in 1.25 ===
* (T73003) No additional code will be generated to try to load CSS-embedded

2
autoload.php
View file

@ -174,6 +174,7 @@ $wgAutoloadLocalClasses = array(
'BloomCacheRedis' => __DIR__ . '/includes/cache/bloom/BloomCacheRedis.php',
'BloomFilterTitleHasLogs' => __DIR__ . '/includes/cache/bloom/BloomFilters.php',
'BmpHandler' => __DIR__ . '/includes/media/BMP.php',
'BufferingStatsdDataFactory' => __DIR__ . '/includes/libs/BufferingStatsdDataFactory.php',
'BrokenRedirectsPage' => __DIR__ . '/includes/specials/SpecialBrokenRedirects.php',
'CLDRPluralRuleConverter' => __DIR__ . '/languages/utils/CLDRPluralRuleConverter.php',
'CLDRPluralRuleConverterExpression' => __DIR__ . '/languages/utils/CLDRPluralRuleConverterExpression.php',
@ -1160,7 +1161,6 @@ $wgAutoloadLocalClasses = array(
'SquidPurgeClientPool' => __DIR__ . '/includes/SquidPurgeClient.php',
'SquidUpdate' => __DIR__ . '/includes/deferred/SquidUpdate.php',
'SrConverter' => __DIR__ . '/languages/classes/LanguageSr.php',
'StatCounter' => __DIR__ . '/includes/StatCounter.php',
'StatsOutput' => __DIR__ . '/maintenance/language/StatOutputs.php',
'Status' => __DIR__ . '/includes/Status.php',
'StatusValue' => __DIR__ . '/includes/libs/StatusValue.php',

1
composer.json
View file

@ -19,6 +19,7 @@
"cssjanus/cssjanus": "1.1.1",
"ext-iconv": "*",
"leafo/lessphp": "0.5.0",
"liuggio/statsd-php-client": "1.0.12",
"oojs/oojs-ui": "0.9.0",
"php": ">=5.3.3",
"psr/log": "1.0.0",

17
includes/GlobalFunctions.php
View file

@ -24,6 +24,9 @@ if ( !defined( 'MEDIAWIKI' ) ) {
die( "This file is part of MediaWiki, it is not a valid entry point" );
}
use Liuggio\StatsdClient\StatsdClient;
use Liuggio\StatsdClient\Sender\SocketSender;
// Hide compatibility functions from Doxygen
/// @cond
@ -1271,7 +1274,16 @@ function wfLogProfilingData() {
global $wgRequestTime, $wgDebugLogGroups, $wgDebugRawPage;
global $wgProfileLimit, $wgUser, $wgRequest;
StatCounter::singleton()->flush();
$context = RequestContext::getMain();
$config = $context->getConfig();
if ( $config->has( 'StatsdServer' ) ) {
$statsdServer = explode( ':', $config->get( 'StatsdServer' ) );
$statsdHost = $statsdServer[0];
$statsdPort = isset( $statsdServer[1] ) ? $statsdServer[1] : 8125;
$statsdSender = new SocketSender( $statsdHost, $statsdPort );
$statsdClient = new StatsdClient( $statsdSender );
$statsdClient->send( $context->getStats()->getBuffer() );
}
$profiler = Profiler::instance();
@ -1346,7 +1358,8 @@ function wfLogProfilingData() {
* @return void
*/
function wfIncrStats( $key, $count = 1 ) {
StatCounter::singleton()->incr( $key, $count );
$stats = RequestContext::getMain()->getStats();
$stats->updateCount( $key, $count );
}
/**

154
includes/StatCounter.php
View file

@ -1,154 +0,0 @@
<?php
/**
* @defgroup StatCounter StatCounter
*
* StatCounter is used to increment arbitrary keys for profiling reasons.
* The key/values are persisted in several possible ways (see $wgStatsMethod).
*/
/**
* Aggregator for wfIncrStats() that batches updates per request.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup StatCounter
* @author Aaron Schulz
*/
/**
* Aggregator for wfIncrStats() that batches updates per request.
* This avoids spamming the collector many times for the same key.
*
* @ingroup StatCounter
*/
class StatCounter {
/** @var array */
protected $deltas = array(); // (key => count)
/** @var Config */
protected $config;
protected function __construct( Config $config ) {
$this->config = $config;
}
/**
* @return StatCounter
*/
public static function singleton() {
static $instance = null;
if ( !$instance ) {
$instance = new self(
ConfigFactory::getDefaultInstance()->makeConfig( 'main' )
);
}
return $instance;
}
/**
* Increment a key by delta $count
*
* @param string $key
* @param int $count
* @return void
*/
public function incr( $key, $count = 1 ) {
$this->deltas[$key] = isset( $this->deltas[$key] ) ? $this->deltas[$key] : 0;
$this->deltas[$key] += $count;
if ( PHP_SAPI === 'cli' ) {
$this->flush();
}
}
/**
* Flush all pending deltas to persistent storage
*
* @return void
*/
public function flush() {
$statsMethod = $this->config->get( 'StatsMethod' );
$deltas = array_filter( $this->deltas ); // remove 0 valued entries
if ( $statsMethod === 'udp' ) {
$this->sendDeltasUDP( $deltas );
} elseif ( $statsMethod === 'cache' ) {
$this->sendDeltasMemc( $deltas );
} else {
// disabled
}
$this->deltas = array();
}
/**
* @param array $deltas
* @return void
*/
protected function sendDeltasUDP( array $deltas ) {
$aggregateStatsID = $this->config->get( 'AggregateStatsID' );
$id = strlen( $aggregateStatsID ) ? $aggregateStatsID : wfWikiID();
$lines = array();
foreach ( $deltas as $key => $count ) {
$lines[] = sprintf( $this->config->get( 'StatsFormatString' ), $id, $count, $key );
}
if ( count( $lines ) ) {
static $socket = null;
if ( !$socket ) {
$socket = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP );
}
$packet = '';
$packets = array();
foreach ( $lines as $line ) {
if ( ( strlen( $packet ) + strlen( $line ) ) > 1450 ) {
$packets[] = $packet;
$packet = '';
}
$packet .= $line;
}
if ( $packet != '' ) {
$packets[] = $packet;
}
foreach ( $packets as $packet ) {
wfSuppressWarnings();
socket_sendto(
$socket,
$packet,
strlen( $packet ),
0,
$this->config->get( 'UDPProfilerHost' ),
$this->config->get( 'UDPProfilerPort' )
);
wfRestoreWarnings();
}
}
}
/**
* @param array $deltas
* @return void
*/
protected function sendDeltasMemc( array $deltas ) {
global $wgMemc;
foreach ( $deltas as $key => $count ) {
$ckey = wfMemcKey( 'stats', $key );
if ( $wgMemc->incr( $ckey, $count ) === null ) {
$wgMemc->add( $ckey, $count );
}
}
}
}

11
includes/context/ContextSource.php
View file

@ -152,6 +152,17 @@ abstract class ContextSource implements IContextSource {
return $this->getContext()->getSkin();
}
/**
* Get the Stats object
*
* @since 1.25
* @return BufferingStatsdDataFactory
*/
public function getStats() {
return $this->getContext()->getStats();
}
/**
* Get a Message object with context set
* Parameters are the same as wfMessage()

13
includes/context/DerivativeContext.php
View file

@ -97,6 +97,19 @@ class DerivativeContext extends ContextSource {
}
}
/**
* Get the stats object
*
* @return BufferingStatsdDataFactory
*/
public function getStats() {
if ( !is_null( $this->stats ) ) {
return $this->stats;
} else {
return $this->getContext()->getStats();
}
}
/**
* Set the WebRequest object
*

8
includes/context/IContextSource.php
View file

@ -121,6 +121,14 @@ interface IContextSource {
*/
public function getConfig();
/**
* Get the stats object
*
* @since 1.25
* @return BufferingStatsdDataFactory
*/
public function getStats();
/**
* Get a Message object with context set
*

21
includes/context/RequestContext.php
View file

@ -61,6 +61,11 @@ class RequestContext implements IContextSource {
*/
private $skin;
/**
* @var StatsdDataFactory
*/
private $stats;
/**
* @var Config
*/
@ -118,6 +123,22 @@ class RequestContext implements IContextSource {
return $this->request;
}
/**
* Get the Stats object
*
* @return BufferingStatsdDataFactory
*/
public function getStats() {
if ( $this->stats === null ) {
$config = $this->getConfig();
$prefix = $config->has( 'StatsdMetricPrefix' )
? rtrim( $config->get( 'StatsdMetricPrefix' ), '.' )
: 'MediaWiki';
$this->stats = new BufferingStatsdDataFactory( $prefix );
}
return $this->stats;
}
/**
* Set the Title object
*

59
includes/libs/BufferingStatsdDataFactory.php Normal file
View file

@ -0,0 +1,59 @@
<?php
/**
* Copyright 2015
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
use Liuggio\StatsdClient\Factory\StatsdDataFactory;
/**
* A factory for application metric data.
*
* This class prepends a context-specific prefix to each metric key and keeps
* a reference to each constructed metric in an internal array buffer.
*
* @since 1.25
*/
class BufferingStatsdDataFactory extends StatsdDataFactory {
protected $buffer = array();
public function __construct( $prefix ) {
parent::__construct();
$this->prefix = $prefix;
}
public function produceStatsdData( $key, $value = 1, $metric = self::STATSD_METRIC_COUNT ) {
$this->buffer[] = $entity = $this->produceStatsdDataEntity();
if ( $key !== null ) {
$prefixedKey = ltrim( $this->prefix . '.' . $key, '.' );
$entity->setKey( $prefixedKey );
}
if ( $value !== null ) {
$entity->setValue( $value );
}
if ( $metric !== null ) {
$entity->setMetric( $metric );
}
return $entity;
}
public function getBuffer() {
return $this->buffer;
}
}

2
includes/page/Article.php
View file

@ -574,7 +574,7 @@ class Article implements Page {
$useParserCache = $this->mPage->isParserCacheUsed( $parserOptions, $oldid );
wfDebug( 'Article::view using parser cache: ' . ( $useParserCache ? 'yes' : 'no' ) . "\n" );
if ( $user->getStubThreshold() ) {
wfIncrStats( 'pcache_miss_stub' );
$this->getContext()->getStats()->increment( 'pcache_miss_stub' );
}
$this->showRedirectedFromHeader();