2014-03-21 04:51:45 +00:00
|
|
|
<?php
|
|
|
|
|
/**
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
|
|
|
* (at your option) any later version.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License along
|
|
|
|
|
* with this program; if not, write to the Free Software Foundation, Inc.,
|
|
|
|
|
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
|
* http://www.gnu.org/copyleft/gpl.html
|
|
|
|
|
*
|
|
|
|
|
* @file
|
|
|
|
|
*/
|
|
|
|
|
|
2015-03-23 00:53:24 +00:00
|
|
|
namespace MediaWiki\Logger;
|
|
|
|
|
|
Produce monolog messages through kafka+avro
This allows a logging channel to be configured to write
directly to kafka. Logs can be serialized either to json
blobs or the more compact apache avro format.
The Kafka handler for monolog needs a list of one of more
kafka servers to query cluster metadata from. This should be
able to use any monolog formatter, although some like
JsonFormatter require you to disable formatBatch as Kafka
protocol would prefer to encode each record independently in
the protocol. This requires the nmred/kafka-php library,
version >= 1.3.0.
Adds a new formatter which serializes to the apache avro
format. This is a compact binary format which uses pre-
defined schemas. This initial implementation is very simple
and takes the plain schemas as a constructor argument.
Adds a new option to MonologSpi to wrap handlers in a
BufferHandler. This doesn't flush until the request shuts
down and prevents any network requests in the logger from
adding latency to web requests.
Related mediawiki/vendor update: Ibfe4bd2036ae8e998e2973f07bd9a6f057691578
The necessary config is something like:
array(
'loggers' => array(
'CirrusSearchRequests' => array(
'handlers' => array( 'kafka' ),
),
),
'handlers' => array(
'kafka' => array(
'factory' => '\\MediaWiki\\Logger\\Monolog\\KafkaHandler::factory',
'args' => array( 'localhost:9092' ),
'formatter' => 'avro',
'buffer' => true,
),
),
'formatters' => array(
'avro' => array(
'class' => '\\MediaWiki\\Logger\\Monolog\\AvroFormatter',
'args' => array(
array(
'CirrusSearchRequests' => array(
'type' => 'record',
'name' => 'CirrusSearchRequests'
'fields' => array( ... )
),
),
),
),
),
)
Bug: T106256
Change-Id: I6ee744b3e5306af0bed70811b558a543eed22840
2015-08-04 18:02:47 +00:00
|
|
|
use MediaWiki\Logger\Monolog\BufferHandler;
|
2015-03-23 00:53:24 +00:00
|
|
|
use Monolog\Logger;
|
2018-02-02 10:27:27 +00:00
|
|
|
use Wikimedia\ObjectFactory;
|
2015-03-23 00:53:24 +00:00
|
|
|
|
2014-03-21 04:51:45 +00:00
|
|
|
/**
|
2015-03-23 00:53:24 +00:00
|
|
|
* LoggerFactory service provider that creates loggers implemented by
|
2015-01-13 23:54:18 +00:00
|
|
|
* Monolog.
|
2014-03-21 04:51:45 +00:00
|
|
|
*
|
|
|
|
|
* Configured using an array of configuration data with the keys 'loggers',
|
|
|
|
|
* 'processors', 'handlers' and 'formatters'.
|
|
|
|
|
*
|
2015-07-20 18:05:02 +00:00
|
|
|
* The ['loggers']['\@default'] configuration will be used to create loggers
|
2014-03-21 04:51:45 +00:00
|
|
|
* for any channel that isn't explicitly named in the 'loggers' configuration
|
|
|
|
|
* section.
|
|
|
|
|
*
|
2014-10-23 03:53:14 +00:00
|
|
|
* Configuration will most typically be provided in the $wgMWLoggerDefaultSpi
|
2015-03-23 00:53:24 +00:00
|
|
|
* global configuration variable used by LoggerFactory to construct its
|
2015-01-13 23:54:18 +00:00
|
|
|
* default SPI provider:
|
2014-03-21 04:51:45 +00:00
|
|
|
* @code
|
2016-08-26 11:36:58 +00:00
|
|
|
* $wgMWLoggerDefaultSpi = [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \MediaWiki\Logger\MonologSpi::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* 'args' => [ [
|
|
|
|
|
* 'loggers' => [
|
|
|
|
|
* '@default' => [
|
|
|
|
|
* 'processors' => [ 'wiki', 'psr', 'pid', 'uid', 'web' ],
|
|
|
|
|
* 'handlers' => [ 'stream' ],
|
|
|
|
|
* ],
|
|
|
|
|
* 'runJobs' => [
|
|
|
|
|
* 'processors' => [ 'wiki', 'psr', 'pid' ],
|
|
|
|
|
* 'handlers' => [ 'stream' ],
|
|
|
|
|
* ]
|
|
|
|
|
* ],
|
|
|
|
|
* 'processors' => [
|
|
|
|
|
* 'wiki' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \MediaWiki\Logger\Monolog\WikiProcessor::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'psr' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Processor\PsrLogMessageProcessor::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'pid' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Processor\ProcessIdProcessor::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'uid' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Processor\UidProcessor::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'web' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Processor\WebProcessor::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* ],
|
|
|
|
|
* 'handlers' => [
|
|
|
|
|
* 'stream' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Handler\StreamHandler::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* 'args' => [ 'path/to/your.log' ],
|
2014-10-23 03:53:14 +00:00
|
|
|
* 'formatter' => 'line',
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'redis' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Handler\RedisHandler::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* 'args' => [ function() {
|
2014-10-23 03:53:14 +00:00
|
|
|
* $redis = new Redis();
|
|
|
|
|
* $redis->connect( '127.0.0.1', 6379 );
|
|
|
|
|
* return $redis;
|
|
|
|
|
* },
|
|
|
|
|
* 'logstash'
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
2014-10-23 03:53:14 +00:00
|
|
|
* 'formatter' => 'logstash',
|
Produce monolog messages through kafka+avro
This allows a logging channel to be configured to write
directly to kafka. Logs can be serialized either to json
blobs or the more compact apache avro format.
The Kafka handler for monolog needs a list of one of more
kafka servers to query cluster metadata from. This should be
able to use any monolog formatter, although some like
JsonFormatter require you to disable formatBatch as Kafka
protocol would prefer to encode each record independently in
the protocol. This requires the nmred/kafka-php library,
version >= 1.3.0.
Adds a new formatter which serializes to the apache avro
format. This is a compact binary format which uses pre-
defined schemas. This initial implementation is very simple
and takes the plain schemas as a constructor argument.
Adds a new option to MonologSpi to wrap handlers in a
BufferHandler. This doesn't flush until the request shuts
down and prevents any network requests in the logger from
adding latency to web requests.
Related mediawiki/vendor update: Ibfe4bd2036ae8e998e2973f07bd9a6f057691578
The necessary config is something like:
array(
'loggers' => array(
'CirrusSearchRequests' => array(
'handlers' => array( 'kafka' ),
),
),
'handlers' => array(
'kafka' => array(
'factory' => '\\MediaWiki\\Logger\\Monolog\\KafkaHandler::factory',
'args' => array( 'localhost:9092' ),
'formatter' => 'avro',
'buffer' => true,
),
),
'formatters' => array(
'avro' => array(
'class' => '\\MediaWiki\\Logger\\Monolog\\AvroFormatter',
'args' => array(
array(
'CirrusSearchRequests' => array(
'type' => 'record',
'name' => 'CirrusSearchRequests'
'fields' => array( ... )
),
),
),
),
),
)
Bug: T106256
Change-Id: I6ee744b3e5306af0bed70811b558a543eed22840
2015-08-04 18:02:47 +00:00
|
|
|
* 'buffer' => true,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'udp2log' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \MediaWiki\Logger\Monolog\LegacyHandler::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* 'args' => [
|
2014-10-23 03:53:14 +00:00
|
|
|
* 'udp://127.0.0.1:8420/mediawiki
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
2014-10-23 03:53:14 +00:00
|
|
|
* 'formatter' => 'line',
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* ],
|
|
|
|
|
* 'formatters' => [
|
|
|
|
|
* 'line' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Formatter\LineFormatter::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* ],
|
|
|
|
|
* 'logstash' => [
|
2018-01-13 00:02:09 +00:00
|
|
|
* 'class' => \Monolog\Formatter\LogstashFormatter::class,
|
2016-08-26 11:36:58 +00:00
|
|
|
* 'args' => [ 'mediawiki', php_uname( 'n' ), null, '', 1 ],
|
|
|
|
|
* ],
|
|
|
|
|
* ],
|
|
|
|
|
* ] ],
|
|
|
|
|
* ];
|
2014-03-21 04:51:45 +00:00
|
|
|
* @endcode
|
|
|
|
|
*
|
|
|
|
|
* @see https://github.com/Seldaek/monolog
|
|
|
|
|
* @since 1.25
|
2017-06-13 16:51:53 +00:00
|
|
|
* @copyright © 2014 Wikimedia Foundation and contributors
|
2014-03-21 04:51:45 +00:00
|
|
|
*/
|
2015-03-23 00:53:24 +00:00
|
|
|
class MonologSpi implements Spi {
|
2014-03-21 04:51:45 +00:00
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @var array $singletons
|
|
|
|
|
*/
|
|
|
|
|
protected $singletons;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Configuration for creating new loggers.
|
|
|
|
|
* @var array $config
|
|
|
|
|
*/
|
|
|
|
|
protected $config;
|
|
|
|
|
|
|
|
|
|
/**
|
2014-10-23 03:53:14 +00:00
|
|
|
* @param array $config Configuration data.
|
2014-03-21 04:51:45 +00:00
|
|
|
*/
|
2014-10-23 03:53:14 +00:00
|
|
|
public function __construct( array $config ) {
|
2016-02-17 09:09:32 +00:00
|
|
|
$this->config = [];
|
2015-07-02 22:38:06 +00:00
|
|
|
$this->mergeConfig( $config );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Merge additional configuration data into the configuration.
|
|
|
|
|
*
|
|
|
|
|
* @since 1.26
|
|
|
|
|
* @param array $config Configuration data.
|
|
|
|
|
*/
|
|
|
|
|
public function mergeConfig( array $config ) {
|
|
|
|
|
foreach ( $config as $key => $value ) {
|
|
|
|
|
if ( isset( $this->config[$key] ) ) {
|
|
|
|
|
$this->config[$key] = array_merge( $this->config[$key], $value );
|
|
|
|
|
} else {
|
|
|
|
|
$this->config[$key] = $value;
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-03-21 04:51:45 +00:00
|
|
|
$this->reset();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Reset internal caches.
|
|
|
|
|
*
|
|
|
|
|
* This is public for use in unit tests. Under normal operation there should
|
|
|
|
|
* be no need to flush the caches.
|
|
|
|
|
*/
|
|
|
|
|
public function reset() {
|
2016-02-17 09:09:32 +00:00
|
|
|
$this->singletons = [
|
|
|
|
|
'loggers' => [],
|
|
|
|
|
'handlers' => [],
|
|
|
|
|
'formatters' => [],
|
|
|
|
|
'processors' => [],
|
|
|
|
|
];
|
2014-03-21 04:51:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get a logger instance.
|
|
|
|
|
*
|
|
|
|
|
* Creates and caches a logger instance based on configuration found in the
|
|
|
|
|
* $wgMWLoggerMonologSpiConfig global. Subsequent request for the same channel
|
|
|
|
|
* name will return the cached instance.
|
|
|
|
|
*
|
|
|
|
|
* @param string $channel Logging channel
|
2015-11-23 22:53:38 +00:00
|
|
|
* @return \Psr\Log\LoggerInterface Logger instance
|
2014-03-21 04:51:45 +00:00
|
|
|
*/
|
|
|
|
|
public function getLogger( $channel ) {
|
|
|
|
|
if ( !isset( $this->singletons['loggers'][$channel] ) ) {
|
|
|
|
|
// Fallback to using the '@default' configuration if an explict
|
|
|
|
|
// configuration for the requested channel isn't found.
|
2017-10-06 22:17:58 +00:00
|
|
|
$spec = $this->config['loggers'][$channel] ?? $this->config['loggers']['@default'];
|
2014-03-21 04:51:45 +00:00
|
|
|
|
2014-10-23 03:53:14 +00:00
|
|
|
$monolog = $this->createLogger( $channel, $spec );
|
2015-01-13 23:54:18 +00:00
|
|
|
$this->singletons['loggers'][$channel] = $monolog;
|
2014-03-21 04:51:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $this->singletons['loggers'][$channel];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Create a logger.
|
|
|
|
|
* @param string $channel Logger channel
|
|
|
|
|
* @param array $spec Configuration
|
2015-11-23 22:53:38 +00:00
|
|
|
* @return \Monolog\Logger
|
2014-03-21 04:51:45 +00:00
|
|
|
*/
|
|
|
|
|
protected function createLogger( $channel, $spec ) {
|
2015-03-23 00:53:24 +00:00
|
|
|
$obj = new Logger( $channel );
|
2014-03-21 04:51:45 +00:00
|
|
|
|
2015-10-25 21:39:44 +00:00
|
|
|
if ( isset( $spec['calls'] ) ) {
|
|
|
|
|
foreach ( $spec['calls'] as $method => $margs ) {
|
2018-06-05 06:24:34 +00:00
|
|
|
$obj->$method( ...$margs );
|
2015-10-25 21:39:44 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-03-21 04:51:45 +00:00
|
|
|
if ( isset( $spec['processors'] ) ) {
|
|
|
|
|
foreach ( $spec['processors'] as $processor ) {
|
|
|
|
|
$obj->pushProcessor( $this->getProcessor( $processor ) );
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( isset( $spec['handlers'] ) ) {
|
|
|
|
|
foreach ( $spec['handlers'] as $handler ) {
|
|
|
|
|
$obj->pushHandler( $this->getHandler( $handler ) );
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return $obj;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Create or return cached processor.
|
|
|
|
|
* @param string $name Processor name
|
|
|
|
|
* @return callable
|
|
|
|
|
*/
|
2014-12-20 21:15:59 +00:00
|
|
|
public function getProcessor( $name ) {
|
2014-03-21 04:51:45 +00:00
|
|
|
if ( !isset( $this->singletons['processors'][$name] ) ) {
|
|
|
|
|
$spec = $this->config['processors'][$name];
|
2014-10-23 03:53:14 +00:00
|
|
|
$processor = ObjectFactory::getObjectFromSpec( $spec );
|
|
|
|
|
$this->singletons['processors'][$name] = $processor;
|
2014-03-21 04:51:45 +00:00
|
|
|
}
|
|
|
|
|
return $this->singletons['processors'][$name];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Create or return cached handler.
|
|
|
|
|
* @param string $name Processor name
|
2015-11-23 22:53:38 +00:00
|
|
|
* @return \Monolog\Handler\HandlerInterface
|
2014-03-21 04:51:45 +00:00
|
|
|
*/
|
2014-12-20 21:15:59 +00:00
|
|
|
public function getHandler( $name ) {
|
2014-03-21 04:51:45 +00:00
|
|
|
if ( !isset( $this->singletons['handlers'][$name] ) ) {
|
|
|
|
|
$spec = $this->config['handlers'][$name];
|
2014-10-23 03:53:14 +00:00
|
|
|
$handler = ObjectFactory::getObjectFromSpec( $spec );
|
2014-12-12 20:29:35 +00:00
|
|
|
if ( isset( $spec['formatter'] ) ) {
|
|
|
|
|
$handler->setFormatter(
|
|
|
|
|
$this->getFormatter( $spec['formatter'] )
|
|
|
|
|
);
|
|
|
|
|
}
|
Produce monolog messages through kafka+avro
This allows a logging channel to be configured to write
directly to kafka. Logs can be serialized either to json
blobs or the more compact apache avro format.
The Kafka handler for monolog needs a list of one of more
kafka servers to query cluster metadata from. This should be
able to use any monolog formatter, although some like
JsonFormatter require you to disable formatBatch as Kafka
protocol would prefer to encode each record independently in
the protocol. This requires the nmred/kafka-php library,
version >= 1.3.0.
Adds a new formatter which serializes to the apache avro
format. This is a compact binary format which uses pre-
defined schemas. This initial implementation is very simple
and takes the plain schemas as a constructor argument.
Adds a new option to MonologSpi to wrap handlers in a
BufferHandler. This doesn't flush until the request shuts
down and prevents any network requests in the logger from
adding latency to web requests.
Related mediawiki/vendor update: Ibfe4bd2036ae8e998e2973f07bd9a6f057691578
The necessary config is something like:
array(
'loggers' => array(
'CirrusSearchRequests' => array(
'handlers' => array( 'kafka' ),
),
),
'handlers' => array(
'kafka' => array(
'factory' => '\\MediaWiki\\Logger\\Monolog\\KafkaHandler::factory',
'args' => array( 'localhost:9092' ),
'formatter' => 'avro',
'buffer' => true,
),
),
'formatters' => array(
'avro' => array(
'class' => '\\MediaWiki\\Logger\\Monolog\\AvroFormatter',
'args' => array(
array(
'CirrusSearchRequests' => array(
'type' => 'record',
'name' => 'CirrusSearchRequests'
'fields' => array( ... )
),
),
),
),
),
)
Bug: T106256
Change-Id: I6ee744b3e5306af0bed70811b558a543eed22840
2015-08-04 18:02:47 +00:00
|
|
|
if ( isset( $spec['buffer'] ) && $spec['buffer'] ) {
|
|
|
|
|
$handler = new BufferHandler( $handler );
|
|
|
|
|
}
|
2014-03-21 04:51:45 +00:00
|
|
|
$this->singletons['handlers'][$name] = $handler;
|
|
|
|
|
}
|
|
|
|
|
return $this->singletons['handlers'][$name];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Create or return cached formatter.
|
|
|
|
|
* @param string $name Formatter name
|
2015-11-23 22:53:38 +00:00
|
|
|
* @return \Monolog\Formatter\FormatterInterface
|
2014-03-21 04:51:45 +00:00
|
|
|
*/
|
2014-12-20 21:15:59 +00:00
|
|
|
public function getFormatter( $name ) {
|
2014-03-21 04:51:45 +00:00
|
|
|
if ( !isset( $this->singletons['formatters'][$name] ) ) {
|
|
|
|
|
$spec = $this->config['formatters'][$name];
|
2014-10-23 03:53:14 +00:00
|
|
|
$formatter = ObjectFactory::getObjectFromSpec( $spec );
|
|
|
|
|
$this->singletons['formatters'][$name] = $formatter;
|
2014-03-21 04:51:45 +00:00
|
|
|
}
|
|
|
|
|
return $this->singletons['formatters'][$name];
|
|
|
|
|
}
|
|
|
|
|
}
|