Provide a JSON recent changes feed.

This introduces a new configuration variable, $wgRCFeeds, which allows the user
to configure multiple destinations for RC notifications. It also allows the
notification format to be customized. Two formats are included by default: the
older IRC format and a new JSON format.

Change-Id: I270bde418a82985c94372ac4579100435b6ee026
This commit is contained in:
Victor Vasiliev 2013-03-04 14:43:22 -05:00 committed by Ori Livneh
parent 1f694007c3
commit 2961884b43
12 changed files with 380 additions and 123 deletions

View file

@ -49,6 +49,11 @@ production.
* The checkbox for staying in HTTPS displayed on the login form when $wgSecureLogin is
enabled has been removed. Instead, whether the user stays in HTTPS will be determined
based on the user's preferences, and whether they came from HTTPS or not.
* $wgRC2UDPAddress, $wgRC2UDPInterwikiPrefix, $wgRC2UDPOmitBots, $wgRC2UDPPort,
and $wgRC2UDPPrefix configuration options have been deprecated in favor of a
$wgRCFeeds configuration array. $wgRCFeeds makes both the format and
destination of recent change notifications customizable, and allows for
multiple destinations to be specified.
=== New features in 1.22 ===
* (bug 44525) mediawiki.jqueryMsg can now parse (whitelisted) HTML elements and attributes.
@ -403,6 +408,11 @@ changes to languages because of Bugzilla reports.
have been deprecated in favour of using mw.hook.
* The 'showjumplinks' user preference has been removed, jump links are now
always included.
* Methods RecentChange::notifyRC2UDP, RecentChange::sendToUDP, and
RecentChange::cleanupForIRC have been deprecated, as it is now the
responsibility of classes implementing the RCFeedFormatter and RCFeedEngine
interfaces to implement the formatting and delivery for recent change
notifications.
== Compatibility ==

View file

@ -835,6 +835,13 @@ $wgAutoloadLocalClasses = array(
'ProfilerStub' => 'includes/profiler/ProfilerStub.php',
'ProfileSection' => 'includes/profiler/Profiler.php',
# includes/rcfeed
'RCFeedEngine' => 'includes/rcfeed/RCFeedEngine.php',
'UDPRCFeedEngine' => 'includes/rcfeed/UDPRCFeedEngine.php',
'RCFeedFormatter' => 'includes/rcfeed/RCFeedFormatter.php',
'IRCColourfulRCFeedFormatter' => 'includes/rcfeed/IRCColourfulRCFeedFormatter.php',
'JSONRCFeedFormatter' => 'includes/rcfeed/JSONRCFeedFormatter.php',
# includes/resourceloader
'ResourceLoader' => 'includes/resourceloader/ResourceLoader.php',
'ResourceLoaderContext' => 'includes/resourceloader/ResourceLoaderContext.php',

View file

@ -5394,11 +5394,15 @@ $wgRCLinkDays = array( 1, 3, 7, 14, 30 );
/**
* Send recent changes updates via UDP. The updates will be formatted for IRC.
* Set this to the IP address of the receiver.
*
* @deprecated since 1.22, use $wgRCFeeds
*/
$wgRC2UDPAddress = false;
/**
* Port number for RC updates
*
* @deprecated since 1.22, use $wgRCFeeds
*/
$wgRC2UDPPort = false;
@ -5407,21 +5411,72 @@ $wgRC2UDPPort = false;
* This can be used to identify the wiki. A script is available called
* mxircecho.py which listens on a UDP port, and uses a prefix ending in a
* tab to identify the IRC channel to send the log line to.
*
* @deprecated since 1.22, use $wgRCFeeds
*/
$wgRC2UDPPrefix = '';
/**
* If this is set to true, $wgLocalInterwiki will be prepended to links in the
* IRC feed. If this is set to a string, that string will be used as the prefix.
*
* @deprecated since 1.22, use $wgRCFeeds
*/
$wgRC2UDPInterwikiPrefix = false;
/**
* Set to true to omit "bot" edits (by users with the bot permission) from the
* UDP feed.
*
* @deprecated since 1.22, use $wgRCFeeds
*/
$wgRC2UDPOmitBots = false;
/**
* Destinations to which notifications about recent changes
* should be sent.
*
* As of MediaWiki 1.22, the only supported 'engine' parameter option in core
* is 'UDPRCFeedEngine', which is used to send recent changes over UDP to the
* specified server.
* The common options are:
* * 'uri' -- the address to which the notices are to be sent.
* * 'formatter' -- the class name (implementing RCFeedFormatter) which will
* produce the text to send.
* * 'omit_bots' -- whether the bot edits should be in the feed
* The IRC-specific options are:
* * 'add_interwiki_prefix' -- whether the titles should be prefixed with
* $wgLocalInterwiki.
* The JSON-specific options are:
* * 'channel' -- if set, the 'channel' parameter is also set in JSON values.
*
* To ensure backwards-compatability, whenever $wgRC2UDPAddress is set, a
* 'default' feed will be created reusing the deprecated $wgRC2UDP* variables.
*
* @example $wgRCFeeds['example'] = array(
* 'formatter' => 'JSONRCFeedFormatter',
* 'uri' => "udp://localhost:1336",
* 'add_interwiki_prefix' => false,
* 'omit_bots' => true,
* );
* @example $wgRCFeeds['exampleirc'] = array(
* 'formatter' => 'IRCColourfulRCFeedFormatter',
* 'uri' => "udp://localhost:1338",
* 'add_interwiki_prefix' => false,
* 'omit_bots' => true,
* );
* @since 1.22
*/
$wgRCFeeds = array();
/**
* Used by RecentChange::getStreamEngine to find the correct engine to use for a given URI protocol.
* Keys are scheme names, values are names of engine classes.
*/
$wgStreamLoggers = array(
'udp' => 'UDPRCFeedEngine',
);
/**
* Enable user search in Special:Newpages
* This is really a temporary hack around an index install bug on some Wikipedias.

View file

@ -263,7 +263,7 @@ class RecentChange {
# Notify external application via UDP
if ( !$noudp ) {
$this->notifyRC2UDP();
$this->notifyRCFeeds();
}
# E-mail notifications
@ -284,54 +284,90 @@ class RecentChange {
}
}
/**
* @deprecated since 1.22, use notifyRCFeeds instead.
*/
public function notifyRC2UDP() {
global $wgRC2UDPAddress, $wgRC2UDPOmitBots;
# Notify external application via UDP
# Omit RC_EXTERNAL changes: bots and tools can get these edits from the feed of the external wiki
if ( $wgRC2UDPAddress && $this->mAttribs['rc_type'] != RC_EXTERNAL &&
( !$this->mAttribs['rc_bot'] || !$wgRC2UDPOmitBots ) ) {
self::sendToUDP( $this->getIRCLine() );
}
wfDeprecated( __METHOD__, '1.22' );
$this->notifyRCFeeds();
}
/**
* Send some text to UDP.
* @see RecentChange::cleanupForIRC
* @param string $line text to send
* @param string $address defaults to $wgRC2UDPAddress.
* @param string $prefix defaults to $wgRC2UDPPrefix.
* @param int $port defaults to $wgRC2UDPPort. (Since 1.17)
* @return Boolean: success
* @deprecated since 1.22
*/
public static function sendToUDP( $line, $address = '', $prefix = '', $port = '' ) {
global $wgRC2UDPAddress, $wgRC2UDPPrefix, $wgRC2UDPPort;
# Assume default for standard RC case
$address = $address ? $address : $wgRC2UDPAddress;
$prefix = $prefix ? $prefix : $wgRC2UDPPrefix;
$port = $port ? $port : $wgRC2UDPPort;
# Notify external application via UDP
if ( $address ) {
$conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP );
if ( $conn ) {
$line = $prefix . $line;
wfDebug( __METHOD__ . ": sending UDP line: $line\n" );
socket_sendto( $conn, $line, strlen( $line ), 0, $address, $port );
socket_close( $conn );
return true;
} else {
wfDebug( __METHOD__ . ": failed to create UDP socket\n" );
}
}
return false;
global $wgRC2UDPPrefix, $wgRC2UDPInterwikiPrefix;
wfDeprecated( __METHOD__, '1.22' );
$engine = new UDPRCFeedEngine();
$feed = array(
'uri' => "udp://$address:$port/$wgRC2UDPPrefix",
'formatter' => 'IRCColourfulRCFeedFormatter',
'add_interwiki_prefix' => $wgRC2UDPInterwikiPrefix,
);
return $engine->send( $feed, $line );
}
/**
* Remove newlines, carriage returns and decode html entities
* @param $text String
* @return String
* Notify all the feeds about the change.
*/
public function notifyRCFeeds() {
global $wgRCFeeds;
foreach ( $wgRCFeeds as $feed ) {
$engine = self::getStreamEngine( $feed['uri'] );
if ( isset( $this->mExtras['actionCommentIRC'] ) ) {
$actionComment = $this->mExtras['actionCommentIRC'];
} else {
$actionComment = null;
}
$omitBots = isset( $feed['omit_bots'] ) ? $feed['omit_bots'] : false;
if (
( $omitBots && $this->mAttribs['rc_bot'] ) ||
$this->mAttribs['rc_type'] == RC_EXTERNAL
) {
continue;
}
$formatter = new $feed['formatter']();
$line = $formatter->getLine( $feed, $this, $actionComment );
$engine->send( $feed, $line );
}
}
/**
* Gets the stream engine object for a given URI from $wgStreamLoggers
*
* @param $uri string URI to get the engine object for
* @return object The engine object
*/
private static function getStreamEngine( $uri ) {
global $wgStreamLoggers;
$scheme = parse_url( $uri, PHP_URL_SCHEME );
if ( !$scheme ) {
throw new MWException( __FUNCTION__ . ": Invalid stream logger URI: '$uri'" );
}
if ( !isset( $wgStreamLoggers[$scheme] ) ) {
throw new MWException( __FUNCTION__ . ": Unknown stream logger URI scheme: $scheme" );
}
return new $wgStreamLoggers[$scheme];
}
/**
* @deprecated since 1.22, moved to IRCColourfulRCFeedFormatter
*/
public static function cleanupForIRC( $text ) {
return Sanitizer::decodeCharReferences( str_replace( array( "\n", "\r" ), array( " ", "" ), $text ) );
wfDeprecated( __METHOD__, '1.22' );
return IRCColourfulRCFeedFormatter::cleanupForIRC( $text );
}
/**
@ -730,89 +766,6 @@ class RecentChange {
return $trail;
}
/**
* @return string
*/
public function getIRCLine() {
global $wgUseRCPatrol, $wgUseNPPatrol, $wgRC2UDPInterwikiPrefix, $wgLocalInterwiki,
$wgCanonicalServer, $wgScript;
if ( $this->mAttribs['rc_type'] == RC_LOG ) {
// Don't use SpecialPage::getTitleFor, backwards compatibility with
// IRC API which expects "Log".
$titleObj = Title::newFromText( 'Log/' . $this->mAttribs['rc_log_type'], NS_SPECIAL );
} else {
$titleObj =& $this->getTitle();
}
$title = $titleObj->getPrefixedText();
$title = self::cleanupForIRC( $title );
if ( $this->mAttribs['rc_type'] == RC_LOG ) {
$url = '';
} else {
$url = $wgCanonicalServer . $wgScript;
if ( $this->mAttribs['rc_type'] == RC_NEW ) {
$query = '?oldid=' . $this->mAttribs['rc_this_oldid'];
} else {
$query = '?diff=' . $this->mAttribs['rc_this_oldid'] . '&oldid=' . $this->mAttribs['rc_last_oldid'];
}
if ( $wgUseRCPatrol || ( $this->mAttribs['rc_type'] == RC_NEW && $wgUseNPPatrol ) ) {
$query .= '&rcid=' . $this->mAttribs['rc_id'];
}
// HACK: We need this hook for WMF's secure server setup
wfRunHooks( 'IRCLineURL', array( &$url, &$query ) );
$url .= $query;
}
if ( $this->mAttribs['rc_old_len'] !== null && $this->mAttribs['rc_new_len'] !== null ) {
$szdiff = $this->mAttribs['rc_new_len'] - $this->mAttribs['rc_old_len'];
if ( $szdiff < -500 ) {
$szdiff = "\002$szdiff\002";
} elseif ( $szdiff >= 0 ) {
$szdiff = '+' . $szdiff;
}
// @todo i18n with parentheses in content language?
$szdiff = '(' . $szdiff . ')';
} else {
$szdiff = '';
}
$user = self::cleanupForIRC( $this->mAttribs['rc_user_text'] );
if ( $this->mAttribs['rc_type'] == RC_LOG ) {
$targetText = $this->getTitle()->getPrefixedText();
$comment = self::cleanupForIRC( str_replace( "[[$targetText]]", "[[\00302$targetText\00310]]", $this->mExtra['actionCommentIRC'] ) );
$flag = $this->mAttribs['rc_log_action'];
} else {
$comment = self::cleanupForIRC( $this->mAttribs['rc_comment'] );
$flag = '';
if ( !$this->mAttribs['rc_patrolled'] && ( $wgUseRCPatrol || $this->mAttribs['rc_type'] == RC_NEW && $wgUseNPPatrol ) ) {
$flag .= '!';
}
$flag .= ( $this->mAttribs['rc_type'] == RC_NEW ? "N" : "" ) . ( $this->mAttribs['rc_minor'] ? "M" : "" ) . ( $this->mAttribs['rc_bot'] ? "B" : "" );
}
if ( $wgRC2UDPInterwikiPrefix === true && $wgLocalInterwiki !== false ) {
$prefix = $wgLocalInterwiki;
} elseif ( $wgRC2UDPInterwikiPrefix ) {
$prefix = $wgRC2UDPInterwikiPrefix;
} else {
$prefix = false;
}
if ( $prefix !== false ) {
$titleString = "\00314[[\00303$prefix:\00307$title\00314]]";
} else {
$titleString = "\00314[[\00307$title\00314]]";
}
# see http://www.irssi.org/documentation/formats for some colour codes. prefix is \003,
# no colour (\003) switches back to the term default
$fullString = "$titleString\0034 $flag\00310 " .
"\00302$url\003 \0035*\003 \00303$user\003 \0035*\003 $szdiff \00310$comment\003\n";
return $fullString;
}
/**
* Returns the change size (HTML).
* The lengths can be given optionally.

View file

@ -393,6 +393,15 @@ if ( $wgCookieSecure === 'detect' ) {
$wgCookieSecure = ( WebRequest::detectProtocol() === 'https' );
}
if ( $wgRC2UDPAddress ) {
$wgRCFeeds['default'] = array(
'formatter' => 'IRCColourfulRCFeedFormatter',
'uri' => "udp://$wgRC2UDPAddress:$wgRC2UDPPort/$wgRC2UDPPrefix",
'add_interwiki_prefix' => &$wgRC2UDPInterwikiPrefix,
'omit_bots' => &$wgRC2UDPOmitBots,
);
}
// Disable MWDebug for command line mode, this prevents MWDebug from eating up
// all the memory from logging SQL queries on maintenance scripts
global $wgCommandLineMode;

View file

@ -544,7 +544,7 @@ class ManualLogEntry extends LogEntryBase {
}
if ( $to === 'udp' || $to === 'rcandudp' ) {
$rc->notifyRC2UDP();
$rc->notifyRCFeeds();
}
}

View file

@ -0,0 +1,99 @@
<?php
class IRCColourfulRCFeedFormatter implements RCFeedFormatter {
/**
* Generates a colourful notification intended for humans on IRC.
* @see RCFeedFormatter::getLine
*/
public function getLine( array $feed, RecentChange $rc, $actionComment ) {
global $wgUseRCPatrol, $wgUseNPPatrol, $wgLocalInterwiki,
$wgCanonicalServer, $wgScript;
$attribs = $rc->getAttributes();
if ( $attribs['rc_type'] == RC_LOG ) {
// Don't use SpecialPage::getTitleFor, backwards compatibility with
// IRC API which expects "Log".
$titleObj = Title::newFromText( 'Log/' . $attribs['rc_log_type'], NS_SPECIAL );
} else {
$titleObj =& $rc->getTitle();
}
$title = $titleObj->getPrefixedText();
$title = self::cleanupForIRC( $title );
if ( $attribs['rc_type'] == RC_LOG ) {
$url = '';
} else {
$url = $wgCanonicalServer . $wgScript;
if ( $attribs['rc_type'] == RC_NEW ) {
$query = '?oldid=' . $attribs['rc_this_oldid'];
} else {
$query = '?diff=' . $attribs['rc_this_oldid'] . '&oldid=' . $attribs['rc_last_oldid'];
}
if ( $wgUseRCPatrol || ( $attribs['rc_type'] == RC_NEW && $wgUseNPPatrol ) ) {
$query .= '&rcid=' . $attribs['rc_id'];
}
// HACK: We need this hook for WMF's secure server setup
wfRunHooks( 'IRCLineURL', array( &$url, &$query ) );
$url .= $query;
}
if ( $attribs['rc_old_len'] !== null && $attribs['rc_new_len'] !== null ) {
$szdiff = $attribs['rc_new_len'] - $attribs['rc_old_len'];
if ( $szdiff < -500 ) {
$szdiff = "\002$szdiff\002";
} elseif ( $szdiff >= 0 ) {
$szdiff = '+' . $szdiff;
}
// @todo i18n with parentheses in content language?
$szdiff = '(' . $szdiff . ')';
} else {
$szdiff = '';
}
$user = self::cleanupForIRC( $attribs['rc_user_text'] );
if ( $attribs['rc_type'] == RC_LOG ) {
$targetText = $rc->getTitle()->getPrefixedText();
$comment = self::cleanupForIRC( str_replace( "[[$targetText]]", "[[\00302$targetText\00310]]", $actionComment ) );
$flag = $attribs['rc_log_action'];
} else {
$comment = self::cleanupForIRC( $attribs['rc_comment'] );
$flag = '';
if ( !$attribs['rc_patrolled'] && ( $wgUseRCPatrol || $attribs['rc_type'] == RC_NEW && $wgUseNPPatrol ) ) {
$flag .= '!';
}
$flag .= ( $attribs['rc_type'] == RC_NEW ? "N" : "" ) . ( $attribs['rc_minor'] ? "M" : "" ) . ( $attribs['rc_bot'] ? "B" : "" );
}
if ( $feed['add_interwiki_prefix'] === true && $wgLocalInterwiki !== false ) {
$prefix = $wgLocalInterwiki;
} elseif ( $feed['add_interwiki_prefix'] ) {
$prefix = $feed['add_interwiki_prefix'];
} else {
$prefix = false;
}
if ( $prefix !== false ) {
$titleString = "\00314[[\00303$prefix:\00307$title\00314]]";
} else {
$titleString = "\00314[[\00307$title\00314]]";
}
# see http://www.irssi.org/documentation/formats for some colour codes. prefix is \003,
# no colour (\003) switches back to the term default
$fullString = "$titleString\0034 $flag\00310 " .
"\00302$url\003 \0035*\003 \00303$user\003 \0035*\003 $szdiff \00310$comment\003\n";
return $fullString;
}
/**
* Remove newlines, carriage returns and decode html entites
* @param string $text
* @return string
*/
public static function cleanupForIRC( $text ) {
return Sanitizer::decodeCharReferences( str_replace(
array( "\n", "\r" ),
array( " ", "" ),
$text
) );
}
}

View file

@ -0,0 +1,89 @@
<?php
class JSONRCFeedFormatter implements RCFeedFormatter {
/**
* Generates a notification that can be easily interpreted by a machine.
* @see RCFeedFormatter::getLine
*/
public function getLine( array $feed, RecentChange $rc, $actionComment ) {
global $wgCanonicalServer, $wgScriptPath, $wgArticlePath, $wgDBname;
$attrib = $rc->getAttributes();
$packet = array(
// Usually, RC ID is exposed only for patrolling purposes,
// but there is no real reason not to expose it in other cases,
// and I can see how this may be potentially useful for clients.
'id' => $attrib['rc_id'],
'type' => $attrib['rc_type'],
'namespace' => $rc->getTitle()->getNamespace(),
'title' => $rc->getTitle()->getPrefixedText(),
'comment' => $attrib['rc_comment'],
'timestamp' => (int)wfTimestamp( TS_UNIX, $attrib['rc_timestamp'] ),
'user' => $attrib['rc_user_text'],
'bot' => (bool)$attrib['rc_bot'],
);
if ( isset( $feed['channel'] ) ) {
$packet['channel'] = $feed['channel'];
}
$type = $attrib['rc_type'];
if ( $type == RC_EDIT || $type == RC_NEW ) {
global $wgUseRCPatrol, $wgUseNPPatrol;
$packet['minor'] = $attrib['rc_minor'];
if ( $wgUseRCPatrol || ( $type == RC_NEW && $wgUseNPPatrol ) ) {
$packet['patrolled'] = $attrib['rc_patrolled'];
}
}
switch ( $type ) {
case RC_EDIT:
$packet['length'] = array( 'old' => $attrib['rc_old_len'], 'new' => $attrib['rc_new_len'] );
$packet['revision'] = array( 'old' => $attrib['rc_last_oldid'], 'new' => $attrib['rc_this_oldid'] );
break;
case RC_NEW:
$packet['length'] = array( 'old' => NULL, 'new' => $attrib['rc_new_len'] );
$packet['revision'] = array( 'old' => NULL, 'new' => $attrib['rc_this_oldid'] );
break;
case RC_LOG:
$packet['log_type'] = $attrib['rc_log_type'];
$packet['log_action'] = $attrib['rc_log_action'];
if ( $attrib['rc_params'] ) {
wfSuppressWarnings();
$params = unserialize( $attrib['rc_params'] );
wfRestoreWarnings();
if (
// If it's an actual serialised false...
$attrib['rc_params'] == serialize( false ) ||
// Or if we did not get false back when trying to unserialise
$params !== false
) {
// From ApiQueryLogEvents::addLogParams
$logParams = array();
// Keys like "4::paramname" can't be used for output so we change them to "paramname"
foreach ( $params as $key => $value ) {
if ( strpos( $key, ':' ) === false ) {
$logParams[$key] = $value;
continue;
}
$logParam = explode( ':', $key, 3 );
$logParams[$logParam[2]] = $value;
}
$packet['log_params'] = $logParams;
} else {
$packet['log_params'] = explode( "\n", $attrib['rc_params'] );
}
}
$packet['log_action_comment'] = $actionComment;
break;
}
$packet['server_url'] = $wgCanonicalServer;
$packet['server_script_path'] = $wgScriptPath ?: '/';
$packet['wiki'] = $wgDBname;
return FormatJson::encode( $packet );
}
}

View file

@ -0,0 +1,12 @@
<?php
interface RCFeedEngine {
/**
* Sends some text to the specified live feed.
*
* @see RecentChange::cleanupForIRC
* @param array $feed The feed, as configured in an associative array.
* @param string $line The text to send.
* @return boolean success
*/
public function send( array $feed, $line );
}

View file

@ -0,0 +1,13 @@
<?php
interface RCFeedFormatter {
/**
* Formats the line for the live feed.
*
* @param array $feed The feed, as configured in an associative array.
* @param RecentChange $rc The RecentChange object showing what sort
* of event has taken place.
* @param string|null $actionComment
* @return string The text to send.
*/
public function getLine( array $feed, RecentChange $rc, $actionComment );
}

View file

@ -0,0 +1,10 @@
<?php
class UDPRCFeedEngine implements RCFeedEngine {
/**
* Sends the notification to the specified host in a UDP packet.
* @see RCFeedEngine::send
*/
public function send( array $feed, $line ) {
wfErrorLog( $line, $feed['uri'] );
}
}

View file

@ -267,8 +267,8 @@ class RecentChangeTest extends MediaWikiTestCase {
$formatter = LogFormatter::newFromEntry( $logEntry );
$formatter->setContext( $this->context );
// Apply the same transformation as done in RecentChange::getIRCLine for rc_comment
$ircRcComment = RecentChange::cleanupForIRC( $formatter->getIRCActionComment() );
// Apply the same transformation as done in IRCColourfulRCFeedFormatter::getLine for rc_comment
$ircRcComment = IRCColourfulRCFeedFormatter::cleanupForIRC( $formatter->getIRCActionComment() );
$this->assertEquals(
$expected,