Add support for JSON i18n files

Implementation for https://www.mediawiki.org/wiki/Requests_for_comment/Localisation_format

Add $wgExtensionMessagesDirs, which tracks the directory
(or directories) where each extension stores it's JSON i18n files.
In this commit only support for messages is implemented, but adding
support for other i18n variables (e.g. magic words) is easy to do later.

To be backwards compatible, an extension can specify both
$wgExtensionMessagesFiles and $wgExtensionMessagesDirs. Older versions
of MediaWiki will just work, and newer versions will use the JSON files
while ignoring the PHP file (except if the PHP file contains non-message
data like magic words).

Misc changes:
* Updated mergeMessageFileList.php to output both
  $wgExtensionMessagesFiles and $wgExtensionMessagesDirs

Change-Id: I8d137e15e1670880a9847263e6ce796c62a4670d

includes/DefaultSettings.php

@@ -5931,6 +5931,16 @@ $wgExtensionFunctions = array();
  * Variables defined in extensions will override conflicting variables defined
  * in the core.
  *
+ * Since MediaWiki 1.23, use of this variable to define messages is discouraged; instead, store
+ * messages in JSON format and use $wgExtensionMessagesDirs. For setting other variables than
+ * $messages, $wgExtensionMessagesFiles should still be used.
+ *
+ * If there is an entry in $wgExtensionMessagesDirs with the same key as one in
+ * $wgExtensionMessagesFiles, then any $messages variables set in the $wgExtensionMessagesFiles file
+ * will be ignored. This means an extension that only provides messages can be backwards compatible
+ * by using both $wgExtensionMessagesFiles and $wgExtensionMessagesDirs, and only one of the two
+ * will be used depending on what the version of MediaWiki supports.
+ *
  * @par Example:
  * @code
  *    $wgExtensionMessagesFiles['ConfirmEdit'] = __DIR__.'/ConfirmEdit.i18n.php';
@@ -5938,6 +5948,32 @@ $wgExtensionFunctions = array();
  */
 $wgExtensionMessagesFiles = array();
 
+/**
+ * Extension messages directories.
+ *
+ * Associative array mapping extension name to the path of the directory where message files can
+ * be found. The message files are expected to be JSON files named for their language code, e.g.
+ * en.json, de.json, etc. Extensions with messages in multiple places may specify an array of
+ * message directories.
+ *
+ * @par Simple example:
+ * @code
+ *    $wgExtensionMessagesDirs['ConfirmEdit'] = __DIR__ . '/i18n';
+ * @endcode
+ *
+ * @par Complex example:
+ * @code
+ *    $wgExtensionMessagesDirs['VisualEditor'] = array(
+ *        __DIR__ . '/i18n',
+ *        __DIR__ . '/modules/ve-core/i18n',
+ *        __DIR__ . '/modules/qunit/localisation',
+ *        __DIR__ . '/modules/oojs-ui/messages',
+ *    )
+ * @endcode
+ * @since 1.23
+ */
+$wgExtensionMessagesDirs = array();
+
 /**
  * Array of files with list(s) of extension entry points to be used in
  * maintenance/mergeMessageFileList.php

includes/cache/LocalisationCache.php

@@ -527,6 +527,36 @@ class LocalisationCache {
 		return $data;
 	}
 
+	/**
+	 * Read a JSON file containing localisation messages.
+	 * @param string $fileName Name of file to read
+	 * @throws MWException if there is a syntax error in the JSON file
+	 * @return array with a 'messages' key, or empty array if the file doesn't exist
+	 */
+	protected function readJSONFile( $fileName ) {
+		wfProfileIn( __METHOD__ );
+		if ( !is_readable( $fileName ) ) {
+			return array();
+		}
+
+		$json = file_get_contents( $fileName );
+		if ( $json === false ) {
+			return array();
+		}
+		$data = FormatJson::decode( $json, true );
+		if ( $data === null ) {
+			throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" );
+		}
+		// Remove keys starting with '@', they're reserved for metadata and non-message data
+		foreach ( $data as $key => $unused ) {
+			if ( $key === '' || $key[0] === '@' ) {
+				unset( $data[$key] );
+			}
+		}
+		// The JSON format only supports messages, none of the other variables, so wrap the data
+		return array( 'messages' => $data );
+	}
+
 	/**
 	 * Get the compiled plural rules for a given language from the XML files.
 	 * @since 1.20
@@ -736,7 +766,7 @@ class LocalisationCache {
 	 * @throws MWException
 	 */
 	public function recache( $code ) {
-		global $wgExtensionMessagesFiles;
+		global $wgExtensionMessagesFiles, $wgExtensionMessagesDirs;
 		wfProfileIn( __METHOD__ );
 
 		if ( !$code ) {
@@ -810,11 +840,33 @@ class LocalisationCache {
 		# like site-specific message overrides.
 		wfProfileIn( __METHOD__ . '-extensions' );
 		$allData = $initialData;
-		foreach ( $wgExtensionMessagesFiles as $fileName ) {
+		foreach ( $wgExtensionMessagesDirs as $dirs ) {
+			foreach ( (array)$dirs as $dir ) {
+				foreach ( $codeSequence as $csCode ) {
+					$fileName = "$dir/$csCode.json";
+					$data = $this->readJSONFile( $fileName );
+
+					foreach ( $data as $key => $item ) {
+						$this->mergeItem( $key, $allData[$key], $item );
+					}
+
+					$deps[] = new FileDependency( $fileName );
+				}
+			}
+		}
+
+		foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) {
 			$data = $this->readPHPFile( $fileName, 'extension' );
 			$used = false;
 
 			foreach ( $data as $key => $item ) {
+				if ( $key === 'messages' && isset( $wgExtensionMessagesDirs[$extension] ) ) {
+					# For backwards compatibility, ignore messages from extensions in
+					# $wgExtensionMessagesFiles that are also present in $wgExtensionMessagesDirs.
+					# This allows extensions to use both and be backwards compatible.
+					# Variables other than $messages still need to be supported though.
+					continue;
+				}
 				if ( $this->mergeExtensionItem( $codeSequence, $key, $allData[$key], $item ) ) {
 					$used = true;
 				}
@@ -833,6 +885,7 @@ class LocalisationCache {
 
 		# Add cache dependencies for any referenced globals
 		$deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' );
+		$deps['wgExtensionMessagesDirs'] = new GlobalDependency( 'wgExtensionMessagesDirs' );
 		$deps['version'] = new ConstantDependency( 'MW_LC_VERSION' );
 
 		# Add dependencies to the cache entry

maintenance/mergeMessageFileList.php

@@ -46,8 +46,8 @@ class MergeMessageFileList extends Maintenance {
 		$this->addOption( 'list-file', 'A file containing a list of extension setup files, one per line.', false, true );
 		$this->addOption( 'extensions-dir', 'Path where extensions can be found.', false, true );
 		$this->addOption( 'output', 'Send output to this file (omit for stdout)', false, true );
-		$this->mDescription = 'Merge $wgExtensionMessagesFiles from various extensions to produce a ' .
-			'single array containing all message files.';
+		$this->mDescription = 'Merge $wgExtensionMessagesFiles and $wgExtensionMessagesDirs from ' .
+			' various extensions to produce a single file listing all message files and dirs.';
 	}
 
 	public function execute() {
@@ -158,7 +158,8 @@ $s =
 	"<" . "?php\n" .
 	"## This file is generated by mergeMessageFileList.php. Do not edit it directly.\n\n" .
 	"if ( defined( 'MW_NO_EXTENSION_MESSAGES' ) ) return;\n\n" .
-	'$wgExtensionMessagesFiles = ' . var_export( $wgExtensionMessagesFiles, true ) . ";\n\n";
+	'$wgExtensionMessagesFiles = ' . var_export( $wgExtensionMessagesFiles, true ) . ";\n\n" .
+	'$wgExtensionMessagesDirs = ' . var_export( $wgExtensionMessagesDirs, true ) . ";\n\n";
 
 $dirs = array(
 	$IP,