wiki.techinc.nl/includes/language/LocalisationCache.php

<?php
/**
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 */

use CLDRPluralRuleParser\Error as CLDRPluralRuleError;
use CLDRPluralRuleParser\Evaluator;
use MediaWiki\Config\ConfigException;
use MediaWiki\Config\ServiceOptions;
use MediaWiki\HookContainer\HookContainer;
use MediaWiki\HookContainer\HookRunner;
use MediaWiki\Json\FormatJson;
use MediaWiki\Languages\LanguageNameUtils;
use MediaWiki\MainConfigNames;
use Psr\Log\LoggerInterface;

/**
 * Caching for the contents of localisation files.
 *
 * Including for i18n JSON files under `/languages/messages`, `Messages*.php`,
 * and `*.i18n.php`.
 *
 * An instance of this class is available using MediaWikiServices.
 *
 * The values retrieved from here are merged, containing items from extension
 * files, core messages files and the language fallback sequence (e.g. zh-cn ->
 * zh-hans -> en ). Some common errors are corrected, for example namespace
 * names with spaces instead of underscores, but heavyweight processing, such
 * as grammatical transformation, is done by the caller.
 *
 * @ingroup Language
 */
class LocalisationCache {
	public const VERSION = 5;

	/** @var ServiceOptions */
	private $options;

	/**
	 * True if re-caching should only be done on an explicit call to recache().
	 * Setting this reduces the overhead of cache freshness checking, which
	 * requires doing a stat() for every extension i18n file.
	 *
	 * @var bool
	 */
	private $manualRecache;

	/**
	 * The cache data. 2/3-d array, where the first key is the language code,
	 * the second key is the item key e.g. 'messages', and the optional third key is
	 * an item specific subkey index. Some items are not arrays, and so for those
	 * items, there are no subkeys.
	 *
	 * @var array<string,array>
	 */
	protected $data = [];

	/**
	 * The source language of cached data items. Only supports messages for now.
	 *
	 * @var array<string,array<string,array<string,string>>>
	 */
	protected $sourceLanguage = [];

	/** @var LCStore */
	private $store;
	/** @var LoggerInterface */
	private $logger;
	/** @var HookRunner */
	private $hookRunner;
	/** @var callable[] See comment for parameter in constructor */
	private $clearStoreCallbacks;
	/** @var LanguageNameUtils */
	private $langNameUtils;

	/**
	 * A 2-d associative array, code/key, where presence indicates that the item
	 * is loaded. Value arbitrary.
	 *
	 * For split items, if set, this indicates that all the subitems have been
	 * loaded.
	 *
	 * @var array<string,array<string,true>>
	 */
	private $loadedItems = [];

	/**
	 * A 3-d associative array, code/key/subkey, where presence indicates that
	 * the subitem is loaded. Only used for the split items, i.e. ,messages.
	 *
	 * @var array<string,array<string,array<string,true>>>
	 */
	private $loadedSubitems = [];

	/**
	 * An array where the presence of a key indicates that that language has been
	 * initialised. Initialisation includes checking for cache expiry and doing
	 * any necessary updates.
	 *
	 * @var array<string,true>
	 */
	private $initialisedLangs = [];

	/**
	 * An array mapping non-existent pseudo-languages to fallback languages. This
	 * is filled by initShallowFallback() when data is requested from a language
	 * that lacks a Messages*.php file.
	 *
	 * @var array<string,string>
	 */
	private $shallowFallbacks = [];

	/**
	 * An array where the keys are codes that have been re-cached by this instance.
	 *
	 * @var array<string,true>
	 */
	private $recachedLangs = [];

	/**
	 * An array indicating whether core data for a language has been loaded.
	 * If the entry for a language code $code is true,
	 * then {@link self::$data} is guaranteed to contain an array for $code,
	 * with at least an entry (possibly null) for each of the {@link self::CORE_ONLY_KEYS},
	 * and all the core-only keys will be marked as loaded in {@link self::$loadedItems} too.
	 * Additionally, there will be a 'deps' entry for $code with the dependencies tracked so far.
	 *
	 * @var array<string,bool>
	 */
	private $coreDataLoaded = [];

	/**
	 * All item keys
	 */
	public const ALL_KEYS = [
		'fallback', 'namespaceNames', 'bookstoreList',
		'magicWords', 'messages', 'rtl',
		'digitTransformTable', 'separatorTransformTable',
		'minimumGroupingDigits', 'fallback8bitEncoding',
		'linkPrefixExtension', 'linkTrail', 'linkPrefixCharset',
		'namespaceAliases', 'dateFormats', 'datePreferences',
		'datePreferenceMigrationMap', 'defaultDateFormat',
		'specialPageAliases', 'imageFiles', 'preloadedMessages',
		'namespaceGenderAliases', 'digitGroupingPattern', 'pluralRules',
		'pluralRuleTypes', 'compiledPluralRules', 'formalityIndex',
	];

	/**
	 * Keys for items that can only be set in the core message files,
	 * not in extensions. Assignments to these keys in extension messages files
	 * are silently ignored.
	 *
	 * @since 1.41
	 */
	private const CORE_ONLY_KEYS = [
		'fallback', 'rtl', 'digitTransformTable', 'separatorTransformTable',
		'minimumGroupingDigits', 'fallback8bitEncoding', 'linkPrefixExtension',
		'linkTrail', 'linkPrefixCharset', 'datePreferences',
		'datePreferenceMigrationMap', 'defaultDateFormat', 'digitGroupingPattern',
		'formalityIndex',
	];

	/**
	 * ALL_KEYS - CORE_ONLY_KEYS. All of these can technically be set
	 * both in core and in extension messages files,
	 * though this is not necessarily useful for all these keys.
	 * Some of these keys are mergeable too.
	 *
	 * @since 1.41
	 */
	private const ALL_EXCEPT_CORE_ONLY_KEYS = [
		'namespaceNames', 'bookstoreList', 'magicWords', 'messages',
		'namespaceAliases', 'dateFormats', 'specialPageAliases',
		'imageFiles', 'preloadedMessages', 'namespaceGenderAliases',
		'pluralRules', 'pluralRuleTypes', 'compiledPluralRules',
	];

	/** Keys for items which can be localized. */
	public const ALL_ALIAS_KEYS = [ 'specialPageAliases' ];

	/**
	 * Keys for items which consist of associative arrays, which may be merged
	 * by a fallback sequence.
	 */
	private const MERGEABLE_MAP_KEYS = [ 'messages', 'namespaceNames',
		'namespaceAliases', 'dateFormats', 'imageFiles', 'preloadedMessages'
	];

	/**
	 * Keys for items which contain an array of arrays of equivalent aliases
	 * for each subitem. The aliases may be merged by a fallback sequence.
	 */
	private const MERGEABLE_ALIAS_LIST_KEYS = [ 'specialPageAliases' ];

	/**
	 * Keys for items which contain an associative array, and may be merged if
	 * the primary value contains the special array key "inherit". That array
	 * key is removed after the first merge.
	 */
	private const OPTIONAL_MERGE_KEYS = [ 'bookstoreList' ];

	/**
	 * Keys for items that are formatted like $magicWords
	 */
	private const MAGIC_WORD_KEYS = [ 'magicWords' ];

	/**
	 * Keys for items where the subitems are stored in the backend separately.
	 */
	private const SPLIT_KEYS = [ 'messages' ];

	/**
	 * Keys for items that will be prefixed with its source language code,
	 * which should be stripped out when loading from cache.
	 */
	private const SOURCE_PREFIX_KEYS = [ 'messages' ];

	/**
	 * Separator for the source language prefix.
	 */
	private const SOURCEPREFIX_SEPARATOR = ':';

	/**
	 * Keys which are loaded automatically by initLanguage()
	 */
	private const PRELOADED_KEYS = [ 'dateFormats', 'namespaceNames' ];

	private const PLURAL_FILES = [
		// Load CLDR plural rules
		MW_INSTALL_PATH . '/languages/data/plurals.xml',
		// Override or extend with MW-specific rules
		MW_INSTALL_PATH . '/languages/data/plurals-mediawiki.xml',
	];

	/**
	 * Associative array of cached plural rules. The key is the language code,
	 * the value is an array of plural rules for that language.
	 *
	 * @var array<string,array<int,string>>|null
	 */
	private static $pluralRules = null;

	/**
	 * Associative array of cached plural rule types. The key is the language
	 * code, the value is an array of plural rule types for that language. For
	 * example, $pluralRuleTypes['ar'] = ['zero', 'one', 'two', 'few', 'many'].
	 * The index for each rule type matches the index for the rule in
	 * $pluralRules, thus allowing correlation between the two. The reason we
	 * don't just use the type names as the keys in $pluralRules is because
	 * Language::convertPlural applies the rules based on numeric order (or
	 * explicit numeric parameter), not based on the name of the rule type. For
	 * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than
	 * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}.
	 *
	 * @var array<string,array<int,string>>|null
	 */
	private static $pluralRuleTypes = null;

	/**
	 * Return a suitable LCStore as specified by the given configuration.
	 *
	 * @since 1.34
	 * @param array $conf In the format of $wgLocalisationCacheConf
	 * @param string|false|null $fallbackCacheDir In case 'storeDirectory' isn't specified
	 * @return LCStore
	 */
	public static function getStoreFromConf( array $conf, $fallbackCacheDir ): LCStore {
		$storeArg = [];
		$storeArg['directory'] =
			$conf['storeDirectory'] ?: $fallbackCacheDir;

		if ( !empty( $conf['storeClass'] ) ) {
			$storeClass = $conf['storeClass'];
		} elseif ( $conf['store'] === 'files' || $conf['store'] === 'file' ||
			( $conf['store'] === 'detect' && $storeArg['directory'] )
		) {
			$storeClass = LCStoreCDB::class;
		} elseif ( $conf['store'] === 'db' || $conf['store'] === 'detect' ) {
			$storeClass = LCStoreDB::class;
			$storeArg['server'] = $conf['storeServer'] ?? [];
		} elseif ( $conf['store'] === 'array' ) {
			$storeClass = LCStoreStaticArray::class;
		} else {
			throw new ConfigException(
				'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.'
			);
		}

		return new $storeClass( $storeArg );
	}

	/**
	 * @internal For use by ServiceWiring
	 */
	public const CONSTRUCTOR_OPTIONS = [
		// True to treat all files as expired until they are regenerated by this object.
		'forceRecache',
		'manualRecache',
		MainConfigNames::ExtensionMessagesFiles,
		MainConfigNames::MessagesDirs,
		MainConfigNames::TranslationAliasesDirs,
	];

	/**
	 * For constructor parameters, @ref \MediaWiki\MainConfigSchema::LocalisationCacheConf.
	 *
	 * @internal Do not construct directly, use MediaWikiServices instead.
	 * @param ServiceOptions $options
	 * @param LCStore $store What backend to use for storage
	 * @param LoggerInterface $logger
	 * @param callable[] $clearStoreCallbacks To be called whenever the cache is cleared. Can be
	 *   used to clear other caches that depend on this one, such as ResourceLoader's
	 *   MessageBlobStore.
	 * @param LanguageNameUtils $langNameUtils
	 * @param HookContainer $hookContainer
	 */
	public function __construct(
		ServiceOptions $options,
		LCStore $store,
		LoggerInterface $logger,
		array $clearStoreCallbacks,
		LanguageNameUtils $langNameUtils,
		HookContainer $hookContainer
	) {
		$options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );

		$this->options = $options;
		$this->store = $store;
		$this->logger = $logger;
		$this->clearStoreCallbacks = $clearStoreCallbacks;
		$this->langNameUtils = $langNameUtils;
		$this->hookRunner = new HookRunner( $hookContainer );

		// Keep this separate from $this->options so that it can be mutable
		$this->manualRecache = $options->get( 'manualRecache' );
	}

	/**
	 * Returns true if the given key is mergeable, that is, if it is an associative
	 * array which can be merged through a fallback sequence.
	 * @param string $key
	 * @return bool
	 */
	private static function isMergeableKey( string $key ): bool {
		static $mergeableKeys;
		$mergeableKeys ??= array_fill_keys( [
			...self::MERGEABLE_MAP_KEYS,
			...self::MERGEABLE_ALIAS_LIST_KEYS,
			...self::OPTIONAL_MERGE_KEYS,
			...self::MAGIC_WORD_KEYS,
		], true );
		return isset( $mergeableKeys[$key] );
	}

	/**
	 * Get a cache item.
	 *
	 * Warning: this may be slow for split items (messages), since it will
	 * need to fetch all the subitems from the cache individually.
	 * @param string $code
	 * @param string $key
	 * @return mixed
	 */
	public function getItem( $code, $key ) {
		if ( !isset( $this->loadedItems[$code][$key] ) ) {
			$this->loadItem( $code, $key );
		}

		if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) {
			return $this->shallowFallbacks[$code];
		}

		// @phan-suppress-next-line PhanTypeArraySuspiciousNullable
		return $this->data[$code][$key];
	}

	/**
	 * Get a subitem, for instance a single message for a given language.
	 * @param string $code
	 * @param string $key
	 * @param string $subkey
	 * @return mixed|null
	 */
	public function getSubitem( $code, $key, $subkey ) {
		if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
			!isset( $this->loadedItems[$code][$key] )
		) {
			$this->loadSubitem( $code, $key, $subkey );
		}

		return $this->data[$code][$key][$subkey] ?? null;
	}

	/**
	 * Get a subitem with its source language. Only supports messages for now.
	 *
	 * @since 1.41
	 * @param string $code
	 * @param string $key
	 * @param string $subkey
	 * @return string[]|null Return [ subitem, sourceLanguage ] if the subitem is defined.
	 */
	public function getSubitemWithSource( $code, $key, $subkey ) {
		$subitem = $this->getSubitem( $code, $key, $subkey );
		// Undefined in the backend.
		if ( $subitem === null ) {
			return null;
		}

		// The source language should have been set, but to avoid a Phan error and to be double sure.
		return [ $subitem, $this->sourceLanguage[$code][$key][$subkey] ?? $code ];
	}

	/**
	 * Get the list of subitem keys for a given item.
	 *
	 * This is faster than array_keys($lc->getItem(...)) for the items listed in
	 * self::SPLIT_KEYS.
	 *
	 * Will return null if the item is not found, or false if the item is not an
	 * array.
	 *
	 * @param string $code
	 * @param string $key
	 * @return bool|null|string|string[]
	 */
	public function getSubitemList( $code, $key ) {
		if ( in_array( $key, self::SPLIT_KEYS ) ) {
			return $this->getSubitem( $code, 'list', $key );
		} else {
			$item = $this->getItem( $code, $key );
			if ( is_array( $item ) ) {
				return array_keys( $item );
			} else {
				return false;
			}
		}
	}

	/**
	 * Load an item into the cache.
	 *
	 * @param string $code
	 * @param string $key
	 */
	private function loadItem( $code, $key ) {
		if ( isset( $this->loadedItems[$code][$key] ) ) {
			return;
		}

		if (
			in_array( $key, self::CORE_ONLY_KEYS, true ) ||
			// "synthetic" keys added by loadCoreData based on "fallback"
			$key === 'fallbackSequence' ||
			$key === 'originalFallbackSequence'
		) {
			if ( $this->langNameUtils->isValidBuiltInCode( $code ) ) {
				$this->loadCoreData( $code );
				return;
			}
		}

		if ( !isset( $this->initialisedLangs[$code] ) ) {
			$this->initLanguage( $code );

			// Check to see if initLanguage() loaded it for us
			if ( isset( $this->loadedItems[$code][$key] ) ) {
				return;
			}
		}

		if ( isset( $this->shallowFallbacks[$code] ) ) {
			$this->loadItem( $this->shallowFallbacks[$code], $key );

			return;
		}

		if ( in_array( $key, self::SPLIT_KEYS ) ) {
			$subkeyList = $this->getSubitem( $code, 'list', $key );
			foreach ( $subkeyList as $subkey ) {
				if ( isset( $this->data[$code][$key][$subkey] ) ) {
					continue;
				}
				$this->loadSubitem( $code, $key, $subkey );
			}
		} else {
			$this->data[$code][$key] = $this->store->get( $code, $key );
		}

		$this->loadedItems[$code][$key] = true;
	}

	/**
	 * Load a subitem into the cache.
	 *
	 * @param string $code
	 * @param string $key
	 * @param string $subkey
	 */
	private function loadSubitem( $code, $key, $subkey ) {
		if ( !in_array( $key, self::SPLIT_KEYS ) ) {
			$this->loadItem( $code, $key );

			return;
		}

		if ( !isset( $this->initialisedLangs[$code] ) ) {
			$this->initLanguage( $code );
		}

		// Check to see if initLanguage() loaded it for us
		if ( isset( $this->loadedItems[$code][$key] ) ||
			isset( $this->loadedSubitems[$code][$key][$subkey] )
		) {
			return;
		}

		if ( isset( $this->shallowFallbacks[$code] ) ) {
			$this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );

			return;
		}

		$value = $this->store->get( $code, "$key:$subkey" );
		if ( $value !== null && in_array( $key, self::SOURCE_PREFIX_KEYS ) ) {
			[
				$this->sourceLanguage[$code][$key][$subkey],
				$this->data[$code][$key][$subkey]
			] = explode( self::SOURCEPREFIX_SEPARATOR, $value, 2 );
		} else {
			$this->data[$code][$key][$subkey] = $value;
		}

		$this->loadedSubitems[$code][$key][$subkey] = true;
	}

	/**
	 * Returns true if the cache identified by $code is missing or expired.
	 *
	 * @param string $code
	 *
	 * @return bool
	 */
	public function isExpired( $code ) {
		if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) {
			$this->logger->debug( __METHOD__ . "($code): forced reload" );

			return true;
		}

		$deps = $this->store->get( $code, 'deps' );
		$keys = $this->store->get( $code, 'list' );
		$preload = $this->store->get( $code, 'preload' );
		// Different keys may expire separately for some stores
		if ( $deps === null || $keys === null || $preload === null ) {
			$this->logger->debug( __METHOD__ . "($code): cache missing, need to make one" );

			return true;
		}

		foreach ( $deps as $dep ) {
			// Because we're unserializing stuff from cache, we
			// could receive objects of classes that don't exist
			// anymore (e.g., uninstalled extensions)
			// When this happens, always expire the cache
			if ( !$dep instanceof CacheDependency || $dep->isExpired() ) {
				$this->logger->debug( __METHOD__ . "($code): cache for $code expired due to " .
					get_class( $dep ) );

				return true;
			}
		}

		return false;
	}

	/**
	 * Initialise a language in this object. Rebuild the cache if necessary.
	 *
	 * @param string $code
	 */
	private function initLanguage( $code ) {
		if ( isset( $this->initialisedLangs[$code] ) ) {
			return;
		}

		$this->initialisedLangs[$code] = true;

		# If the code is of the wrong form for a Messages*.php file, do a shallow fallback
		if ( !$this->langNameUtils->isValidBuiltInCode( $code ) ) {
			$this->initShallowFallback( $code, 'en' );

			return;
		}

		# Re-cache the data if necessary
		if ( !$this->manualRecache && $this->isExpired( $code ) ) {
			if ( $this->langNameUtils->isSupportedLanguage( $code ) ) {
				$this->recache( $code );
			} elseif ( $code === 'en' ) {
				throw new RuntimeException( 'MessagesEn.php is missing.' );
			} else {
				$this->initShallowFallback( $code, 'en' );
			}

			return;
		}

		# Preload some stuff
		$preload = $this->getItem( $code, 'preload' );
		if ( $preload === null ) {
			if ( $this->manualRecache ) {
				// No Messages*.php file. Do shallow fallback to en.
				if ( $code === 'en' ) {
					throw new RuntimeException( 'No localisation cache found for English. ' .
						'Please run maintenance/rebuildLocalisationCache.php.' );
				}
				$this->initShallowFallback( $code, 'en' );

				return;
			} else {
				throw new RuntimeException( 'Invalid or missing localisation cache.' );
			}
		}

		foreach ( self::SOURCE_PREFIX_KEYS as $key ) {
			if ( !isset( $preload[$key] ) ) {
				continue;
			}
			foreach ( $preload[$key] as $subkey => $value ) {
				if ( $value !== null ) {
					[
						$this->sourceLanguage[$code][$key][$subkey],
						$preload[$key][$subkey]
					] = explode( self::SOURCEPREFIX_SEPARATOR, $value, 2 );
				} else {
					$preload[$key][$subkey] = null;
				}
			}
		}

		if ( isset( $this->data[$code] ) ) {
			foreach ( $preload as $key => $value ) {
				// @phan-suppress-next-line PhanTypeArraySuspiciousNullable -- see isset() above
				$this->mergeItem( $key, $this->data[$code][$key], $value );
			}
		} else {
			$this->data[$code] = $preload;
		}
		foreach ( $preload as $key => $item ) {
			if ( in_array( $key, self::SPLIT_KEYS ) ) {
				foreach ( $item as $subkey => $subitem ) {
					$this->loadedSubitems[$code][$key][$subkey] = true;
				}
			} else {
				$this->loadedItems[$code][$key] = true;
			}
		}
	}

	/**
	 * Create a fallback from one language to another, without creating a
	 * complete persistent cache.
	 *
	 * @param string $primaryCode
	 * @param string $fallbackCode
	 */
	private function initShallowFallback( $primaryCode, $fallbackCode ) {
		$this->data[$primaryCode] =& $this->data[$fallbackCode];
		$this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode];
		$this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode];
		$this->shallowFallbacks[$primaryCode] = $fallbackCode;
		$this->coreDataLoaded[$primaryCode] =& $this->coreDataLoaded[$fallbackCode];
	}

	/**
	 * Read a PHP file containing localisation data.
	 *
	 * @param string $_fileName
	 * @param string $_fileType
	 * @return array
	 */
	protected function readPHPFile( $_fileName, $_fileType ) {
		include $_fileName;

		$data = [];
		if ( $_fileType == 'core' ) {
			foreach ( self::ALL_KEYS as $key ) {
				// Not all keys are set in language files, so
				// check they exist first
				if ( isset( $$key ) ) {
					$data[$key] = $$key;
				}
			}
		} elseif ( $_fileType == 'extension' ) {
			foreach ( self::ALL_EXCEPT_CORE_ONLY_KEYS as $key ) {
				if ( isset( $$key ) ) {
					$data[$key] = $$key;
				}
			}
		} elseif ( $_fileType == 'aliases' ) {
			// @phan-suppress-next-line PhanImpossibleCondition May be set in the included file
			if ( isset( $aliases ) ) {
				$data['aliases'] = $aliases;
			}
		} else {
			throw new InvalidArgumentException( __METHOD__ . ": Invalid file type: $_fileType" );
		}

		return $data;
	}

	/**
	 * Read a JSON file containing localisation messages.
	 *
	 * @param string $fileName Name of file to read
	 * @return array Array with a 'messages' key, or empty array if the file doesn't exist
	 */
	private function readJSONFile( $fileName ) {
		if ( !is_readable( $fileName ) ) {
			return [];
		}

		$json = file_get_contents( $fileName );
		if ( $json === false ) {
			return [];
		}

		$data = FormatJson::decode( $json, true );
		if ( $data === null ) {
			throw new RuntimeException( __METHOD__ . ": Invalid JSON file: $fileName" );
		}

		// Remove keys starting with '@'; they are reserved for metadata and non-message data
		foreach ( $data as $key => $unused ) {
			if ( $key === '' || $key[0] === '@' ) {
				unset( $data[$key] );
			}
		}

		return $data;
	}

	/**
	 * Get the compiled plural rules for a given language from the XML files.
	 *
	 * @since 1.20
	 * @param string $code
	 * @return array<int,string>|null
	 */
	private function getCompiledPluralRules( $code ) {
		$rules = $this->getPluralRules( $code );
		if ( $rules === null ) {
			return null;
		}
		try {
			$compiledRules = Evaluator::compile( $rules );
		} catch ( CLDRPluralRuleError $e ) {
			$this->logger->debug( $e->getMessage() );

			return [];
		}

		return $compiledRules;
	}

	/**
	 * Get the plural rules for a given language from the XML files.
	 *
	 * Cached.
	 *
	 * @since 1.20
	 * @param string $code
	 * @return array<int,string>|null
	 */
	private function getPluralRules( $code ) {
		if ( self::$pluralRules === null ) {
			self::loadPluralFiles();
		}
		return self::$pluralRules[$code] ?? null;
	}

	/**
	 * Get the plural rule types for a given language from the XML files.
	 *
	 * Cached.
	 *
	 * @since 1.22
	 * @param string $code
	 * @return array<int,string>|null
	 */
	private function getPluralRuleTypes( $code ) {
		if ( self::$pluralRuleTypes === null ) {
			self::loadPluralFiles();
		}
		return self::$pluralRuleTypes[$code] ?? null;
	}

	/**
	 * Load the plural XML files.
	 */
	private static function loadPluralFiles() {
		foreach ( self::PLURAL_FILES as $fileName ) {
			self::loadPluralFile( $fileName );
		}
	}

	/**
	 * Load a plural XML file with the given filename, compile the relevant
	 * rules, and save the compiled rules in a process-local cache.
	 *
	 * @param string $fileName
	 */
	private static function loadPluralFile( $fileName ) {
		// Use file_get_contents instead of DOMDocument::load (T58439)
		$xml = file_get_contents( $fileName );
		if ( !$xml ) {
			throw new RuntimeException( "Unable to read plurals file $fileName" );
		}
		$doc = new DOMDocument;
		$doc->loadXML( $xml );
		$rulesets = $doc->getElementsByTagName( "pluralRules" );
		foreach ( $rulesets as $ruleset ) {
			$codes = $ruleset->getAttribute( 'locales' );
			$rules = [];
			$ruleTypes = [];
			$ruleElements = $ruleset->getElementsByTagName( "pluralRule" );
			foreach ( $ruleElements as $elt ) {
				$ruleType = $elt->getAttribute( 'count' );
				if ( $ruleType === 'other' ) {
					// Don't record "other" rules, which have an empty condition
					continue;
				}
				$rules[] = $elt->nodeValue;
				$ruleTypes[] = $ruleType;
			}
			foreach ( explode( ' ', $codes ) as $code ) {
				self::$pluralRules[$code] = $rules;
				self::$pluralRuleTypes[$code] = $ruleTypes;
			}
		}
	}

	/**
	 * Read the data from the source files for a given language, and register
	 * the relevant dependencies in the $deps array.
	 *
	 * @param string $code
	 * @param array &$deps
	 * @return array
	 */
	private function readSourceFilesAndRegisterDeps( $code, &$deps ) {
		// This reads in the PHP i18n file with non-messages l10n data
		$fileName = $this->langNameUtils->getMessagesFileName( $code );
		if ( !is_file( $fileName ) ) {
			$data = [];
		} else {
			$deps[] = new FileDependency( $fileName );
			$data = $this->readPHPFile( $fileName, 'core' );
		}

		return $data;
	}

	/**
	 * Read and compile the plural data for a given language,
	 * and register the relevant dependencies in the $deps array.
	 *
	 * @param string $code
	 * @param array &$deps
	 * @return array
	 */
	private function readPluralFilesAndRegisterDeps( $code, &$deps ) {
		$data = [
			// Load CLDR plural rules for JavaScript
			'pluralRules' => $this->getPluralRules( $code ),
			// And for PHP
			'compiledPluralRules' => $this->getCompiledPluralRules( $code ),
			// Load plural rule types
			'pluralRuleTypes' => $this->getPluralRuleTypes( $code ),
		];

		foreach ( self::PLURAL_FILES as $fileName ) {
			$deps[] = new FileDependency( $fileName );
		}

		return $data;
	}

	/**
	 * Merge two localisation values, a primary and a fallback, overwriting the
	 * primary value in place.
	 *
	 * @param string $key
	 * @param mixed &$value
	 * @param mixed $fallbackValue
	 */
	private function mergeItem( $key, &$value, $fallbackValue ) {
		if ( $value !== null ) {
			if ( $fallbackValue !== null ) {
				if ( in_array( $key, self::MERGEABLE_MAP_KEYS ) ) {
					$value += $fallbackValue;
				} elseif ( in_array( $key, self::MERGEABLE_ALIAS_LIST_KEYS ) ) {
					$value = array_merge_recursive( $value, $fallbackValue );
				} elseif ( in_array( $key, self::OPTIONAL_MERGE_KEYS ) ) {
					if ( !empty( $value['inherit'] ) ) {
						$value = array_merge( $fallbackValue, $value );
					}

					unset( $value['inherit'] );
				} elseif ( in_array( $key, self::MAGIC_WORD_KEYS ) ) {
					$this->mergeMagicWords( $value, $fallbackValue );
				}
			}
		} else {
			$value = $fallbackValue;
		}
	}

	/**
	 * @param array &$value
	 * @param array $fallbackValue
	 */
	private function mergeMagicWords( array &$value, array $fallbackValue ): void {
		foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
			if ( !isset( $value[$magicName] ) ) {
				$value[$magicName] = $fallbackInfo;
			} else {
				$value[$magicName] = [
					$fallbackInfo[0],
					...array_unique( [
						// First value is 1 if the magic word is case-sensitive, 0 if not
						...array_slice( $value[$magicName], 1 ),
						...array_slice( $fallbackInfo, 1 ),
					] )
				];
			}
		}
	}

	/**
	 * Gets the combined list of messages dirs from
	 * core and extensions
	 *
	 * @since 1.25
	 * @return array
	 */
	public function getMessagesDirs() {
		global $IP;

		return [
			'core' => "$IP/languages/i18n",
			'codex' => "$IP/languages/i18n/codex",
			'exif' => "$IP/languages/i18n/exif",
			'preferences' => "$IP/languages/i18n/preferences",
			'api' => "$IP/includes/api/i18n",
			'rest' => "$IP/includes/Rest/i18n",
			'oojs-ui' => "$IP/resources/lib/ooui/i18n",
			'paramvalidator' => "$IP/includes/libs/ParamValidator/i18n",
		] + $this->options->get( MainConfigNames::MessagesDirs );
	}

	/**
	 * Load the core localisation data for a given language code,
	 * without extensions, using only the process cache.
	 * See {@link self::$coreDataLoaded} for what this guarantees.
	 *
	 * In addition to the core-only keys,
	 * {@link self::$data} may contain additional entries for $code,
	 * but those must not be used outside of {@link self::recache()}
	 * (and accordingly, they are not marked as loaded yet).
	 */
	private function loadCoreData( string $code ) {
		if ( !$code ) {
			throw new InvalidArgumentException( "Invalid language code requested" );
		}
		if ( $this->coreDataLoaded[$code] ?? false ) {
			return;
		}

		$coreData = array_fill_keys( self::CORE_ONLY_KEYS, null );
		$deps = [];

		# Load the primary localisation from the source file
		$data = $this->readSourceFilesAndRegisterDeps( $code, $deps );
		$this->logger->debug( __METHOD__ . ": got localisation for $code from source" );

		# Merge primary localisation
		foreach ( $data as $key => $value ) {
			$this->mergeItem( $key, $coreData[ $key ], $value );
		}

		# Fill in the fallback if it's not there already
		// @phan-suppress-next-line PhanSuspiciousValueComparison
		if ( ( $coreData['fallback'] === null || $coreData['fallback'] === false ) && $code === 'en' ) {
			$coreData['fallback'] = false;
			$coreData['originalFallbackSequence'] = $coreData['fallbackSequence'] = [];
		} else {
			if ( $coreData['fallback'] !== null ) {
				$coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) );
			} else {
				$coreData['fallbackSequence'] = [];
			}
			$len = count( $coreData['fallbackSequence'] );

			# Before we add the 'en' fallback for messages, keep a copy of
			# the original fallback sequence
			$coreData['originalFallbackSequence'] = $coreData['fallbackSequence'];

			# Ensure that the sequence ends at 'en' for messages
			if ( !$len || $coreData['fallbackSequence'][$len - 1] !== 'en' ) {
				$coreData['fallbackSequence'][] = 'en';
			}
		}

		foreach ( $coreData['fallbackSequence'] as $fbCode ) {
			// load core fallback data
			$fbData = $this->readSourceFilesAndRegisterDeps( $fbCode, $deps );
			foreach ( self::CORE_ONLY_KEYS as $key ) {
				// core-only keys are not mergeable, only set if not present in core data yet
				if ( isset( $fbData[$key] ) && !isset( $coreData[$key] ) ) {
					$coreData[$key] = $fbData[$key];
				}
			}
		}

		$coreData['deps'] = $deps;
		foreach ( $coreData as $key => $item ) {
			$this->data[$code][$key] ??= null;
			// @phan-suppress-next-line PhanTypeArraySuspiciousNullable -- we just set a default null
			$this->mergeItem( $key, $this->data[$code][$key], $item );
			if (
				in_array( $key, self::CORE_ONLY_KEYS, true ) ||
				// "synthetic" keys based on "fallback" (see above)
				$key === 'fallbackSequence' ||
				$key === 'originalFallbackSequence'
			) {
				// only mark core-only keys as loaded;
				// we may have loaded additional ones from the source file,
				// but they are not fully loaded yet, since recache()
				// may have to merge in additional values from fallback languages
				$this->loadedItems[$code][$key] = true;
			}
		}

		$this->coreDataLoaded[$code] = true;
	}

	/**
	 * Load localisation data for a given language for both core and extensions
	 * and save it to the persistent cache store and the process cache.
	 *
	 * @param string $code
	 */
	public function recache( $code ) {
		if ( !$code ) {
			throw new InvalidArgumentException( "Invalid language code requested" );
		}
		$this->recachedLangs[ $code ] = true;

		# Initial values
		$initialData = array_fill_keys( self::ALL_KEYS, null );
		$this->data[$code] = [];
		$this->loadedItems[$code] = [];
		$this->loadedSubitems[$code] = [];
		$this->coreDataLoaded[$code] = false;
		$this->loadCoreData( $code );
		$coreData = $this->data[$code];
		// @phan-suppress-next-line PhanTypeArraySuspiciousNullable -- guaranteed by loadCoreData()
		$deps = $coreData['deps'];
		$coreData += $this->readPluralFilesAndRegisterDeps( $code, $deps );

		$codeSequence = array_merge( [ $code ], $coreData['fallbackSequence'] );
		$messageDirs = $this->getMessagesDirs();
		$translationAliasesDirs = $this->options->get( MainConfigNames::TranslationAliasesDirs );

		# Load non-JSON localisation data for extensions
		$extensionData = array_fill_keys( $codeSequence, $initialData );
		foreach ( $this->options->get( MainConfigNames::ExtensionMessagesFiles ) as $extension => $fileName ) {
			if ( isset( $messageDirs[$extension] ) || isset( $translationAliasesDirs[$extension] ) ) {
				# This extension has JSON message data; skip the PHP shim
				continue;
			}

			$data = $this->readPHPFile( $fileName, 'extension' );
			$used = false;

			foreach ( $data as $key => $item ) {
				foreach ( $codeSequence as $csCode ) {
					if ( isset( $item[$csCode] ) ) {
						// Keep the behaviour the same as for json messages.
						// TODO: Consider deprecating using a PHP file for messages.
						if ( in_array( $key, self::SOURCE_PREFIX_KEYS ) ) {
							foreach ( $item[$csCode] as $subkey => $_ ) {
								$this->sourceLanguage[$code][$key][$subkey] ??= $csCode;
							}
						}
						$this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
						$used = true;
					}
				}
			}

			if ( $used ) {
				$deps[] = new FileDependency( $fileName );
			}
		}

		# Load the localisation data for each fallback, then merge it into the full array
		$allData = $initialData;
		foreach ( $codeSequence as $csCode ) {
			$csData = $initialData;

			# Load core messages and the extension localisations.
			foreach ( $messageDirs as $dirs ) {
				foreach ( (array)$dirs as $dir ) {
					$fileName = "$dir/$csCode.json";
					$messages = $this->readJSONFile( $fileName );

					foreach ( $messages as $subkey => $_ ) {
						$this->sourceLanguage[$code]['messages'][$subkey] ??= $csCode;
					}
					$this->mergeItem( 'messages', $csData['messages'], $messages );

					$deps[] = new FileDependency( $fileName );
				}
			}

			foreach ( $translationAliasesDirs as $dirs ) {
				foreach ( (array)$dirs as $dir ) {
					$fileName = "$dir/$csCode.json";
					$data = $this->readJSONFile( $fileName );

					foreach ( $data as $key => $item ) {
						// We allow the key in the JSON to be specified in PascalCase similar to key definitions in
						// extension.json, but eventually they are stored in camelCase
						$normalizedKey = lcfirst( $key );

						if ( $normalizedKey === '@metadata' ) {
							// Don't store @metadata information in extension data.
							continue;
						}

						if ( !in_array( $normalizedKey, self::ALL_ALIAS_KEYS ) ) {
							throw new UnexpectedValueException(
								"Invalid key: \"$key\" for " . MainConfigNames::TranslationAliasesDirs . ". " .
								'Valid keys: ' . implode( ', ', self::ALL_ALIAS_KEYS )
							);
						}

						$this->mergeItem( $normalizedKey, $extensionData[$csCode][$normalizedKey], $item );
					}

					$deps[] = new FileDependency( $fileName );
				}
			}

			# Merge non-JSON extension data
			if ( isset( $extensionData[$csCode] ) ) {
				foreach ( $extensionData[$csCode] as $key => $item ) {
					$this->mergeItem( $key, $csData[$key], $item );
				}
			}

			if ( $csCode === $code ) {
				# Merge core data into extension data
				foreach ( $coreData as $key => $item ) {
					$this->mergeItem( $key, $csData[$key], $item );
				}
			} else {
				# Load the secondary localisation from the source file to
				# avoid infinite cycles on cyclic fallbacks
				$fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
				$fbData += $this->readPluralFilesAndRegisterDeps( $csCode, $deps );
				# Only merge the keys that make sense to merge
				foreach ( self::ALL_KEYS as $key ) {
					if ( !isset( $fbData[ $key ] ) ) {
						continue;
					}

					if ( !isset( $coreData[ $key ] ) || self::isMergeableKey( $key ) ) {
						$this->mergeItem( $key, $csData[ $key ], $fbData[ $key ] );
					}
				}
			}

			# Allow extensions an opportunity to adjust the data for this fallback
			$this->hookRunner->onLocalisationCacheRecacheFallback( $this, $csCode, $csData );

			# Merge the data for this fallback into the final array
			if ( $csCode === $code ) {
				$allData = $csData;
			} else {
				foreach ( self::ALL_KEYS as $key ) {
					if ( !isset( $csData[$key] ) ) {
						continue;
					}

					// @phan-suppress-next-line PhanTypeArraySuspiciousNullable
					if ( $allData[$key] === null || self::isMergeableKey( $key ) ) {
						$this->mergeItem( $key, $allData[$key], $csData[$key] );
					}
				}
			}
		}

		if ( !isset( $allData['rtl'] ) ) {
			throw new RuntimeException( __METHOD__ . ': Localisation data failed validation check! ' .
				'Check that your languages/messages/MessagesEn.php file is intact.' );
		}

		// Add cache dependencies for any referenced configs
		// We use the keys prefixed with 'wg' for historical reasons.
		$deps['wgExtensionMessagesFiles'] =
			new MainConfigDependency( MainConfigNames::ExtensionMessagesFiles );
		$deps['wgMessagesDirs'] =
			new MainConfigDependency( MainConfigNames::MessagesDirs );
		$deps['version'] = new ConstantDependency( self::class . '::VERSION' );

		# Add dependencies to the cache entry
		$allData['deps'] = $deps;

		# Replace spaces with underscores in namespace names
		$allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] );

		# And do the same for special page aliases. $page is an array.
		foreach ( $allData['specialPageAliases'] as &$page ) {
			$page = str_replace( ' ', '_', $page );
		}
		# Decouple the reference to prevent accidental damage
		unset( $page );

		# If there were no plural rules, return an empty array
		$allData['pluralRules'] ??= [];
		$allData['compiledPluralRules'] ??= [];
		# If there were no plural rule types, return an empty array
		$allData['pluralRuleTypes'] ??= [];

		# Set the list keys
		$allData['list'] = [];
		foreach ( self::SPLIT_KEYS as $key ) {
			$allData['list'][$key] = array_keys( $allData[$key] );
		}
		# Run hooks
		$unused = true; // Used to be $purgeBlobs, removed in 1.34
		$this->hookRunner->onLocalisationCacheRecache( $this, $code, $allData, $unused );

		# Save to the process cache and register the items loaded
		$this->data[$code] = $allData;
		$this->loadedItems[$code] = [];
		$this->loadedSubitems[$code] = [];
		foreach ( $allData as $key => $item ) {
			$this->loadedItems[$code][$key] = true;
		}

		# Prefix each item with its source language code before save
		foreach ( self::SOURCE_PREFIX_KEYS as $key ) {
			// @phan-suppress-next-line PhanTypeArraySuspiciousNullable
			foreach ( $allData[$key] as $subKey => $value ) {
				// The source language should have been set, but to avoid Phan error and be double sure.
				$allData[$key][$subKey] = ( $this->sourceLanguage[$code][$key][$subKey] ?? $code ) .
					self::SOURCEPREFIX_SEPARATOR . $value;
			}
		}

		# Set the preload key
		$allData['preload'] = $this->buildPreload( $allData );

		# Save to the persistent cache
		$this->store->startWrite( $code );
		foreach ( $allData as $key => $value ) {
			if ( in_array( $key, self::SPLIT_KEYS ) ) {
				foreach ( $value as $subkey => $subvalue ) {
					$this->store->set( "$key:$subkey", $subvalue );
				}
			} else {
				$this->store->set( $key, $value );
			}
		}
		$this->store->finishWrite();

		# Clear out the MessageBlobStore
		# HACK: If using a null (i.e., disabled) storage backend, we
		# can't write to the MessageBlobStore either
		if ( !$this->store instanceof LCStoreNull ) {
			foreach ( $this->clearStoreCallbacks as $callback ) {
				$callback();
			}
		}
	}

	/**
	 * Build the preload item from the given pre-cache data.
	 *
	 * The preload item will be loaded automatically, improving performance
	 * for the commonly requested items it contains.
	 *
	 * @param array $data
	 * @return array
	 */
	private function buildPreload( $data ) {
		$preload = [ 'messages' => [] ];
		foreach ( self::PRELOADED_KEYS as $key ) {
			$preload[$key] = $data[$key];
		}

		foreach ( $data['preloadedMessages'] as $subkey ) {
			$subitem = $data['messages'][$subkey] ?? null;
			$preload['messages'][$subkey] = $subitem;
		}

		return $preload;
	}

	/**
	 * Unload the data for a given language from the object cache.
	 *
	 * Reduces memory usage.
	 *
	 * @param string $code
	 */
	public function unload( $code ) {
		unset( $this->data[$code] );
		unset( $this->loadedItems[$code] );
		unset( $this->loadedSubitems[$code] );
		unset( $this->initialisedLangs[$code] );
		unset( $this->shallowFallbacks[$code] );
		unset( $this->sourceLanguage[$code] );
		unset( $this->coreDataLoaded[$code] );

		foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) {
			if ( $fbCode === $code ) {
				$this->unload( $shallowCode );
			}
		}
	}

	/**
	 * Unload all data
	 */
	public function unloadAll() {
		foreach ( $this->initialisedLangs as $lang => $unused ) {
			$this->unload( $lang );
		}
	}

	/**
	 * Disable the storage backend
	 */
	public function disableBackend() {
		$this->store = new LCStoreNull;
		$this->manualRecache = false;
	}
}