diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 6090a9a8646..727fd8f4eba 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -41,6 +41,15 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN appropriate privileges. Creating this user with web-install page requires oci8.privileged_connect set to On in php.ini. * Removed UserrightsChangeableGroups hook introduced in 1.14 +* Added $wgCacheDirectory, to replace $wgFileCacheDirectory, + $wgLocalMessageCache, and any other local caches which need a place to put + files. +* $wgFileCacheDirectory is no longer set to anything by default, and so either + needs to be set explicitly, or $wgCacheDirectory needs to be set instead. +* $wgLocalMessageCache has been removed. Instead, set $wgUseLocalMessageCache + to true +* Removed $wgEnableSerializedMessages and $wgCheckSerialized. Similar + functionality is now available via $wgLocalisationCacheConf. === New features in 1.16 === @@ -93,6 +102,12 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN the DBA extension is not available. * (bug 14611) Added support showing the version of the image thumbnailing engine and diff/diff3 engine. +* Introduced a new system for localisation caching. The system is based around + fast fetches of individual messages, minimising memory overhead and startup + time in the typical case. The database backend will be used by default, but + set $wgCacheDirectory to get a faster CDB-based implementation. +* Expanded the number of variables which can be set in the extension messages + files. === Bug fixes in 1.16 === diff --git a/cache/.htaccess b/cache/.htaccess new file mode 100644 index 00000000000..3a428827887 --- /dev/null +++ b/cache/.htaccess @@ -0,0 +1 @@ +Deny from all diff --git a/config/index.php b/config/index.php index b6d944b20da..c529a9133ad 100644 --- a/config/index.php +++ b/config/index.php @@ -1924,6 +1924,11 @@ if ( \$wgCommandLineMode ) { ## you can enable inline LaTeX equations: \$wgUseTeX = false; +## Set \$wgCacheDirectory to a writable directory on the web server +## to make your wiki go slightly faster. The directory should not +## be publically accessible from the web. +#\$wgCacheDirectory = \"\$IP/cache\"; + \$wgLocalInterwiki = strtolower( \$wgSitename ); \$wgLanguageCode = \"{$slconf['LanguageCode']}\"; diff --git a/docs/hooks.txt b/docs/hooks.txt index 8b82fd1f8ed..85b4963d8bd 100644 --- a/docs/hooks.txt +++ b/docs/hooks.txt @@ -833,13 +833,15 @@ $password: The password entered by the user &$result: Set this to either true (passes) or the key for a message error $user: User the password is being validated for -'LanguageGetMagic': Use this to define synonyms of magic words depending -of the language +'LanguageGetMagic': DEPRECATED, use $magicWords in a file listed in +$wgExtensionMessagesFiles instead. +Use this to define synonyms of magic words depending of the language $magicExtensions: associative array of magic words synonyms $lang: laguage code (string) -'LanguageGetSpecialPageAliases': Use to define aliases of special pages -names depending of the language +'LanguageGetSpecialPageAliases': DEPRECATED, use $specialPageAliases in a file +listed in $wgExtensionMessagesFiles instead. +Use to define aliases of special pages names depending of the language $specialPageAliases: associative array of magic words synonyms $lang: laguage code (string) @@ -900,10 +902,6 @@ completed 'ListDefinedTags': When trying to find all defined tags. &$tags: The list of tags. -'LoadAllMessages': called by MessageCache::loadAllMessages() to load extensions -messages -&$messageCache: The MessageCache object - 'LoadExtensionSchemaUpdates': called by maintenance/updaters.inc when upgrading database schema @@ -1000,13 +998,6 @@ Useful for updating caches. $title: name of the page changed. $text: new contents of the page. -'MessageNotInMwNs': When trying to get a message that isn't found in the -MediaWiki namespace (but before checking the message files) -&$message: message's content; can be changed -$lckey: message's name -$langcode: language code -$isFullKey: specifies whether $lckey is a two part key "msg/lang" - 'MonoBookTemplateToolboxEnd': Called by Monobook skin after toolbox links have been rendered (useful for adding more) Note: this is only run for the Monobook skin. To add items to the toolbox diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 0035dacd84b..f2192088ed1 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -115,6 +115,8 @@ $wgAutoloadLocalClasses = array( 'Interwiki' => 'includes/Interwiki.php', 'IP' => 'includes/IP.php', 'Job' => 'includes/JobQueue.php', + 'LCStore_DB' => 'includes/LocalisationCache.php', + 'LCStore_CDB' => 'includes/LocalisationCache.php', 'License' => 'includes/Licenses.php', 'Licenses' => 'includes/Licenses.php', 'LinkBatch' => 'includes/LinkBatch.php', @@ -122,6 +124,8 @@ $wgAutoloadLocalClasses = array( 'Linker' => 'includes/Linker.php', 'LinkFilter' => 'includes/LinkFilter.php', 'LinksUpdate' => 'includes/LinksUpdate.php', + 'LocalisationCache' => 'includes/LocalisationCache.php', + 'LocalisationCache_BulkLoad' => 'includes/LocalisationCache.php', 'LogPage' => 'includes/LogPage.php', 'LogPager' => 'includes/LogEventsList.php', 'LogEventsList' => 'includes/LogEventsList.php', diff --git a/includes/CacheDependency.php b/includes/CacheDependency.php index b050c46d95c..8bd0be49e69 100644 --- a/includes/CacheDependency.php +++ b/includes/CacheDependency.php @@ -134,6 +134,11 @@ class FileDependency extends CacheDependency { $this->timestamp = $timestamp; } + function __sleep() { + $this->loadDependencyValues(); + return array( 'filename', 'timestamp' ); + } + function loadDependencyValues() { if ( is_null( $this->timestamp ) ) { if ( !file_exists( $this->filename ) ) { diff --git a/includes/Cdb.php b/includes/Cdb.php index 20cb7e3e6e8..e7c2c00b8a8 100644 --- a/includes/Cdb.php +++ b/includes/Cdb.php @@ -13,7 +13,7 @@ abstract class CdbReader { if ( self::haveExtension() ) { return new CdbReader_DBA( $fileName ); } else { - wfDebug( 'Warning: no dba extension found, using emulation.' ); + wfDebug( "Warning: no dba extension found, using emulation.\n" ); return new CdbReader_PHP( $fileName ); } } @@ -61,7 +61,7 @@ abstract class CdbWriter { if ( CdbReader::haveExtension() ) { return new CdbWriter_DBA( $fileName ); } else { - wfDebug( 'Warning: no dba extension found, using emulation.' ); + wfDebug( "Warning: no dba extension found, using emulation.\n" ); return new CdbWriter_PHP( $fileName ); } } diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 43962857b37..afc9dd9637e 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -164,6 +164,12 @@ $wgTmpDirectory = false; ///< defaults to "{$wgUploadDirectory}/tmp" $wgUploadBaseUrl = ""; /**@}*/ +/** + * Directory for caching data in the local filesystem. Should not be accessible + * from the web.Set this to false to not use any local caches. + */ +$wgCacheDirectory = false; + /** * Default value for chmoding of new directories. */ @@ -755,15 +761,35 @@ $wgMemCachedPersistent = false; /**@}*/ /** - * Directory for local copy of message cache, for use in addition to memcached + * Set this to true to make a local copy of the message cache, for use in + * addition to memcached. The files will be put in $wgCacheDirectory. */ -$wgLocalMessageCache = false; +$wgUseLocalMessageCache = false; + /** - * Defines format of local cache - * true - Serialized object - * false - PHP source file (Warning - security risk) + * Localisation cache configuration. Associative array with keys: + * class: The class to use. May be overridden by extensions. + * + * store: The location to store cache data. May be 'files', 'db' or + * 'detect'. If set to "files", data will be in CDB files in + * the directory specified by $wgCacheDirectory. If set to "db", + * data will be stored to the database. If set to "detect", files + * will be used if $wgCacheDirectory is set, otherwise the + * database will be used. + * + * storeClass: The class name for the underlying storage. If set to a class + * name, it overrides the "store" setting. + * + * manualRecache: Set this to true to disable cache updates on web requests. + * Use maintenance/rebuildLocalisationCache.php instead. */ -$wgLocalMessageCacheSerialized = true; +$wgLocalisationCacheConf = array( + 'class' => 'LocalisationCache', + 'store' => 'detect', + 'storeClass' => false, + 'manualRecache' => false, +); + # Language settings # @@ -872,20 +898,6 @@ $wgMsgCacheExpiry = 86400; */ $wgMaxMsgCacheEntrySize = 10000; -/** - * If true, serialized versions of the messages arrays will be - * read from the 'serialized' subdirectory if they are present. - * Set to false to always use the Messages files, regardless of - * whether they are up to date or not. - */ -$wgEnableSerializedMessages = true; - -/** - * Set to false if you are thorough system admin who always remembers to keep - * serialized files up to date to save few mtime calls. - */ -$wgCheckSerialized = true; - /** Whether to enable language variant conversion. */ $wgDisableLangConversion = false; @@ -1509,7 +1521,7 @@ $wgStyleVersion = '228'; $wgUseFileCache = false; /** Directory where the cached page will be saved */ -$wgFileCacheDirectory = false; ///< defaults to "{$wgUploadDirectory}/cache"; +$wgFileCacheDirectory = false; ///< defaults to "$wgCacheDirectory/html"; /** * When using the file cache, we can store the cached HTML gzipped to save disk @@ -2550,10 +2562,15 @@ $wgExtensionFunctions = array(); $wgSkinExtensionFunctions = array(); /** - * Extension messages files - * Associative array mapping extension name to the filename where messages can be found. - * The file must create a variable called $messages. - * When the messages are needed, the extension should call wfLoadExtensionMessages(). + * Extension messages files. + * + * Associative array mapping extension name to the filename where messages can be + * found. The file should contain variable assignments. Any of the variables + * present in languages/messages/MessagesEn.php may be defined, but $messages + * is the most common. + * + * Variables defined in extensions will override conflicting variables defined + * in the core. * * Example: * $wgExtensionMessagesFiles['ConfirmEdit'] = dirname(__FILE__).'/ConfirmEdit.i18n.php'; @@ -2563,13 +2580,7 @@ $wgExtensionMessagesFiles = array(); /** * Aliases for special pages provided by extensions. - * Associative array mapping special page to array of aliases. First alternative - * for each special page will be used as the normalised name for it. English - * aliases will be added to the end of the list so that they always work. The - * file must define a variable $aliases. - * - * Example: - * $wgExtensionAliasesFiles['Translate'] = dirname(__FILE__).'/Translate.alias.php'; + * @deprecated Use $specialPageAliases in a file referred to by $wgExtensionMessagesFiles */ $wgExtensionAliasesFiles = array(); diff --git a/includes/Exception.php b/includes/Exception.php index dc5b72d43af..b2d668c804d 100644 --- a/includes/Exception.php +++ b/includes/Exception.php @@ -8,13 +8,13 @@ * @ingroup Exception */ class MWException extends Exception { - /** * Should the exception use $wgOut to output the error ? * @return bool */ function useOutputPage() { - return !empty( $GLOBALS['wgFullyInitialised'] ) && + return $this->useMessageCache() && + !empty( $GLOBALS['wgFullyInitialised'] ) && ( !empty( $GLOBALS['wgArticle'] ) || ( !empty( $GLOBALS['wgOut'] ) && !$GLOBALS['wgOut']->isArticle() ) ) && !empty( $GLOBALS['wgTitle'] ); } @@ -25,6 +25,11 @@ class MWException extends Exception { */ function useMessageCache() { global $wgLang; + foreach ( $this->getTrace() as $frame ) { + if ( $frame['class'] == 'LocalisationCache' ) { + return false; + } + } return is_object( $wgLang ); } diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php index 20f21b4b33a..1f6002c08d4 100644 --- a/includes/GlobalFunctions.php +++ b/includes/GlobalFunctions.php @@ -2932,42 +2932,9 @@ function wfBoolToStr( $value ) { /** * Load an extension messages file - * - * @param string $extensionName Name of extension to load messages from\for. - * @param string $langcode Language to load messages for, or false for default - * behvaiour (en, content language and user language). - * @since r24808 (v1.11) Using this method of loading extension messages will not work - * on MediaWiki prior to that + * @deprecated */ function wfLoadExtensionMessages( $extensionName, $langcode = false ) { - global $wgExtensionMessagesFiles, $wgMessageCache, $wgLang, $wgContLang; - - #For recording whether extension message files have been loaded in a given language. - static $loaded = array(); - - if( !array_key_exists( $extensionName, $loaded ) ) { - $loaded[$extensionName] = array(); - } - - if ( !isset($wgExtensionMessagesFiles[$extensionName]) ) { - throw new MWException( "Messages file for extensions $extensionName is not defined" ); - } - - if( !$langcode && !array_key_exists( '*', $loaded[$extensionName] ) ) { - # Just do en, content language and user language. - $wgMessageCache->loadMessagesFile( $wgExtensionMessagesFiles[$extensionName], false ); - # Mark that they have been loaded. - $loaded[$extensionName]['en'] = true; - $loaded[$extensionName][$wgLang->getCode()] = true; - $loaded[$extensionName][$wgContLang->getCode()] = true; - # Mark that this part has been done to avoid weird if statements. - $loaded[$extensionName]['*'] = true; - } elseif( is_string( $langcode ) && !array_key_exists( $langcode, $loaded[$extensionName] ) ) { - # Load messages for specified language. - $wgMessageCache->loadMessagesFile( $wgExtensionMessagesFiles[$extensionName], $langcode ); - # Mark that they have been loaded. - $loaded[$extensionName][$langcode] = true; - } } /** diff --git a/includes/HTMLFileCache.php b/includes/HTMLFileCache.php index 68cafa242c1..d205624e1ae 100644 --- a/includes/HTMLFileCache.php +++ b/includes/HTMLFileCache.php @@ -14,6 +14,7 @@ * - $wgCachePages * - $wgCacheEpoch * - $wgUseFileCache + * - $wgCacheDirectory * - $wgFileCacheDirectory * - $wgUseGzip * @@ -30,7 +31,16 @@ class HTMLFileCache { public function fileCacheName() { if( !$this->mFileCache ) { - global $wgFileCacheDirectory, $wgRequest; + global $wgCacheDirectory, $wgFileCacheDirectory, $wgRequest; + + if ( $wgFileCacheDirectory ) { + $dir = $wgFileCacheDirectory; + } elseif ( $wgCacheDirectory ) { + $dir = "$wgCacheDirectory/html"; + } else { + throw new MWException( 'Please set $wgCacheDirectory in LocalSettings.php if you wish to use the HTML file cache' ); + } + # Store raw pages (like CSS hits) elsewhere $subdir = ($this->mType === 'raw') ? 'raw/' : ''; $key = $this->mTitle->getPrefixedDbkey(); diff --git a/includes/Hooks.php b/includes/Hooks.php index a05f732b40a..faf7fb961f0 100644 --- a/includes/Hooks.php +++ b/includes/Hooks.php @@ -32,15 +32,16 @@ function wfRunHooks($event, $args = array()) { global $wgHooks; + // Return quickly in the most common case + if ( !isset( $wgHooks[$event] ) ) { + return true; + } + if (!is_array($wgHooks)) { throw new MWException("Global hooks array is not an array!\n"); return false; } - if (!array_key_exists($event, $wgHooks)) { - return true; - } - if (!is_array($wgHooks[$event])) { throw new MWException("Hooks array for event '$event' is not an array!\n"); return false; diff --git a/includes/LocalisationCache.php b/includes/LocalisationCache.php new file mode 100644 index 00000000000..79330075835 --- /dev/null +++ b/includes/LocalisationCache.php @@ -0,0 +1,880 @@ + + * zh-hans -> en ). Some common errors are corrected, for example namespace + * names with spaces instead of underscores, but heavyweight processing, such + * as grammatical transformation, is done by the caller. + */ +class LocalisationCache { + /** Configuration associative array */ + var $conf; + + /** + * True if recaching should only be done on an explicit call to recache(). + * Setting this reduces the overhead of cache freshness checking, which + * requires doing a stat() for every extension i18n file. + */ + var $manualRecache = false; + + /** + * True to treat all files as expired until they are regenerated by this object. + */ + var $forceRecache = false; + + /** + * The cache data. 3-d array, where the first key is the language code, + * the second key is the item key e.g. 'messages', and the third key is + * an item specific subkey index. Some items are not arrays and so for those + * items, there are no subkeys. + */ + var $data = array(); + + /** + * The persistent store object. An instance of LCStore. + */ + var $store; + + /** + * A 2-d associative array, code/key, where presence indicates that the item + * is loaded. Value arbitrary. + * + * For split items, if set, this indicates that all of the subitems have been + * loaded. + */ + var $loadedItems = array(); + + /** + * A 3-d associative array, code/key/subkey, where presence indicates that + * the subitem is loaded. Only used for the split items, i.e. messages. + */ + var $loadedSubitems = array(); + + /** + * An array where presence of a key indicates that that language has been + * initialised. Initialisation includes checking for cache expiry and doing + * any necessary updates. + */ + var $initialisedLangs = array(); + + /** + * An array mapping non-existent pseudo-languages to fallback languages. This + * is filled by initShallowFallback() when data is requested from a language + * that lacks a Messages*.php file. + */ + var $shallowFallbacks = array(); + + /** + * An array where the keys are codes that have been recached by this instance. + */ + var $recachedLangs = array(); + + /** + * All item keys + */ + static public $allKeys = array( + 'fallback', 'namespaceNames', 'mathNames', 'bookstoreList', + 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns', 'digitTransformTable', + 'separatorTransformTable', 'fallback8bitEncoding', 'linkPrefixExtension', + 'defaultUserOptionOverrides', 'linkTrail', 'namespaceAliases', + 'dateFormats', 'datePreferences', 'datePreferenceMigrationMap', + 'defaultDateFormat', 'extraUserToggles', 'specialPageAliases', + 'imageFiles', 'preloadedMessages', + ); + + /** + * Keys for items which consist of associative arrays, which may be merged + * by a fallback sequence. + */ + static public $mergeableMapKeys = array( 'messages', 'namespaceNames', 'mathNames', + 'dateFormats', 'defaultUserOptionOverrides', 'magicWords', 'imageFiles', + 'preloadedMessages', + ); + + /** + * Keys for items which are a numbered array. + */ + static public $mergeableListKeys = array( 'extraUserToggles' ); + + /** + * Keys for items which contain an array of arrays of equivalent aliases + * for each subitem. The aliases may be merged by a fallback sequence. + */ + static public $mergeableAliasListKeys = array( 'specialPageAliases' ); + + /** + * Keys for items which contain an associative array, and may be merged if + * the primary value contains the special array key "inherit". That array + * key is removed after the first merge. + */ + static public $optionalMergeKeys = array( 'bookstoreList' ); + + /** + * Keys for items where the subitems are stored in the backend separately. + */ + static public $splitKeys = array( 'messages' ); + + /** + * Keys which are loaded automatically by initLanguage() + */ + static public $preloadedKeys = array( 'dateFormats', 'namespaceNames', + 'defaultUserOptionOverrides' ); + + /** + * Constructor. + * For constructor parameters, see the documentation in DefaultSettings.php + * for $wgLocalisationCacheConf. + */ + function __construct( $conf ) { + global $wgCacheDirectory; + + $this->conf = $conf; + $this->data = array(); + $this->loadedItems = array(); + $this->loadedSubitems = array(); + $this->initialisedLangs = array(); + if ( !empty( $conf['storeClass'] ) ) { + $storeClass = $conf['storeClass']; + } else { + switch ( $conf['store'] ) { + case 'files': + case 'file': + $storeClass = 'LCStore_CDB'; + break; + case 'db': + $storeClass = 'LCStore_DB'; + break; + case 'detect': + $storeClass = $wgCacheDirectory ? 'LCStore_CDB' : 'LCStore_DB'; + break; + default: + throw new MWException( + 'Please set $wgLocalisationConf[\'store\'] to something sensible.' ); + } + } + + wfDebug( get_class( $this ) . ": using store $storeClass\n" ); + $this->store = new $storeClass; + foreach ( array( 'manualRecache', 'forceRecache' ) as $var ) { + if ( isset( $conf[$var] ) ) { + $this->$var = $conf[$var]; + } + } + } + + /** + * Returns true if the given key is mergeable, that is, if it is an associative + * array which can be merged through a fallback sequence. + */ + public function isMergeableKey( $key ) { + if ( !isset( $this->mergeableKeys ) ) { + $this->mergeableKeys = array_flip( array_merge( + self::$mergeableMapKeys, + self::$mergeableListKeys, + self::$mergeableAliasListKeys, + self::$optionalMergeKeys + ) ); + } + return isset( $this->mergeableKeys[$key] ); + } + + /** + * Get a cache item. + * + * Warning: this may be slow for split items (messages), since it will + * need to fetch all of the subitems from the cache individually. + */ + public function getItem( $code, $key ) { + if ( !isset( $this->loadedItems[$code][$key] ) ) { + wfProfileIn( __METHOD__.'-load' ); + $this->loadItem( $code, $key ); + wfProfileOut( __METHOD__.'-load' ); + } + if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) { + return $this->shallowFallbacks[$code]; + } + return $this->data[$code][$key]; + } + + /** + * Get a subitem, for instance a single message for a given language. + */ + public function getSubitem( $code, $key, $subkey ) { + if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) ) { + if ( isset( $this->loadedItems[$code][$key] ) ) { + if ( isset( $this->data[$code][$key][$subkey] ) ) { + return $this->data[$code][$key][$subkey]; + } else { + return null; + } + } else { + wfProfileIn( __METHOD__.'-load' ); + $this->loadSubitem( $code, $key, $subkey ); + wfProfileOut( __METHOD__.'-load' ); + } + } + return $this->data[$code][$key][$subkey]; + } + + /** + * Load an item into the cache. + */ + protected function loadItem( $code, $key ) { + if ( !isset( $this->initialisedLangs[$code] ) ) { + $this->initLanguage( $code ); + } + // Check to see if initLanguage() loaded it for us + if ( isset( $this->loadedItems[$code][$key] ) ) { + return; + } + if ( isset( $this->shallowFallbacks[$code] ) ) { + $this->loadItem( $this->shallowFallbacks[$code], $key ); + return; + } + if ( in_array( $key, self::$splitKeys ) ) { + $subkeyList = $this->getSubitem( $code, 'list', $key ); + foreach ( $subkeyList as $subkey ) { + if ( isset( $this->data[$code][$key][$subkey] ) ) { + continue; + } + $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey ); + } + } else { + $this->data[$code][$key] = $this->store->get( $code, $key ); + } + $this->loadedItems[$code][$key] = true; + } + + /** + * Load a subitem into the cache + */ + protected function loadSubitem( $code, $key, $subkey ) { + if ( !in_array( $key, self::$splitKeys ) ) { + $this->loadItem( $code, $key ); + return; + } + if ( !isset( $this->initialisedLangs[$code] ) ) { + $this->initLanguage( $code ); + } + // Check to see if initLanguage() loaded it for us + if ( isset( $this->loadedSubitems[$code][$key][$subkey] ) ) { + return; + } + if ( isset( $this->shallowFallbacks[$code] ) ) { + $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey ); + return; + } + $value = $this->store->get( $code, "$key:$subkey" ); + $this->data[$code][$key][$subkey] = $value; + $this->loadedSubitems[$code][$key][$subkey] = true; + } + + /** + * Returns true if the cache identified by $code is missing or expired. + */ + public function isExpired( $code ) { + if ( $this->forceRecache && !isset( $this->recachedLangs[$code] ) ) { + wfDebug( __METHOD__."($code): forced reload\n" ); + return true; + } + + $deps = $this->store->get( $code, 'deps' ); + if ( $deps === null ) { + wfDebug( __METHOD__."($code): cache missing, need to make one\n" ); + return true; + } + foreach ( $deps as $dep ) { + if ( $dep->isExpired() ) { + wfDebug( __METHOD__."($code): cache for $code expired due to " . + get_class( $dep ) . "\n" ); + return true; + } + } + return false; + } + + /** + * Initialise a language in this object. Rebuild the cache if necessary. + */ + protected function initLanguage( $code ) { + if ( isset( $this->initialisedLangs[$code] ) ) { + return; + } + $this->initialisedLangs[$code] = true; + + # Recache the data if necessary + if ( !$this->manualRecache && $this->isExpired( $code ) ) { + if ( file_exists( Language::getMessagesFileName( $code ) ) ) { + $this->recache( $code ); + } elseif ( $code === 'en' ) { + throw new MWException( 'MessagesEn.php is missing.' ); + } else { + $this->initShallowFallback( $code, 'en' ); + } + return; + } + + # Preload some stuff + $preload = $this->getItem( $code, 'preload' ); + if ( $preload === null ) { + if ( $this->manualRecache ) { + // No Messages*.php file. Do shallow fallback to en. + if ( $code === 'en' ) { + throw new MWException( 'No localisation cache found for English. ' . + 'Please run maintenance/rebuildLocalisationCache.php.' ); + } + $this->initShallowFallback( $code, 'en' ); + return; + } else { + throw new MWException( 'Invalid or missing localisation cache.' ); + } + } + $this->data[$code] = $preload; + foreach ( $preload as $key => $item ) { + if ( in_array( $key, self::$splitKeys ) ) { + foreach ( $item as $subkey => $subitem ) { + $this->loadedSubitems[$code][$key][$subkey] = true; + } + } else { + $this->loadedItems[$code][$key] = true; + } + } + } + + /** + * Create a fallback from one language to another, without creating a + * complete persistent cache. + */ + public function initShallowFallback( $primaryCode, $fallbackCode ) { + $this->data[$primaryCode] =& $this->data[$fallbackCode]; + $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode]; + $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode]; + $this->shallowFallbacks[$primaryCode] = $fallbackCode; + } + + /** + * Read a PHP file containing localisation data. + */ + protected function readPHPFile( $_fileName, $_fileType ) { + // Disable APC caching + $_apcEnabled = ini_set( 'apc.enabled', '0' ); + include( $_fileName ); + ini_set( 'apc.enabled', $_apcEnabled ); + + if ( $_fileType == 'core' || $_fileType == 'extension' ) { + $data = compact( self::$allKeys ); + } elseif ( $_fileType == 'aliases' ) { + $data = compact( 'aliases' ); + } else { + throw new MWException( __METHOD__.": Invalid file type: $_fileType" ); + } + return $data; + } + + /** + * Merge two localisation values, a primary and a fallback, overwriting the + * primary value in place. + */ + protected function mergeItem( $key, &$value, $fallbackValue ) { + if ( !is_null( $value ) ) { + if ( !is_null( $fallbackValue ) ) { + if ( in_array( $key, self::$mergeableMapKeys ) ) { + $value = $value + $fallbackValue; + } elseif ( in_array( $key, self::$mergeableListKeys ) ) { + $value = array_unique( array_merge( $fallbackValue, $value ) ); + } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) { + $value = array_merge_recursive( $value, $fallbackValue ); + } elseif ( in_array( $key, self::$optionalMergeKeys ) ) { + if ( !empty( $value['inherit'] ) ) { + $value = array_merge( $fallbackValue, $value ); + } + if ( isset( $value['inherit'] ) ) { + unset( $value['inherit'] ); + } + } + } + } else { + $value = $fallbackValue; + } + } + + /** + * Given an array mapping language code to localisation value, such as is + * found in extension *.i18n.php files, iterate through a fallback sequence + * to merge the given data with an existing primary value. + * + * Returns true if any data from the extension array was used, false + * otherwise. + */ + protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) { + $used = false; + foreach ( $codeSequence as $code ) { + if ( isset( $fallbackValue[$code] ) ) { + $this->mergeItem( $key, $value, $fallbackValue[$code] ); + $used = true; + } + } + return $used; + } + + /** + * Load localisation data for a given language for both core and extensions + * and save it to the persistent cache store and the process cache + */ + public function recache( $code ) { + static $recursionGuard = array(); + global $wgExtensionMessagesFiles, $wgExtensionAliasesFiles; + wfProfileIn( __METHOD__ ); + + if ( !$code ) { + throw new MWException( "Invalid language code requested" ); + } + $this->recachedLangs[$code] = true; + + # Initial values + $initialData = array_combine( + self::$allKeys, + array_fill( 0, count( self::$allKeys ), null ) ); + $coreData = $initialData; + $deps = array(); + + # Load the primary localisation from the source file + $fileName = Language::getMessagesFileName( $code ); + if ( !file_exists( $fileName ) ) { + wfDebug( __METHOD__.": no localisation file for $code, using fallback to en\n" ); + $coreData['fallback'] = 'en'; + } else { + $deps[] = new FileDependency( $fileName ); + $data = $this->readPHPFile( $fileName, 'core' ); + wfDebug( __METHOD__.": got localisation for $code from source\n" ); + + # Merge primary localisation + foreach ( $data as $key => $value ) { + $this->mergeItem( $key, $coreData[$key], $value ); + } + } + + # Fill in the fallback if it's not there already + if ( is_null( $coreData['fallback'] ) ) { + $coreData['fallback'] = $code === 'en' ? false : 'en'; + } + + if ( $coreData['fallback'] !== false ) { + # Guard against circular references + if ( isset( $recursionGuard[$code] ) ) { + throw new MWException( "Error: Circular fallback reference in language code $code" ); + } + $recursionGuard[$code] = true; + + # Load the fallback localisation item by item and merge it + $deps = array_merge( $deps, $this->getItem( $coreData['fallback'], 'deps' ) ); + foreach ( self::$allKeys as $key ) { + if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) { + $fallbackValue = $this->getItem( $coreData['fallback'], $key ); + $this->mergeItem( $key, $coreData[$key], $fallbackValue ); + } + } + $fallbackSequence = $this->getItem( $coreData['fallback'], 'fallbackSequence' ); + array_unshift( $fallbackSequence, $coreData['fallback'] ); + $coreData['fallbackSequence'] = $fallbackSequence; + unset( $recursionGuard[$code] ); + } else { + $coreData['fallbackSequence'] = array(); + } + $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] ); + + # Load the extension localisations + # This is done after the core because we know the fallback sequence now. + # But it has a higher precedence for merging so that we can support things + # like site-specific message overrides. + $allData = $initialData; + foreach ( $wgExtensionMessagesFiles as $fileName ) { + $data = $this->readPHPFile( $fileName, 'extension' ); + $used = false; + foreach ( $data as $key => $item ) { + $used = $used || + $this->mergeExtensionItem( $codeSequence, $key, $allData[$key], $item ); + } + if ( $used ) { + $deps[] = new FileDependency( $fileName ); + } + } + + # Load deprecated $wgExtensionAliasesFiles + foreach ( $wgExtensionAliasesFiles as $fileName ) { + $data = $this->readPHPFile( $fileName, 'aliases' ); + if ( !isset( $data['aliases'] ) ) { + continue; + } + $used = $this->mergeExtensionItem( $codeSequence, 'specialPageAliases', + $allData['specialPageAliases'], $data['aliases'] ); + if ( $used ) { + $deps[] = new FileDependency( $fileName ); + } + } + + # Merge core data into extension data + foreach ( $coreData as $key => $item ) { + $this->mergeItem( $key, $allData[$key], $item ); + } + + # Add cache dependencies for any referenced globals + $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' ); + $deps['wgExtensionAliasesFiles'] = new GlobalDependency( 'wgExtensionAliasesFiles' ); + $deps['version'] = new ConstantDependency( 'MW_LC_VERSION' ); + + # Add dependencies to the cache entry + $allData['deps'] = $deps; + + # Replace spaces with underscores in namespace names + $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] ); + + # And do the same for special page aliases. $page is an array. + foreach ( $allData['specialPageAliases'] as &$page ) { + $page = str_replace( ' ', '_', $page ); + } + # Decouple the reference to prevent accidental damage + unset($page); + + # Fix broken defaultUserOptionOverrides + if ( !is_array( $allData['defaultUserOptionOverrides'] ) ) { + $allData['defaultUserOptionOverrides'] = array(); + } + + # Set the preload key + $allData['preload'] = $this->buildPreload( $allData ); + + # Set the list keys + $allData['list'] = array(); + foreach ( self::$splitKeys as $key ) { + $allData['list'][$key] = array_keys( $allData[$key] ); + } + + # Run hooks + wfRunHooks( 'LocalisationCacheRecache', array( $this, $code, &$allData ) ); + + if ( is_null( $allData['defaultUserOptionOverrides'] ) ) { + throw new MWException( __METHOD__.': Localisation data failed sanity check! ' . + 'Check that your languages/messages/MessagesEn.php file is intact.' ); + } + + # Save to the process cache and register the items loaded + $this->data[$code] = $allData; + foreach ( $allData as $key => $item ) { + $this->loadedItems[$code][$key] = true; + } + + # Save to the persistent cache + $this->store->startWrite( $code ); + foreach ( $allData as $key => $value ) { + if ( in_array( $key, self::$splitKeys ) ) { + foreach ( $value as $subkey => $subvalue ) { + $this->store->set( "$key:$subkey", $subvalue ); + } + } else { + $this->store->set( $key, $value ); + } + } + $this->store->finishWrite(); + + wfProfileOut( __METHOD__ ); + } + + /** + * Build the preload item from the given pre-cache data. + * + * The preload item will be loaded automatically, improving performance + * for the commonly-requested items it contains. + */ + protected function buildPreload( $data ) { + $preload = array( 'messages' => array() ); + foreach ( self::$preloadedKeys as $key ) { + $preload[$key] = $data[$key]; + } + foreach ( $data['preloadedMessages'] as $subkey ) { + if ( isset( $data['messages'][$subkey] ) ) { + $subitem = $data['messages'][$subkey]; + } else { + $subitem = null; + } + $preload['messages'][$subkey] = $subitem; + } + return $preload; + } + + /** + * Unload the data for a given language from the object cache. + * Reduces memory usage. + */ + public function unload( $code ) { + unset( $this->data[$code] ); + unset( $this->loadedItems[$code] ); + unset( $this->loadedSubitems[$code] ); + unset( $this->initialisedLangs[$code] ); + foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) { + if ( $fbCode === $code ) { + $this->unload( $shallowCode ); + } + } + } +} + +/** + * Interface for the persistence layer of LocalisationCache. + * + * The persistence layer is two-level hierarchical cache. The first level + * is the language, the second level is the item or subitem. + * + * Since the data for a whole language is rebuilt in one operation, it needs + * to have a fast and atomic method for deleting or replacing all of the + * current data for a given language. The interface reflects this bulk update + * operation. Callers writing to the cache must first call startWrite(), then + * will call set() a couple of thousand times, then will call finishWrite() + * to commit the operation. When finishWrite() is called, the cache is + * expected to delete all data previously stored for that language. + * + * The values stored are PHP variables suitable for serialize(). Implementations + * of LCStore are responsible for serializing and unserializing. + */ +interface LCStore { + /** + * Get a value. + * @param $code Language code + * @param $key Cache key + */ + public function get( $code, $key ); + + /** + * Start a write transaction. + * @param $code Language code + */ + public function startWrite( $code ); + + /** + * Finish a write transaction. + */ + public function finishWrite(); + + /** + * Set a key to a given value. startWrite() must be called before this + * is called, and finishWrite() must be called afterwards. + */ + public function set( $key, $value ); + +} + +/** + * LCStore implementation which uses the standard DB functions to store data. + * This will work on any MediaWiki installation. + */ +class LCStore_DB implements LCStore { + var $currentLang; + var $writesDone = false; + var $dbw, $batch; + + public function get( $code, $key ) { + if ( $this->writesDone ) { + $db = wfGetDB( DB_MASTER ); + } else { + $db = wfGetDB( DB_SLAVE ); + } + $row = $db->selectRow( 'l10n_cache', array( 'lc_value' ), + array( 'lc_lang' => $code, 'lc_key' => $key ), __METHOD__ ); + if ( $row ) { + return unserialize( $row->lc_value ); + } else { + return null; + } + } + + public function startWrite( $code ) { + if ( !$code ) { + throw new MWException( __METHOD__.": Invalid language \"$code\"" ); + } + $this->dbw = wfGetDB( DB_MASTER ); + $this->dbw->begin(); + $this->dbw->delete( 'l10n_cache', array( 'lc_lang' => $code ), __METHOD__ ); + $this->currentLang = $code; + $this->batch = array(); + } + + public function finishWrite() { + if ( $this->batch ) { + $this->dbw->insert( 'l10n_cache', $this->batch, __METHOD__ ); + } + $this->dbw->commit(); + $this->currentLang = null; + $this->dbw = null; + $this->batch = array(); + $this->writesDone = true; + } + + public function set( $key, $value ) { + if ( is_null( $this->currentLang ) ) { + throw new MWException( __CLASS__.': must call startWrite() before calling set()' ); + } + $this->batch[] = array( + 'lc_lang' => $this->currentLang, + 'lc_key' => $key, + 'lc_value' => serialize( $value ) ); + if ( count( $this->batch ) >= 100 ) { + $this->dbw->insert( 'l10n_cache', $this->batch, __METHOD__ ); + $this->batch = array(); + } + } +} + +/** + * LCStore implementation which stores data as a collection of CDB files in the + * directory given by $wgCacheDirectory. If $wgCacheDirectory is not set, this + * will throw an exception. + * + * Profiling indicates that on Linux, this implementation outperforms MySQL if + * the directory is on a local filesystem and there is ample kernel cache + * space. The performance advantage is greater when the DBA extension is + * available than it is with the PHP port. + * + * See Cdb.php and http://cr.yp.to/cdb.html + */ +class LCStore_CDB implements LCStore { + var $readers, $writer, $currentLang; + + public function get( $code, $key ) { + if ( !isset( $this->readers[$code] ) ) { + $fileName = $this->getFileName( $code ); + if ( !file_exists( $fileName ) ) { + $this->readers[$code] = false; + } else { + $this->readers[$code] = CdbReader::open( $fileName ); + } + } + if ( !$this->readers[$code] ) { + return null; + } else { + $value = $this->readers[$code]->get( $key ); + if ( $value === false ) { + return null; + } + return unserialize( $value ); + } + } + + public function startWrite( $code ) { + $this->writer = CdbWriter::open( $this->getFileName( $code ) ); + $this->currentLang = $code; + } + + public function finishWrite() { + // Close the writer + $this->writer->close(); + $this->writer = null; + + // Reopen the reader + if ( !empty( $this->readers[$this->currentLang] ) ) { + $this->readers[$this->currentLang]->close(); + } + unset( $this->readers[$this->currentLang] ); + $this->currentLang = null; + } + + public function set( $key, $value ) { + if ( is_null( $this->writer ) ) { + throw new MWException( __CLASS__.': must call startWrite() before calling set()' ); + } + $this->writer->set( $key, serialize( $value ) ); + } + + protected function getFileName( $code ) { + global $wgCacheDirectory; + if ( !$code || strpos( $code, '/' ) !== false ) { + throw new MWException( __METHOD__.": Invalid language \"$code\"" ); + } + return "$wgCacheDirectory/l10n_cache-$code.cdb"; + } +} + +/** + * A localisation cache optimised for loading large amounts of data for many + * languages. Used by rebuildLocalisationCache.php. + */ +class LocalisationCache_BulkLoad extends LocalisationCache { + /** + * A cache of the contents of data files. + * Core files are serialized to avoid using ~1GB of RAM during a recache. + */ + var $fileCache = array(); + + /** + * Most recently used languages. Uses the linked-list aspect of PHP hashtables + * to keep the most recently used language codes at the end of the array, and + * the language codes that are ready to be deleted at the beginning. + */ + var $mruLangs = array(); + + /** + * Maximum number of languages that may be loaded into $this->data + */ + var $maxLoadedLangs = 10; + + protected function readPHPFile( $fileName, $fileType ) { + $serialize = $fileType === 'core'; + if ( !isset( $this->fileCache[$fileName][$fileType] ) ) { + $data = parent::readPHPFile( $fileName, $fileType ); + if ( $serialize ) { + $encData = serialize( $data ); + } else { + $encData = $data; + } + $this->fileCache[$fileName][$fileType] = $encData; + return $data; + } elseif ( $serialize ) { + return unserialize( $this->fileCache[$fileName][$fileType] ); + } else { + return $this->fileCache[$fileName][$fileType]; + } + } + + public function getItem( $code, $key ) { + unset( $this->mruLangs[$code] ); + $this->mruLangs[$code] = true; + return parent::getItem( $code, $key ); + } + + public function getSubitem( $code, $key, $subkey ) { + unset( $this->mruLangs[$code] ); + $this->mruLangs[$code] = true; + return parent::getSubitem( $code, $key, $subkey ); + } + + public function recache( $code ) { + parent::recache( $code ); + unset( $this->mruLangs[$code] ); + $this->mruLangs[$code] = true; + $this->trimCache(); + } + + public function unload( $code ) { + unset( $this->mruLangs[$code] ); + parent::unload( $code ); + } + + /** + * Unload cached languages until there are less than $this->maxLoadedLangs + */ + protected function trimCache() { + while ( count( $this->data ) > $this->maxLoadedLangs && count( $this->mruLangs ) ) { + reset( $this->mruLangs ); + $code = key( $this->mruLangs ); + wfDebug( __METHOD__.": unloading $code\n" ); + $this->unload( $code ); + } + } +} diff --git a/includes/MagicWord.php b/includes/MagicWord.php index b69d57e8be5..f618d9f9654 100644 --- a/includes/MagicWord.php +++ b/includes/MagicWord.php @@ -185,7 +185,7 @@ class MagicWord { */ static function &get( $id ) { wfProfileIn( __METHOD__ ); - if (!array_key_exists( $id, self::$mObjects ) ) { + if ( !isset( self::$mObjects[$id] ) ) { $mw = new MagicWord(); $mw->load( $id ); self::$mObjects[$id] = $mw; diff --git a/includes/MessageCache.php b/includes/MessageCache.php index b831f330935..b354b3b236b 100644 --- a/includes/MessageCache.php +++ b/includes/MessageCache.php @@ -23,9 +23,6 @@ class MessageCache { var $mUseCache, $mDisable, $mExpiry; var $mKeys, $mParserOptions, $mParser; - var $mExtensionMessages = array(); - var $mInitialised = false; - var $mAllMessagesLoaded = array(); // Extension messages // Variable for tracking which variables are loaded var $mLoadedLanguages = array(); @@ -37,7 +34,6 @@ class MessageCache { $this->mExpiry = $expiry; $this->mDisableTransform = false; $this->mKeys = false; # initialised on demand - $this->mInitialised = true; $this->mParser = null; } @@ -62,9 +58,9 @@ class MessageCache { * @return false on failure. */ function loadFromLocal( $hash, $code ) { - global $wgLocalMessageCache, $wgLocalMessageCacheSerialized; + global $wgCacheDirectory, $wgLocalMessageCacheSerialized; - $filename = "$wgLocalMessageCache/messages-" . wfWikiID() . "-$code"; + $filename = "$wgCacheDirectory/messages-" . wfWikiID() . "-$code"; # Check file existence wfSuppressWarnings(); @@ -106,10 +102,10 @@ class MessageCache { * Save the cache to a local file. */ function saveToLocal( $serialized, $hash, $code ) { - global $wgLocalMessageCache; + global $wgCacheDirectory; - $filename = "$wgLocalMessageCache/messages-" . wfWikiID() . "-$code"; - wfMkdirParents( $wgLocalMessageCache ); // might fail + $filename = "$wgCacheDirectory/messages-" . wfWikiID() . "-$code"; + wfMkdirParents( $wgCacheDirectory ); // might fail wfSuppressWarnings(); $file = fopen( $filename, 'w' ); @@ -126,11 +122,11 @@ class MessageCache { } function saveToScript( $array, $hash, $code ) { - global $wgLocalMessageCache; + global $wgCacheDirectory; - $filename = "$wgLocalMessageCache/messages-" . wfWikiID() . "-$code"; + $filename = "$wgCacheDirectory/messages-" . wfWikiID() . "-$code"; $tempFilename = $filename . '.tmp'; - wfMkdirParents( $wgLocalMessageCache ); // might fail + wfMkdirParents( $wgCacheDirectory ); // might fail wfSuppressWarnings(); $file = fopen( $tempFilename, 'w'); @@ -174,7 +170,7 @@ class MessageCache { /** * Loads messages from caches or from database in this order: - * (1) local message cache (if $wgLocalMessageCache is enabled) + * (1) local message cache (if $wgUseLocalMessageCache is enabled) * (2) memcached * (3) from the database. * @@ -191,7 +187,7 @@ class MessageCache { * @param $code String: language to which load messages */ function load( $code = false ) { - global $wgLocalMessageCache; + global $wgUseLocalMessageCache; if ( !$this->mUseCache ) { return true; @@ -227,7 +223,7 @@ class MessageCache { # (1) local cache # Hash of the contents is stored in memcache, to detect if local cache goes # out of date (due to update in other thread?) - if ( $wgLocalMessageCache !== false ) { + if ( $wgUseLocalMessageCache ) { wfProfileIn( __METHOD__ . '-fromlocal' ); $hash = $this->mMemc->get( wfMemcKey( 'messages', $code, 'hash' ) ); @@ -423,7 +419,7 @@ class MessageCache { */ protected function saveToCaches( $cache, $memc = true, $code = false ) { wfProfileIn( __METHOD__ ); - global $wgLocalMessageCache, $wgLocalMessageCacheSerialized; + global $wgUseLocalMessageCache, $wgLocalMessageCacheSerialized; $cacheKey = wfMemcKey( 'messages', $code ); @@ -440,7 +436,7 @@ class MessageCache { } # Save to local cache - if ( $wgLocalMessageCache !== false ) { + if ( $wgUseLocalMessageCache ) { $serialized = serialize( $cache ); $hash = md5( $serialized ); $this->mMemc->set( wfMemcKey( 'messages', $code, 'hash' ), $hash, $this->mExpiry ); @@ -508,35 +504,21 @@ class MessageCache { $lang = wfGetLangObj( $langcode ); $langcode = $lang->getCode(); - # If uninitialised, someone is trying to call this halfway through Setup.php - if( !$this->mInitialised ) { - return '<' . htmlspecialchars($key) . '>'; - } - $message = false; # Normalise title-case input - $lckey = $wgContLang->lcfirst( $key ); - $lckey = str_replace( ' ', '_', $lckey ); + $lckey = str_replace( ' ', '_', $key ); + $lckey[0] = strtolower( $lckey[0] ); + $uckey = ucfirst( $lckey ); # Try the MediaWiki namespace if( !$this->mDisable && $useDB ) { - $title = $wgContLang->ucfirst( $lckey ); + $title = $uckey; if(!$isFullKey && ( $langcode != $wgContLanguageCode ) ) { $title .= '/' . $langcode; } $message = $this->getMsgFromNamespace( $title, $langcode ); } - if( $message === false ) - wfRunHooks( 'MessageNotInMwNs', array( &$message, $lckey, $langcode, $isFullKey ) ); - - # Try the extension array - if ( $message === false && isset( $this->mExtensionMessages[$langcode][$lckey] ) ) { - $message = $this->mExtensionMessages[$langcode][$lckey]; - } - if ( $message === false && isset( $this->mExtensionMessages['en'][$lckey] ) ) { - $message = $this->mExtensionMessages['en'][$lckey]; - } # Try the array in the language object if ( $message === false ) { @@ -547,19 +529,15 @@ class MessageCache { } # Try the array of another language - $pos = strrpos( $lckey, '/' ); - if( $message === false && $pos !== false) { - $mkey = substr( $lckey, 0, $pos ); - $code = substr( $lckey, $pos+1 ); - if ( $code ) { - # We may get calls for things that are http-urls from sidebar - # Let's not load nonexistent languages for those - $validCodes = array_keys( Language::getLanguageNames() ); - if ( in_array( $code, $validCodes ) ) { - $message = Language::getMessageFor( $mkey, $code ); - if ( is_null( $message ) ) { - $message = false; - } + if( $message === false ) { + $parts = explode( '/', $lckey ); + # We may get calls for things that are http-urls from sidebar + # Let's not load nonexistent languages for those + # They usually have more than one slash. + if ( count( $parts ) == 2 && $parts[1] !== '' ) { + $message = Language::getMessageFor( $parts[0], $parts[1] ); + if ( is_null( $message ) ) { + $message = false; } } } @@ -568,7 +546,7 @@ class MessageCache { if( ($message === false || $message === '-' ) && !$this->mDisable && $useDB && !$isFullKey && ($langcode != $wgContLanguageCode) ) { - $message = $this->getMsgFromNamespace( $wgContLang->ucfirst( $lckey ), $wgContLanguageCode ); + $message = $this->getMsgFromNamespace( $uckey, $wgContLanguageCode ); } # Final fallback @@ -662,7 +640,7 @@ class MessageCache { } function transform( $message, $interface = false, $language = null ) { - // Avoid creating parser if nothing to transfrom + // Avoid creating parser if nothing to transform if( strpos( $message, '{{' ) === false ) { return $message; } @@ -708,71 +686,6 @@ class MessageCache { return false; } - /** - * Add a message to the cache - * - * @param mixed $key - * @param mixed $value - * @param string $lang The messages language, English by default - */ - function addMessage( $key, $value, $lang = 'en' ) { - global $wgContLang; - # Normalise title-case input - $lckey = str_replace( ' ', '_', $wgContLang->lcfirst( $key ) ); - $this->mExtensionMessages[$lang][$lckey] = $value; - } - - /** - * Add an associative array of message to the cache - * - * @param array $messages An associative array of key => values to be added - * @param string $lang The messages language, English by default - */ - function addMessages( $messages, $lang = 'en' ) { - wfProfileIn( __METHOD__ ); - if ( !is_array( $messages ) ) { - throw new MWException( __METHOD__.': Invalid message array' ); - } - if ( isset( $this->mExtensionMessages[$lang] ) ) { - $this->mExtensionMessages[$lang] = $messages + $this->mExtensionMessages[$lang]; - } else { - $this->mExtensionMessages[$lang] = $messages; - } - wfProfileOut( __METHOD__ ); - } - - /** - * Add a 2-D array of messages by lang. Useful for extensions. - * - * @param array $messages The array to be added - */ - function addMessagesByLang( $messages ) { - wfProfileIn( __METHOD__ ); - foreach ( $messages as $key => $value ) { - $this->addMessages( $value, $key ); - } - wfProfileOut( __METHOD__ ); - } - - /** - * Get the extension messages for a specific language. Only English, interface - * and content language are guaranteed to be loaded. - * - * @param string $lang The messages language, English by default - */ - function getExtensionMessagesFor( $lang = 'en' ) { - wfProfileIn( __METHOD__ ); - $messages = array(); - if ( isset( $this->mExtensionMessages[$lang] ) ) { - $messages = $this->mExtensionMessages[$lang]; - } - if ( $lang != 'en' ) { - $messages = $messages + $this->mExtensionMessages['en']; - } - wfProfileOut( __METHOD__ ); - return $messages; - } - /** * Clear all stored messages. Mainly used after a mass rebuild. */ @@ -788,81 +701,16 @@ class MessageCache { } } + /** + * @deprecated + */ function loadAllMessages( $lang = false ) { - global $wgExtensionMessagesFiles; - $key = $lang === false ? '*' : $lang; - if ( isset( $this->mAllMessagesLoaded[$key] ) ) { - return; - } - $this->mAllMessagesLoaded[$key] = true; - - # Some extensions will load their messages when you load their class file - wfLoadAllExtensions(); - # Others will respond to this hook - wfRunHooks( 'LoadAllMessages', array( $this ) ); - # Some register their messages in $wgExtensionMessagesFiles - foreach ( $wgExtensionMessagesFiles as $name => $file ) { - wfLoadExtensionMessages( $name, $lang ); - } - # Still others will respond to neither, they are EVIL. We sometimes need to know! } /** - * Load messages from a given file - * - * @param string $filename Filename of file to load. - * @param string $langcode Language to load messages for, or false for - * default behvaiour (en, content language and user - * language). + * @deprecated */ function loadMessagesFile( $filename, $langcode = false ) { - global $wgLang, $wgContLang; - wfProfileIn( __METHOD__ ); - $messages = $magicWords = false; - require( $filename ); - - $validCodes = Language::getLanguageNames(); - if( is_string( $langcode ) && array_key_exists( $langcode, $validCodes ) ) { - # Load messages for given language code. - $this->processMessagesArray( $messages, $langcode ); - } elseif( is_string( $langcode ) && !array_key_exists( $langcode, $validCodes ) ) { - wfDebug( "Invalid language '$langcode' code passed to MessageCache::loadMessagesFile()" ); - } else { - # Load only languages that are usually used, and merge all - # fallbacks, except English. - $langs = array_unique( array( 'en', $wgContLang->getCode(), $wgLang->getCode() ) ); - foreach( $langs as $code ) { - $this->processMessagesArray( $messages, $code ); - } - } - - if ( $magicWords !== false ) { - global $wgContLang; - $wgContLang->addMagicWordsByLang( $magicWords ); - } - wfProfileOut( __METHOD__ ); - } - - /** - * Process an array of messages, loading it into the message cache. - * - * @param array $messages Messages array. - * @param string $langcode Language code to process. - */ - function processMessagesArray( $messages, $langcode ) { - wfProfileIn( __METHOD__ ); - $fallbackCode = $langcode; - $mergedMessages = array(); - do { - if ( isset($messages[$fallbackCode]) ) { - $mergedMessages += $messages[$fallbackCode]; - } - $fallbackCode = Language::getFallbackfor( $fallbackCode ); - } while( $fallbackCode && $fallbackCode !== 'en' ); - - if ( !empty($mergedMessages) ) - $this->addMessages( $mergedMessages, $langcode ); - wfProfileOut( __METHOD__ ); } public function figureMessage( $key ) { diff --git a/includes/api/ApiQuerySiteinfo.php b/includes/api/ApiQuerySiteinfo.php index 4a06b3540ff..c4c86b75bf6 100644 --- a/includes/api/ApiQuerySiteinfo.php +++ b/includes/api/ApiQuerySiteinfo.php @@ -185,8 +185,7 @@ class ApiQuerySiteinfo extends ApiQueryBase { protected function appendNamespaceAliases( $property ) { global $wgNamespaceAliases, $wgContLang; - $wgContLang->load(); - $aliases = array_merge( $wgNamespaceAliases, $wgContLang->namespaceAliases ); + $aliases = array_merge( $wgNamespaceAliases, $wgContLang->getNamespaceAliases() ); $namespaces = $wgContLang->getNamespaces(); $data = array(); foreach( $aliases as $title => $ns ) { diff --git a/includes/specials/SpecialAllmessages.php b/includes/specials/SpecialAllmessages.php index f81e49aa219..6470261e3f7 100644 --- a/includes/specials/SpecialAllmessages.php +++ b/includes/specials/SpecialAllmessages.php @@ -29,8 +29,7 @@ function wfSpecialAllmessages() { $wgMessageCache->loadAllMessages(); - $sortedArray = array_merge( Language::getMessagesFor( 'en' ), - $wgMessageCache->getExtensionMessagesFor( 'en' ) ); + $sortedArray = Language::getMessagesFor( 'en' ); ksort( $sortedArray ); $messages = array(); diff --git a/languages/Language.php b/languages/Language.php index 65c62a3284e..6987129941b 100644 --- a/languages/Language.php +++ b/languages/Language.php @@ -57,24 +57,12 @@ class Language { var $mConverter, $mVariants, $mCode, $mLoaded = false; var $mMagicExtensions = array(), $mMagicHookDone = false; - static public $mLocalisationKeys = array( - 'fallback', 'namespaceNames', 'mathNames', 'bookstoreList', - 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns', 'digitTransformTable', - 'separatorTransformTable', 'fallback8bitEncoding', 'linkPrefixExtension', - 'defaultUserOptionOverrides', 'linkTrail', 'namespaceAliases', - 'dateFormats', 'datePreferences', 'datePreferenceMigrationMap', - 'defaultDateFormat', 'extraUserToggles', 'specialPageAliases', - 'imageFiles' - ); + var $mNamespaceIds, $namespaceNames, $namespaceAliases; + var $dateFormatStrings = array(); + var $minSearchLength; + var $mExtendedSpecialPageAliases; - static public $mMergeableMapKeys = array( 'messages', 'namespaceNames', 'mathNames', - 'dateFormats', 'defaultUserOptionOverrides', 'magicWords', 'imageFiles' ); - - static public $mMergeableListKeys = array( 'extraUserToggles' ); - - static public $mMergeableAliasListKeys = array( 'specialPageAliases' ); - - static public $mLocalisationCache = array(); + static public $dataCache; static public $mLangObjCache = array(); static public $mWeekdayMsgs = array( @@ -180,6 +168,15 @@ class Language { return $lang; } + public static function getLocalisationCache() { + if ( is_null( self::$dataCache ) ) { + global $wgLocalisationCacheConf; + $class = $wgLocalisationCacheConf['class']; + self::$dataCache = new $class( $wgLocalisationCacheConf ); + } + return self::$dataCache; + } + function __construct() { $this->mConverter = new FakeConverter($this); // Set the code to the name of the descendant @@ -188,6 +185,7 @@ class Language { } else { $this->mCode = str_replace( '_', '-', strtolower( substr( get_class( $this ), 8 ) ) ); } + self::getLocalisationCache(); } /** @@ -215,7 +213,11 @@ class Language { } function getFallbackLanguageCode() { - return self::getFallbackFor( $this->mCode ); + if ( $this->mCode === 'en' ) { + return false; + } else { + return self::$dataCache->getItem( $this->mCode, 'fallback' ); + } } /** @@ -223,15 +225,34 @@ class Language { * @return array */ function getBookstoreList() { - $this->load(); - return $this->bookstoreList; + return self::$dataCache->getItem( $this->mCode, 'bookstoreList' ); } /** * @return array */ function getNamespaces() { - $this->load(); + if ( is_null( $this->namespaceNames ) ) { + global $wgExtraNamespaces, $wgMetaNamespace, $wgMetaNamespaceTalk; + + $this->namespaceNames = self::$dataCache->getItem( $this->mCode, 'namespaceNames' ); + if ( $wgExtraNamespaces ) { + $this->namespaceNames = $wgExtraNamespaces + $this->namespaceNames; + } + + $this->namespaceNames[NS_PROJECT] = $wgMetaNamespace; + if ( $wgMetaNamespaceTalk ) { + $this->namespaceNames[NS_PROJECT_TALK] = $wgMetaNamespaceTalk; + } else { + $talk = $this->namespaceNames[NS_PROJECT_TALK]; + $this->namespaceNames[NS_PROJECT_TALK] = + $this->fixVariableInNamespace( $talk ); + } + + # The above mixing may leave namespaces out of canonical order. + # Re-order by namespace ID number... + ksort( $this->namespaceNames ); + } return $this->namespaceNames; } @@ -287,11 +308,54 @@ class Language { * @return mixed An integer if $text is a valid value otherwise false */ function getLocalNsIndex( $text ) { - $this->load(); $lctext = $this->lc($text); - return isset( $this->mNamespaceIds[$lctext] ) ? $this->mNamespaceIds[$lctext] : false; + $ids = $this->getNamespaceIds(); + return isset( $ids[$lctext] ) ? $ids[$lctext] : false; } + function getNamespaceAliases() { + if ( is_null( $this->namespaceAliases ) ) { + $aliases = self::$dataCache->getItem( $this->mCode, 'namespaceAliases' ); + if ( !$aliases ) { + $aliases = array(); + } else { + foreach ( $aliases as $name => $index ) { + if ( $index === NS_PROJECT_TALK ) { + unset( $aliases[$name] ); + $name = $this->fixVariableInNamespace( $name ); + $aliases[$name] = $index; + } + } + } + $this->namespaceAliases = $aliases; + } + return $this->namespaceAliases; + } + + function getNamespaceIds() { + if ( is_null( $this->mNamespaceIds ) ) { + global $wgNamespaceAliases; + # Put namespace names and aliases into a hashtable. + # If this is too slow, then we should arrange it so that it is done + # before caching. The catch is that at pre-cache time, the above + # class-specific fixup hasn't been done. + $this->mNamespaceIds = array(); + foreach ( $this->getNamespaces() as $index => $name ) { + $this->mNamespaceIds[$this->lc($name)] = $index; + } + foreach ( $this->getNamespaceAliases() as $name => $index ) { + $this->mNamespaceIds[$this->lc($name)] = $index; + } + if ( $wgNamespaceAliases ) { + foreach ( $wgNamespaceAliases as $name => $index ) { + $this->mNamespaceIds[$this->lc($name)] = $index; + } + } + } + return $this->mNamespaceIds; + } + + /** * Get a namespace key by value, case insensitive. Canonical namespace * names override custom ones defined for the current language. @@ -300,10 +364,12 @@ class Language { * @return mixed An integer if $text is a valid value otherwise false */ function getNsIndex( $text ) { - $this->load(); $lctext = $this->lc($text); - if( ( $ns = MWNamespace::getCanonicalIndex( $lctext ) ) !== null ) return $ns; - return isset( $this->mNamespaceIds[$lctext] ) ? $this->mNamespaceIds[$lctext] : false; + if ( ( $ns = MWNamespace::getCanonicalIndex( $lctext ) ) !== null ) { + return $ns; + } + $ids = $this->getNamespaceIds(); + return isset( $ids[$lctext] ) ? $ids[$lctext] : false; } /** @@ -335,48 +401,41 @@ class Language { } function getMathNames() { - $this->load(); - return $this->mathNames; + return self::$dataCache->getItem( $this->mCode, 'mathNames' ); } function getDatePreferences() { - $this->load(); - return $this->datePreferences; + return self::$dataCache->getItem( $this->mCode, 'datePreferences' ); } function getDateFormats() { - $this->load(); - return $this->dateFormats; + return self::$dataCache->getItem( $this->mCode, 'dateFormats' ); } function getDefaultDateFormat() { - $this->load(); - return $this->defaultDateFormat; - } - - function getDatePreferenceMigrationMap() { - $this->load(); - return $this->datePreferenceMigrationMap; - } - - function getImageFile( $image ) { - $this->load(); - return $this->imageFiles[$image]; - } - - function getDefaultUserOptionOverrides() { - $this->load(); - # XXX - apparently some languageas get empty arrays, didn't get to it yet -- midom - if (is_array($this->defaultUserOptionOverrides)) { - return $this->defaultUserOptionOverrides; + $df = self::$dataCache->getItem( $this->mCode, 'defaultDateFormat' ); + if ( $df === 'dmy or mdy' ) { + global $wgAmericanDates; + return $wgAmericanDates ? 'mdy' : 'dmy'; } else { - return array(); + return $df; } } + function getDatePreferenceMigrationMap() { + return self::$dataCache->getItem( $this->mCode, 'datePreferenceMigrationMap' ); + } + + function getImageFile( $image ) { + return self::$dataCache->getSubitem( $this->mCode, 'imageFiles', $image ); + } + + function getDefaultUserOptionOverrides() { + return self::$dataCache->getItem( $this->mCode, 'defaultUserOptionOverrides' ); + } + function getExtraUserToggles() { - $this->load(); - return $this->extraUserToggles; + return self::$dataCache->getItem( $this->mCode, 'extraUserToggles' ); } function getUserToggle( $tog ) { @@ -1318,6 +1377,28 @@ class Language { return $datePreference; } + /** + * Get a format string for a given type and preference + * @param $type May be date, time or both + * @param $pref The format name as it appears in Messages*.php + */ + function getDateFormatString( $type, $pref ) { + if ( !isset( $this->dateFormatStrings[$type][$pref] ) ) { + if ( $pref == 'default' ) { + $pref = $this->getDefaultDateFormat(); + $df = self::$dataCache->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); + } else { + $df = self::$dataCache->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); + if ( is_null( $df ) ) { + $pref = $this->getDefaultDateFormat(); + $df = self::$dataCache->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); + } + } + $this->dateFormatStrings[$type][$pref] = $df; + } + return $this->dateFormatStrings[$type][$pref]; + } + /** * @param $ts Mixed: the time format which needs to be turned into a * date('YmdHis') format with wfTimestamp(TS_MW,$ts) @@ -1329,16 +1410,11 @@ class Language { * @return string */ function date( $ts, $adj = false, $format = true, $timecorrection = false ) { - $this->load(); if ( $adj ) { $ts = $this->userAdjust( $ts, $timecorrection ); } - - $pref = $this->dateFormat( $format ); - if( $pref == 'default' || !isset( $this->dateFormats["$pref date"] ) ) { - $pref = $this->defaultDateFormat; - } - return $this->sprintfDate( $this->dateFormats["$pref date"], $ts ); + $df = $this->getDateFormatString( 'date', $this->dateFormat( $format ) ); + return $this->sprintfDate( $df, $ts ); } /** @@ -1352,16 +1428,11 @@ class Language { * @return string */ function time( $ts, $adj = false, $format = true, $timecorrection = false ) { - $this->load(); if ( $adj ) { $ts = $this->userAdjust( $ts, $timecorrection ); } - - $pref = $this->dateFormat( $format ); - if( $pref == 'default' || !isset( $this->dateFormats["$pref time"] ) ) { - $pref = $this->defaultDateFormat; - } - return $this->sprintfDate( $this->dateFormats["$pref time"], $ts ); + $df = $this->getDateFormatString( 'time', $this->dateFormat( $format ) ); + return $this->sprintfDate( $df, $ts ); } /** @@ -1376,30 +1447,20 @@ class Language { * @return string */ function timeanddate( $ts, $adj = false, $format = true, $timecorrection = false) { - $this->load(); - $ts = wfTimestamp( TS_MW, $ts ); - if ( $adj ) { $ts = $this->userAdjust( $ts, $timecorrection ); } - - $pref = $this->dateFormat( $format ); - if( $pref == 'default' || !isset( $this->dateFormats["$pref both"] ) ) { - $pref = $this->defaultDateFormat; - } - - return $this->sprintfDate( $this->dateFormats["$pref both"], $ts ); + $df = $this->getDateFormatString( 'both', $this->dateFormat( $format ) ); + return $this->sprintfDate( $df, $ts ); } function getMessage( $key ) { - $this->load(); - return isset( $this->messages[$key] ) ? $this->messages[$key] : null; + return self::$dataCache->getSubitem( $this->mCode, 'messages', $key ); } function getAllMessages() { - $this->load(); - return $this->messages; + return self::$dataCache->getItem( $this->mCode, 'messages' ); } function iconv( $in, $out, $string ) { @@ -1590,8 +1651,7 @@ class Language { } function fallback8bitEncoding() { - $this->load(); - return $this->fallback8bitEncoding; + return self::$dataCache->getItem( $this->mCode, 'fallback8bitEncoding' ); } /** @@ -1669,7 +1729,7 @@ class Language { * if we need to pad short words... */ protected function minSearchLength() { - if( !isset( $this->minSearchLength ) ) { + if( is_null( $this->minSearchLength ) ) { $sql = "show global variables like 'ft\\_min\\_word\\_len'"; $dbr = wfGetDB( DB_SLAVE ); $result = $dbr->query( $sql ); @@ -1789,8 +1849,7 @@ class Language { * @return bool */ function isRTL() { - $this->load(); - return $this->rtl; + return self::$dataCache->getItem( $this->mCode, 'rtl' ); } /** @@ -1803,8 +1862,7 @@ class Language { } function capitalizeAllNouns() { - $this->load(); - return $this->capitalizeAllNouns; + return self::$dataCache->getItem( $this->mCode, 'capitalizeAllNouns' ); } /** @@ -1822,13 +1880,11 @@ class Language { * @return bool */ function linkPrefixExtension() { - $this->load(); - return $this->linkPrefixExtension; + return self::$dataCache->getItem( $this->mCode, 'linkPrefixExtension' ); } - function &getMagicWords() { - $this->load(); - return $this->magicWords; + function getMagicWords() { + return self::$dataCache->getItem( $this->mCode, 'magicWords' ); } # Fill a MagicWord object with data from here @@ -1840,16 +1896,11 @@ class Language { if ( isset( $this->mMagicExtensions[$mw->mId] ) ) { $rawEntry = $this->mMagicExtensions[$mw->mId]; } else { - $magicWords =& $this->getMagicWords(); + $magicWords = $this->getMagicWords(); if ( isset( $magicWords[$mw->mId] ) ) { $rawEntry = $magicWords[$mw->mId]; } else { - # Fall back to English if local list is incomplete - $magicWords =& Language::getMagicWords(); - if ( !isset($magicWords[$mw->mId]) ) { - throw new MWException("Magic word '{$mw->mId}' not found" ); - } - $rawEntry = $magicWords[$mw->mId]; + $rawEntry = false; } } @@ -1887,43 +1938,11 @@ class Language { * case folded alias => real name */ function getSpecialPageAliases() { - $this->load(); - // Cache aliases because it may be slow to load them - if ( !isset( $this->mExtendedSpecialPageAliases ) ) { - + if ( is_null( $this->mExtendedSpecialPageAliases ) ) { // Initialise array - $this->mExtendedSpecialPageAliases = $this->specialPageAliases; - - global $wgExtensionAliasesFiles; - foreach ( $wgExtensionAliasesFiles as $file ) { - - // Fail fast - if ( !file_exists($file) ) - throw new MWException( "Aliases file does not exist: $file" ); - - $aliases = array(); - require($file); - - // Check the availability of aliases - if ( !isset($aliases['en']) ) - throw new MWException( "Malformed aliases file: $file" ); - - // Merge all aliases in fallback chain - $code = $this->getCode(); - do { - if ( !isset($aliases[$code]) ) continue; - - $aliases[$code] = $this->fixSpecialPageAliases( $aliases[$code] ); - /* Merge the aliases, THIS will break if there is special page name - * which looks like a numerical key, thanks to PHP... - * See the array_merge_recursive manual entry */ - $this->mExtendedSpecialPageAliases = array_merge_recursive( - $this->mExtendedSpecialPageAliases, $aliases[$code] ); - - } while ( $code = self::getFallbackFor( $code ) ); - } - + $this->mExtendedSpecialPageAliases = + self::$dataCache->getItem( $this->mCode, 'specialPageAliases' ); wfRunHooks( 'LanguageGetSpecialPageAliases', array( &$this->mExtendedSpecialPageAliases, $this->getCode() ) ); } @@ -1931,20 +1950,6 @@ class Language { return $this->mExtendedSpecialPageAliases; } - /** - * Function to fix special page aliases. Will convert the first letter to - * upper case and spaces to underscores. Can be given a full aliases array, - * in which case it will recursively fix all aliases. - */ - public function fixSpecialPageAliases( $mixed ) { - // Work recursively until in string level - if ( is_array($mixed) ) { - $callback = array( $this, 'fixSpecialPageAliases' ); - return array_map( $callback, $mixed ); - } - return str_replace( ' ', '_', $this->ucfirst( $mixed ) ); - } - /** * Italic is unsuitable for some languages * @@ -2017,13 +2022,11 @@ class Language { } function digitTransformTable() { - $this->load(); - return $this->digitTransformTable; + return self::$dataCache->getItem( $this->mCode, 'digitTransformTable' ); } function separatorTransformTable() { - $this->load(); - return $this->separatorTransformTable; + return self::$dataCache->getItem( $this->mCode, 'separatorTransformTable' ); } @@ -2380,8 +2383,7 @@ class Language { * @return string */ function linkTrail() { - $this->load(); - return $this->linkTrail; + return self::$dataCache->getItem( $this->mCode, 'linkTrail' ); } function getLangObj() { @@ -2413,306 +2415,31 @@ class Language { return self::getFileName( "$IP/languages/classes/Language", $code, '.php' ); } - static function getLocalisationArray( $code, $disableCache = false ) { - self::loadLocalisation( $code, $disableCache ); - return self::$mLocalisationCache[$code]; - } - - /** - * Load localisation data for a given code into the static cache - * - * @return array Dependencies, map of filenames to mtimes - */ - static function loadLocalisation( $code, $disableCache = false ) { - static $recursionGuard = array(); - global $wgMemc, $wgEnableSerializedMessages, $wgCheckSerialized; - - if ( !$code ) { - throw new MWException( "Invalid language code requested" ); - } - - if ( !$disableCache ) { - # Try the per-process cache - if ( isset( self::$mLocalisationCache[$code] ) ) { - return self::$mLocalisationCache[$code]['deps']; - } - - wfProfileIn( __METHOD__ ); - - # Try the serialized directory - if( $wgEnableSerializedMessages ) { - $cache = wfGetPrecompiledData( self::getFileName( "Messages", $code, '.ser' ) ); - if ( $cache ) { - if ( $wgCheckSerialized && self::isLocalisationOutOfDate( $cache ) ) { - $cache = false; - wfDebug( "Language::loadLocalisation(): precompiled data file for $code is out of date\n" ); - } else { - self::$mLocalisationCache[$code] = $cache; - wfDebug( "Language::loadLocalisation(): got localisation for $code from precompiled data file\n" ); - wfProfileOut( __METHOD__ ); - return self::$mLocalisationCache[$code]['deps']; - } - } - } else { - $cache = false; - } - - # Try the global cache - $memcKey = wfMemcKey('localisation', $code ); - $fbMemcKey = wfMemcKey('fallback', $cache['fallback'] ); - $cache = $wgMemc->get( $memcKey ); - if ( $cache ) { - if ( self::isLocalisationOutOfDate( $cache ) ) { - $wgMemc->delete( $memcKey ); - $wgMemc->delete( $fbMemcKey ); - $cache = false; - wfDebug( "Language::loadLocalisation(): localisation cache for $code had expired\n" ); - } else { - self::$mLocalisationCache[$code] = $cache; - wfDebug( "Language::loadLocalisation(): got localisation for $code from cache\n" ); - wfProfileOut( __METHOD__ ); - return $cache['deps']; - } - } - } else { - wfProfileIn( __METHOD__ ); - } - - # Default fallback, may be overridden when the messages file is included - if ( $code != 'en' ) { - $fallback = 'en'; - } else { - $fallback = false; - } - - # Load the primary localisation from the source file - $filename = self::getMessagesFileName( $code ); - if ( !file_exists( $filename ) ) { - wfDebug( "Language::loadLocalisation(): no localisation file for $code, using implicit fallback to en\n" ); - $cache = compact( self::$mLocalisationKeys ); // Set correct fallback - $deps = array(); - } else { - $deps = array( $filename => filemtime( $filename ) ); - require( $filename ); - $cache = compact( self::$mLocalisationKeys ); - wfDebug( "Language::loadLocalisation(): got localisation for $code from source\n" ); - } - - # Load magic word source file - global $IP; - $filename = "$IP/includes/MagicWord.php"; - $newDeps = array( $filename => filemtime( $filename ) ); - $deps = array_merge( $deps, $newDeps ); - - if ( !empty( $fallback ) ) { - # Load the fallback localisation, with a circular reference guard - if ( isset( $recursionGuard[$code] ) ) { - throw new MWException( "Error: Circular fallback reference in language code $code" ); - } - $recursionGuard[$code] = true; - $newDeps = self::loadLocalisation( $fallback, $disableCache ); - unset( $recursionGuard[$code] ); - - $secondary = self::$mLocalisationCache[$fallback]; - $deps = array_merge( $deps, $newDeps ); - - # Merge the fallback localisation with the current localisation - foreach ( self::$mLocalisationKeys as $key ) { - if ( isset( $cache[$key] ) ) { - if ( isset( $secondary[$key] ) ) { - if ( in_array( $key, self::$mMergeableMapKeys ) ) { - $cache[$key] = $cache[$key] + $secondary[$key]; - } elseif ( in_array( $key, self::$mMergeableListKeys ) ) { - $cache[$key] = array_merge( $secondary[$key], $cache[$key] ); - } elseif ( in_array( $key, self::$mMergeableAliasListKeys ) ) { - $cache[$key] = array_merge_recursive( $cache[$key], $secondary[$key] ); - } - } - } else { - $cache[$key] = $secondary[$key]; - } - } - - # Merge bookstore lists if requested - if ( !empty( $cache['bookstoreList']['inherit'] ) ) { - $cache['bookstoreList'] = array_merge( $cache['bookstoreList'], $secondary['bookstoreList'] ); - } - if ( isset( $cache['bookstoreList']['inherit'] ) ) { - unset( $cache['bookstoreList']['inherit'] ); - } - } - - # Add dependencies to the cache entry - $cache['deps'] = $deps; - - # Replace spaces with underscores in namespace names - $cache['namespaceNames'] = str_replace( ' ', '_', $cache['namespaceNames'] ); - - # And do the same for specialpage aliases. $page is an array. - foreach ( $cache['specialPageAliases'] as &$page ) { - $page = str_replace( ' ', '_', $page ); - } - # Decouple the reference to prevent accidental damage - unset($page); - - # Save to both caches - self::$mLocalisationCache[$code] = $cache; - if ( !$disableCache ) { - $wgMemc->set( $memcKey, $cache ); - $wgMemc->set( $fbMemcKey, (string) $cache['fallback'] ); - } - - wfProfileOut( __METHOD__ ); - return $deps; - } - - /** - * Test if a given localisation cache is out of date with respect to the - * source Messages files. This is done automatically for the global cache - * in $wgMemc, but is only done on certain occasions for the serialized - * data file. - * - * @param $cache mixed Either a language code or a cache array - */ - static function isLocalisationOutOfDate( $cache ) { - if ( !is_array( $cache ) ) { - self::loadLocalisation( $cache ); - $cache = self::$mLocalisationCache[$cache]; - } - // At least one language file and the MagicWord file needed - if( count($cache['deps']) < 2 ) { - return true; - } - $expired = false; - foreach ( $cache['deps'] as $file => $mtime ) { - if ( !file_exists( $file ) || filemtime( $file ) > $mtime ) { - $expired = true; - break; - } - } - return $expired; - } - /** * Get the fallback for a given language */ static function getFallbackFor( $code ) { - // Shortcut - if ( $code === 'en' ) return false; - - // Local cache - static $cache = array(); - // Quick return - if ( isset($cache[$code]) ) return $cache[$code]; - - // Try memcache - global $wgMemc; - $memcKey = wfMemcKey( 'fallback', $code ); - $fbcode = $wgMemc->get( $memcKey ); - - if ( is_string($fbcode) ) { - // False is stored as a string to detect failures in memcache properly - if ( $fbcode === '' ) $fbcode = false; - - // Update local cache and return - $cache[$code] = $fbcode; - return $fbcode; + if ( $code === 'en' ) { + // Shortcut + return false; + } else { + return self::getLocalisationCache()->getItem( $code, 'fallback' ); } - - // Nothing in caches, load and and update both caches - self::loadLocalisation( $code ); - $fbcode = self::$mLocalisationCache[$code]['fallback']; - - $cache[$code] = $fbcode; - $wgMemc->set( $memcKey, (string) $fbcode ); - - return $fbcode; } /** * Get all messages for a given language + * WARNING: this may take a long time */ static function getMessagesFor( $code ) { - self::loadLocalisation( $code ); - return self::$mLocalisationCache[$code]['messages']; + return self::getLocalisationCache()->getItem( $code, 'messages' ); } /** * Get a message for a given language */ static function getMessageFor( $key, $code ) { - self::loadLocalisation( $code ); - return isset( self::$mLocalisationCache[$code]['messages'][$key] ) ? self::$mLocalisationCache[$code]['messages'][$key] : null; - } - - /** - * Load localisation data for this object - */ - function load() { - if ( !$this->mLoaded ) { - self::loadLocalisation( $this->getCode() ); - $cache =& self::$mLocalisationCache[$this->getCode()]; - foreach ( self::$mLocalisationKeys as $key ) { - $this->$key = $cache[$key]; - } - $this->mLoaded = true; - - $this->fixUpSettings(); - } - } - - /** - * Do any necessary post-cache-load settings adjustment - */ - function fixUpSettings() { - global $wgExtraNamespaces, $wgMetaNamespace, $wgMetaNamespaceTalk, - $wgNamespaceAliases, $wgAmericanDates; - wfProfileIn( __METHOD__ ); - if ( $wgExtraNamespaces ) { - $this->namespaceNames = $wgExtraNamespaces + $this->namespaceNames; - } - - $this->namespaceNames[NS_PROJECT] = $wgMetaNamespace; - if ( $wgMetaNamespaceTalk ) { - $this->namespaceNames[NS_PROJECT_TALK] = $wgMetaNamespaceTalk; - } else { - $talk = $this->namespaceNames[NS_PROJECT_TALK]; - $this->namespaceNames[NS_PROJECT_TALK] = - $this->fixVariableInNamespace( $talk ); - } - - # The above mixing may leave namespaces out of canonical order. - # Re-order by namespace ID number... - ksort( $this->namespaceNames ); - - # Put namespace names and aliases into a hashtable. - # If this is too slow, then we should arrange it so that it is done - # before caching. The catch is that at pre-cache time, the above - # class-specific fixup hasn't been done. - $this->mNamespaceIds = array(); - foreach ( $this->namespaceNames as $index => $name ) { - $this->mNamespaceIds[$this->lc($name)] = $index; - } - if ( $this->namespaceAliases ) { - foreach ( $this->namespaceAliases as $name => $index ) { - if ( $index === NS_PROJECT_TALK ) { - unset( $this->namespaceAliases[$name] ); - $name = $this->fixVariableInNamespace( $name ); - $this->namespaceAliases[$name] = $index; - } - $this->mNamespaceIds[$this->lc($name)] = $index; - } - } - if ( $wgNamespaceAliases ) { - foreach ( $wgNamespaceAliases as $name => $index ) { - $this->mNamespaceIds[$this->lc($name)] = $index; - } - } - - if ( $this->defaultDateFormat == 'dmy or mdy' ) { - $this->defaultDateFormat = $wgAmericanDates ? 'mdy' : 'dmy'; - } - wfProfileOut( __METHOD__ ); + return self::getLocalisationCache()->getSubitem( $code, 'messages', $key ); } function fixVariableInNamespace( $talk ) { diff --git a/languages/messages/MessagesEn.php b/languages/messages/MessagesEn.php index 18ec5062599..347338aafe6 100644 --- a/languages/messages/MessagesEn.php +++ b/languages/messages/MessagesEn.php @@ -474,6 +474,109 @@ $imageFiles = array( 'button-hr' => 'button_hr.png', ); +/** + * A list of messages to preload for each request. + * We add messages here which are needed for a typical anonymous parser cache hit. + */ +$preloadedMessages = array( + 'aboutpage', + 'aboutsite', + 'accesskey-ca-edit', + 'accesskey-ca-history', + 'accesskey-ca-nstab-main', + 'accesskey-ca-talk', + 'accesskey-n-currentevents', + 'accesskey-n-help', + 'accesskey-n-mainpage-description', + 'accesskey-n-portal', + 'accesskey-n-randompage', + 'accesskey-n-recentchanges', + 'accesskey-n-sitesupport', + 'accesskey-p-logo', + 'accesskey-pt-login', + 'accesskey-search', + 'accesskey-search-fulltext', + 'accesskey-search-go', + 'accesskey-t-permalink', + 'accesskey-t-print', + 'accesskey-t-recentchangeslinked', + 'accesskey-t-specialpages', + 'accesskey-t-whatlinkshere', + 'anonnotice', + 'catseparator', + 'colon-separator', + 'currentevents', + 'currentevents-url', + 'disclaimerpage', + 'disclaimers', + 'edit', + 'help', + 'helppage', + 'history_short', + 'jumpto', + 'jumptonavigation', + 'jumptosearch', + 'lastmodifiedat', + 'mainpage', + 'mainpage-description', + 'nav-login-createaccount', + 'navigation', + 'nstab-main', + 'opensearch-desc', + 'pagecategories', + 'pagecategorieslink', + 'pagetitle', + 'pagetitle-view-mainpage', + 'permalink', + 'personaltools', + 'portal', + 'portal-url', + 'printableversion', + 'privacy', + 'privacypage', + 'randompage', + 'randompage-url', + 'recentchanges', + 'recentchanges-url', + 'recentchangeslinked-toolbox', + 'retrievedfrom', + 'search', + 'searcharticle', + 'searchbutton', + 'sidebar', + 'site-atom-feed', + 'site-rss-feed', + 'sitenotice', + 'specialpages', + 'tagline', + 'talk', + 'toolbox', + 'tooltip-ca-edit', + 'tooltip-ca-history', + 'tooltip-ca-nstab-main', + 'tooltip-ca-talk', + 'tooltip-n-currentevents', + 'tooltip-n-help', + 'tooltip-n-mainpage-description', + 'tooltip-n-portal', + 'tooltip-n-randompage', + 'tooltip-n-recentchanges', + 'tooltip-n-sitesupport', + 'tooltip-p-logo', + 'tooltip-p-navigation', + 'tooltip-pt-login', + 'tooltip-search', + 'tooltip-search-fulltext', + 'tooltip-search-go', + 'tooltip-t-permalink', + 'tooltip-t-print', + 'tooltip-t-recentchangeslinked', + 'tooltip-t-specialpages', + 'tooltip-t-whatlinkshere', + 'views', + 'whatlinkshere', +); + #------------------------------------------------------------------- # Default messages #------------------------------------------------------------------- diff --git a/maintenance/archives/patch-l10n_cache.sql b/maintenance/archives/patch-l10n_cache.sql new file mode 100644 index 00000000000..32a04f994b6 --- /dev/null +++ b/maintenance/archives/patch-l10n_cache.sql @@ -0,0 +1,8 @@ +-- Table for storing localisation data +CREATE TABLE /*_*/l10n_cache ( + lc_lang varbinary(32) NOT NULL, + lc_key varchar(255) NOT NULL, + lc_value mediumblob NOT NULL +); +CREATE INDEX /*i*/lc_lang_key ON /*_*/l10n_cache (lc_lang, lc_key); + diff --git a/maintenance/rebuildLocalisationCache.php b/maintenance/rebuildLocalisationCache.php new file mode 100644 index 00000000000..fba69a4ee49 --- /dev/null +++ b/maintenance/rebuildLocalisationCache.php @@ -0,0 +1,41 @@ +isExpired( $code ) ) { + echo "Rebuilding $code...\n"; + $lc->recache( $code ); + $numRebuilt++; + } +} +echo "$numRebuilt languages rebuilt out of " . count( $codes ) . ".\n"; +if ( $numRebuilt == 0 ) { + echo "Use --force to rebuild the caches which are still fresh.\n"; +} + + + diff --git a/maintenance/tables.sql b/maintenance/tables.sql index a52d338f9b6..52855ad1f75 100644 --- a/maintenance/tables.sql +++ b/maintenance/tables.sql @@ -1310,4 +1310,15 @@ CREATE TABLE /*_*/valid_tag ( vt_tag varchar(255) NOT NULL PRIMARY KEY ) /*$wgDBTableOptions*/; +-- Table for storing localisation data +CREATE TABLE /*_*/l10n_cache ( + -- Language code + lc_lang varbinary(32) NOT NULL, + -- Cache key + lc_key varchar(255) NOT NULL, + -- Value + lc_value mediumblob NOT NULL +); +CREATE INDEX /*i*/lc_lang_key ON /*_*/l10n_cache (lc_lang, lc_key); + -- vim: sw=2 sts=2 et diff --git a/maintenance/updaters.inc b/maintenance/updaters.inc index 4b77af4723c..5352d06578e 100644 --- a/maintenance/updaters.inc +++ b/maintenance/updaters.inc @@ -161,6 +161,7 @@ $wgUpdates = array( array( 'add_table', 'log_search', 'patch-log_search.sql' ), array( 'do_log_search_population' ), array( 'add_field', 'logging', 'log_user_text', 'patch-log_user_text.sql' ), + array( 'add_table', 'l10n_cache', 'patch-l10n_cache.sql' ), ), 'sqlite' => array( @@ -180,6 +181,7 @@ $wgUpdates = array( array( 'add_table', 'log_search', 'patch-log_search.sql' ), array( 'do_log_search_population' ), array( 'add_field', 'redirect', 'rd_interwiki', 'patch-rd_interwiki.sql' ), + array( 'add_table', 'l10n_cache', 'patch-l10n_cache.sql' ), ), ); diff --git a/serialized/Makefile b/serialized/Makefile index fcdcbff7473..062155b6844 100644 --- a/serialized/Makefile +++ b/serialized/Makefile @@ -1,20 +1,12 @@ -MESSAGE_SOURCES=$(wildcard ../languages/messages/Messages*.php) -MESSAGE_TARGETS=$(patsubst ../languages/messages/Messages%.php, Messages%.ser, $(MESSAGE_SOURCES)) SPECIAL_TARGETS=Utf8Case.ser -ALL_TARGETS=$(MESSAGE_TARGETS) $(SPECIAL_TARGETS) -DIST_TARGETS=$(SPECIAL_TARGETS) \ - MessagesDe.ser \ - MessagesEn.ser \ - MessagesFr.ser \ - MessagesJa.ser \ - MessagesNl.ser \ - MessagesPl.ser \ - MessagesSv.ser +ALL_TARGETS=$(SPECIAL_TARGETS) +DIST_TARGETS=$(SPECIAL_TARGETS) .PHONY: all dist clean all: $(ALL_TARGETS) + @echo 'Warning: messages are no longer serialized by this makefile.' dist: $(DIST_TARGETS) @@ -24,5 +16,3 @@ clean: Utf8Case.ser : ../includes/normal/Utf8Case.php php serialize.php -o $@ $< -Messages%.ser : ../languages/messages/Messages%.php ../languages/messages/MessagesEn.php - php serialize-localisation.php -o $@ $< diff --git a/serialized/README b/serialized/README deleted file mode 100644 index eae9c527372..00000000000 --- a/serialized/README +++ /dev/null @@ -1,37 +0,0 @@ -This directory contains data files in the format of PHP's serialize() function. -The source data are typically array literals in PHP source files. We have -observed that unserialize(file_get_contents(...)) is faster than executing such -a file from an oparray cache like APC, and very much faster than loading it by -parsing the source file without such a cache. It should also be faster than -loading the data across the network with memcached, as long as you are careful -to put your MediaWiki root directory on a local hard drive rather than on NFS. -This is a good idea for performance in any case. - -To generate all data files: - - cd /path/to/wiki/serialized - make - -This requires GNU Make. At present, the only serialized data file which is -strictly required is Utf8Case.ser. This contains UTF-8 case conversion tables, -which have essentially never changed since MediaWiki was invented. - -The Messages*.ser files are localisation files, containing user interface text -and various other data related to language-specific behaviour. Because they -are merged with the fallback language (usually English) before caching, they -are all quite large, about 140 KB each at the time of writing. If you generate -all of them, they take up about 20 MB. Hence, I don't expect we will include -all of them in the release tarballs. However, to obtain optimum performance, -YOU SHOULD GENERATE ALL THE LOCALISATION FILES THAT YOU WILL BE USING ON YOUR -WIKIS. - -You can generate individual files by typing a command such as: - cd /path/to/wiki/serialized - make MessagesAr.ser - -If you change a Messages*.php source file, you must recompile any serialized -data files which are present. If you change MessagesEn.php, this will -invalidate *all* Messages*.ser files. - -I think we should distribute a few Messages*.ser files in the release tarballs, -specifically the ones created by "make dist". diff --git a/serialized/serialize-localisation.php b/serialized/serialize-localisation.php deleted file mode 100644 index 9801b823bcd..00000000000 --- a/serialized/serialize-localisation.php +++ /dev/null @@ -1,35 +0,0 @@ -