wiki.techinc.nl/includes/cache/localisation/LCStoreStaticArray.php
Timo Tijhof 55db848b77 localisation: Release data from memory in LCStoreStaticArray::finishWrite
With this change, the memory behaviour of LCStoreStaticArray
matches the other LCStore implementations. Specifically, that when
mass-rebuilding LocalisationCache entries for all language codes,
the computed data should be released from memory after
calling LCStore::finishWrite().

This doesn't affect user-facing web requests, even in the case
of stock MW where every once in a while a user request can lazy-
regenerate the LCStore, there is a process-cache in front of LCStore
in the LocalisationCache class.

The rebuildLocalisationCache.php clears that via
LocalisationCacheBulkLoad::unload(), but due to LCStoreStaticArray
internally holding on to the data, it was still leaking.

The leak was found by @Nikerabbit as part of testing for T218207.

To test this, amend rebuildLocalisationCache.php and add the
following on line 161, as the first line of the doRebuild/foreach/if
block:

  echo "[$code-start-mem] " . round(memory_get_usage(true)/1024/1024, 2) . " MB\n";

If you then have LocalSettings.php configured like so:

  $wgCacheDirectory = $wgTmpDirectory;
  $wgLocalisationCacheConf['store'] = 'array';

Then before this patch, running rebuildLocalisationCache.php,
shows memory starting at 12 MB and growing 2-3 MB for every language
until the very end, closing with 970 MB memory use.

After this patch, it starts at 12 MB and stops growing at 32 MB.

When configuring as `['store'] = 'files'`, which uses LCStoreCDB,
the memory starts at 12 MB and stops growing at 44 MB, both before
and after this patch.

Bug: T218207
Change-Id: I0d215efee5b31766776a068b16811d52f9879312
2019-09-04 21:35:10 +01:00

144 lines
3.9 KiB
PHP

<?php
/**
* Localisation cache storage based on PHP files and static arrays.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
use Wikimedia\StaticArrayWriter;
/**
* @since 1.26
*/
class LCStoreStaticArray implements LCStore {
/** @var string|null Current language code. */
private $currentLang = null;
/** @var array Localisation data. */
private $data = [];
/** @var string File name. */
private $fname = null;
/** @var string Directory for cache files. */
private $directory;
public function __construct( $conf = [] ) {
$this->directory = $conf['directory'];
}
public function startWrite( $code ) {
if ( !file_exists( $this->directory ) && !wfMkdirParents( $this->directory, null, __METHOD__ ) ) {
throw new MWException( "Unable to create the localisation store " .
"directory \"{$this->directory}\"" );
}
$this->currentLang = $code;
$this->fname = $this->directory . '/' . $code . '.l10n.php';
$this->data[$code] = [];
if ( file_exists( $this->fname ) ) {
$this->data[$code] = require $this->fname;
}
}
public function set( $key, $value ) {
$this->data[$this->currentLang][$key] = self::encode( $value );
}
/**
* Encodes a value into an array format
*
* @param mixed $value
* @return array|mixed
* @throws RuntimeException
*/
public static function encode( $value ) {
if ( is_array( $value ) ) {
// [a]rray
return [ 'a', array_map( 'LCStoreStaticArray::encode', $value ) ];
}
if ( is_object( $value ) ) {
// [s]erialized
return [ 's', serialize( $value ) ];
}
if ( is_scalar( $value ) || $value === null ) {
// Scalar value, written directly without array
return $value;
}
throw new RuntimeException( 'Cannot encode ' . var_export( $value, true ) );
}
/**
* Decode something that was encoded with encode
*
* @param mixed $encoded
* @return array|mixed
* @throws RuntimeException
*/
public static function decode( $encoded ) {
if ( !is_array( $encoded ) ) {
// Scalar values are written directly without array
return $encoded;
}
list( $type, $data ) = $encoded;
switch ( $type ) {
case 'a':
return array_map( 'LCStoreStaticArray::decode', $data );
case 's':
return unserialize( $data );
case 'v':
// Support: MediaWiki 1.32 and earlier
// Backward compatibility with older file format
return $data;
default:
throw new RuntimeException(
'Unable to decode ' . var_export( $encoded, true ) );
}
}
public function finishWrite() {
$writer = new StaticArrayWriter();
$out = $writer->create(
$this->data[$this->currentLang],
'Generated by LCStoreStaticArray.php -- do not edit!'
);
file_put_contents( $this->fname, $out );
// Release the data to manage the memory in rebuildLocalisationCache
unset( $this->data[$this->currentLang] );
$this->currentLang = null;
$this->fname = null;
}
public function get( $code, $key ) {
if ( !array_key_exists( $code, $this->data ) ) {
$fname = $this->directory . '/' . $code . '.l10n.php';
if ( !file_exists( $fname ) ) {
return null;
}
$this->data[$code] = require $fname;
}
$data = $this->data[$code];
if ( array_key_exists( $key, $data ) ) {
return self::decode( $data[$key] );
}
return null;
}
}