Split SiteLookup interface from SiteStore

* SiteLookup interface is added, and SiteStore extends
  it. (any SiteStore type hints can be changed to use
  SiteLookup if all they need is lookup functionality)
* Memcached based SiteStore code is split from the
  database SiteStore, and SiteSQLStore is deprecated.
  If no caching is desired when using a SiteStore, then
  use a SiteDBStore instance, instead of passing $source
  parameter in SiteStore::getSite and SiteStore::getSites.
* SiteListFileCache renamed to FileBasedSiteLookup and
  implements SiteLookup.

Bug: T77990
Change-Id: I36b599884c211580ea6806a8a190c65c4f9087cf
This commit is contained in:
aude 2015-02-05 21:00:26 -05:00
parent 1f4a23e58f
commit aded554d70
18 changed files with 996 additions and 562 deletions

View file

@ -188,6 +188,7 @@ $wgAutoloadLocalClasses = array(
'CacheHelper' => __DIR__ . '/includes/cache/CacheHelper.php',
'CacheTime' => __DIR__ . '/includes/parser/CacheTime.php',
'CachedAction' => __DIR__ . '/includes/actions/CachedAction.php',
'CachingSiteStore' => __DIR__ . '/includes/site/CachingSiteStore.php',
'CapsCleanup' => __DIR__ . '/maintenance/cleanupCaps.php',
'Category' => __DIR__ . '/includes/Category.php',
'CategoryFinder' => __DIR__ . '/includes/CategoryFinder.php',
@ -276,6 +277,7 @@ $wgAutoloadLocalClasses = array(
'DBMasterPos' => __DIR__ . '/includes/db/DatabaseUtility.php',
'DBObject' => __DIR__ . '/includes/db/DatabaseUtility.php',
'DBQueryError' => __DIR__ . '/includes/db/DatabaseError.php',
'DBSiteStore' => __DIR__ . '/includes/site/DBSiteStore.php',
'DBUnexpectedError' => __DIR__ . '/includes/db/DatabaseError.php',
'DataUpdate' => __DIR__ . '/includes/deferred/DataUpdate.php',
'DatabaseBase' => __DIR__ . '/includes/db/Database.php',
@ -417,6 +419,7 @@ $wgAutoloadLocalClasses = array(
'FileBackendStoreShardDirIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php',
'FileBackendStoreShardFileIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php',
'FileBackendStoreShardListIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php',
'FileBasedSiteLookup' => __DIR__ . '/includes/site/FileBasedSiteLookup.php',
'FileCacheBase' => __DIR__ . '/includes/cache/FileCacheBase.php',
'FileDeleteForm' => __DIR__ . '/includes/FileDeleteForm.php',
'FileDependency' => __DIR__ . '/includes/cache/CacheDependency.php',
@ -1060,14 +1063,14 @@ $wgAutoloadLocalClasses = array(
'SiteExporter' => __DIR__ . '/includes/site/SiteExporter.php',
'SiteImporter' => __DIR__ . '/includes/site/SiteImporter.php',
'SiteList' => __DIR__ . '/includes/site/SiteList.php',
'SiteListFileCache' => __DIR__ . '/includes/site/SiteListFileCache.php',
'SiteListFileCacheBuilder' => __DIR__ . '/includes/site/SiteListFileCacheBuilder.php',
'SiteLookup' => __DIR__ . '/includes/site/SiteLookup.php',
'SiteObject' => __DIR__ . '/includes/site/Site.php',
'SiteSQLStore' => __DIR__ . '/includes/site/SiteSQLStore.php',
'SiteStats' => __DIR__ . '/includes/SiteStats.php',
'SiteStatsInit' => __DIR__ . '/includes/SiteStats.php',
'SiteStatsUpdate' => __DIR__ . '/includes/deferred/SiteStatsUpdate.php',
'SiteStore' => __DIR__ . '/includes/site/SiteStore.php',
'SitesCacheFileBuilder' => __DIR__ . '/includes/site/SitesCacheFileBuilder.php',
'Skin' => __DIR__ . '/includes/skins/Skin.php',
'SkinApi' => __DIR__ . '/includes/skins/SkinApi.php',
'SkinApiTemplate' => __DIR__ . '/includes/skins/SkinApiTemplate.php',

42
docs/sitescache.txt Normal file
View file

@ -0,0 +1,42 @@
MediaWiki's SiteStore can be cached and stored in a flat file,
in a json format. If the SiteStore is frequently accessed, the
file cache may provide a performance benefit over a database
store, even with memcached in front of it.
Configuration:
File-based caching can be enabled by setting $wgSitesCacheFile
to the file path of the cache file.
The file can then be generated with the rebuildSitesCache.php
maintenance script.
Format:
In the sites cache file, sites are listed in a key-value
map, with the key being the site's global id (e.g. "enwiki")
and a key-value map as the value. The site list is wrapped
with in a "sites" key.
Example:
"sites": {
"aawiktionary": {
"globalid": "aawiktionary",
"type": "mediawiki",
"group": "wiktionary",
"source": "local",
"language": "aa",
"localids": [],
"config": [],
"data": {
"paths": {
"file_path": "http:\/\/aa.wiktionary.org\/w\/$1",
"page_path": "http:\/\/aa.wiktionary.org\/wiki\/$1"
}
},
"forward": false,
"internalid": 2666,
"identifiers": []
}
}

View file

@ -3788,7 +3788,7 @@ $wgInterwikiFallbackSite = 'wiki';
*/
/**
* Specify the file location for the SiteStore json cache file.
* Specify the file location for the Sites json cache file.
*/
$wgSitesCacheFile = false;

View file

@ -0,0 +1,195 @@
<?php
/**
* Represents the site configuration of a wiki.
* Holds a list of sites (ie SiteList), with a caching layer.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @since 1.25
*
* @file
* @ingroup Site
*
* @license GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
* @author Katie Filbert < aude.wiki@gmail.com >
*/
class CachingSiteStore implements SiteStore {
/**
* @var SiteList|null
*/
private $sites = null;
/**
* @var string|null
*/
private $cacheKey;
/**
* @var int
*/
private $cacheTimeout;
/**
* @var BagOStuff
*/
private $cache;
/**
* @var SiteStore
*/
private $siteStore;
/**
* @param SiteStore $siteStore
* @param BagOStuff $cache
* @param string|null $cacheKey
* @param int $cacheTimeout
*/
public function __construct(
SiteStore $siteStore,
BagOStuff $cache,
$cacheKey = null,
$cacheTimeout = 3600
) {
$this->siteStore = $siteStore;
$this->cache = $cache;
$this->cacheKey = $cacheKey;
$this->cacheTimeout = $cacheTimeout;
}
/**
* Constructs a cache key to use for caching the list of sites.
*
* This includes the concrete class name of the site list as well as a version identifier
* for the list's serialization, to avoid problems when unserializing site lists serialized
* by an older version, e.g. when reading from a cache.
*
* The cache key also includes information about where the sites were loaded from, e.g.
* the name of a database table.
*
* @see SiteList::getSerialVersionId
*
* @return string The cache key.
*/
private function getCacheKey() {
if ( $this->cacheKey === null ) {
$type = 'SiteList#' . SiteList::getSerialVersionId();
$this->cacheKey = wfMemcKey( "sites/$type" );
}
return $this->cacheKey;
}
/**
* @see SiteStore::getSites
*
* @since 1.25
*
* @return SiteList
*/
public function getSites() {
if ( $this->sites === null ) {
$this->sites = $this->cache->get( $this->getCacheKey() );
if ( !is_object( $this->sites ) ) {
$this->sites = $this->siteStore->getSites();
$this->cache->set( $this->getCacheKey(), $this->sites, $this->cacheTimeout );
}
}
return $this->sites;
}
/**
* @see SiteStore::getSite
*
* @since 1.25
*
* @param string $globalId
*
* @return Site|null
*/
public function getSite( $globalId ) {
$sites = $this->getSites();
return $sites->hasSite( $globalId ) ? $sites->getSite( $globalId ) : null;
}
/**
* @see SiteStore::saveSite
*
* @since 1.25
*
* @param Site $site
*
* @return bool Success indicator
*/
public function saveSite( Site $site ) {
return $this->saveSites( array( $site ) );
}
/**
* @see SiteStore::saveSites
*
* @since 1.25
*
* @param Site[] $sites
*
* @return bool Success indicator
*/
public function saveSites( array $sites ) {
if ( empty( $sites ) ) {
return true;
}
$success = $this->siteStore->saveSites( $sites );
// purge cache
$this->reset();
return $success;
}
/**
* Purges the internal and external cache of the site list, forcing the list
* of sites to be reloaded.
*
* @since 1.25
*/
public function reset() {
// purge cache
$this->cache->delete( $this->getCacheKey() );
$this->sites = null;
}
/**
* Clears the list of sites stored.
*
* @see SiteStore::clear()
*
* @return bool Success
*/
public function clear() {
$this->reset();
return $this->siteStore->clear();
}
}

View file

@ -0,0 +1,345 @@
<?php
/**
* Represents the site configuration of a wiki.
* Holds a list of sites (ie SiteList), stored in the database.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @since 1.25
*
* @file
* @ingroup Site
*
* @license GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
*/
class DBSiteStore implements SiteStore {
/**
* @var SiteList|null
*/
protected $sites = null;
/**
* @var ORMTable
*/
protected $sitesTable;
/**
* @since 1.25
*
* @param ORMTable|null $sitesTable
*/
public function __construct( ORMTable $sitesTable = null ) {
if ( $sitesTable === null ) {
$sitesTable = $this->newSitesTable();
}
$this->sitesTable = $sitesTable;
}
/**
* @see SiteStore::getSites
*
* @since 1.25
*
* @return SiteList
*/
public function getSites() {
$this->loadSites();
return $this->sites;
}
/**
* Returns a new Site object constructed from the provided ORMRow.
*
* @since 1.25
*
* @param ORMRow $siteRow
*
* @return Site
*/
protected function siteFromRow( ORMRow $siteRow ) {
$site = Site::newForType( $siteRow->getField( 'type', Site::TYPE_UNKNOWN ) );
$site->setGlobalId( $siteRow->getField( 'global_key' ) );
$site->setInternalId( $siteRow->getField( 'id' ) );
if ( $siteRow->hasField( 'forward' ) ) {
$site->setForward( $siteRow->getField( 'forward' ) );
}
if ( $siteRow->hasField( 'group' ) ) {
$site->setGroup( $siteRow->getField( 'group' ) );
}
if ( $siteRow->hasField( 'language' ) ) {
$site->setLanguageCode( $siteRow->getField( 'language' ) === ''
? null
: $siteRow->getField( 'language' )
);
}
if ( $siteRow->hasField( 'source' ) ) {
$site->setSource( $siteRow->getField( 'source' ) );
}
if ( $siteRow->hasField( 'data' ) ) {
$site->setExtraData( $siteRow->getField( 'data' ) );
}
if ( $siteRow->hasField( 'config' ) ) {
$site->setExtraConfig( $siteRow->getField( 'config' ) );
}
return $site;
}
/**
* Get a new ORMRow from a Site object
*
* @since 1.25
*
* @param Site $site
*
* @return ORMRow
*/
protected function getRowFromSite( Site $site ) {
$fields = array(
// Site data
'global_key' => $site->getGlobalId(), // TODO: check not null
'type' => $site->getType(),
'group' => $site->getGroup(),
'source' => $site->getSource(),
'language' => $site->getLanguageCode() === null ? '' : $site->getLanguageCode(),
'protocol' => $site->getProtocol(),
'domain' => strrev( $site->getDomain() ) . '.',
'data' => $site->getExtraData(),
// Site config
'forward' => $site->shouldForward(),
'config' => $site->getExtraConfig(),
);
if ( $site->getInternalId() !== null ) {
$fields['id'] = $site->getInternalId();
}
return new ORMRow( $this->sitesTable, $fields );
}
/**
* Fetches the site from the database and loads them into the sites field.
*
* @since 1.25
*/
protected function loadSites() {
$this->sites = new SiteList();
foreach ( $this->sitesTable->select() as $siteRow ) {
$this->sites[] = $this->siteFromRow( $siteRow );
}
// Batch load the local site identifiers.
$ids = wfGetDB( $this->sitesTable->getReadDb() )->select(
'site_identifiers',
array(
'si_site',
'si_type',
'si_key',
),
array(),
__METHOD__
);
foreach ( $ids as $id ) {
if ( $this->sites->hasInternalId( $id->si_site ) ) {
$site = $this->sites->getSiteByInternalId( $id->si_site );
$site->addLocalId( $id->si_type, $id->si_key );
$this->sites->setSite( $site );
}
}
}
/**
* @see SiteStore::getSite
*
* @since 1.25
*
* @param string $globalId
*
* @return Site|null
*/
public function getSite( $globalId ) {
if ( $this->sites === null ) {
$this->sites = $this->getSites();
}
return $this->sites->hasSite( $globalId ) ? $this->sites->getSite( $globalId ) : null;
}
/**
* @see SiteStore::saveSite
*
* @since 1.25
*
* @param Site $site
*
* @return bool Success indicator
*/
public function saveSite( Site $site ) {
return $this->saveSites( array( $site ) );
}
/**
* @see SiteStore::saveSites
*
* @since 1.25
*
* @param Site[] $sites
*
* @return bool Success indicator
*/
public function saveSites( array $sites ) {
if ( empty( $sites ) ) {
return true;
}
$dbw = $this->sitesTable->getWriteDbConnection();
$dbw->startAtomic( __METHOD__ );
$success = true;
$internalIds = array();
$localIds = array();
foreach ( $sites as $site ) {
if ( $site->getInternalId() !== null ) {
$internalIds[] = $site->getInternalId();
}
$siteRow = $this->getRowFromSite( $site );
$success = $siteRow->save( __METHOD__ ) && $success;
foreach ( $site->getLocalIds() as $idType => $ids ) {
foreach ( $ids as $id ) {
$localIds[] = array( $siteRow->getId(), $idType, $id );
}
}
}
if ( $internalIds !== array() ) {
$dbw->delete(
'site_identifiers',
array( 'si_site' => $internalIds ),
__METHOD__
);
}
foreach ( $localIds as $localId ) {
$dbw->insert(
'site_identifiers',
array(
'si_site' => $localId[0],
'si_type' => $localId[1],
'si_key' => $localId[2],
),
__METHOD__
);
}
$dbw->endAtomic( __METHOD__ );
$this->reset();
return $success;
}
/**
* Resets the SiteList
*
* @since 1.25
*/
public function reset() {
$this->sites = null;
}
/**
* Clears the list of sites stored in the database.
*
* @see SiteStore::clear()
*
* @return bool Success
*/
public function clear() {
$dbw = $this->sitesTable->getWriteDbConnection();
$dbw->startAtomic( __METHOD__ );
$ok = $dbw->delete( 'sites', '*', __METHOD__ );
$ok = $dbw->delete( 'site_identifiers', '*', __METHOD__ ) && $ok;
$dbw->endAtomic( __METHOD__ );
$this->reset();
return $ok;
}
/**
* @since 1.25
*
* @return ORMTable
*/
protected function newSitesTable() {
return new ORMTable(
'sites',
array(
'id' => 'id',
// Site data
'global_key' => 'str',
'type' => 'str',
'group' => 'str',
'source' => 'str',
'language' => 'str',
'protocol' => 'str',
'domain' => 'str',
'data' => 'array',
// Site config
'forward' => 'bool',
'config' => 'array',
),
array(
'type' => Site::TYPE_UNKNOWN,
'group' => Site::GROUP_NONE,
'source' => Site::SOURCE_LOCAL,
'data' => array(),
'forward' => false,
'config' => array(),
'language' => '',
),
'ORMRow',
'site_'
);
}
}

View file

@ -21,14 +21,16 @@
*/
/**
* Provides a file-based cache of a SiteStore, stored as a json file.
* Provides a file-based cache of a SiteStore. The sites are stored in
* a json file. (see docs/sitescache.txt regarding format)
*
* The cache can be built with the rebuildSitesCache.php maintenance script,
* and a MediaWiki instance can be setup to use this by setting the
* 'wgSitesCacheFile' configuration to the cache file location.
*
* @since 1.25
*/
class SiteListFileCache {
class FileBasedSiteLookup implements SiteLookup {
/**
* @var SiteList
@ -91,7 +93,7 @@ class SiteListFileCache {
/**
* @throws MWException
* @return array
* @return array see docs/sitescache.txt for format of the array.
*/
private function loadJsonFile() {
if ( !is_readable( $this->cacheFile ) ) {
@ -101,7 +103,9 @@ class SiteListFileCache {
$contents = file_get_contents( $this->cacheFile );
$data = json_decode( $contents, true );
if ( !is_array( $data ) || !array_key_exists( 'sites', $data ) ) {
if ( !is_array( $data ) || !is_array( $data['sites'] )
|| !array_key_exists( 'sites', $data )
) {
throw new MWException( 'SiteStore json cache data is invalid.' );
}
@ -118,7 +122,6 @@ class SiteListFileCache {
$site = Site::newForType( $siteType );
$site->setGlobalId( $data['globalid'] );
$site->setInternalId( $data['internalid'] );
$site->setForward( $data['forward'] );
$site->setGroup( $data['group'] );
$site->setLanguageCode( $data['language'] );

View file

@ -54,6 +54,8 @@ class HashSiteStore implements SiteStore {
*/
public function saveSite( Site $site ) {
$this->sites[$site->getGlobalId()] = $site;
return true;
}
/**
@ -69,6 +71,8 @@ class HashSiteStore implements SiteStore {
foreach ( $sites as $site ) {
$this->saveSite( $site );
}
return true;
}
/**
@ -112,6 +116,8 @@ class HashSiteStore implements SiteStore {
*/
public function clear() {
$this->sites = array();
return true;
}
}

View file

@ -0,0 +1,50 @@
<?php
/**
* Interface for service objects providing a lookup of Site objects.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @since 1.25
*
* @file
* @ingroup Site
*
* @license GNU GPL v2+
*/
interface SiteLookup {
/**
* Returns the site with provided global id, or null if there is no such site.
*
* @since 1.25
*
* @param string $globalId
*
* @return Site|null
*/
public function getSite( $globalId );
/**
* Returns a list of all sites.
*
* @since 1.25
*
* @return SiteList
*/
public function getSites();
}

View file

@ -28,36 +28,11 @@
* @license GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
*/
class SiteSQLStore implements SiteStore {
/**
* @since 1.21
*
* @var SiteList|null
*/
protected $sites = null;
/**
* @var ORMTable
*/
protected $sitesTable;
/**
* @var string|null
*/
private $cacheKey = null;
/**
* @var int
*/
private $cacheTimeout = 3600;
/**
* @var BagOStuff
*/
private $cache;
class SiteSQLStore extends CachingSiteStore {
/**
* @since 1.21
* @deprecated 1.25 Construct a SiteStore instance directly instead.
*
* @param ORMTable|null $sitesTable
* @param BagOStuff|null $cache
@ -69,370 +44,9 @@ class SiteSQLStore implements SiteStore {
$cache = wfGetMainCache();
}
return new static( $cache, $sitesTable );
}
$siteStore = new DBSiteStore();
/**
* Constructor.
*
* @since 1.21
*
* @param BagOStuff $cache
* @param ORMTable|null $sitesTable
*/
protected function __construct( BagOStuff $cache, ORMTable $sitesTable = null ) {
if ( $sitesTable === null ) {
$sitesTable = $this->newSitesTable();
}
$this->cache = $cache;
$this->sitesTable = $sitesTable;
}
/**
* Constructs a cache key to use for caching the list of sites.
*
* This includes the concrete class name of the site list as well as a version identifier
* for the list's serialization, to avoid problems when unserializing site lists serialized
* by an older version, e.g. when reading from a cache.
*
* The cache key also includes information about where the sites were loaded from, e.g.
* the name of a database table.
*
* @see SiteList::getSerialVersionId
*
* @return string The cache key.
*/
protected function getCacheKey() {
if ( $this->cacheKey === null ) {
$type = 'SiteList#' . SiteList::getSerialVersionId();
$source = $this->sitesTable->getName();
if ( $this->sitesTable->getTargetWiki() !== false ) {
$source = $this->sitesTable->getTargetWiki() . '.' . $source;
}
$this->cacheKey = wfMemcKey( "$source/$type" );
}
return $this->cacheKey;
}
/**
* @see SiteStore::getSites
*
* @since 1.21
*
* @param string $source Either 'cache' or 'recache'
*
* @return SiteList
*/
public function getSites( $source = 'cache' ) {
if ( $source === 'cache' ) {
if ( $this->sites === null ) {
$sites = $this->cache->get( $this->getCacheKey() );
if ( is_object( $sites ) ) {
$this->sites = $sites;
} else {
$this->loadSites();
}
}
}
else {
$this->loadSites();
}
return $this->sites;
}
/**
* Returns a new Site object constructed from the provided ORMRow.
*
* @since 1.21
*
* @param ORMRow $siteRow
*
* @return Site
*/
protected function siteFromRow( ORMRow $siteRow ) {
$site = Site::newForType( $siteRow->getField( 'type', Site::TYPE_UNKNOWN ) );
$site->setGlobalId( $siteRow->getField( 'global_key' ) );
$site->setInternalId( $siteRow->getField( 'id' ) );
if ( $siteRow->hasField( 'forward' ) ) {
$site->setForward( $siteRow->getField( 'forward' ) );
}
if ( $siteRow->hasField( 'group' ) ) {
$site->setGroup( $siteRow->getField( 'group' ) );
}
if ( $siteRow->hasField( 'language' ) ) {
$site->setLanguageCode( $siteRow->getField( 'language' ) === ''
? null
: $siteRow->getField( 'language' )
);
}
if ( $siteRow->hasField( 'source' ) ) {
$site->setSource( $siteRow->getField( 'source' ) );
}
if ( $siteRow->hasField( 'data' ) ) {
$site->setExtraData( $siteRow->getField( 'data' ) );
}
if ( $siteRow->hasField( 'config' ) ) {
$site->setExtraConfig( $siteRow->getField( 'config' ) );
}
return $site;
}
/**
* Get a new ORMRow from a Site object
*
* @since 1.22
*
* @param Site $site
*
* @return ORMRow
*/
protected function getRowFromSite( Site $site ) {
$fields = array(
// Site data
'global_key' => $site->getGlobalId(), // TODO: check not null
'type' => $site->getType(),
'group' => $site->getGroup(),
'source' => $site->getSource(),
'language' => $site->getLanguageCode() === null ? '' : $site->getLanguageCode(),
'protocol' => $site->getProtocol(),
'domain' => strrev( $site->getDomain() ) . '.',
'data' => $site->getExtraData(),
// Site config
'forward' => $site->shouldForward(),
'config' => $site->getExtraConfig(),
);
if ( $site->getInternalId() !== null ) {
$fields['id'] = $site->getInternalId();
}
return new ORMRow( $this->sitesTable, $fields );
}
/**
* Fetches the site from the database and loads them into the sites field.
*
* @since 1.21
*/
protected function loadSites() {
$this->sites = new SiteList();
foreach ( $this->sitesTable->select() as $siteRow ) {
$this->sites[] = $this->siteFromRow( $siteRow );
}
// Batch load the local site identifiers.
$ids = wfGetDB( $this->sitesTable->getReadDb() )->select(
'site_identifiers',
array(
'si_site',
'si_type',
'si_key',
),
array(),
__METHOD__
);
foreach ( $ids as $id ) {
if ( $this->sites->hasInternalId( $id->si_site ) ) {
$site = $this->sites->getSiteByInternalId( $id->si_site );
$site->addLocalId( $id->si_type, $id->si_key );
$this->sites->setSite( $site );
}
}
$this->cache->set( $this->getCacheKey(), $this->sites, $this->cacheTimeout );
}
/**
* @see SiteStore::getSite
*
* @since 1.21
*
* @param string $globalId
* @param string $source
*
* @return Site|null
*/
public function getSite( $globalId, $source = 'cache' ) {
$sites = $this->getSites( $source );
return $sites->hasSite( $globalId ) ? $sites->getSite( $globalId ) : null;
}
/**
* @see SiteStore::saveSite
*
* @since 1.21
*
* @param Site $site
*
* @return bool Success indicator
*/
public function saveSite( Site $site ) {
return $this->saveSites( array( $site ) );
}
/**
* @see SiteStore::saveSites
*
* @since 1.21
*
* @param Site[] $sites
*
* @return bool Success indicator
*/
public function saveSites( array $sites ) {
if ( empty( $sites ) ) {
return true;
}
$dbw = $this->sitesTable->getWriteDbConnection();
$dbw->startAtomic( __METHOD__ );
$success = true;
$internalIds = array();
$localIds = array();
foreach ( $sites as $site ) {
if ( $site->getInternalId() !== null ) {
$internalIds[] = $site->getInternalId();
}
$siteRow = $this->getRowFromSite( $site );
$success = $siteRow->save( __METHOD__ ) && $success;
foreach ( $site->getLocalIds() as $idType => $ids ) {
foreach ( $ids as $id ) {
$localIds[] = array( $siteRow->getId(), $idType, $id );
}
}
}
if ( $internalIds !== array() ) {
$dbw->delete(
'site_identifiers',
array( 'si_site' => $internalIds ),
__METHOD__
);
}
foreach ( $localIds as $localId ) {
$dbw->insert(
'site_identifiers',
array(
'si_site' => $localId[0],
'si_type' => $localId[1],
'si_key' => $localId[2],
),
__METHOD__
);
}
$dbw->endAtomic( __METHOD__ );
// purge cache
$this->reset();
return $success;
}
/**
* Purges the internal and external cache of the site list, forcing the list
* of sites to be re-read from the database.
*
* @since 1.21
*/
public function reset() {
// purge cache
$this->cache->delete( $this->getCacheKey() );
$this->sites = null;
}
/**
* Clears the list of sites stored in the database.
*
* @see SiteStore::clear()
*
* @return bool Success
*/
public function clear() {
$dbw = $this->sitesTable->getWriteDbConnection();
$dbw->startAtomic( __METHOD__ );
$ok = $dbw->delete( 'sites', '*', __METHOD__ );
$ok = $dbw->delete( 'site_identifiers', '*', __METHOD__ ) && $ok;
$dbw->endAtomic( __METHOD__ );
$this->reset();
return $ok;
}
/**
* @since 1.21
*
* @return ORMTable
*/
protected function newSitesTable() {
return new ORMTable(
'sites',
array(
'id' => 'id',
// Site data
'global_key' => 'str',
'type' => 'str',
'group' => 'str',
'source' => 'str',
'language' => 'str',
'protocol' => 'str',
'domain' => 'str',
'data' => 'array',
// Site config
'forward' => 'bool',
'config' => 'array',
),
array(
'type' => Site::TYPE_UNKNOWN,
'group' => Site::GROUP_NONE,
'source' => Site::SOURCE_LOCAL,
'data' => array(),
'forward' => false,
'config' => array(),
'language' => '',
),
'ORMRow',
'site_'
);
return new static( $siteStore, $cache );
}
}

View file

@ -26,7 +26,7 @@
* @license GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
*/
interface SiteStore {
interface SiteStore extends SiteLookup {
/**
* Saves the provided site.
@ -50,33 +50,6 @@ interface SiteStore {
*/
public function saveSites( array $sites );
/**
* Returns the site with provided global id, or null if there is no such site.
*
* @since 1.21
*
* @param string $globalId
* @param string $source Either 'cache' or 'recache'.
* If 'cache', the values are allowed (but not obliged) to come from a cache.
*
* @return Site|null
*/
public function getSite( $globalId, $source = 'cache' );
/**
* Returns a list of all sites. By default this site is
* fetched from the cache, which can be changed to loading
* the list from the database using the $useCache parameter.
*
* @since 1.21
*
* @param string $source Either 'cache' or 'recache'.
* If 'cache', the values are allowed (but not obliged) to come from a cache.
*
* @return SiteList
*/
public function getSites( $source = 'cache' );
/**
* Deletes all sites from the database. After calling clear(), getSites() will return an empty
* list and getSite() will return null until saveSite() or saveSites() is called.

View file

@ -22,12 +22,12 @@
*
* @license GNU GPL v2+
*/
class SiteListFileCacheBuilder {
class SitesCacheFileBuilder {
/**
* @var SiteStore
* @var SiteLookup
*/
private $siteStore;
private $siteLookup;
/**
* @var string
@ -35,16 +35,16 @@ class SiteListFileCacheBuilder {
private $cacheFile;
/**
* @param SiteStore $siteStore
* @param SiteLookup $siteLookup
* @param string $cacheFile
*/
public function __construct( SiteStore $siteStore, $cacheFile ) {
$this->siteStore = $siteStore;
public function __construct( SiteLookup $siteLookup, $cacheFile ) {
$this->siteLookup = $siteLookup;
$this->cacheFile = $cacheFile;
}
public function build() {
$this->sites = $this->siteStore->getSites( 'recache' );
$this->sites = $this->siteLookup->getSites();
$this->cacheSites( $this->sites->getArrayCopy() );
}

View file

@ -23,7 +23,7 @@
require_once __DIR__ . '/Maintenance.php';
/**
* Maintenance script to dump the SiteStore as a static json file.
* Maintenance script to dump a SiteStore as a static json file.
*
* @ingroup Maintenance
*/
@ -32,17 +32,17 @@ class RebuildSitesCache extends Maintenance {
public function __construct() {
parent::__construct();
$this->mDescription = "Dumps site store as json";
$this->mDescription = "Cache sites as json for file-based lookup.";
$this->addOption( 'file', 'File to output the json to', false, true );
}
public function execute() {
$siteListFileCacheBuilder = new SiteListFileCacheBuilder(
SiteSQLStore::newInstance(),
$sitesCacheFileBuilder = new SitesCacheFileBuilder(
new DBSiteStore(),
$this->getCacheFile()
);
$siteListFileCacheBuilder->build();
$sitesCacheFileBuilder->build();
}
/**
@ -55,7 +55,7 @@ class RebuildSitesCache extends Maintenance {
$jsonFile = $this->getConfig()->get( 'SitesCacheFile' );
if ( $jsonFile === false ) {
$this->error( 'Error: No sites cache file is set in configuration.', 1 );
$this->error( 'Error: No file set in configuration for SitesCacheFile.', 1 );
}
}

View file

@ -0,0 +1,162 @@
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @since 1.25
*
* @ingroup Site
* @ingroup Test
*
* @group Site
* @group Database
*
* @licence GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
*/
class CachingSiteStoreTest extends MediaWikiTestCase {
/**
* @covers CachingSiteStore::getSites
*/
public function testGetSites() {
$testSites = TestSites::getSites();
$store = new CachingSiteStore(
$this->getHashSiteStore( $testSites ),
wfGetMainCache()
);
$sites = $store->getSites();
$this->assertInstanceOf( 'SiteList', $sites );
/**
* @var Site $site
*/
foreach ( $sites as $site ) {
$this->assertInstanceOf( 'Site', $site );
}
foreach ( $testSites as $site ) {
if ( $site->getGlobalId() !== null ) {
$this->assertTrue( $sites->hasSite( $site->getGlobalId() ) );
}
}
}
/**
* @covers CachingSiteStore::saveSites
*/
public function testSaveSites() {
$store = new CachingSiteStore( new HashSiteStore(), wfGetMainCache() );
$sites = array();
$site = new Site();
$site->setGlobalId( 'ertrywuutr' );
$site->setLanguageCode( 'en' );
$sites[] = $site;
$site = new MediaWikiSite();
$site->setGlobalId( 'sdfhxujgkfpth' );
$site->setLanguageCode( 'nl' );
$sites[] = $site;
$this->assertTrue( $store->saveSites( $sites ) );
$site = $store->getSite( 'ertrywuutr' );
$this->assertInstanceOf( 'Site', $site );
$this->assertEquals( 'en', $site->getLanguageCode() );
$site = $store->getSite( 'sdfhxujgkfpth' );
$this->assertInstanceOf( 'Site', $site );
$this->assertEquals( 'nl', $site->getLanguageCode() );
}
/**
* @covers CachingSiteStore::reset
*/
public function testReset() {
$dbSiteStore = $this->getMockBuilder( 'SiteStore' )
->disableOriginalConstructor()
->getMock();
// php 5.3 compatibility!
$self = $this;
$dbSiteStore->expects( $this->any() )
->method( 'getSite' )
->will( $this->returnValue( $self->getTestSite() ) );
$dbSiteStore->expects( $this->any() )
->method( 'getSites' )
->will( $this->returnCallback( function() use( $self ) {
$siteList = new SiteList();
$siteList->setSite( $self->getTestSite() );
return $siteList;
} ) );
$store = new CachingSiteStore( $dbSiteStore, wfGetMainCache() );
// initialize internal cache
$this->assertGreaterThan( 0, $store->getSites()->count(), 'count sites' );
$store->getSite( 'enwiki' )->setLanguageCode( 'en-ca' );
// sanity check: $store should have the new language code for 'enwiki'
$this->assertEquals( 'en-ca', $store->getSite( 'enwiki' )->getLanguageCode(), 'sanity check' );
// purge cache
$store->reset();
// the internal cache of $store should be updated, and now pulling
// the site from the 'fallback' DBSiteStore with the original language code.
$this->assertEquals( 'en', $store->getSite( 'enwiki' )->getLanguageCode(), 'reset' );
}
public function getTestSite() {
$enwiki = new MediaWikiSite();
$enwiki->setGlobalId( 'enwiki' );
$enwiki->setLanguageCode( 'en' );
return $enwiki;
}
/**
* @covers CachingSiteStore::clear
*/
public function testClear() {
$store = new CachingSiteStore( new HashSiteStore(), wfGetMainCache() );
$this->assertTrue( $store->clear() );
$site = $store->getSite( 'enwiki' );
$this->assertNull( $site );
$sites = $store->getSites();
$this->assertEquals( 0, $sites->count() );
}
private function getHashSiteStore( array $sites ) {
$siteStore = new HashSiteStore();
$siteStore->saveSites( $sites );
return $siteStore;
}
}

View file

@ -0,0 +1,134 @@
<?php
/**
* Tests for the DBSiteStore class.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @since 1.21
*
* @ingroup Site
* @ingroup Test
*
* @group Site
* @group Database
*
* @licence GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
*/
class DBSiteStoreTest extends MediaWikiTestCase {
/**
* @covers DBSiteStore::getSites
*/
public function testGetSites() {
$expectedSites = TestSites::getSites();
TestSites::insertIntoDb();
$store = new DBSiteStore();
$sites = $store->getSites();
$this->assertInstanceOf( 'SiteList', $sites );
/**
* @var Site $site
*/
foreach ( $sites as $site ) {
$this->assertInstanceOf( 'Site', $site );
}
foreach ( $expectedSites as $site ) {
if ( $site->getGlobalId() !== null ) {
$this->assertTrue( $sites->hasSite( $site->getGlobalId() ) );
}
}
}
/**
* @covers DBSiteStore::saveSites
*/
public function testSaveSites() {
$store = new DBSiteStore();
$sites = array();
$site = new Site();
$site->setGlobalId( 'ertrywuutr' );
$site->setLanguageCode( 'en' );
$sites[] = $site;
$site = new MediaWikiSite();
$site->setGlobalId( 'sdfhxujgkfpth' );
$site->setLanguageCode( 'nl' );
$sites[] = $site;
$this->assertTrue( $store->saveSites( $sites ) );
$site = $store->getSite( 'ertrywuutr' );
$this->assertInstanceOf( 'Site', $site );
$this->assertEquals( 'en', $site->getLanguageCode() );
$this->assertTrue( is_integer( $site->getInternalId() ) );
$this->assertTrue( $site->getInternalId() >= 0 );
$site = $store->getSite( 'sdfhxujgkfpth' );
$this->assertInstanceOf( 'Site', $site );
$this->assertEquals( 'nl', $site->getLanguageCode() );
$this->assertTrue( is_integer( $site->getInternalId() ) );
$this->assertTrue( $site->getInternalId() >= 0 );
}
/**
* @covers DBSiteStore::reset
*/
public function testReset() {
$store1 = new DBSiteStore();
$store2 = new DBSiteStore();
// initialize internal cache
$this->assertGreaterThan( 0, $store1->getSites()->count() );
$this->assertGreaterThan( 0, $store2->getSites()->count() );
// Clear actual data. Will purge the external cache and reset the internal
// cache in $store1, but not the internal cache in store2.
$this->assertTrue( $store1->clear() );
// sanity check: $store2 should have a stale cache now
$this->assertNotNull( $store2->getSite( 'enwiki' ) );
// purge cache
$store2->reset();
// ...now the internal cache of $store2 should be updated and thus empty.
$site = $store2->getSite( 'enwiki' );
$this->assertNull( $site );
}
/**
* @covers DBSiteStore::clear
*/
public function testClear() {
$store = new DBSiteStore();
$this->assertTrue( $store->clear() );
$site = $store->getSite( 'enwiki' );
$this->assertNull( $site );
$sites = $store->getSites();
$this->assertEquals( 0, $sites->count() );
}
}

View file

@ -22,13 +22,13 @@
* @ingroup Site
* @ingroup Test
*
* @covers SiteListFileCache
* @covers FileBasedSiteLookup
* @group Site
*
* @licence GNU GPL v2+
* @author Katie Filbert < aude.wiki@gmail.com >
*/
class SiteListFileCacheTest extends PHPUnit_Framework_TestCase {
class FileBasedSiteLookupTest extends PHPUnit_Framework_TestCase {
protected function setUp() {
$this->cacheFile = $this->getCacheFile();
@ -40,40 +40,40 @@ class SiteListFileCacheTest extends PHPUnit_Framework_TestCase {
public function testGetSites() {
$sites = $this->getSites();
$cacheBuilder = $this->newSiteListFileCacheBuilder( $sites );
$cacheBuilder = $this->newSitesCacheFileBuilder( $sites );
$cacheBuilder->build();
$cache = new SiteListFileCache( $this->cacheFile );
$cache = new FileBasedSiteLookup( $this->cacheFile );
$this->assertEquals( $sites, $cache->getSites() );
}
public function testGetSite() {
$sites = $this->getSites();
$cacheBuilder = $this->newSiteListFileCacheBuilder( $sites );
$cacheBuilder = $this->newSitesCacheFileBuilder( $sites );
$cacheBuilder->build();
$cache = new SiteListFileCache( $this->cacheFile );
$cache = new FileBasedSiteLookup( $this->cacheFile );
$this->assertEquals( $sites->getSite( 'enwiktionary' ), $cache->getSite( 'enwiktionary' ) );
}
private function newSiteListFileCacheBuilder( SiteList $sites ) {
return new SiteListFileCacheBuilder(
$this->getSiteSQLStore( $sites ),
private function newSitesCacheFileBuilder( SiteList $sites ) {
return new SitesCacheFileBuilder(
$this->getSiteLookup( $sites ),
$this->cacheFile
);
}
private function getSiteSQLStore( SiteList $sites ) {
$siteSQLStore = $this->getMockBuilder( 'SiteSQLStore' )
private function getSiteLookup( SiteList $sites ) {
$siteLookup = $this->getMockBuilder( 'SiteLookup' )
->disableOriginalConstructor()
->getMock();
$siteSQLStore->expects( $this->any() )
$siteLookup->expects( $this->any() )
->method( 'getSites' )
->will( $this->returnValue( $sites ) );
return $siteSQLStore;
return $siteLookup;
}
private function getSites() {

View file

@ -1,8 +1,6 @@
<?php
/**
* Tests for the SiteSQLStore class.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@ -19,7 +17,7 @@
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @since 1.21
* @since 1.25
*
* @ingroup Site
* @ingroup Test
@ -28,107 +26,16 @@
* @group Database
*
* @licence GNU GPL v2+
* @author Jeroen De Dauw < jeroendedauw@gmail.com >
* @author Katie Filbert < aude.wiki@gmail.com >
*/
class SiteSQLStoreTest extends MediaWikiTestCase {
/**
* @covers SiteSQLStore::getSites
* @covers SiteSQLStore::newInstance
*/
public function testGetSites() {
$expectedSites = TestSites::getSites();
TestSites::insertIntoDb();
$store = SiteSQLStore::newInstance();
$sites = $store->getSites();
$this->assertInstanceOf( 'SiteList', $sites );
/**
* @var Site $site
*/
foreach ( $sites as $site ) {
$this->assertInstanceOf( 'Site', $site );
}
foreach ( $expectedSites as $site ) {
if ( $site->getGlobalId() !== null ) {
$this->assertTrue( $sites->hasSite( $site->getGlobalId() ) );
}
}
public function testNewInstance() {
$siteStore = SiteSQLStore::newInstance();
$this->assertInstanceOf( 'SiteSQLStore', $siteStore );
}
/**
* @covers SiteSQLStore::saveSites
*/
public function testSaveSites() {
$store = SiteSQLStore::newInstance();
$sites = array();
$site = new Site();
$site->setGlobalId( 'ertrywuutr' );
$site->setLanguageCode( 'en' );
$sites[] = $site;
$site = new MediaWikiSite();
$site->setGlobalId( 'sdfhxujgkfpth' );
$site->setLanguageCode( 'nl' );
$sites[] = $site;
$this->assertTrue( $store->saveSites( $sites ) );
$site = $store->getSite( 'ertrywuutr' );
$this->assertInstanceOf( 'Site', $site );
$this->assertEquals( 'en', $site->getLanguageCode() );
$this->assertTrue( is_integer( $site->getInternalId() ) );
$this->assertTrue( $site->getInternalId() >= 0 );
$site = $store->getSite( 'sdfhxujgkfpth' );
$this->assertInstanceOf( 'Site', $site );
$this->assertEquals( 'nl', $site->getLanguageCode() );
$this->assertTrue( is_integer( $site->getInternalId() ) );
$this->assertTrue( $site->getInternalId() >= 0 );
}
/**
* @covers SiteSQLStore::reset
*/
public function testReset() {
$store1 = SiteSQLStore::newInstance();
$store2 = SiteSQLStore::newInstance();
// initialize internal cache
$this->assertGreaterThan( 0, $store1->getSites()->count() );
$this->assertGreaterThan( 0, $store2->getSites()->count() );
// Clear actual data. Will purge the external cache and reset the internal
// cache in $store1, but not the internal cache in store2.
$this->assertTrue( $store1->clear() );
// sanity check: $store2 should have a stale cache now
$this->assertNotNull( $store2->getSite( 'enwiki' ) );
// purge cache
$store2->reset();
// ...now the internal cache of $store2 should be updated and thus empty.
$site = $store2->getSite( 'enwiki' );
$this->assertNull( $site );
}
/**
* @covers SiteSQLStore::clear
*/
public function testClear() {
$store = SiteSQLStore::newInstance();
$this->assertTrue( $store->clear() );
$site = $store->getSite( 'enwiki' );
$this->assertNull( $site );
$sites = $store->getSites();
$this->assertEquals( 0, $sites->count() );
}
}

View file

@ -22,13 +22,13 @@
* @ingroup Site
* @ingroup Test
*
* @covers SiteListFileCacheBuilder
* @covers SitesCacheFileBuilder
* @group Site
*
* @licence GNU GPL v2+
* @author Katie Filbert < aude.wiki@gmail.com >
*/
class SiteListFileCacheBuilderTest extends PHPUnit_Framework_TestCase {
class SitesCacheFileBuilderTest extends PHPUnit_Framework_TestCase {
protected function setUp() {
$this->cacheFile = $this->getCacheFile();
@ -39,7 +39,7 @@ class SiteListFileCacheBuilderTest extends PHPUnit_Framework_TestCase {
}
public function testBuild() {
$cacheBuilder = $this->newSiteListFileCacheBuilder( $this->getSites() );
$cacheBuilder = $this->newSitesCacheFileBuilder( $this->getSites() );
$cacheBuilder->build();
$contents = file_get_contents( $this->cacheFile );
@ -91,23 +91,23 @@ class SiteListFileCacheBuilderTest extends PHPUnit_Framework_TestCase {
);
}
private function newSiteListFileCacheBuilder( SiteList $sites ) {
return new SiteListFileCacheBuilder(
$this->getSiteSQLStore( $sites ),
private function newSitesCacheFileBuilder( SiteList $sites ) {
return new SitesCacheFileBuilder(
$this->getSiteLookup( $sites ),
$this->cacheFile
);
}
private function getSiteSQLStore( SiteList $sites ) {
$siteSQLStore = $this->getMockBuilder( 'SiteSQLStore' )
private function getSiteLookup( SiteList $sites ) {
$siteLookup = $this->getMockBuilder( 'SiteLookup' )
->disableOriginalConstructor()
->getMock();
$siteSQLStore->expects( $this->any() )
$siteLookup->expects( $this->any() )
->method( 'getSites' )
->will( $this->returnValue( $sites ) );
return $siteSQLStore;
return $siteLookup;
}
private function getSites() {

View file

@ -108,7 +108,7 @@ class TestSites {
* @since 0.1
*/
public static function insertIntoDb() {
$sitesTable = SiteSQLStore::newInstance();
$sitesTable = new DBSiteStore();
$sitesTable->clear();
$sitesTable->saveSites( TestSites::getSites() );
}