ae63b9ae10
Callers often tend to end up calling getFileStat(), at least indirectly, or in various successive function on the same path. This created RTTs when the file didn't exist since negatives were not cached. This change does the following: * Cache definitive negatives (404s) in the process cache. Nothing is cached on failure (like network problems). * Ignore process cache entries after a brief time period so long running scripts do not have overly stale entries. Change-Id: I356bd9f48281e3c7e7a273778b2aca59c521a0c7
1877 lines
62 KiB
PHP
1877 lines
62 KiB
PHP
<?php
|
|
/**
|
|
* Base class for all backends using particular storage medium.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along
|
|
* with this program; if not, write to the Free Software Foundation, Inc.,
|
|
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
* http://www.gnu.org/copyleft/gpl.html
|
|
*
|
|
* @file
|
|
* @ingroup FileBackend
|
|
* @author Aaron Schulz
|
|
*/
|
|
|
|
/**
|
|
* @brief Base class for all backends using particular storage medium.
|
|
*
|
|
* This class defines the methods as abstract that subclasses must implement.
|
|
* Outside callers should *not* use functions with "Internal" in the name.
|
|
*
|
|
* The FileBackend operations are implemented using basic functions
|
|
* such as storeInternal(), copyInternal(), deleteInternal() and the like.
|
|
* This class is also responsible for path resolution and sanitization.
|
|
*
|
|
* @ingroup FileBackend
|
|
* @since 1.19
|
|
*/
|
|
abstract class FileBackendStore extends FileBackend {
|
|
/** @var BagOStuff */
|
|
protected $memCache;
|
|
/** @var ProcessCacheLRU */
|
|
protected $cheapCache; // Map of paths to small (RAM/disk) cache items
|
|
/** @var ProcessCacheLRU */
|
|
protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
|
|
|
|
/** @var Array Map of container names to sharding settings */
|
|
protected $shardViaHashLevels = array(); // (container name => config array)
|
|
|
|
protected $maxFileSize = 4294967296; // integer bytes (4GiB)
|
|
|
|
const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
|
|
|
|
/**
|
|
* @see FileBackend::__construct()
|
|
*
|
|
* @param $config Array
|
|
*/
|
|
public function __construct( array $config ) {
|
|
parent::__construct( $config );
|
|
$this->memCache = new EmptyBagOStuff(); // disabled by default
|
|
$this->cheapCache = new ProcessCacheLRU( 300 );
|
|
$this->expensiveCache = new ProcessCacheLRU( 5 );
|
|
}
|
|
|
|
/**
|
|
* Get the maximum allowable file size given backend
|
|
* medium restrictions and basic performance constraints.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* @return integer Bytes
|
|
*/
|
|
final public function maxFileSizeInternal() {
|
|
return $this->maxFileSize;
|
|
}
|
|
|
|
/**
|
|
* Check if a file can be created or changed at a given storage path.
|
|
* FS backends should check if the parent directory exists, files can be
|
|
* written under it, and that any file already there is writable.
|
|
* Backends using key/value stores should check if the container exists.
|
|
*
|
|
* @param $storagePath string
|
|
* @return bool
|
|
*/
|
|
abstract public function isPathUsableInternal( $storagePath );
|
|
|
|
/**
|
|
* Create a file in the backend with the given contents.
|
|
* This will overwrite any file that exists at the destination.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* $params include:
|
|
* - content : the raw file contents
|
|
* - dst : destination storage path
|
|
* - disposition : Content-Disposition header value for the destination
|
|
* - async : Status will be returned immediately if supported.
|
|
* If the status is OK, then its value field will be
|
|
* set to a FileBackendStoreOpHandle object.
|
|
*
|
|
* @param $params Array
|
|
* @return Status
|
|
*/
|
|
final public function createInternal( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
|
|
$status = Status::newFatal( 'backend-fail-maxsize',
|
|
$params['dst'], $this->maxFileSizeInternal() );
|
|
} else {
|
|
$status = $this->doCreateInternal( $params );
|
|
$this->clearCache( array( $params['dst'] ) );
|
|
$this->deleteFileCache( $params['dst'] ); // persistent cache
|
|
}
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::createInternal()
|
|
*/
|
|
abstract protected function doCreateInternal( array $params );
|
|
|
|
/**
|
|
* Store a file into the backend from a file on disk.
|
|
* This will overwrite any file that exists at the destination.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* $params include:
|
|
* - src : source path on disk
|
|
* - dst : destination storage path
|
|
* - disposition : Content-Disposition header value for the destination
|
|
* - async : Status will be returned immediately if supported.
|
|
* If the status is OK, then its value field will be
|
|
* set to a FileBackendStoreOpHandle object.
|
|
*
|
|
* @param $params Array
|
|
* @return Status
|
|
*/
|
|
final public function storeInternal( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
|
|
$status = Status::newFatal( 'backend-fail-maxsize',
|
|
$params['dst'], $this->maxFileSizeInternal() );
|
|
} else {
|
|
$status = $this->doStoreInternal( $params );
|
|
$this->clearCache( array( $params['dst'] ) );
|
|
$this->deleteFileCache( $params['dst'] ); // persistent cache
|
|
}
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::storeInternal()
|
|
*/
|
|
abstract protected function doStoreInternal( array $params );
|
|
|
|
/**
|
|
* Copy a file from one storage path to another in the backend.
|
|
* This will overwrite any file that exists at the destination.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* $params include:
|
|
* - src : source storage path
|
|
* - dst : destination storage path
|
|
* - ignoreMissingSource : do nothing if the source file does not exist
|
|
* - disposition : Content-Disposition header value for the destination
|
|
* - async : Status will be returned immediately if supported.
|
|
* If the status is OK, then its value field will be
|
|
* set to a FileBackendStoreOpHandle object.
|
|
*
|
|
* @param $params Array
|
|
* @return Status
|
|
*/
|
|
final public function copyInternal( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = $this->doCopyInternal( $params );
|
|
$this->clearCache( array( $params['dst'] ) );
|
|
$this->deleteFileCache( $params['dst'] ); // persistent cache
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::copyInternal()
|
|
*/
|
|
abstract protected function doCopyInternal( array $params );
|
|
|
|
/**
|
|
* Delete a file at the storage path.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* $params include:
|
|
* - src : source storage path
|
|
* - ignoreMissingSource : do nothing if the source file does not exist
|
|
* - async : Status will be returned immediately if supported.
|
|
* If the status is OK, then its value field will be
|
|
* set to a FileBackendStoreOpHandle object.
|
|
*
|
|
* @param $params Array
|
|
* @return Status
|
|
*/
|
|
final public function deleteInternal( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = $this->doDeleteInternal( $params );
|
|
$this->clearCache( array( $params['src'] ) );
|
|
$this->deleteFileCache( $params['src'] ); // persistent cache
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::deleteInternal()
|
|
*/
|
|
abstract protected function doDeleteInternal( array $params );
|
|
|
|
/**
|
|
* Move a file from one storage path to another in the backend.
|
|
* This will overwrite any file that exists at the destination.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* $params include:
|
|
* - src : source storage path
|
|
* - dst : destination storage path
|
|
* - ignoreMissingSource : do nothing if the source file does not exist
|
|
* - disposition : Content-Disposition header value for the destination
|
|
* - async : Status will be returned immediately if supported.
|
|
* If the status is OK, then its value field will be
|
|
* set to a FileBackendStoreOpHandle object.
|
|
*
|
|
* @param $params Array
|
|
* @return Status
|
|
*/
|
|
final public function moveInternal( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = $this->doMoveInternal( $params );
|
|
$this->clearCache( array( $params['src'], $params['dst'] ) );
|
|
$this->deleteFileCache( $params['src'] ); // persistent cache
|
|
$this->deleteFileCache( $params['dst'] ); // persistent cache
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::moveInternal()
|
|
* @return Status
|
|
*/
|
|
protected function doMoveInternal( array $params ) {
|
|
unset( $params['async'] ); // two steps, won't work here :)
|
|
// Copy source to dest
|
|
$status = $this->copyInternal( $params );
|
|
if ( $status->isOK() ) {
|
|
// Delete source (only fails due to races or medium going down)
|
|
$status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
|
|
$status->setResult( true, $status->value ); // ignore delete() errors
|
|
}
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* No-op file operation that does nothing.
|
|
* Do not call this function from places outside FileBackend and FileOp.
|
|
*
|
|
* @param $params Array
|
|
* @return Status
|
|
*/
|
|
final public function nullInternal( array $params ) {
|
|
return Status::newGood();
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::concatenate()
|
|
* @return Status
|
|
*/
|
|
final public function concatenate( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
// Try to lock the source files for the scope of this function
|
|
$scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
|
|
if ( $status->isOK() ) {
|
|
// Actually do the file concatenation...
|
|
$start_time = microtime( true );
|
|
$status->merge( $this->doConcatenate( $params ) );
|
|
$sec = microtime( true ) - $start_time;
|
|
if ( !$status->isOK() ) {
|
|
wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
|
|
count( $params['srcs'] ) . " file(s) [$sec sec]" );
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::concatenate()
|
|
* @return Status
|
|
*/
|
|
protected function doConcatenate( array $params ) {
|
|
$status = Status::newGood();
|
|
$tmpPath = $params['dst']; // convenience
|
|
unset( $params['latest'] ); // sanity
|
|
|
|
// Check that the specified temp file is valid...
|
|
wfSuppressWarnings();
|
|
$ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
|
|
wfRestoreWarnings();
|
|
if ( !$ok ) { // not present or not empty
|
|
$status->fatal( 'backend-fail-opentemp', $tmpPath );
|
|
return $status;
|
|
}
|
|
|
|
// Get local FS versions of the chunks needed for the concatenation...
|
|
$fsFiles = $this->getLocalReferenceMulti( $params );
|
|
foreach ( $fsFiles as $path => &$fsFile ) {
|
|
if ( !$fsFile ) { // chunk failed to download?
|
|
$fsFile = $this->getLocalReference( array( 'src' => $path ) );
|
|
if ( !$fsFile ) { // retry failed?
|
|
$status->fatal( 'backend-fail-read', $path );
|
|
return $status;
|
|
}
|
|
}
|
|
}
|
|
unset( $fsFile ); // unset reference so we can reuse $fsFile
|
|
|
|
// Get a handle for the destination temp file
|
|
$tmpHandle = fopen( $tmpPath, 'ab' );
|
|
if ( $tmpHandle === false ) {
|
|
$status->fatal( 'backend-fail-opentemp', $tmpPath );
|
|
return $status;
|
|
}
|
|
|
|
// Build up the temp file using the source chunks (in order)...
|
|
foreach ( $fsFiles as $virtualSource => $fsFile ) {
|
|
// Get a handle to the local FS version
|
|
$sourceHandle = fopen( $fsFile->getPath(), 'rb' );
|
|
if ( $sourceHandle === false ) {
|
|
fclose( $tmpHandle );
|
|
$status->fatal( 'backend-fail-read', $virtualSource );
|
|
return $status;
|
|
}
|
|
// Append chunk to file (pass chunk size to avoid magic quotes)
|
|
if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
|
|
fclose( $sourceHandle );
|
|
fclose( $tmpHandle );
|
|
$status->fatal( 'backend-fail-writetemp', $tmpPath );
|
|
return $status;
|
|
}
|
|
fclose( $sourceHandle );
|
|
}
|
|
if ( !fclose( $tmpHandle ) ) {
|
|
$status->fatal( 'backend-fail-closetemp', $tmpPath );
|
|
return $status;
|
|
}
|
|
|
|
clearstatcache(); // temp file changed
|
|
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::doPrepare()
|
|
* @return Status
|
|
*/
|
|
final protected function doPrepare( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
|
|
$status = Status::newGood();
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) {
|
|
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status; // invalid storage path
|
|
}
|
|
|
|
if ( $shard !== null ) { // confined to a single container/shard
|
|
$status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
|
|
} else { // directory is on several shards
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
|
|
$status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::doPrepare()
|
|
* @return Status
|
|
*/
|
|
protected function doPrepareInternal( $container, $dir, array $params ) {
|
|
return Status::newGood();
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::doSecure()
|
|
* @return Status
|
|
*/
|
|
final protected function doSecure( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) {
|
|
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status; // invalid storage path
|
|
}
|
|
|
|
if ( $shard !== null ) { // confined to a single container/shard
|
|
$status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
|
|
} else { // directory is on several shards
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
|
|
$status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::doSecure()
|
|
* @return Status
|
|
*/
|
|
protected function doSecureInternal( $container, $dir, array $params ) {
|
|
return Status::newGood();
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::doPublish()
|
|
* @return Status
|
|
*/
|
|
final protected function doPublish( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) {
|
|
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status; // invalid storage path
|
|
}
|
|
|
|
if ( $shard !== null ) { // confined to a single container/shard
|
|
$status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
|
|
} else { // directory is on several shards
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
|
|
$status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::doPublish()
|
|
* @return Status
|
|
*/
|
|
protected function doPublishInternal( $container, $dir, array $params ) {
|
|
return Status::newGood();
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::doClean()
|
|
* @return Status
|
|
*/
|
|
final protected function doClean( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
// Recursive: first delete all empty subdirs recursively
|
|
if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
|
|
$subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
|
|
if ( $subDirsRel !== null ) { // no errors
|
|
foreach ( $subDirsRel as $subDirRel ) {
|
|
$subDir = $params['dir'] . "/{$subDirRel}"; // full path
|
|
$status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
|
|
}
|
|
}
|
|
}
|
|
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) {
|
|
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status; // invalid storage path
|
|
}
|
|
|
|
// Attempt to lock this directory...
|
|
$filesLockEx = array( $params['dir'] );
|
|
$scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
|
|
if ( !$status->isOK() ) {
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status; // abort
|
|
}
|
|
|
|
if ( $shard !== null ) { // confined to a single container/shard
|
|
$status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
|
|
$this->deleteContainerCache( $fullCont ); // purge cache
|
|
} else { // directory is on several shards
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
|
|
$status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
|
|
$this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::doClean()
|
|
* @return Status
|
|
*/
|
|
protected function doCleanInternal( $container, $dir, array $params ) {
|
|
return Status::newGood();
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::fileExists()
|
|
* @return bool|null
|
|
*/
|
|
final public function fileExists( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$stat = $this->getFileStat( $params );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return ( $stat === null ) ? null : (bool)$stat; // null => failure
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getFileTimestamp()
|
|
* @return bool
|
|
*/
|
|
final public function getFileTimestamp( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$stat = $this->getFileStat( $params );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $stat ? $stat['mtime'] : false;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getFileSize()
|
|
* @return bool
|
|
*/
|
|
final public function getFileSize( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$stat = $this->getFileStat( $params );
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $stat ? $stat['size'] : false;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getFileStat()
|
|
* @return bool
|
|
*/
|
|
final public function getFileStat( array $params ) {
|
|
$path = self::normalizeStoragePath( $params['src'] );
|
|
if ( $path === null ) {
|
|
return false; // invalid storage path
|
|
}
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$latest = !empty( $params['latest'] ); // use latest data?
|
|
if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
|
|
$this->primeFileCache( array( $path ) ); // check persistent cache
|
|
}
|
|
if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
|
|
$stat = $this->cheapCache->get( $path, 'stat' );
|
|
// If we want the latest data, check that this cached
|
|
// value was in fact fetched with the latest available data
|
|
// (the process cache is ignored if it contains a negative).
|
|
if ( !$latest || ( is_array( $stat ) && $stat['latest'] ) ) {
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $stat;
|
|
}
|
|
}
|
|
wfProfileIn( __METHOD__ . '-miss' );
|
|
wfProfileIn( __METHOD__ . '-miss-' . $this->name );
|
|
$stat = $this->doGetFileStat( $params );
|
|
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
|
|
wfProfileOut( __METHOD__ . '-miss' );
|
|
if ( is_array( $stat ) ) { // file exists
|
|
$stat['latest'] = $latest;
|
|
$this->cheapCache->set( $path, 'stat', $stat );
|
|
$this->setFileCache( $path, $stat ); // update persistent cache
|
|
if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
|
|
$this->cheapCache->set( $path, 'sha1',
|
|
array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
|
|
}
|
|
} elseif ( $stat === false ) { // file does not exist
|
|
$this->cheapCache->set( $path, 'stat', false );
|
|
wfDebug( __METHOD__ . ": File $path does not exist.\n" );
|
|
} else { // an error occurred
|
|
wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
|
|
}
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $stat;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::getFileStat()
|
|
*/
|
|
abstract protected function doGetFileStat( array $params );
|
|
|
|
/**
|
|
* @see FileBackend::getFileContentsMulti()
|
|
* @return Array
|
|
*/
|
|
public function getFileContentsMulti( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
|
|
$params = $this->setConcurrencyFlags( $params );
|
|
$contents = $this->doGetFileContentsMulti( $params );
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $contents;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::getFileContentsMulti()
|
|
* @return Array
|
|
*/
|
|
protected function doGetFileContentsMulti( array $params ) {
|
|
$contents = array();
|
|
foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
|
|
wfSuppressWarnings();
|
|
$contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
|
|
wfRestoreWarnings();
|
|
}
|
|
return $contents;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getFileSha1Base36()
|
|
* @return bool|string
|
|
*/
|
|
final public function getFileSha1Base36( array $params ) {
|
|
$path = self::normalizeStoragePath( $params['src'] );
|
|
if ( $path === null ) {
|
|
return false; // invalid storage path
|
|
}
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$latest = !empty( $params['latest'] ); // use latest data?
|
|
if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
|
|
$stat = $this->cheapCache->get( $path, 'sha1' );
|
|
// If we want the latest data, check that this cached
|
|
// value was in fact fetched with the latest available data.
|
|
if ( !$latest || $stat['latest'] ) {
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $stat['hash'];
|
|
}
|
|
}
|
|
wfProfileIn( __METHOD__ . '-miss' );
|
|
wfProfileIn( __METHOD__ . '-miss-' . $this->name );
|
|
$hash = $this->doGetFileSha1Base36( $params );
|
|
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
|
|
wfProfileOut( __METHOD__ . '-miss' );
|
|
if ( $hash ) { // don't cache negatives
|
|
$this->cheapCache->set( $path, 'sha1',
|
|
array( 'hash' => $hash, 'latest' => $latest ) );
|
|
}
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $hash;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::getFileSha1Base36()
|
|
* @return bool|string
|
|
*/
|
|
protected function doGetFileSha1Base36( array $params ) {
|
|
$fsFile = $this->getLocalReference( $params );
|
|
if ( !$fsFile ) {
|
|
return false;
|
|
} else {
|
|
return $fsFile->getSha1Base36();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getFileProps()
|
|
* @return Array
|
|
*/
|
|
final public function getFileProps( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$fsFile = $this->getLocalReference( $params );
|
|
$props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $props;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getLocalReferenceMulti()
|
|
* @return Array
|
|
*/
|
|
final public function getLocalReferenceMulti( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
|
|
$params = $this->setConcurrencyFlags( $params );
|
|
|
|
$fsFiles = array(); // (path => FSFile)
|
|
$latest = !empty( $params['latest'] ); // use latest data?
|
|
// Reuse any files already in process cache...
|
|
foreach ( $params['srcs'] as $src ) {
|
|
$path = self::normalizeStoragePath( $src );
|
|
if ( $path === null ) {
|
|
$fsFiles[$src] = null; // invalid storage path
|
|
} elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
|
|
$val = $this->expensiveCache->get( $path, 'localRef' );
|
|
// If we want the latest data, check that this cached
|
|
// value was in fact fetched with the latest available data.
|
|
if ( !$latest || $val['latest'] ) {
|
|
$fsFiles[$src] = $val['object'];
|
|
}
|
|
}
|
|
}
|
|
// Fetch local references of any remaning files...
|
|
$params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
|
|
foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
|
|
$fsFiles[$path] = $fsFile;
|
|
if ( $fsFile ) { // update the process cache...
|
|
$this->expensiveCache->set( $path, 'localRef',
|
|
array( 'object' => $fsFile, 'latest' => $latest ) );
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $fsFiles;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::getLocalReferenceMulti()
|
|
* @return Array
|
|
*/
|
|
protected function doGetLocalReferenceMulti( array $params ) {
|
|
return $this->doGetLocalCopyMulti( $params );
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getLocalCopyMulti()
|
|
* @return Array
|
|
*/
|
|
final public function getLocalCopyMulti( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
|
|
$params = $this->setConcurrencyFlags( $params );
|
|
$tmpFiles = $this->doGetLocalCopyMulti( $params );
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $tmpFiles;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::getLocalCopyMulti()
|
|
* @return Array
|
|
*/
|
|
abstract protected function doGetLocalCopyMulti( array $params );
|
|
|
|
/**
|
|
* @see FileBackend::getFileHttpUrl()
|
|
* @return string|null
|
|
*/
|
|
public function getFileHttpUrl( array $params ) {
|
|
return null; // not supported
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::streamFile()
|
|
* @return Status
|
|
*/
|
|
final public function streamFile( array $params ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
$info = $this->getFileStat( $params );
|
|
if ( !$info ) { // let StreamFile handle the 404
|
|
$status->fatal( 'backend-fail-notexists', $params['src'] );
|
|
}
|
|
|
|
// Set output buffer and HTTP headers for stream
|
|
$extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
|
|
$res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
|
|
if ( $res == StreamFile::NOT_MODIFIED ) {
|
|
// do nothing; client cache is up to date
|
|
} elseif ( $res == StreamFile::READY_STREAM ) {
|
|
wfProfileIn( __METHOD__ . '-send' );
|
|
wfProfileIn( __METHOD__ . '-send-' . $this->name );
|
|
$status = $this->doStreamFile( $params );
|
|
wfProfileOut( __METHOD__ . '-send-' . $this->name );
|
|
wfProfileOut( __METHOD__ . '-send' );
|
|
if ( !$status->isOK() ) {
|
|
// Per bug 41113, nasty things can happen if bad cache entries get
|
|
// stuck in cache. It's also possible that this error can come up
|
|
// with simple race conditions. Clear out the stat cache to be safe.
|
|
$this->clearCache( array( $params['src'] ) );
|
|
$this->deleteFileCache( $params['src'] );
|
|
trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
|
|
}
|
|
} else {
|
|
$status->fatal( 'backend-fail-stream', $params['src'] );
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::streamFile()
|
|
* @return Status
|
|
*/
|
|
protected function doStreamFile( array $params ) {
|
|
$status = Status::newGood();
|
|
|
|
$fsFile = $this->getLocalReference( $params );
|
|
if ( !$fsFile ) {
|
|
$status->fatal( 'backend-fail-stream', $params['src'] );
|
|
} elseif ( !readfile( $fsFile->getPath() ) ) {
|
|
$status->fatal( 'backend-fail-stream', $params['src'] );
|
|
}
|
|
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::directoryExists()
|
|
* @return bool|null
|
|
*/
|
|
final public function directoryExists( array $params ) {
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) {
|
|
return false; // invalid storage path
|
|
}
|
|
if ( $shard !== null ) { // confined to a single container/shard
|
|
return $this->doDirectoryExists( $fullCont, $dir, $params );
|
|
} else { // directory is on several shards
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
$res = false; // response
|
|
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
|
|
$exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
|
|
if ( $exists ) {
|
|
$res = true;
|
|
break; // found one!
|
|
} elseif ( $exists === null ) { // error?
|
|
$res = null; // if we don't find anything, it is indeterminate
|
|
}
|
|
}
|
|
return $res;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::directoryExists()
|
|
*
|
|
* @param $container string Resolved container name
|
|
* @param $dir string Resolved path relative to container
|
|
* @param $params Array
|
|
* @return bool|null
|
|
*/
|
|
abstract protected function doDirectoryExists( $container, $dir, array $params );
|
|
|
|
/**
|
|
* @see FileBackend::getDirectoryList()
|
|
* @return Traversable|Array|null Returns null on failure
|
|
*/
|
|
final public function getDirectoryList( array $params ) {
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) { // invalid storage path
|
|
return null;
|
|
}
|
|
if ( $shard !== null ) {
|
|
// File listing is confined to a single container/shard
|
|
return $this->getDirectoryListInternal( $fullCont, $dir, $params );
|
|
} else {
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
// File listing spans multiple containers/shards
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
return new FileBackendStoreShardDirIterator( $this,
|
|
$fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Do not call this function from places outside FileBackend
|
|
*
|
|
* @see FileBackendStore::getDirectoryList()
|
|
*
|
|
* @param $container string Resolved container name
|
|
* @param $dir string Resolved path relative to container
|
|
* @param $params Array
|
|
* @return Traversable|Array|null Returns null on failure
|
|
*/
|
|
abstract public function getDirectoryListInternal( $container, $dir, array $params );
|
|
|
|
/**
|
|
* @see FileBackend::getFileList()
|
|
* @return Traversable|Array|null Returns null on failure
|
|
*/
|
|
final public function getFileList( array $params ) {
|
|
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
|
|
if ( $dir === null ) { // invalid storage path
|
|
return null;
|
|
}
|
|
if ( $shard !== null ) {
|
|
// File listing is confined to a single container/shard
|
|
return $this->getFileListInternal( $fullCont, $dir, $params );
|
|
} else {
|
|
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
|
|
// File listing spans multiple containers/shards
|
|
list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
|
|
return new FileBackendStoreShardFileIterator( $this,
|
|
$fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Do not call this function from places outside FileBackend
|
|
*
|
|
* @see FileBackendStore::getFileList()
|
|
*
|
|
* @param $container string Resolved container name
|
|
* @param $dir string Resolved path relative to container
|
|
* @param $params Array
|
|
* @return Traversable|Array|null Returns null on failure
|
|
*/
|
|
abstract public function getFileListInternal( $container, $dir, array $params );
|
|
|
|
/**
|
|
* Return a list of FileOp objects from a list of operations.
|
|
* Do not call this function from places outside FileBackend.
|
|
*
|
|
* The result must have the same number of items as the input.
|
|
* An exception is thrown if an unsupported operation is requested.
|
|
*
|
|
* @param $ops Array Same format as doOperations()
|
|
* @return Array List of FileOp objects
|
|
* @throws MWException
|
|
*/
|
|
final public function getOperationsInternal( array $ops ) {
|
|
$supportedOps = array(
|
|
'store' => 'StoreFileOp',
|
|
'copy' => 'CopyFileOp',
|
|
'move' => 'MoveFileOp',
|
|
'delete' => 'DeleteFileOp',
|
|
'create' => 'CreateFileOp',
|
|
'null' => 'NullFileOp'
|
|
);
|
|
|
|
$performOps = array(); // array of FileOp objects
|
|
// Build up ordered array of FileOps...
|
|
foreach ( $ops as $operation ) {
|
|
$opName = $operation['op'];
|
|
if ( isset( $supportedOps[$opName] ) ) {
|
|
$class = $supportedOps[$opName];
|
|
// Get params for this operation
|
|
$params = $operation;
|
|
// Append the FileOp class
|
|
$performOps[] = new $class( $this, $params );
|
|
} else {
|
|
throw new MWException( "Operation '$opName' is not supported." );
|
|
}
|
|
}
|
|
|
|
return $performOps;
|
|
}
|
|
|
|
/**
|
|
* Get a list of storage paths to lock for a list of operations
|
|
* Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
|
|
* each corresponding to a list of storage paths to be locked.
|
|
* All returned paths are normalized.
|
|
*
|
|
* @param $performOps Array List of FileOp objects
|
|
* @return Array ('sh' => list of paths, 'ex' => list of paths)
|
|
*/
|
|
final public function getPathsToLockForOpsInternal( array $performOps ) {
|
|
// Build up a list of files to lock...
|
|
$paths = array( 'sh' => array(), 'ex' => array() );
|
|
foreach ( $performOps as $fileOp ) {
|
|
$paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
|
|
$paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
|
|
}
|
|
// Optimization: if doing an EX lock anyway, don't also set an SH one
|
|
$paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
|
|
// Get a shared lock on the parent directory of each path changed
|
|
$paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
|
|
|
|
return $paths;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::getScopedLocksForOps()
|
|
* @return Array
|
|
*/
|
|
public function getScopedLocksForOps( array $ops, Status $status ) {
|
|
$paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
|
|
return array(
|
|
$this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
|
|
$this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::doOperationsInternal()
|
|
* @return Status
|
|
*/
|
|
final protected function doOperationsInternal( array $ops, array $opts ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
// Build up a list of FileOps...
|
|
$performOps = $this->getOperationsInternal( $ops );
|
|
|
|
// Acquire any locks as needed...
|
|
if ( empty( $opts['nonLocking'] ) ) {
|
|
// Build up a list of files to lock...
|
|
$paths = $this->getPathsToLockForOpsInternal( $performOps );
|
|
// Try to lock those files for the scope of this function...
|
|
$scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
|
|
$scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
|
|
if ( !$status->isOK() ) {
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status; // abort
|
|
}
|
|
}
|
|
|
|
// Clear any file cache entries (after locks acquired)
|
|
if ( empty( $opts['preserveCache'] ) ) {
|
|
$this->clearCache();
|
|
}
|
|
|
|
// Load from the persistent file and container caches
|
|
$this->primeFileCache( $performOps );
|
|
$this->primeContainerCache( $performOps );
|
|
|
|
// Actually attempt the operation batch...
|
|
$opts = $this->setConcurrencyFlags( $opts );
|
|
$subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
|
|
|
|
// Merge errors into status fields
|
|
$status->merge( $subStatus );
|
|
$status->success = $subStatus->success; // not done in merge()
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::doQuickOperationsInternal()
|
|
* @return Status
|
|
* @throws MWException
|
|
*/
|
|
final protected function doQuickOperationsInternal( array $ops ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
$status = Status::newGood();
|
|
|
|
$supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
|
|
$async = ( $this->parallelize === 'implicit' );
|
|
$maxConcurrency = $this->concurrency; // throttle
|
|
|
|
$statuses = array(); // array of (index => Status)
|
|
$fileOpHandles = array(); // list of (index => handle) arrays
|
|
$curFileOpHandles = array(); // current handle batch
|
|
// Perform the sync-only ops and build up op handles for the async ops...
|
|
foreach ( $ops as $index => $params ) {
|
|
if ( !in_array( $params['op'], $supportedOps ) ) {
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
throw new MWException( "Operation '{$params['op']}' is not supported." );
|
|
}
|
|
$method = $params['op'] . 'Internal'; // e.g. "storeInternal"
|
|
$subStatus = $this->$method( array( 'async' => $async ) + $params );
|
|
if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
|
|
if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
|
|
$fileOpHandles[] = $curFileOpHandles; // push this batch
|
|
$curFileOpHandles = array();
|
|
}
|
|
$curFileOpHandles[$index] = $subStatus->value; // keep index
|
|
} else { // error or completed
|
|
$statuses[$index] = $subStatus; // keep index
|
|
}
|
|
}
|
|
if ( count( $curFileOpHandles ) ) {
|
|
$fileOpHandles[] = $curFileOpHandles; // last batch
|
|
}
|
|
// Do all the async ops that can be done concurrently...
|
|
foreach ( $fileOpHandles as $fileHandleBatch ) {
|
|
$statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
|
|
}
|
|
// Marshall and merge all the responses...
|
|
foreach ( $statuses as $index => $subStatus ) {
|
|
$status->merge( $subStatus );
|
|
if ( $subStatus->isOK() ) {
|
|
$status->success[$index] = true;
|
|
++$status->successCount;
|
|
} else {
|
|
$status->success[$index] = false;
|
|
++$status->failCount;
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $status;
|
|
}
|
|
|
|
/**
|
|
* Execute a list of FileBackendStoreOpHandle handles in parallel.
|
|
* The resulting Status object fields will correspond
|
|
* to the order in which the handles where given.
|
|
*
|
|
* @param $handles Array List of FileBackendStoreOpHandle objects
|
|
* @return Array Map of Status objects
|
|
* @throws MWException
|
|
*/
|
|
final public function executeOpHandlesInternal( array $fileOpHandles ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
foreach ( $fileOpHandles as $fileOpHandle ) {
|
|
if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
|
|
throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
|
|
} elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
|
|
throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
|
|
}
|
|
}
|
|
$res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
|
|
foreach ( $fileOpHandles as $fileOpHandle ) {
|
|
$fileOpHandle->closeResources();
|
|
}
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
return $res;
|
|
}
|
|
|
|
/**
|
|
* @see FileBackendStore::executeOpHandlesInternal()
|
|
* @param array $fileOpHandles
|
|
* @throws MWException
|
|
* @return Array List of corresponding Status objects
|
|
*/
|
|
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
|
|
foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
|
|
throw new MWException( "This backend supports no asynchronous operations." );
|
|
}
|
|
return array();
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::preloadCache()
|
|
*/
|
|
final public function preloadCache( array $paths ) {
|
|
$fullConts = array(); // full container names
|
|
foreach ( $paths as $path ) {
|
|
list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
|
|
$fullConts[] = $fullCont;
|
|
}
|
|
// Load from the persistent file and container caches
|
|
$this->primeContainerCache( $fullConts );
|
|
$this->primeFileCache( $paths );
|
|
}
|
|
|
|
/**
|
|
* @see FileBackend::clearCache()
|
|
*/
|
|
final public function clearCache( array $paths = null ) {
|
|
if ( is_array( $paths ) ) {
|
|
$paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
|
|
$paths = array_filter( $paths, 'strlen' ); // remove nulls
|
|
}
|
|
if ( $paths === null ) {
|
|
$this->cheapCache->clear();
|
|
$this->expensiveCache->clear();
|
|
} else {
|
|
foreach ( $paths as $path ) {
|
|
$this->cheapCache->clear( $path );
|
|
$this->expensiveCache->clear( $path );
|
|
}
|
|
}
|
|
$this->doClearCache( $paths );
|
|
}
|
|
|
|
/**
|
|
* Clears any additional stat caches for storage paths
|
|
*
|
|
* @see FileBackend::clearCache()
|
|
*
|
|
* @param $paths Array Storage paths (optional)
|
|
* @return void
|
|
*/
|
|
protected function doClearCache( array $paths = null ) {}
|
|
|
|
/**
|
|
* Is this a key/value store where directories are just virtual?
|
|
* Virtual directories exists in so much as files exists that are
|
|
* prefixed with the directory path followed by a forward slash.
|
|
*
|
|
* @return bool
|
|
*/
|
|
abstract protected function directoriesAreVirtual();
|
|
|
|
/**
|
|
* Check if a container name is valid.
|
|
* This checks for for length and illegal characters.
|
|
*
|
|
* @param $container string
|
|
* @return bool
|
|
*/
|
|
final protected static function isValidContainerName( $container ) {
|
|
// This accounts for Swift and S3 restrictions while leaving room
|
|
// for things like '.xxx' (hex shard chars) or '.seg' (segments).
|
|
// This disallows directory separators or traversal characters.
|
|
// Note that matching strings URL encode to the same string;
|
|
// in Swift, the length restriction is *after* URL encoding.
|
|
return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
|
|
}
|
|
|
|
/**
|
|
* Splits a storage path into an internal container name,
|
|
* an internal relative file name, and a container shard suffix.
|
|
* Any shard suffix is already appended to the internal container name.
|
|
* This also checks that the storage path is valid and within this backend.
|
|
*
|
|
* If the container is sharded but a suffix could not be determined,
|
|
* this means that the path can only refer to a directory and can only
|
|
* be scanned by looking in all the container shards.
|
|
*
|
|
* @param $storagePath string
|
|
* @return Array (container, path, container suffix) or (null, null, null) if invalid
|
|
*/
|
|
final protected function resolveStoragePath( $storagePath ) {
|
|
list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
|
|
if ( $backend === $this->name ) { // must be for this backend
|
|
$relPath = self::normalizeContainerPath( $relPath );
|
|
if ( $relPath !== null ) {
|
|
// Get shard for the normalized path if this container is sharded
|
|
$cShard = $this->getContainerShard( $container, $relPath );
|
|
// Validate and sanitize the relative path (backend-specific)
|
|
$relPath = $this->resolveContainerPath( $container, $relPath );
|
|
if ( $relPath !== null ) {
|
|
// Prepend any wiki ID prefix to the container name
|
|
$container = $this->fullContainerName( $container );
|
|
if ( self::isValidContainerName( $container ) ) {
|
|
// Validate and sanitize the container name (backend-specific)
|
|
$container = $this->resolveContainerName( "{$container}{$cShard}" );
|
|
if ( $container !== null ) {
|
|
return array( $container, $relPath, $cShard );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return array( null, null, null );
|
|
}
|
|
|
|
/**
|
|
* Like resolveStoragePath() except null values are returned if
|
|
* the container is sharded and the shard could not be determined.
|
|
*
|
|
* @see FileBackendStore::resolveStoragePath()
|
|
*
|
|
* @param $storagePath string
|
|
* @return Array (container, path) or (null, null) if invalid
|
|
*/
|
|
final protected function resolveStoragePathReal( $storagePath ) {
|
|
list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
|
|
if ( $cShard !== null ) {
|
|
return array( $container, $relPath );
|
|
}
|
|
return array( null, null );
|
|
}
|
|
|
|
/**
|
|
* Get the container name shard suffix for a given path.
|
|
* Any empty suffix means the container is not sharded.
|
|
*
|
|
* @param $container string Container name
|
|
* @param $relPath string Storage path relative to the container
|
|
* @return string|null Returns null if shard could not be determined
|
|
*/
|
|
final protected function getContainerShard( $container, $relPath ) {
|
|
list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
|
|
if ( $levels == 1 || $levels == 2 ) {
|
|
// Hash characters are either base 16 or 36
|
|
$char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
|
|
// Get a regex that represents the shard portion of paths.
|
|
// The concatenation of the captures gives us the shard.
|
|
if ( $levels === 1 ) { // 16 or 36 shards per container
|
|
$hashDirRegex = '(' . $char . ')';
|
|
} else { // 256 or 1296 shards per container
|
|
if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
|
|
$hashDirRegex = $char . '/(' . $char . '{2})';
|
|
} else { // short hash dir format (e.g. "a/b/c")
|
|
$hashDirRegex = '(' . $char . ')/(' . $char . ')';
|
|
}
|
|
}
|
|
// Allow certain directories to be above the hash dirs so as
|
|
// to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
|
|
// They must be 2+ chars to avoid any hash directory ambiguity.
|
|
$m = array();
|
|
if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
|
|
return '.' . implode( '', array_slice( $m, 1 ) );
|
|
}
|
|
return null; // failed to match
|
|
}
|
|
return ''; // no sharding
|
|
}
|
|
|
|
/**
|
|
* Check if a storage path maps to a single shard.
|
|
* Container dirs like "a", where the container shards on "x/xy",
|
|
* can reside on several shards. Such paths are tricky to handle.
|
|
*
|
|
* @param $storagePath string Storage path
|
|
* @return bool
|
|
*/
|
|
final public function isSingleShardPathInternal( $storagePath ) {
|
|
list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
|
|
return ( $shard !== null );
|
|
}
|
|
|
|
/**
|
|
* Get the sharding config for a container.
|
|
* If greater than 0, then all file storage paths within
|
|
* the container are required to be hashed accordingly.
|
|
*
|
|
* @param $container string
|
|
* @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
|
|
*/
|
|
final protected function getContainerHashLevels( $container ) {
|
|
if ( isset( $this->shardViaHashLevels[$container] ) ) {
|
|
$config = $this->shardViaHashLevels[$container];
|
|
$hashLevels = (int)$config['levels'];
|
|
if ( $hashLevels == 1 || $hashLevels == 2 ) {
|
|
$hashBase = (int)$config['base'];
|
|
if ( $hashBase == 16 || $hashBase == 36 ) {
|
|
return array( $hashLevels, $hashBase, $config['repeat'] );
|
|
}
|
|
}
|
|
}
|
|
return array( 0, 0, false ); // no sharding
|
|
}
|
|
|
|
/**
|
|
* Get a list of full container shard suffixes for a container
|
|
*
|
|
* @param $container string
|
|
* @return Array
|
|
*/
|
|
final protected function getContainerSuffixes( $container ) {
|
|
$shards = array();
|
|
list( $digits, $base ) = $this->getContainerHashLevels( $container );
|
|
if ( $digits > 0 ) {
|
|
$numShards = pow( $base, $digits );
|
|
for ( $index = 0; $index < $numShards; $index++ ) {
|
|
$shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
|
|
}
|
|
}
|
|
return $shards;
|
|
}
|
|
|
|
/**
|
|
* Get the full container name, including the wiki ID prefix
|
|
*
|
|
* @param $container string
|
|
* @return string
|
|
*/
|
|
final protected function fullContainerName( $container ) {
|
|
if ( $this->wikiId != '' ) {
|
|
return "{$this->wikiId}-$container";
|
|
} else {
|
|
return $container;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Resolve a container name, checking if it's allowed by the backend.
|
|
* This is intended for internal use, such as encoding illegal chars.
|
|
* Subclasses can override this to be more restrictive.
|
|
*
|
|
* @param $container string
|
|
* @return string|null
|
|
*/
|
|
protected function resolveContainerName( $container ) {
|
|
return $container;
|
|
}
|
|
|
|
/**
|
|
* Resolve a relative storage path, checking if it's allowed by the backend.
|
|
* This is intended for internal use, such as encoding illegal chars or perhaps
|
|
* getting absolute paths (e.g. FS based backends). Note that the relative path
|
|
* may be the empty string (e.g. the path is simply to the container).
|
|
*
|
|
* @param $container string Container name
|
|
* @param $relStoragePath string Storage path relative to the container
|
|
* @return string|null Path or null if not valid
|
|
*/
|
|
protected function resolveContainerPath( $container, $relStoragePath ) {
|
|
return $relStoragePath;
|
|
}
|
|
|
|
/**
|
|
* Get the cache key for a container
|
|
*
|
|
* @param $container string Resolved container name
|
|
* @return string
|
|
*/
|
|
private function containerCacheKey( $container ) {
|
|
return wfMemcKey( 'backend', $this->getName(), 'container', $container );
|
|
}
|
|
|
|
/**
|
|
* Set the cached info for a container
|
|
*
|
|
* @param $container string Resolved container name
|
|
* @param $val mixed Information to cache
|
|
*/
|
|
final protected function setContainerCache( $container, $val ) {
|
|
$this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 );
|
|
}
|
|
|
|
/**
|
|
* Delete the cached info for a container.
|
|
* The cache key is salted for a while to prevent race conditions.
|
|
*
|
|
* @param $container string Resolved container name
|
|
*/
|
|
final protected function deleteContainerCache( $container ) {
|
|
if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
|
|
trigger_error( "Unable to delete stat cache for container $container." );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Do a batch lookup from cache for container stats for all containers
|
|
* used in a list of container names, storage paths, or FileOp objects.
|
|
* This loads the persistent cache values into the process cache.
|
|
*
|
|
* @param $items Array
|
|
* @return void
|
|
*/
|
|
final protected function primeContainerCache( array $items ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
|
|
$paths = array(); // list of storage paths
|
|
$contNames = array(); // (cache key => resolved container name)
|
|
// Get all the paths/containers from the items...
|
|
foreach ( $items as $item ) {
|
|
if ( $item instanceof FileOp ) {
|
|
$paths = array_merge( $paths, $item->storagePathsRead() );
|
|
$paths = array_merge( $paths, $item->storagePathsChanged() );
|
|
} elseif ( self::isStoragePath( $item ) ) {
|
|
$paths[] = $item;
|
|
} elseif ( is_string( $item ) ) { // full container name
|
|
$contNames[$this->containerCacheKey( $item )] = $item;
|
|
}
|
|
}
|
|
// Get all the corresponding cache keys for paths...
|
|
foreach ( $paths as $path ) {
|
|
list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
|
|
if ( $fullCont !== null ) { // valid path for this backend
|
|
$contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
|
|
}
|
|
}
|
|
|
|
$contInfo = array(); // (resolved container name => cache value)
|
|
// Get all cache entries for these container cache keys...
|
|
$values = $this->memCache->getMulti( array_keys( $contNames ) );
|
|
foreach ( $values as $cacheKey => $val ) {
|
|
$contInfo[$contNames[$cacheKey]] = $val;
|
|
}
|
|
|
|
// Populate the container process cache for the backend...
|
|
$this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
}
|
|
|
|
/**
|
|
* Fill the backend-specific process cache given an array of
|
|
* resolved container names and their corresponding cached info.
|
|
* Only containers that actually exist should appear in the map.
|
|
*
|
|
* @param $containerInfo Array Map of resolved container names to cached info
|
|
* @return void
|
|
*/
|
|
protected function doPrimeContainerCache( array $containerInfo ) {}
|
|
|
|
/**
|
|
* Get the cache key for a file path
|
|
*
|
|
* @param $path string Normalized storage path
|
|
* @return string
|
|
*/
|
|
private function fileCacheKey( $path ) {
|
|
return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
|
|
}
|
|
|
|
/**
|
|
* Set the cached stat info for a file path.
|
|
* Negatives (404s) are not cached. By not caching negatives, we can skip cache
|
|
* salting for the case when a file is created at a path were there was none before.
|
|
*
|
|
* @param $path string Storage path
|
|
* @param $val mixed Information to cache
|
|
*/
|
|
final protected function setFileCache( $path, $val ) {
|
|
$path = FileBackend::normalizeStoragePath( $path );
|
|
if ( $path === null ) {
|
|
return; // invalid storage path
|
|
}
|
|
$this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
|
|
}
|
|
|
|
/**
|
|
* Delete the cached stat info for a file path.
|
|
* The cache key is salted for a while to prevent race conditions.
|
|
*
|
|
* @param $path string Storage path
|
|
*/
|
|
final protected function deleteFileCache( $path ) {
|
|
$path = FileBackend::normalizeStoragePath( $path );
|
|
if ( $path === null ) {
|
|
return; // invalid storage path
|
|
}
|
|
if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
|
|
trigger_error( "Unable to delete stat cache for file $path." );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Do a batch lookup from cache for file stats for all paths
|
|
* used in a list of storage paths or FileOp objects.
|
|
* This loads the persistent cache values into the process cache.
|
|
*
|
|
* @param $items Array List of storage paths or FileOps
|
|
* @return void
|
|
*/
|
|
final protected function primeFileCache( array $items ) {
|
|
wfProfileIn( __METHOD__ );
|
|
wfProfileIn( __METHOD__ . '-' . $this->name );
|
|
|
|
$paths = array(); // list of storage paths
|
|
$pathNames = array(); // (cache key => storage path)
|
|
// Get all the paths/containers from the items...
|
|
foreach ( $items as $item ) {
|
|
if ( $item instanceof FileOp ) {
|
|
$paths = array_merge( $paths, $item->storagePathsRead() );
|
|
$paths = array_merge( $paths, $item->storagePathsChanged() );
|
|
} elseif ( self::isStoragePath( $item ) ) {
|
|
$paths[] = FileBackend::normalizeStoragePath( $item );
|
|
}
|
|
}
|
|
// Get rid of any paths that failed normalization...
|
|
$paths = array_filter( $paths, 'strlen' ); // remove nulls
|
|
// Get all the corresponding cache keys for paths...
|
|
foreach ( $paths as $path ) {
|
|
list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
|
|
if ( $rel !== null ) { // valid path for this backend
|
|
$pathNames[$this->fileCacheKey( $path )] = $path;
|
|
}
|
|
}
|
|
// Get all cache entries for these container cache keys...
|
|
$values = $this->memCache->getMulti( array_keys( $pathNames ) );
|
|
foreach ( $values as $cacheKey => $val ) {
|
|
if ( is_array( $val ) ) {
|
|
$path = $pathNames[$cacheKey];
|
|
$this->cheapCache->set( $path, 'stat', $val );
|
|
if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
|
|
$this->cheapCache->set( $path, 'sha1',
|
|
array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
|
|
}
|
|
}
|
|
}
|
|
|
|
wfProfileOut( __METHOD__ . '-' . $this->name );
|
|
wfProfileOut( __METHOD__ );
|
|
}
|
|
|
|
/**
|
|
* Set the 'concurrency' option from a list of operation options
|
|
*
|
|
* @param $opts array Map of operation options
|
|
* @return Array
|
|
*/
|
|
final protected function setConcurrencyFlags( array $opts ) {
|
|
$opts['concurrency'] = 1; // off
|
|
if ( $this->parallelize === 'implicit' ) {
|
|
if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
|
|
$opts['concurrency'] = $this->concurrency;
|
|
}
|
|
} elseif ( $this->parallelize === 'explicit' ) {
|
|
if ( !empty( $opts['parallelize'] ) ) {
|
|
$opts['concurrency'] = $this->concurrency;
|
|
}
|
|
}
|
|
return $opts;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* FileBackendStore helper class for performing asynchronous file operations.
|
|
*
|
|
* For example, calling FileBackendStore::createInternal() with the "async"
|
|
* param flag may result in a Status that contains this object as a value.
|
|
* This class is largely backend-specific and is mostly just "magic" to be
|
|
* passed to FileBackendStore::executeOpHandlesInternal().
|
|
*/
|
|
abstract class FileBackendStoreOpHandle {
|
|
/** @var Array */
|
|
public $params = array(); // params to caller functions
|
|
/** @var FileBackendStore */
|
|
public $backend;
|
|
/** @var Array */
|
|
public $resourcesToClose = array();
|
|
|
|
public $call; // string; name that identifies the function called
|
|
|
|
/**
|
|
* Close all open file handles
|
|
*
|
|
* @return void
|
|
*/
|
|
public function closeResources() {
|
|
array_map( 'fclose', $this->resourcesToClose );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* FileBackendStore helper function to handle listings that span container shards.
|
|
* Do not use this class from places outside of FileBackendStore.
|
|
*
|
|
* @ingroup FileBackend
|
|
*/
|
|
abstract class FileBackendStoreShardListIterator implements Iterator {
|
|
/** @var FileBackendStore */
|
|
protected $backend;
|
|
/** @var Array */
|
|
protected $params;
|
|
/** @var Array */
|
|
protected $shardSuffixes;
|
|
protected $container; // string; full container name
|
|
protected $directory; // string; resolved relative path
|
|
|
|
/** @var Traversable */
|
|
protected $iter;
|
|
protected $curShard = 0; // integer
|
|
protected $pos = 0; // integer
|
|
|
|
/** @var Array */
|
|
protected $multiShardPaths = array(); // (rel path => 1)
|
|
|
|
/**
|
|
* @param $backend FileBackendStore
|
|
* @param $container string Full storage container name
|
|
* @param $dir string Storage directory relative to container
|
|
* @param $suffixes Array List of container shard suffixes
|
|
* @param $params Array
|
|
*/
|
|
public function __construct(
|
|
FileBackendStore $backend, $container, $dir, array $suffixes, array $params
|
|
) {
|
|
$this->backend = $backend;
|
|
$this->container = $container;
|
|
$this->directory = $dir;
|
|
$this->shardSuffixes = $suffixes;
|
|
$this->params = $params;
|
|
}
|
|
|
|
/**
|
|
* @see Iterator::key()
|
|
* @return integer
|
|
*/
|
|
public function key() {
|
|
return $this->pos;
|
|
}
|
|
|
|
/**
|
|
* @see Iterator::valid()
|
|
* @return bool
|
|
*/
|
|
public function valid() {
|
|
if ( $this->iter instanceof Iterator ) {
|
|
return $this->iter->valid();
|
|
} elseif ( is_array( $this->iter ) ) {
|
|
return ( current( $this->iter ) !== false ); // no paths can have this value
|
|
}
|
|
return false; // some failure?
|
|
}
|
|
|
|
/**
|
|
* @see Iterator::current()
|
|
* @return string|bool String or false
|
|
*/
|
|
public function current() {
|
|
return ( $this->iter instanceof Iterator )
|
|
? $this->iter->current()
|
|
: current( $this->iter );
|
|
}
|
|
|
|
/**
|
|
* @see Iterator::next()
|
|
* @return void
|
|
*/
|
|
public function next() {
|
|
++$this->pos;
|
|
( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
|
|
do {
|
|
$continue = false; // keep scanning shards?
|
|
$this->filterViaNext(); // filter out duplicates
|
|
// Find the next non-empty shard if no elements are left
|
|
if ( !$this->valid() ) {
|
|
$this->nextShardIteratorIfNotValid();
|
|
$continue = $this->valid(); // re-filter unless we ran out of shards
|
|
}
|
|
} while ( $continue );
|
|
}
|
|
|
|
/**
|
|
* @see Iterator::rewind()
|
|
* @return void
|
|
*/
|
|
public function rewind() {
|
|
$this->pos = 0;
|
|
$this->curShard = 0;
|
|
$this->setIteratorFromCurrentShard();
|
|
do {
|
|
$continue = false; // keep scanning shards?
|
|
$this->filterViaNext(); // filter out duplicates
|
|
// Find the next non-empty shard if no elements are left
|
|
if ( !$this->valid() ) {
|
|
$this->nextShardIteratorIfNotValid();
|
|
$continue = $this->valid(); // re-filter unless we ran out of shards
|
|
}
|
|
} while ( $continue );
|
|
}
|
|
|
|
/**
|
|
* Filter out duplicate items by advancing to the next ones
|
|
*/
|
|
protected function filterViaNext() {
|
|
while ( $this->valid() ) {
|
|
$rel = $this->iter->current(); // path relative to given directory
|
|
$path = $this->params['dir'] . "/{$rel}"; // full storage path
|
|
if ( $this->backend->isSingleShardPathInternal( $path ) ) {
|
|
break; // path is only on one shard; no issue with duplicates
|
|
} elseif ( isset( $this->multiShardPaths[$rel] ) ) {
|
|
// Don't keep listing paths that are on multiple shards
|
|
( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
|
|
} else {
|
|
$this->multiShardPaths[$rel] = 1;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* If the list iterator for this container shard is out of items,
|
|
* then move on to the next container that has items.
|
|
* If there are none, then it advances to the last container.
|
|
*/
|
|
protected function nextShardIteratorIfNotValid() {
|
|
while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) {
|
|
$this->setIteratorFromCurrentShard();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Set the list iterator to that of the current container shard
|
|
*/
|
|
protected function setIteratorFromCurrentShard() {
|
|
$this->iter = $this->listFromShard(
|
|
$this->container . $this->shardSuffixes[$this->curShard],
|
|
$this->directory, $this->params );
|
|
// Start loading results so that current() works
|
|
if ( $this->iter ) {
|
|
( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the list for a given container shard
|
|
*
|
|
* @param $container string Resolved container name
|
|
* @param $dir string Resolved path relative to container
|
|
* @param $params Array
|
|
* @return Traversable|Array|null
|
|
*/
|
|
abstract protected function listFromShard( $container, $dir, array $params );
|
|
}
|
|
|
|
/**
|
|
* Iterator for listing directories
|
|
*/
|
|
class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
|
|
/**
|
|
* @see FileBackendStoreShardListIterator::listFromShard()
|
|
* @return Array|null|Traversable
|
|
*/
|
|
protected function listFromShard( $container, $dir, array $params ) {
|
|
return $this->backend->getDirectoryListInternal( $container, $dir, $params );
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Iterator for listing regular files
|
|
*/
|
|
class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
|
|
/**
|
|
* @see FileBackendStoreShardListIterator::listFromShard()
|
|
* @return Array|null|Traversable
|
|
*/
|
|
protected function listFromShard( $container, $dir, array $params ) {
|
|
return $this->backend->getFileListInternal( $container, $dir, $params );
|
|
}
|
|
}
|