Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Depedencency cleanups to SwiftFileBackend

Browse source 
* Avoid wf* function MediaWiki dependencies.
* Don't bother with getLocalClusterInstance() caching for the CLI case.
* Give FileBackend a default 'resetOutputBuffer' callback.

Change-Id: I359da1ad77c62880ea799b65cd3a16ad673a64eb
This commit is contained in:
Aaron Schulz 2016-09-21 16:12:27 -07:00
parent 016c5801ce
commit dc522cf0a3
4 changed files with 45 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
includes/filebackend/FileBackendGroup.php
View file

@ -168,6 +168,7 @@ class FileBackendGroup {
? FileJournal::factory( $config['fileJournal'], $name ) ? FileJournal::factory( $config['fileJournal'], $name )
: FileJournal::factory( [ 'class' => 'NullFileJournal' ], $name ); : FileJournal::factory( [ 'class' => 'NullFileJournal' ], $name );
$config['wanCache'] = ObjectCache::getMainWANInstance(); $config['wanCache'] = ObjectCache::getMainWANInstance();
$config['srvCache'] = ObjectCache::getLocalServerInstance( 'hash' );
$config['statusWrapper'] = [ 'Status', 'wrap' ]; $config['statusWrapper'] = [ 'Status', 'wrap' ];
$config['tmpDirectory'] = wfTempDir(); $config['tmpDirectory'] = wfTempDir();
$config['logger'] = LoggerFactory::getInstance( 'FileOperation' ); $config['logger'] = LoggerFactory::getInstance( 'FileOperation' );

57
includes/filebackend/SwiftFileBackend.php
View file

@ -134,14 +134,8 @@ class SwiftFileBackend extends FileBackendStore {
// Process cache for container info // Process cache for container info
$this->containerStatCache = new ProcessCacheLRU( 300 ); $this->containerStatCache = new ProcessCacheLRU( 300 );
// Cache auth token information to avoid RTTs // Cache auth token information to avoid RTTs
if ( !empty( $config['cacheAuthInfo'] ) ) { if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
if ( PHP_SAPI === 'cli' ) { $this->srvCache = $config['srvCache'];
// Preferrably memcached
$this->srvCache = ObjectCache::getLocalClusterInstance();
} else {
// Look for APC, XCache, WinCache, ect...
$this->srvCache = ObjectCache::getLocalServerInstance( CACHE_NONE );
}
} else { } else {
$this->srvCache = new EmptyBagOStuff(); $this->srvCache = new EmptyBagOStuff();
} }
@ -576,7 +570,7 @@ class SwiftFileBackend extends FileBackendStore {
return $status; // already there return $status; // already there
} elseif ( $stat === null ) { } elseif ( $stat === null ) {
$status->fatal( 'backend-fail-internal', $this->name ); $status->fatal( 'backend-fail-internal', $this->name );
wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); $this->logger->error( __METHOD__ . ': cannot get container stat' );
return $status; return $status;
} }
@ -608,7 +602,7 @@ class SwiftFileBackend extends FileBackendStore {
$status->fatal( 'backend-fail-usable', $params['dir'] ); $status->fatal( 'backend-fail-usable', $params['dir'] );
} else { } else {
$status->fatal( 'backend-fail-internal', $this->name ); $status->fatal( 'backend-fail-internal', $this->name );
wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); $this->logger->error( __METHOD__ . ': cannot get container stat' );
} }
return $status; return $status;
@ -629,7 +623,7 @@ class SwiftFileBackend extends FileBackendStore {
$status->fatal( 'backend-fail-usable', $params['dir'] ); $status->fatal( 'backend-fail-usable', $params['dir'] );
} else { } else {
$status->fatal( 'backend-fail-internal', $this->name ); $status->fatal( 'backend-fail-internal', $this->name );
wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); $this->logger->error( __METHOD__ . ': cannot get container stat' );
} }
return $status; return $status;
@ -649,7 +643,7 @@ class SwiftFileBackend extends FileBackendStore {
return $status; // ok, nothing to do return $status; // ok, nothing to do
} elseif ( !is_array( $stat ) ) { } elseif ( !is_array( $stat ) ) {
$status->fatal( 'backend-fail-internal', $this->name ); $status->fatal( 'backend-fail-internal', $this->name );
wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); $this->logger->error( __METHOD__ . ': cannot get container stat' );
return $status; return $status;
} }
@ -704,8 +698,8 @@ class SwiftFileBackend extends FileBackendStore {
} }
/** @noinspection PhpUnusedLocalVariableInspection */ /** @noinspection PhpUnusedLocalVariableInspection */
$ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
wfDebugLog( 'SwiftBackend', __METHOD__ . ": $path was not stored with SHA-1 metadata." ); $this->logger->error( __METHOD__ . ": $path was not stored with SHA-1 metadata." );
$objHdrs['x-object-meta-sha1base36'] = false; $objHdrs['x-object-meta-sha1base36'] = false;
@ -745,7 +739,7 @@ class SwiftFileBackend extends FileBackendStore {
} }
} }
wfDebugLog( 'SwiftBackend', __METHOD__ . ": unable to set SHA-1 metadata for $path" ); $this->logger->error( __METHOD__ . ": unable to set SHA-1 metadata for $path" );
return $objHdrs; // failed return $objHdrs; // failed
} }
@ -848,14 +842,14 @@ class SwiftFileBackend extends FileBackendStore {
return $dirs; // nothing more return $dirs; // nothing more
} }
$ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
$prefix = ( $dir == '' ) ? null : "{$dir}/"; $prefix = ( $dir == '' ) ? null : "{$dir}/";
// Non-recursive: only list dirs right under $dir // Non-recursive: only list dirs right under $dir
if ( !empty( $params['topOnly'] ) ) { if ( !empty( $params['topOnly'] ) ) {
$status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
if ( !$status->isOK() ) { if ( !$status->isOK() ) {
throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" ); throw new FileBackendError( "Iterator page I/O error." );
} }
$objects = $status->value; $objects = $status->value;
foreach ( $objects as $object ) { // files and directories foreach ( $objects as $object ) { // files and directories
@ -874,7 +868,7 @@ class SwiftFileBackend extends FileBackendStore {
$status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
if ( !$status->isOK() ) { if ( !$status->isOK() ) {
throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" ); throw new FileBackendError( "Iterator page I/O error." );
} }
$objects = $status->value; $objects = $status->value;
@ -928,7 +922,7 @@ class SwiftFileBackend extends FileBackendStore {
return $files; // nothing more return $files; // nothing more
} }
$ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
$prefix = ( $dir == '' ) ? null : "{$dir}/"; $prefix = ( $dir == '' ) ? null : "{$dir}/";
// $objects will contain a list of unfiltered names or CF_Object items // $objects will contain a list of unfiltered names or CF_Object items
@ -950,7 +944,7 @@ class SwiftFileBackend extends FileBackendStore {
// Reformat this list into a list of (name, stat array or null) entries // Reformat this list into a list of (name, stat array or null) entries
if ( !$status->isOK() ) { if ( !$status->isOK() ) {
throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" ); throw new FileBackendError( "Iterator page I/O error." );
} }
$objects = $status->value; $objects = $status->value;
@ -1079,7 +1073,7 @@ class SwiftFileBackend extends FileBackendStore {
if ( empty( $params['allowOB'] ) ) { if ( empty( $params['allowOB'] ) ) {
// Cancel output buffering and gzipping if set // Cancel output buffering and gzipping if set
wfResetOutputBuffers(); call_user_func( $this->obResetFunc );
} }
$handle = fopen( 'php://output', 'wb' ); $handle = fopen( 'php://output', 'wb' );
@ -1109,6 +1103,7 @@ class SwiftFileBackend extends FileBackendStore {
} }
protected function doGetLocalCopyMulti( array $params ) { protected function doGetLocalCopyMulti( array $params ) {
/** @var TempFSFile[] $tmpFiles */
$tmpFiles = []; $tmpFiles = [];
$auth = $this->getAuthentication(); $auth = $this->getAuthentication();
@ -1216,14 +1211,14 @@ class SwiftFileBackend extends FileBackendStore {
) ); ) );
// See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
// Note: adding a newline for empty CanonicalizedAmzHeaders does not work. // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
return wfAppendQuery( // Note: S3 API is the rgw default; remove the /swift/ URL bit.
str_replace( '/swift/v1', '', // S3 API is the rgw default return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
$this->storageUrl( $auth ) . $spath ), '?' .
[ http_build_query( [
'Signature' => $signature, 'Signature' => $signature,
'Expires' => $expires, 'Expires' => $expires,
'AWSAccessKeyId' => $this->rgwS3AccessKey ] 'AWSAccessKeyId' => $this->rgwS3AccessKey
); ] );
} }
} }
@ -1257,6 +1252,7 @@ class SwiftFileBackend extends FileBackendStore {
* @return StatusValue[] * @return StatusValue[]
*/ */
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
/** @var $statuses StatusValue[] */
$statuses = []; $statuses = [];
$auth = $this->getAuthentication(); $auth = $this->getAuthentication();
@ -1271,6 +1267,7 @@ class SwiftFileBackend extends FileBackendStore {
// Split the HTTP requests into stages that can be done concurrently // Split the HTTP requests into stages that can be done concurrently
$httpReqsByStage = []; // map of (stage => index => HTTP request) $httpReqsByStage = []; // map of (stage => index => HTTP request)
foreach ( $fileOpHandles as $index => $fileOpHandle ) { foreach ( $fileOpHandles as $index => $fileOpHandle ) {
/** @var SwiftFileOpHandle $fileOpHandle */
$reqs = $fileOpHandle->httpOp; $reqs = $fileOpHandle->httpOp;
// Convert the 'url' parameter to an actual URL using $auth // Convert the 'url' parameter to an actual URL using $auth
foreach ( $reqs as $stage => &$req ) { foreach ( $reqs as $stage => &$req ) {
@ -1348,7 +1345,7 @@ class SwiftFileBackend extends FileBackendStore {
if ( $rcode != 204 && $rcode !== 202 ) { if ( $rcode != 204 && $rcode !== 202 ) {
$status->fatal( 'backend-fail-internal', $this->name ); $status->fatal( 'backend-fail-internal', $this->name );
wfDebugLog( 'SwiftBackend', __METHOD__ . ': unexpected rcode value (' . $rcode . ')' ); $this->logger->error( __METHOD__ . ': unexpected rcode value (' . $rcode . ')' );
} }
return $status; return $status;
@ -1363,7 +1360,7 @@ class SwiftFileBackend extends FileBackendStore {
* @return array|bool|null False on 404, null on failure * @return array|bool|null False on 404, null on failure
*/ */
protected function getContainerStat( $container, $bypassCache = false ) { protected function getContainerStat( $container, $bypassCache = false ) {
$ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
if ( $bypassCache ) { // purge cache if ( $bypassCache ) { // purge cache
$this->containerStatCache->clear( $container ); $this->containerStatCache->clear( $container );
@ -1755,7 +1752,7 @@ class SwiftFileBackend extends FileBackendStore {
if ( $code == 401 ) { // possibly a stale token if ( $code == 401 ) { // possibly a stale token
$this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) ); $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
} }
wfDebugLog( 'SwiftBackend', $this->logger->error(
"HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
( $err ? ": $err" : "" ) ( $err ? ": $err" : "" )
); );

14
includes/libs/filebackend/FileBackend.php
View file

@ -185,7 +185,9 @@ abstract class FileBackend implements LoggerAwareInterface {
$this->concurrency = isset( $config['concurrency'] ) $this->concurrency = isset( $config['concurrency'] )
? (int)$config['concurrency'] ? (int)$config['concurrency']
: 50; : 50;
$this->obResetFunc = isset( $params['obResetFunc'] ) ? $params['obResetFunc'] : null; $this->obResetFunc = isset( $params['obResetFunc'] )
? $params['obResetFunc']
: [ $this, 'resetOutputBuffer' ];
$this->streamMimeFunc = isset( $params['streamMimeFunc'] ) $this->streamMimeFunc = isset( $params['streamMimeFunc'] )
? $params['streamMimeFunc'] ? $params['streamMimeFunc']
: null; : null;
@ -1623,4 +1625,14 @@ abstract class FileBackend implements LoggerAwareInterface {
return null; return null;
} }
protected function resetOutputBuffer() {
while ( ob_get_status() ) {
if ( !ob_end_clean() ) {
// Could not remove output buffer handler; abort now
// to avoid getting in some kind of infinite loop.
break;
}
}
}
} }

4
includes/libs/filebackend/FileBackendStore.php
View file

@ -38,6 +38,8 @@
abstract class FileBackendStore extends FileBackend { abstract class FileBackendStore extends FileBackend {
/** @var WANObjectCache */ /** @var WANObjectCache */
protected $memCache; protected $memCache;
/** @var BagOStuff */
protected $srvCache;
/** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */ /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */
protected $cheapCache; protected $cheapCache;
/** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */ /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */
@ -58,6 +60,7 @@ abstract class FileBackendStore extends FileBackend {
/** /**
* @see FileBackend::__construct() * @see FileBackend::__construct()
* Additional $config params include: * Additional $config params include:
* - srvCache : BagOStuff cache to APC/XCache or the like.
* - wanCache : WANObjectCache object to use for persistent caching. * - wanCache : WANObjectCache object to use for persistent caching.
* - mimeCallback : Callback that takes (storage path, content, file system path) and * - mimeCallback : Callback that takes (storage path, content, file system path) and
* returns the MIME type of the file or 'unknown/unknown'. The file * returns the MIME type of the file or 'unknown/unknown'. The file
@ -70,6 +73,7 @@ abstract class FileBackendStore extends FileBackend {
$this->mimeCallback = isset( $config['mimeCallback'] ) $this->mimeCallback = isset( $config['mimeCallback'] )
? $config['mimeCallback'] ? $config['mimeCallback']
: null; : null;
$this->srvCache = new EmptyBagOStuff(); // disabled by default
$this->memCache = WANObjectCache::newEmpty(); // disabled by default $this->memCache = WANObjectCache::newEmpty(); // disabled by default
$this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE ); $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
$this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE ); $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );