Use database to track uploaded chunks and concatenate at the end.

with i18n documentation dont break phpunit

follow up r93720
This commit is contained in:
Jan Gerber 2011-11-30 14:56:40 +00:00
parent 9cd2609361
commit 0095c08ed2
15 changed files with 447 additions and 111 deletions

View file

@ -840,6 +840,7 @@ $wgAutoloadLocalClasses = array(
# includes/upload
'UploadBase' => 'includes/upload/UploadBase.php',
'UploadFromFile' => 'includes/upload/UploadFromFile.php',
'UploadFromChunks' => 'includes/upload/UploadFromChunks.php',
'UploadFromStash' => 'includes/upload/UploadFromStash.php',
'UploadFromUrl' => 'includes/upload/UploadFromUrl.php',
'UploadStash' => 'includes/upload/UploadStash.php',

View file

@ -89,8 +89,7 @@ class ApiUpload extends ApiBase {
} else {
$this->verifyUpload();
}
// Check if the user has the rights to modify or overwrite the requested title
// (This check is irrelevant if stashing is already requested, since the errors
// can always be fixed by changing the title)
@ -100,57 +99,8 @@ class ApiUpload extends ApiBase {
$this->dieRecoverableError( $permErrors[0], 'filename' );
}
}
// Prepare the API result
$result = array();
$warnings = $this->getApiWarnings();
if ( $warnings ) {
$result['result'] = 'Warning';
$result['warnings'] = $warnings;
// in case the warnings can be fixed with some further user action, let's stash this upload
// and return a key they can use to restart it
try {
$result['filekey'] = $this->performStash();
$result['sessionkey'] = $result['filekey']; // backwards compatibility
} catch ( MWException $e ) {
$result['warnings']['stashfailed'] = $e->getMessage();
}
} elseif ( $this->mParams['chunk'] ) {
$result['result'] = 'Continue';
$chunk = $request->getFileTempName( 'chunk' );
$chunkSize = $request->getUpload( 'chunk' )->getSize();
if ($this->mParams['offset'] == 0) {
$result['filekey'] = $this->performStash();
} else {
$status = $this->mUpload->appendChunk($chunk, $chunkSize,
$this->mParams['offset']);
if ( !$status->isGood() ) {
$this->dieUsage( $status->getWikiText(), 'stashfailed' );
} else {
$result['filekey'] = $this->mParams['filekey'];
if($this->mParams['offset'] + $chunkSize == $this->mParams['filesize']) {
$this->mUpload->finalizeFile();
$result['result'] = 'Success';
}
}
}
$result['offset'] = $this->mParams['offset'] + $chunkSize;
} elseif ( $this->mParams['stash'] ) {
// Some uploads can request they be stashed, so as not to publish them immediately.
// In this case, a failure to stash ought to be fatal
try {
$result['result'] = 'Success';
$result['filekey'] = $this->performStash();
$result['sessionkey'] = $result['filekey']; // backwards compatibility
} catch ( MWException $e ) {
$this->dieUsage( $e->getMessage(), 'stashfailed' );
}
} else {
// This is the most common case -- a normal upload with no warnings
// $result will be formatted properly for the API already, with a status
$result = $this->performUpload();
}
// Get the result based on the current upload context:
$result = $this->getContextResult();
if ( $result['result'] === 'Success' ) {
$result['imageinfo'] = $this->mUpload->getImageInfo( $this->getResult() );
@ -161,7 +111,93 @@ class ApiUpload extends ApiBase {
// Cleanup any temporary mess
$this->mUpload->cleanupTempFile();
}
/**
* Get an uplaod result based on upload context
*/
private function getContextResult(){
$warnings = $this->getApiWarnings();
if ( $warnings ) {
// Get warnings formated in result array format
return $this->getWarningsResult( $warnings );
} elseif ( $this->mParams['chunk'] ) {
// Add chunk, and get result
return $this->getChunkResult();
} elseif ( $this->mParams['stash'] ) {
// Stash the file and get stash result
return $this->getStashResult();
}
// This is the most common case -- a normal upload with no warnings
// performUpload will return a formatted properly for the API with status
return $this->performUpload();
}
/**
* Get Stash Result, throws an expetion if the file could not be stashed.
*/
private function getStashResult(){
$result = array ();
// Some uploads can request they be stashed, so as not to publish them immediately.
// In this case, a failure to stash ought to be fatal
try {
$result['result'] = 'Success';
$result['filekey'] = $this->performStash();
$result['sessionkey'] = $result['filekey']; // backwards compatibility
} catch ( MWException $e ) {
$this->dieUsage( $e->getMessage(), 'stashfailed' );
}
return $result;
}
/**
* Get Warnings Result
* @param $warnings Array of Api upload warnings
*/
private function getWarningsResult( $warnings ){
$result = array();
$result['result'] = 'Warning';
$result['warnings'] = $warnings;
// in case the warnings can be fixed with some further user action, let's stash this upload
// and return a key they can use to restart it
try {
$result['filekey'] = $this->performStash();
$result['sessionkey'] = $result['filekey']; // backwards compatibility
} catch ( MWException $e ) {
$result['warnings']['stashfailed'] = $e->getMessage();
}
return $result;
}
/**
* Get the result of a chunk upload.
*/
private function getChunkResult(){
$result = array();
$result['result'] = 'Continue';
$request = $this->getMain()->getRequest();
$chunkPath = $request->getFileTempname( 'chunk' );
$chunkSize = $request->getUpload( 'chunk' )->getSize();
if ($this->mParams['offset'] == 0) {
$result['filekey'] = $this->performStash();
} else {
$status = $this->mUpload->addChunk($chunkPath, $chunkSize,
$this->mParams['offset']);
if ( !$status->isGood() ) {
$this->dieUsage( $status->getWikiText(), 'stashfailed' );
return ;
}
$result['filekey'] = $this->mParams['filekey'];
// Check we added the last chunk:
if( $this->mParams['offset'] + $chunkSize == $this->mParams['filesize'] ) {
$status = $this->mUpload->concatenateChunks();
if ( !$status->isGood() ) {
$this->dieUsage( $status->getWikiText(), 'stashfailed' );
return ;
}
$result['result'] = 'Success';
}
}
$result['offset'] = $this->mParams['offset'] + $chunkSize;
return $result;
}
/**
* Stash the file and return the file key
* Also re-raises exceptions with slightly more informative message strings (useful for API)
@ -244,7 +280,24 @@ class ApiUpload extends ApiBase {
$this->dieUsageMsg( array( 'missingparam', 'filename' ) );
}
if ( $this->mParams['filekey'] ) {
if ( $this->mParams['chunk'] ) {
// Chunk upload
$this->mUpload = new UploadFromChunks();
if( isset( $this->mParams['filekey'] ) ){
// handle new chunk
$this->mUpload->continueChunks(
$this->mParams['filename'],
$this->mParams['filekey'],
$request->getUpload( 'chunk' )
);
} else {
// handle first chunk
$this->mUpload->initialize(
$this->mParams['filename'],
$request->getUpload( 'chunk' )
);
}
} elseif ( isset( $this->mParams['filekey'] ) ) {
// Upload stashed in a previous request
if ( !UploadFromStash::isValidKey( $this->mParams['filekey'] ) ) {
$this->dieUsageMsg( 'invalid-file-key' );
@ -253,14 +306,6 @@ class ApiUpload extends ApiBase {
$this->mUpload = new UploadFromStash( $this->getUser() );
$this->mUpload->initialize( $this->mParams['filekey'], $this->mParams['filename'] );
} elseif ( isset( $this->mParams['chunk'] ) ) {
// Start new Chunk upload
$this->mUpload = new UploadFromFile();
$this->mUpload->initialize(
$this->mParams['filename'],
$request->getUpload( 'chunk' )
);
} elseif ( isset( $this->mParams['file'] ) ) {
$this->mUpload = new UploadFromFile();
$this->mUpload->initialize(

View file

@ -313,11 +313,63 @@ class FSRepo extends FileRepo {
wfRestoreWarnings();
}
}
/**
* Concatenate a list of files into a target file location.
*
* @param $fileList array of files
* @param $targetFile String target path
* @param $flags Integer: bitwise combination of the following flags:
* self::FILES_ONLY Mark file as existing only if it is a file (not directory)
*/
function concatenate( $fileList, $targetPath, $flags = 0 ){
$status = $this->newGood();
// Resolve the virtual URL for taget:
if ( self::isVirtualUrl( $targetPath ) ) {
$targetPath = $this->resolveVirtualUrl( $targetPath );
// empty out the target file:
if ( is_file( $targetPath ) ){
unlink( $targetPath );
}
}
foreach( $fileList as $sourcePath ){
// Resolve the virtual URL for source:
if ( self::isVirtualUrl( $sourcePath ) ) {
$sourcePath = $this->resolveVirtualUrl( $sourcePath );
}
if ( !is_file( $sourcePath ) )
$status->fatal( 'filenotfound', $sourcePath );
if ( !$status->isOk() ){
return $status;
}
// Do the append
$chunk = file_get_contents( $sourcePath );
if( $chunk === false ) {
$status->fatal( 'fileconcatenateerrorread', $sourcePath );
return $status;
}
if( $status->isOk() ) {
if ( file_put_contents( $targetPath, $chunk, FILE_APPEND ) ) {
$status->value = $targetPath;
} else {
$status->fatal( 'fileconcatenateerror', $sourcePath, $targetPath);
}
}
if ( $flags & self::DELETE_SOURCE ) {
unlink( $sourcePath );
}
}
return $status;
}
/**
* @deprecated 1.19
*
* @return Status
*/
function append( $srcPath, $toAppendPath, $flags = 0 ) {
wfDeprecated(__METHOD__);
$status = $this->newGood();
// Resolve the virtual URL

View file

@ -420,6 +420,14 @@ abstract class FileRepo {
abstract function storeTemp( $originalName, $srcPath );
/**
* Concatenate and array of file sources.
* @param $fileList Array of file sources
* @param $targetPath String target destination for file.
* @throws MWException
*/
abstract function concatenate( $fileList, $targetPath, $flags = 0 );
/**
* Append the contents of the source path to the given file, OR queue
* the appending operation in anticipation of a later appendFinish() call.

View file

@ -44,4 +44,7 @@ class NullRepo extends FileRepo {
function findFile( $title, $options = array() ) {
return false;
}
function concatenate( $fileList, $targetPath, $flags = 0 ) {
return false;
}
}

View file

@ -190,6 +190,7 @@ class MysqlUpdater extends DatabaseUpdater {
array( 'addField', 'archive', 'ar_sha1', 'patch-ar_sha1.sql' ),
array( 'addIndex', 'page', 'page_redirect_namespace_len', 'patch-page_redirect_namespace_len.sql' ),
array( 'modifyField', 'user', 'ug_group', 'patch-ug_group-length-increase.sql' ),
array( 'addField', 'uploadstash', 'us_chunk_inx', 'patch-uploadstash_chunk.sql' ),
);
}

View file

@ -199,23 +199,31 @@ abstract class UploadBase {
/**
* Append a file to the Repo file
*
* @deprecated since 1.19
*
* @param $srcPath String: path to source file
* @param $toAppendPath String: path to the Repo file that will be appended to.
* @return Status Status
*/
protected function appendToUploadFile( $srcPath, $toAppendPath ) {
wfDeprecated(__METHOD__);
$repo = RepoGroup::singleton()->getLocalRepo();
$status = $repo->append( $srcPath, $toAppendPath );
return $status;
}
/**
* Finish appending to the Repo file
*
*
* @deprecated since 1.19
*
* @param $toAppendPath String: path to the Repo file that will be appended to.
* @return Status Status
*/
protected function appendFinish( $toAppendPath ) {
wfDeprecated(__METHOD__);
$repo = RepoGroup::singleton()->getLocalRepo();
$status = $repo->appendFinish( $toAppendPath );
return $status;
@ -760,7 +768,6 @@ abstract class UploadBase {
public function stashFile() {
// was stashSessionFile
$stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash();
$file = $stash->stashFile( $this->mTempPath, $this->getSourceType() );
$this->mLocalFile = $file;
return $file;

View file

@ -0,0 +1,253 @@
<?php
/**
* Implements uploading from chunks
*
* @file
* @ingroup upload
* @author Michael Dale
*/
class UploadFromChunks extends UploadFromFile {
protected $mOffset, $mChunkIndex, $mFileKey, $mVirtualTempPath;
/**
* Setup local pointers to stash, repo and user ( similar to UploadFromStash )
*
* @param $user User
* @param $stash UploadStash
* @param $repo FileRepo
*/
public function __construct( $user = false, $stash = false, $repo = false ) {
// user object. sometimes this won't exist, as when running from cron.
$this->user = $user;
if( $repo ) {
$this->repo = $repo;
} else {
$this->repo = RepoGroup::singleton()->getLocalRepo();
}
if( $stash ) {
$this->stash = $stash;
} else {
if( $user ) {
wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() . "\n" );
} else {
wfDebug( __METHOD__ . " creating new UploadFromChunks instance with no user\n" );
}
$this->stash = new UploadStash( $this->repo, $this->user );
}
return true;
}
/**
* Calls the parent stashFile and updates the uploadsession table to handle "chunks"
*
* @return UploadStashFile stashed file
*/
public function stashFile() {
// Stash file is the called on creating a new chunk session:
$this->mChunkIndex = 0;
$this->mOffset = 0;
// Create a local stash target
$this->mLocalFile = parent::stashFile();
// Update the initial file offset ( based on file size )
$this->mOffset = $this->mLocalFile->getSize();
$this->mFileKey = $this->mLocalFile->getFileKey();
// Output a copy of this first to chunk 0 location:
$status = $this->outputChunk( $this->mLocalFile->getPath() );
// Update db table to reflect initial "chunk" state
$this->updateChunkStatus();
return $this->mLocalFile;
}
/**
* Continue chunk uploading
*/
public function continueChunks( $name, $key, $webRequestUpload ) {
$this->mFileKey = $key;
$this->mUpload = $webRequestUpload;
// Get the chunk status form the db:
$this->getChunkStatus();
$metadata = $this->stash->getMetadata( $key );
$this->initializePathInfo( $name,
$this->getRealPath ( $metadata['us_path'] ),
$metadata['us_size'],
false
);
}
/**
* Append the final chunk and ready file for parent::performUpload()
* @return void
*/
public function concatenateChunks() {
wfDebug( __METHOD__ . " concatenate {$this->mChunkIndex} chunks:" .
$this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
// Concatenate all the chunks to mVirtualTempPath
$fileList = Array();
// The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
for( $i = 0; $i <= $this->getChunkIndex(); $i++ ){
$fileList[] = $this->getVirtualChunkLocation( $i );
}
// Concatinate into the mVirtualTempPath location;
$status = $this->repo->concatenate( $fileList, $this->mVirtualTempPath, FileRepo::DELETE_SOURCE );
if( !$status->isOk() ){
return $status;
}
// Update the mTempPath variable ( for FileUpload or normal Stash to take over )
$this->mTempPath = $this->getRealPath( $this->mVirtualTempPath );
return $status;
}
/**
* Returns the virtual chunk location:
* @param unknown_type $index
*/
function getVirtualChunkLocation( $index ){
return $this->repo->getVirtualUrl( 'temp' ) .
'/' .
$this->repo->getHashPath(
$this->getChunkFileKey( $index )
) .
$this->getChunkFileKey( $index );
}
/**
* Add a chunk to the temporary directory
*
* @param $chunkPath path to temporary chunk file
* @param $chunkSize size of the current chunk
* @param $offset offset of current chunk ( mutch match database chunk offset )
* @return Status
*/
public function addChunk( $chunkPath, $chunkSize, $offset ) {
// Get the offset before we add the chunk to the file system
$preAppendOffset = $this->getOffset();
if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize()) {
$status = Status::newFatal( 'file-too-large' );
} else {
// Make sure the client is uploading the correct chunk with a matching offset.
if ( $preAppendOffset == $offset ) {
// Update local chunk index for the current chunk
$this->mChunkIndex++;
$status = $this->outputChunk( $chunkPath );
if( $status->isGood() ){
// Update local offset:
$this->mOffset = $preAppendOffset + $chunkSize;
// Update chunk table status db
$this->updateChunkStatus();
}
} else {
$status = Status::newFatal( 'invalid-chunk-offset' );
}
}
return $status;
}
/**
* Update the chunk db table with the current status:
*/
private function updateChunkStatus(){
wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" .
$this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
$dbw = $this->repo->getMasterDb();
$dbw->update(
'uploadstash',
array(
'us_status' => 'chunks',
'us_chunk_inx' => $this->getChunkIndex(),
'us_size' => $this->getOffset()
),
array( 'us_key' => $this->mFileKey ),
__METHOD__
);
}
/**
* Get the chunk db state and populate update relevant local values
*/
private function getChunkStatus(){
$dbr = $this->repo->getSlaveDb();
$row = $dbr->selectRow(
'uploadstash',
array(
'us_chunk_inx',
'us_size',
'us_path',
),
array( 'us_key' => $this->mFileKey ),
__METHOD__
);
// Handle result:
if ( $row ) {
$this->mChunkIndex = $row->us_chunk_inx;
$this->mOffset = $row->us_size;
$this->mVirtualTempPath = $row->us_path;
}
}
/**
* Get the current Chunk index
* @return Integer index of the current chunk
*/
private function getChunkIndex(){
if( $this->mChunkIndex !== null ){
return $this->mChunkIndex;
}
return 0;
}
/**
* Gets the current offset in fromt the stashedupload table
* @return Integer current byte offset of the chunk file set
*/
private function getOffset(){
if ( $this->mOffset !== null ){
return $this->mOffset;
}
return 0;
}
/**
* Output the chunk to disk
*
* @param $chunk
* @param unknown_type $path
*/
private function outputChunk( $chunkPath ){
// Key is fileKey + chunk index
$fileKey = $this->getChunkFileKey();
// Store the chunk per its indexed fileKey:
$hashPath = $this->repo->getHashPath( $fileKey );
$storeStatus = $this->repo->store( $chunkPath, 'temp', "$hashPath$fileKey" );
// Check for error in stashing the chunk:
if ( ! $storeStatus->isOK() ) {
$error = $storeStatus->getErrorsArray();
$error = reset( $error );
if ( ! count( $error ) ) {
$error = $storeStatus->getWarningsArray();
$error = reset( $error );
if ( ! count( $error ) ) {
$error = array( 'unknown', 'no error recorded' );
}
}
throw new UploadChunkFileException( "error storing file in '$path': " . implode( '; ', $error ) );
}
return $storeStatus;
}
private function getChunkFileKey( $index = null ){
if( $index === null ){
$index = $this->getChunkIndex();
}
return $this->mFileKey . '.' . $index ;
}
}
class UploadChunkZeroLengthFileException extends MWException {};
class UploadChunkFileException extends MWException {};

View file

@ -75,12 +75,4 @@ class UploadFromFile extends UploadBase {
return parent::verifyUpload();
}
/**
* Get the path to the file underlying the upload
* @return String path to file
*/
public function getFileTempname() {
return $this->mUpload->getTempname();
}
}

View file

@ -161,40 +161,4 @@ class UploadFromStash extends UploadBase {
$this->unsaveUploadedFile();
return $rv;
}
/**
* Append a chunk to the temporary file.
*
* @param $chunk
* @param $chunkSize
* @param $offset
* @return Status
*/
public function appendChunk( $chunk, $chunkSize, $offset ) {
//to use $this->getFileSize() here, db needs to be updated
//in appendToUploadFile for that
$fileSize = $this->stash->getFile( $this->mFileKey )->getSize();
if ( $fileSize + $chunkSize > $this->getMaxUploadSize()) {
$status = Status::newFatal( 'file-too-large' );
} else {
//append chunk
if ( $fileSize == $offset ) {
$status = $this->appendToUploadFile( $chunk,
$this->mVirtualTempPath );
} else {
$status = Status::newFatal( 'invalid-chunk-offset' );
}
}
return $status;
}
/**
* Append the final chunk and ready file for parent::performUpload()
* @return void
*/
public function finalizeFile() {
$this->appendFinish ( $this->mVirtualTempPath );
$this->cleanupTempFile();
$this->mTempPath = $this->getRealPath( $this->mVirtualTempPath );
}
}

View file

@ -2252,6 +2252,7 @@ It cannot be properly checked for security.',
'uploadstash-badtoken' => 'Performing of that action was unsuccessful, perhaps because your editing credentials expired. Try again.',
'uploadstash-errclear' => 'Clearing the files was unsuccessful.',
'uploadstash-refresh' => 'Refresh the list of files',
'invalid-chunk-offset' => 'Invalid chunk offset',
# img_auth script messages
'img-auth-accessdenied' => 'Access denied',

View file

@ -1867,6 +1867,9 @@ Extensions making use of it:
{{Identical|Internal error}}',
'invalid-chunk-offset' => 'Error that can happen if chunkd get uploaded out of order.
As a result of this error, clients can continue from offset provided or restart upload.
Used on [[Special:UploadWizard]]',
# ZipDirectoryReader
'zip-unsupported' => "Perhaps translations of 'software' can be used instead of 'features' and 'understood' or 'handled' instead of 'supported'.",

View file

@ -0,0 +1,3 @@
-- Adding us_chunk_inx field
ALTER TABLE /*$wgDBprefix*/uploadstash
ADD us_chunk_inx int unsigned NULL;

View file

@ -1362,6 +1362,7 @@ $wgMessageStructure = array(
'uploadstash-badtoken',
'uploadstash-errclear',
'uploadstash-refresh',
'invalid-chunk-offset',
),
'img-auth' => array(

View file

@ -966,6 +966,8 @@ CREATE TABLE /*_*/uploadstash (
us_timestamp varbinary(14) not null,
us_status varchar(50) not null,
us_chunk_inx int unsigned NULL,
-- file properties from File::getPropsFromPath. these may prove unnecessary.
--