Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added async upload-from-stash support.

Browse source 
* This works similarly to the async concatenation code.

Change-Id: Iae38b93d65182158561b92168af51a1e9f50374c
This commit is contained in:
Aaron Schulz 2012-12-03 18:18:48 -08:00 committed by Gerrit Code Review
parent d47f1cbea4
commit a31d0f8edd
4 changed files with 237 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
includes/api/ApiUpload.php
View file

@ -51,6 +51,8 @@ class ApiUpload extends ApiBase {
// Parameter handling
$this->mParams = $this->extractRequestParams();
$request = $this->getMain()->getRequest();
// Check if async mode is actually supported
$this->mParams['async'] = ( $this->mParams['async'] && !wfIsWindows() );
// Add the uploaded file to the params array
$this->mParams['file'] = $request->getFileName( 'file' );
$this->mParams['chunk'] = $request->getFileName( 'chunk' );
@ -62,17 +64,15 @@ class ApiUpload extends ApiBase {
// Select an upload module
if ( !$this->selectUploadModule() ) {
// This is not a true upload, but a status request or similar
return;
}
if ( !isset( $this->mUpload ) ) {
return; // not a true upload, but a status request or similar
} elseif ( !isset( $this->mUpload ) ) {
$this->dieUsage( 'No upload module set', 'nomodule' );
}
// First check permission to upload
$this->checkPermissions( $user );
// Fetch the file
// Fetch the file (usually a no-op)
$status = $this->mUpload->fetchFile();
if ( !$status->isGood() ) {
$errors = $status->getErrorsArray();
@ -89,6 +89,8 @@ class ApiUpload extends ApiBase {
if ( !$this->mUpload->getTitle() ) {
$this->dieUsage( 'Invalid file title supplied', 'internal-error' );
}
} elseif ( $this->mParams['async'] ) {
// defer verification to background process
} else {
$this->verifyUpload();
}
@ -96,15 +98,15 @@ class ApiUpload extends ApiBase {
// Check if the user has the rights to modify or overwrite the requested title
// (This check is irrelevant if stashing is already requested, since the errors
// can always be fixed by changing the title)
if ( ! $this->mParams['stash'] ) {
if ( !$this->mParams['stash'] ) {
$permErrors = $this->mUpload->verifyTitlePermissions( $user );
if ( $permErrors !== true ) {
$this->dieRecoverableError( $permErrors[0], 'filename' );
}
}
// Get the result based on the current upload context:
$result = $this->getContextResult();
if ( $result['result'] === 'Success' ) {
$result['imageinfo'] = $this->mUpload->getImageInfo( $this->getResult() );
}
@ -135,6 +137,7 @@ class ApiUpload extends ApiBase {
// performUpload will return a formatted properly for the API with status
return $this->performUpload( $warnings );
}
/**
* Get Stash Result, throws an expetion if the file could not be stashed.
* @param $warnings array Array of Api upload warnings
@ -156,6 +159,7 @@ class ApiUpload extends ApiBase {
}
return $result;
}
/**
* Get Warnings Result
* @param $warnings array Array of Api upload warnings
@ -175,6 +179,7 @@ class ApiUpload extends ApiBase {
}
return $result;
}
/**
* Get the result of a chunk upload.
* @param $warnings array Array of Api upload warnings
@ -206,7 +211,7 @@ class ApiUpload extends ApiBase {
if( $this->mParams['offset'] + $chunkSize == $this->mParams['filesize'] ) {
if ( $this->mParams['async'] && !wfIsWindows() ) {
$progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
if ( $progress && $progress['result'] !== 'Failed' ) {
if ( $progress && $progress['result'] === 'Poll' ) {
$this->dieUsage( "Chunk assembly already in progress.", 'stashfailed' );
}
UploadBase::setSessionStatus(
@ -266,7 +271,7 @@ class ApiUpload extends ApiBase {
* @throws MWException
* @return String file key
*/
function performStash() {
private function performStash() {
try {
$stashFile = $this->mUpload->stashFile();
@ -291,7 +296,7 @@ class ApiUpload extends ApiBase {
* @param $data array Optional extra data to pass to the user
* @throws UsageException
*/
function dieRecoverableError( $error, $parameter, $data = array() ) {
private function dieRecoverableError( $error, $parameter, $data = array() ) {
try {
$data['filekey'] = $this->performStash();
$data['sessionkey'] = $data['filekey'];
@ -320,6 +325,7 @@ class ApiUpload extends ApiBase {
'filekey', 'file', 'url', 'statuskey' );
}
// Status report for "upload to stash"/"upload from stash"
if ( $this->mParams['filekey'] && $this->mParams['checkstatus'] ) {
$progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
if ( !$progress ) {
@ -327,6 +333,9 @@ class ApiUpload extends ApiBase {
} elseif ( !$progress['status']->isGood() ) {
$this->dieUsage( $progress['status']->getWikiText(), 'stashfailed' );
}
if ( isset( $progress['status']->value['verification'] ) ) {
$this->checkVerification( $progress['status']->value['verification'] );
}
unset( $progress['status'] ); // remove Status object
$this->getResult()->addValue( null, $this->getModuleName(), $progress );
return false;
@ -377,8 +386,11 @@ class ApiUpload extends ApiBase {
}
$this->mUpload = new UploadFromStash( $this->getUser() );
$this->mUpload->initialize( $this->mParams['filekey'], $this->mParams['filename'] );
// This will not download the temp file in initialize() in async mode.
// We still have enough information to call checkWarnings() and such.
$this->mUpload->initialize(
$this->mParams['filekey'], $this->mParams['filename'], !$this->mParams['async']
);
} elseif ( isset( $this->mParams['file'] ) ) {
$this->mUpload = new UploadFromFile();
$this->mUpload->initialize(
@ -440,12 +452,19 @@ class ApiUpload extends ApiBase {
* Performs file verification, dies on error.
*/
protected function verifyUpload( ) {
global $wgFileExtensions;
$verification = $this->mUpload->verifyUpload( );
if ( $verification['status'] === UploadBase::OK ) {
return;
} else {
return $this->checkVerification( $verification );
}
}
/**
* Performs file verification, dies on error.
*/
protected function checkVerification( array $verification ) {
global $wgFileExtensions;
// TODO: Move them to ApiBase's message map
switch( $verification['status'] ) {
@ -555,6 +574,8 @@ class ApiUpload extends ApiBase {
* @return array
*/
protected function performUpload( $warnings ) {
global $IP;
// Use comment as initial page text by default
if ( is_null( $this->mParams['text'] ) ) {
$this->mParams['text'] = $this->mParams['comment'];
@ -569,29 +590,63 @@ class ApiUpload extends ApiBase {
}
// No errors, no warnings: do the upload
$status = $this->mUpload->performUpload( $this->mParams['comment'],
$this->mParams['text'], $watch, $this->getUser() );
if ( !$status->isGood() ) {
$error = $status->getErrorsArray();
if ( count( $error ) == 1 && $error[0][0] == 'async' ) {
// The upload can not be performed right now, because the user
// requested so
return array(
'result' => 'Queued',
'statuskey' => $error[0][1],
);
} else {
$this->getResult()->setIndexedTagName( $error, 'error' );
$this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error );
if ( $this->mParams['async'] ) {
$progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
if ( $progress && $progress['result'] === 'Poll' ) {
$this->dieUsage( "Upload from stash already in progress.", 'publishfailed' );
}
UploadBase::setSessionStatus(
$this->mParams['filekey'],
array( 'result' => 'Poll', 'stage' => 'queued', 'status' => Status::newGood() )
);
$retVal = 1;
$cmd = wfShellWikiCmd(
"$IP/includes/upload/PublishStashedFile.php",
array(
'--wiki', wfWikiID(),
'--filename', $this->mParams['filename'],
'--filekey', $this->mParams['filekey'],
'--userid', $this->getUser()->getId(),
'--comment', $this->mParams['comment'],
'--text', $this->mParams['text'],
'--watch', $watch,
'--sessionid', session_id(),
'--quiet'
)
) . " < " . wfGetNull() . " > " . wfGetNull() . " 2>&1 &";
// Start a process in the background. Enforce the time limits via PHP
// since ulimit4.sh seems to often not work for this particular usage.
wfShellExec( $cmd, $retVal, array(), array( 'time' => 0, 'memory' => 0 ) );
if ( $retVal == 0 ) {
$result['result'] = 'Poll';
} else {
UploadBase::setSessionStatus( $this->mParams['filekey'], false );
$this->dieUsage(
"Failed to start PublishStashedFile.php", 'publishfailed' );
}
} else {
$status = $this->mUpload->performUpload( $this->mParams['comment'],
$this->mParams['text'], $watch, $this->getUser() );
if ( !$status->isGood() ) {
$error = $status->getErrorsArray();
if ( count( $error ) == 1 && $error[0][0] == 'async' ) {
// The upload can not be performed right now, because the user
// requested so
return array(
'result' => 'Queued',
'statuskey' => $error[0][1],
);
} else {
$this->getResult()->setIndexedTagName( $error, 'error' );
$this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error );
}
}
$result['result'] = 'Success';
}
$file = $this->mUpload->getLocalFile();
$result['result'] = 'Success';
$result['filename'] = $file->getName();
if ( $warnings && count( $warnings ) > 0 ) {
$result['warnings'] = $warnings;
@ -759,6 +814,7 @@ class ApiUpload extends ApiBase {
array( 'code' => 'filename-tooshort', 'info' => 'The filename is too short' ),
array( 'code' => 'overwrite', 'info' => 'Overwriting an existing file is not allowed' ),
array( 'code' => 'stashfailed', 'info' => 'Stashing temporary file failed' ),
array( 'code' => 'publishfailed', 'info' => 'Publishing of stashed file failed' ),
array( 'code' => 'internal-error', 'info' => 'An internal error occurred' ),
array( 'code' => 'asynccopyuploaddisabled', 'info' => 'Asynchronous copy uploads disabled' ),
array( 'fileexists-forbidden' ),

125
includes/upload/PublishStashedFile.php Normal file
View file

@ -0,0 +1,125 @@
<?php
/**
* Upload a file from the upload stash into the local file repo.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup Maintenance
*/
require_once( __DIR__ . '/../../maintenance/Maintenance.php' );
set_time_limit( 3600 ); // 1 hour
/**
* Upload a file from the upload stash into the local file repo.
*
* @ingroup Maintenance
*/
class PublishStashedFile extends Maintenance {
public function __construct() {
parent::__construct();
$this->mDescription = "Upload stashed file into the local file repo";
$this->addOption( 'filename', "Desired file name", true, true );
$this->addOption( 'filekey', "Upload stash file key", true, true );
$this->addOption( 'userid', "Upload owner user ID", true, true );
$this->addOption( 'comment', "Upload comment", true, true );
$this->addOption( 'text', "Upload description", true, true );
$this->addOption( 'watch', "Whether the uploader should watch the page", true, true );
$this->addOption( 'sessionid', "Upload owner session ID", true, true );
}
public function execute() {
wfSetupSession( $this->getOption( 'sessionid' ) );
try {
$user = User::newFromId( $this->getOption( 'userid' ) );
if ( !$user ) {
throw new MWException( "No user with ID " . $this->getOption( 'userid' ) . "." );
}
UploadBase::setSessionStatus(
$this->getOption( 'filekey' ),
array( 'result' => 'Poll', 'stage' => 'publish', 'status' => Status::newGood() )
);
$upload = new UploadFromStash( $user );
// @TODO: initialize() causes a GET, ideally we could frontload the antivirus
// checks and anything else to the stash stage (which includes concatenation and
// the local file is thus already there). That way, instead of GET+PUT, there could
// just be a COPY operation from the stash to the public zone.
$upload->initialize( $this->getOption( 'filekey' ), $this->getOption( 'filename' ) );
// Check if the local file checks out (this is generally a no-op)
$verification = $upload->verifyUpload();
if ( $verification['status'] !== UploadBase::OK ) {
$status = Status::newFatal( 'verification-error' );
$status->value = array( 'verification' => $verification );
UploadBase::setSessionStatus(
$this->getOption( 'filekey' ),
array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status )
);
$this->error( "Could not verify upload.\n", 1 ); // die
}
// Upload the stashed file to a permanent location
$status = $upload->performUpload(
$this->getOption( 'comment' ),
$this->getOption( 'text' ),
$this->getOption( 'watch' ),
$user
);
if ( !$status->isGood() ) {
UploadBase::setSessionStatus(
$this->getOption( 'filekey' ),
array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status )
);
$this->error( $status->getWikiText() . "\n", 1 ); // die
}
// Build the image info array while we have the local reference handy
$apiMain = new ApiMain(); // dummy object (XXX)
$imageInfo = $upload->getImageInfo( $apiMain->getResult() );
// Cleanup any temporary local file
$upload->cleanupTempFile();
// Cache the info so the user doesn't have to wait forever to get the final info
UploadBase::setSessionStatus(
$this->getOption( 'filekey' ),
array(
'result' => 'Success',
'stage' => 'publish',
'filename' => $upload->getLocalFile()->getName(),
'imageinfo' => $imageInfo,
'status' => Status::newGood()
)
);
} catch ( MWException $e ) {
UploadBase::setSessionStatus(
$this->getOption( 'filekey' ),
array(
'result' => 'Failure',
'stage' => 'publish',
'status' => Status::newFatal( 'api-error-stashfailed' )
)
);
throw $e;
}
session_write_close();
}
}
$maintClass = "PublishStashedFile";
require_once( RUN_MAINTENANCE_IF_MAIN );

14
includes/upload/UploadBase.php
View file

@ -235,6 +235,14 @@ abstract class UploadBase {
return $this->mFileSize;
}
/**
* Get the base 36 SHA1 of the file
* @return string
*/
protected function getTempFileSha1Base36() {
return FSFile::getSha1Base36FromPath( $this->mTempPath );
}
/**
* @param $srcPath String: the source path
* @return string the real path if it was a virtual URL
@ -546,7 +554,9 @@ abstract class UploadBase {
}
/**
* Check for non fatal problems with the file
* Check for non fatal problems with the file.
*
* This should not assume that mTempPath is set.
*
* @return Array of warnings
*/
@ -594,7 +604,7 @@ abstract class UploadBase {
}
// Check dupes against existing files
$hash = FSFile::getSha1Base36FromPath( $this->mTempPath );
$hash = $this->getTempFileSha1Base36();
$dupes = RepoGroup::singleton()->findBySha1( $hash );
$title = $this->getTitle();
// Remove all matches against self

12
includes/upload/UploadFromStash.php
View file

@ -89,7 +89,7 @@ class UploadFromStash extends UploadBase {
* @param $key string
* @param $name string
*/
public function initialize( $key, $name = 'upload_file' ) {
public function initialize( $key, $name = 'upload_file', $initTempFile = true ) {
/**
* Confirming a temporarily stashed upload.
* We don't want path names to be forged, so we keep
@ -98,7 +98,7 @@ class UploadFromStash extends UploadBase {
*/
$metadata = $this->stash->getMetadata( $key );
$this->initializePathInfo( $name,
$this->getRealPath( $metadata['us_path'] ),
$initTempFile ? $this->getRealPath( $metadata['us_path'] ) : false,
$metadata['us_size'],
false
);
@ -129,6 +129,14 @@ class UploadFromStash extends UploadBase {
return $this->mSourceType;
}
/**
* Get the base 36 SHA1 of the file
* @return string
*/
protected function getTempFileSha1Base36() {
return $this->mFileProps['sha1'];
}
/**
* File has been previously verified so no need to do so again.
*