diff --git a/RELEASE-NOTES-1.24 b/RELEASE-NOTES-1.24 index a62a560e4de..9c605d04e38 100644 --- a/RELEASE-NOTES-1.24 +++ b/RELEASE-NOTES-1.24 @@ -151,6 +151,9 @@ production. * The deprecated action=parse&prop=languageshtml has been removed. * (bug 48071) action=setnotificationtimestamp no longer throws PHP or database errors when no pages are given. +* (bug 60734) Actions that use ApiPageSet (e.g. purge, watch, + setnotificationtimestamp) will now include continuation information when + using a generator. === Languages updated in 1.24 === diff --git a/includes/api/ApiImageRotate.php b/includes/api/ApiImageRotate.php index b8e16ab7240..c932a74bad9 100644 --- a/includes/api/ApiImageRotate.php +++ b/includes/api/ApiImageRotate.php @@ -52,6 +52,8 @@ class ApiImageRotate extends ApiBase { $params = $this->extractRequestParams(); $rotation = $params['rotation']; + $this->getResult()->beginContinuation( $params['continue'], array(), array() ); + $pageSet = $this->getPageSet(); $pageSet->execute(); @@ -131,6 +133,7 @@ class ApiImageRotate extends ApiBase { $apiResult = $this->getResult(); $apiResult->setIndexedTagName( $result, 'page' ); $apiResult->addValue( null, $this->getModuleName(), $result ); + $apiResult->endContinuation(); } /** @@ -185,6 +188,7 @@ class ApiImageRotate extends ApiBase { ApiBase::PARAM_TYPE => 'string', ApiBase::PARAM_REQUIRED => true ), + 'continue' => '', ); if ( $flags ) { $result += $this->getPageSet()->getFinalParams( $flags ); @@ -199,6 +203,7 @@ class ApiImageRotate extends ApiBase { return $pageSet->getFinalParamDescription() + array( 'rotation' => 'Degrees to rotate image clockwise', 'token' => 'Edit token. You can get one of these through action=tokens', + 'continue' => 'When more results are available, use this to continue', ); } diff --git a/includes/api/ApiPurge.php b/includes/api/ApiPurge.php index 981dc18f030..33998431c48 100644 --- a/includes/api/ApiPurge.php +++ b/includes/api/ApiPurge.php @@ -38,6 +38,8 @@ class ApiPurge extends ApiBase { public function execute() { $params = $this->extractRequestParams(); + $this->getResult()->beginContinuation( $params['continue'], array(), array() ); + $forceLinkUpdate = $params['forcelinkupdate']; $forceRecursiveLinkUpdate = $params['forcerecursivelinkupdate']; $pageSet = $this->getPageSet(); @@ -102,6 +104,8 @@ class ApiPurge extends ApiBase { if ( $values ) { $apiResult->addValue( null, 'redirects', $values ); } + + $apiResult->endContinuation(); } /** @@ -128,7 +132,8 @@ class ApiPurge extends ApiBase { public function getAllowedParams( $flags = 0 ) { $result = array( 'forcelinkupdate' => false, - 'forcerecursivelinkupdate' => false + 'forcerecursivelinkupdate' => false, + 'continue' => '', ); if ( $flags ) { $result += $this->getPageSet()->getFinalParams( $flags ); @@ -143,6 +148,7 @@ class ApiPurge extends ApiBase { 'forcelinkupdate' => 'Update the links tables', 'forcerecursivelinkupdate' => 'Update the links table, and update ' . 'the links tables for any page that uses this page as a template', + 'continue' => 'When more results are available, use this to continue', ); } diff --git a/includes/api/ApiQuery.php b/includes/api/ApiQuery.php index a2f4121e2c1..cd6a8402373 100644 --- a/includes/api/ApiQuery.php +++ b/includes/api/ApiQuery.php @@ -118,8 +118,6 @@ class ApiQuery extends ApiBase { private $mParams; private $mNamedDB = array(); private $mModuleMgr; - private $mGeneratorContinue; - private $mUseLegacyContinue; /** * @param ApiMain $main @@ -245,23 +243,24 @@ class ApiQuery extends ApiBase { public function execute() { $this->mParams = $this->extractRequestParams(); - // $pagesetParams is a array of parameter names used by the pageset generator - // or null if pageset has already finished and is no longer needed - // $completeModules is a set of complete modules with the name as key - $this->initContinue( $pagesetParams, $completeModules ); - // Instantiate requested modules $allModules = array(); $this->instantiateModules( $allModules, 'prop' ); - $propModules = $allModules; // Keep a copy + $propModules = array_keys( $allModules ); $this->instantiateModules( $allModules, 'list' ); $this->instantiateModules( $allModules, 'meta' ); // Filter modules based on continue parameter - $modules = $this->initModules( $allModules, $completeModules, $pagesetParams !== null ); + list( $generatorDone, $modules ) = $this->getResult()->beginContinuation( + $this->mParams['continue'], $allModules, $propModules + ); - // Execute pageset if in legacy mode or if pageset is not done - if ( $completeModules === null || $pagesetParams !== null ) { + if ( !$generatorDone ) { + // Query modules may optimize data requests through the $this->getPageSet() + // object by adding extra fields from the page table. + foreach ( $modules as $module ) { + $module->requestExtraData( $this->mPageSet ); + } // Populate page/revision information $this->mPageSet->execute(); // Record page information (title, namespace, if exists, etc) @@ -287,135 +286,10 @@ class ApiQuery extends ApiBase { // Set the cache mode $this->getMain()->setCacheMode( $cacheMode ); - if ( $completeModules === null ) { - return; // Legacy continue, we are done - } - - // Reformat query-continue result section - $result = $this->getResult(); - $qc = $result->getData(); - if ( isset( $qc['query-continue'] ) ) { - $qc = $qc['query-continue']; - $result->unsetValue( null, 'query-continue' ); - } elseif ( $this->mGeneratorContinue !== null ) { - $qc = array(); - } else { - // no more "continue"s, we are done! - return; - } - - // we are done with all the modules that do not have result in query-continue - $completeModules = array_merge( $completeModules, array_diff_key( $modules, $qc ) ); - if ( $pagesetParams !== null ) { - // The pageset is still in use, check if all props have finished - $incompleteProps = array_intersect_key( $propModules, $qc ); - if ( count( $incompleteProps ) > 0 ) { - // Properties are not done, continue with the same pageset state - copy current parameters - $main = $this->getMain(); - $contValues = array(); - foreach ( $pagesetParams as $param ) { - // The param name is already prefix-encoded - $contValues[$param] = $main->getVal( $param ); - } - } elseif ( $this->mGeneratorContinue !== null ) { - // Move to the next set of pages produced by pageset, properties need to be restarted - $contValues = $this->mGeneratorContinue; - $pagesetParams = array_keys( $contValues ); - $completeModules = array_diff_key( $completeModules, $propModules ); - } else { - // Done with the pageset, finish up with the the lists and meta modules - $pagesetParams = null; - } - } - - $continue = '||' . implode( '|', array_keys( $completeModules ) ); - if ( $pagesetParams !== null ) { - // list of all pageset parameters to use in the next request - $continue = implode( '|', $pagesetParams ) . $continue; - } else { - // we are done with the pageset - $contValues = array(); - $continue = '-' . $continue; - } - $contValues['continue'] = $continue; - foreach ( $qc as $qcModule ) { - foreach ( $qcModule as $qcKey => $qcValue ) { - $contValues[$qcKey] = $qcValue; - } - } - $this->getResult()->addValue( null, 'continue', $contValues ); - } - - /** - * Parse 'continue' parameter into the list of complete modules and a list of generator parameters - * @param array|null $pagesetParams Returns list of generator params or null if pageset is done - * @param array|null $completeModules Returns list of finished modules (as keys), or null if legacy - */ - private function initContinue( &$pagesetParams, &$completeModules ) { - $pagesetParams = array(); - $continue = $this->mParams['continue']; - if ( $continue !== null ) { - $this->mUseLegacyContinue = false; - if ( $continue !== '' ) { - // Format: ' pagesetParam1 | pagesetParam2 || module1 | module2 | module3 | ... - // If pageset is done, use '-' - $continue = explode( '||', $continue ); - $this->dieContinueUsageIf( count( $continue ) !== 2 ); - if ( $continue[0] === '-' ) { - $pagesetParams = null; // No need to execute pageset - } elseif ( $continue[0] !== '' ) { - // list of pageset params that might need to be repeated - $pagesetParams = explode( '|', $continue[0] ); - } - $continue = $continue[1]; - } - if ( $continue !== '' ) { - $completeModules = array_flip( explode( '|', $continue ) ); - } else { - $completeModules = array(); - } - } else { - $this->mUseLegacyContinue = true; - $completeModules = null; - } - } - - /** - * Validate sub-modules, filter out completed ones, and do requestExtraData() - * @param array $allModules An dict of name=>instance of all modules requested by the client - * @param array|null $completeModules List of finished modules, or null if legacy continue - * @param bool $usePageset True if pageset will be executed - * @return array Array of modules to be processed during this execution - */ - private function initModules( $allModules, $completeModules, $usePageset ) { - $modules = $allModules; - $tmp = $completeModules; - $wasPosted = $this->getRequest()->wasPosted(); - - /** @var $module ApiQueryBase */ - foreach ( $allModules as $moduleName => $module ) { - if ( !$wasPosted && $module->mustBePosted() ) { - $this->dieUsageMsgOrDebug( array( 'mustbeposted', $moduleName ) ); - } - if ( $completeModules !== null && array_key_exists( $moduleName, $completeModules ) ) { - // If this module is done, mark all its params as used - $module->extractRequestParams(); - // Make sure this module is not used during execution - unset( $modules[$moduleName] ); - unset( $tmp[$moduleName] ); - } elseif ( $completeModules === null || $usePageset ) { - // Query modules may optimize data requests through the $this->getPageSet() - // object by adding extra fields from the page table. - // This function will gather all the extra request fields from the modules. - $module->requestExtraData( $this->mPageSet ); - } else { - // Error - this prop module must have finished before generator is done - $this->dieContinueUsageIf( $this->mModuleMgr->getModuleGroup( $moduleName ) === 'prop' ); - } - } - $this->dieContinueUsageIf( $completeModules !== null && count( $tmp ) !== 0 ); - - return $modules; + // Write the continuation data into the result + $this->getResult()->endContinuation( + $this->mParams['continue'] === null ? 'raw' : 'standard' + ); } /** @@ -447,12 +321,16 @@ class ApiQuery extends ApiBase { * @param string $param Parameter name to read modules from */ private function instantiateModules( &$modules, $param ) { + $wasPosted = $this->getRequest()->wasPosted(); if ( isset( $this->mParams[$param] ) ) { foreach ( $this->mParams[$param] as $moduleName ) { $instance = $this->mModuleMgr->getModule( $moduleName, $param ); if ( $instance === null ) { ApiBase::dieDebug( __METHOD__, 'Error instantiating module' ); } + if ( !$wasPosted && $instance->mustBePosted() ) { + $this->dieUsageMsgOrDebug( array( 'mustbeposted', $moduleName ) ); + } // Ignore duplicates. TODO 2.0: die()? if ( !array_key_exists( $moduleName, $modules ) ) { $modules[$moduleName] = $instance; @@ -563,22 +441,16 @@ class ApiQuery extends ApiBase { * This method is called by the generator base when generator in the smart-continue * mode tries to set 'query-continue' value. ApiQuery stores those values separately * until the post-processing when it is known if the generation should continue or repeat. + * @deprecated @since 1.24 * @param ApiQueryGeneratorBase $module Generator module * @param string $paramName * @param mixed $paramValue * @return bool True if processed, false if this is a legacy continue */ public function setGeneratorContinue( $module, $paramName, $paramValue ) { - if ( $this->mUseLegacyContinue ) { - return false; - } - $paramName = $module->encodeParamName( $paramName ); - if ( $this->mGeneratorContinue === null ) { - $this->mGeneratorContinue = array(); - } - $this->mGeneratorContinue[$paramName] = $paramValue; - - return true; + wfDeprecated( __METHOD__, '1.24' ); + $this->getResult()->setGeneratorContinueParam( $module, $paramName, $paramValue ); + return $this->getParameter( 'continue' ) !== null; } /** diff --git a/includes/api/ApiQueryBase.php b/includes/api/ApiQueryBase.php index 8e014dff957..2fd8597a802 100644 --- a/includes/api/ApiQueryBase.php +++ b/includes/api/ApiQueryBase.php @@ -398,15 +398,10 @@ abstract class ApiQueryBase extends ApiBase { /** * Set a query-continue value * @param string $paramName Parameter name - * @param string $paramValue Parameter value + * @param string|array $paramValue Parameter value */ protected function setContinueEnumParameter( $paramName, $paramValue ) { - $paramName = $this->encodeParamName( $paramName ); - $msg = array( $paramName => $paramValue ); - $result = $this->getResult(); - $result->disableSizeCheck(); - $result->addValue( 'query-continue', $this->getModuleName(), $msg, ApiResult::ADD_ON_TOP ); - $result->enableSizeCheck(); + $this->getResult()->setContinueParam( $this, $paramName, $paramValue ); } /** @@ -667,16 +662,14 @@ abstract class ApiQueryGeneratorBase extends ApiQueryBase { } /** - * Overrides base in case of generator & smart continue to - * notify ApiQueryMain instead of adding them to the result right away. + * Overridden to set the generator param if in generator mode * @param string $paramName Parameter name - * @param string $paramValue Parameter value + * @param string|array $paramValue Parameter value */ protected function setContinueEnumParameter( $paramName, $paramValue ) { - // If this is a generator and query->setGeneratorContinue() returns false, treat as before - if ( $this->mGeneratorPageSet === null - || !$this->getQuery()->setGeneratorContinue( $this, $paramName, $paramValue ) - ) { + if ( $this->mGeneratorPageSet !== null ) { + $this->getResult()->setGeneratorContinueParam( $this, $paramName, $paramValue ); + } else { parent::setContinueEnumParameter( $paramName, $paramValue ); } } diff --git a/includes/api/ApiResult.php b/includes/api/ApiResult.php index 7d0a15a9a52..b30d9ddd756 100644 --- a/includes/api/ApiResult.php +++ b/includes/api/ApiResult.php @@ -60,6 +60,13 @@ class ApiResult extends ApiBase { private $mData, $mIsRawMode, $mSize, $mCheckingSize; + private $continueAllModules = array(); + private $continueGeneratedModules = array(); + private $continuationData = array(); + private $generatorContinuationData = array(); + private $generatorParams = array(); + private $generatorDone = false; + /** * @param ApiMain $main */ @@ -437,4 +444,181 @@ class ApiResult extends ApiBase { public function execute() { ApiBase::dieDebug( __METHOD__, 'execute() is not supported on Result object' ); } + + /** + * Parse a 'continue' parameter and return status information. + * + * This must be balanced by a call to endContinuation(). + * + * @since 1.24 + * @param string|null $continue The "continue" parameter, if any + * @param array $allModules Contains ApiBase instances that will be executed + * @param array $generatedModules Names of modules that depend on the generator + * @return array Two elements: a boolean indicating if the generator is done, + * and an array of modules to actually execute. + */ + public function beginContinuation( + $continue, array $allModules = array(), array $generatedModules = array() + ) { + $this->continueGeneratedModules = $generatedModules + ? array_combine( $generatedModules, $generatedModules ) + : array(); + $this->continuationData = array(); + $this->generatorContinuationData = array(); + $this->generatorParams = array(); + + $skip = array(); + if ( is_string( $continue ) && $continue !== '' ) { + $continue = explode( '||', $continue ); + $this->dieContinueUsageIf( count( $continue ) !== 2 ); + $this->generatorDone = ( $continue[0] === '-' ); + if ( !$this->generatorDone ) { + $this->generatorParams = explode( '|', $continue[0] ); + } + $skip = explode( '|', $continue[1] ); + } + + $this->continueAllModules = array(); + $runModules = array(); + foreach ( $allModules as $module ) { + $name = $module->getModuleName(); + if ( in_array( $name, $skip ) ) { + $this->continueAllModules[$name] = false; + // Prevent spurious "unused parameter" warnings + $module->extractRequestParams(); + } else { + $this->continueAllModules[$name] = true; + $runModules[] = $module; + } + } + + return array( + $this->generatorDone, + $runModules, + ); + } + + /** + * Set the continuation parameter for a module + * + * @since 1.24 + * @param ApiBase $module + * @param string $paramName + * @param string|array $paramValue + */ + public function setContinueParam( ApiBase $module, $paramName, $paramValue ) { + $name = $module->getModuleName(); + if ( !isset( $this->continueAllModules[$name] ) ) { + throw new MWException( + "Module '$name' called ApiResult::setContinueParam but was not " . + 'passed to ApiResult::beginContinuation' + ); + } + if ( !$this->continueAllModules[$name] ) { + throw new MWException( + "Module '$name' was not supposed to have been executed, but " . + 'it was executed anyway' + ); + } + $paramName = $module->encodeParamName( $paramName ); + if ( is_array( $paramValue ) ) { + $paramValue = join( '|', $paramValue ); + } + $this->continuationData[$name][$paramName] = $paramValue; + } + + /** + * Set the continuation parameter for the generator module + * + * @since 1.24 + * @param ApiBase $module + * @param string $paramName + * @param string|array $paramValue + */ + public function setGeneratorContinueParam( ApiBase $module, $paramName, $paramValue ) { + $name = $module->getModuleName(); + $paramName = $module->encodeParamName( $paramName ); + if ( is_array( $paramValue ) ) { + $paramValue = join( '|', $paramValue ); + } + $this->generatorContinuationData[$name][$paramName] = $paramValue; + } + + /** + * Close continuation, writing the data into the result + * + * @since 1.24 + * @param string $style 'standard' for the new style since 1.21, 'raw' for + * the style used in 1.20 and earlier. + */ + public function endContinuation( $style = 'standard' ) { + if ( $style === 'raw' ) { + $key = 'query-continue'; + $data = array_merge_recursive( + $this->continuationData, $this->generatorContinuationData + ); + } else { + $key = 'continue'; + $data = array(); + + $finishedModules = array_diff( + array_keys( $this->continueAllModules ), + array_keys( $this->continuationData ) + ); + + // First, grab the non-generator-using continuation data + $continuationData = array_diff_key( + $this->continuationData, $this->continueGeneratedModules + ); + foreach ( $continuationData as $module => $kvp ) { + $data += $kvp; + } + + // Next, handle the generator-using continuation data + $continuationData = array_intersect_key( + $this->continuationData, $this->continueGeneratedModules + ); + if ( $continuationData ) { + // Some modules are unfinished: include those params, and copy + // the generator params. + foreach ( $continuationData as $module => $kvp ) { + $data += $kvp; + } + $data += array_intersect_key( + $this->getMain()->getRequest()->getValues(), + array_flip( $this->generatorParams ) + ); + } else if ( $this->generatorContinuationData ) { + // All the generator-using modules are complete, but the + // generator isn't. Continue the generator and restart the + // generator-using modules + $this->generatorParams = array(); + foreach ( $this->generatorContinuationData as $kvp ) { + $this->generatorParams = array_merge( + $this->generatorParams, array_keys( $kvp ) + ); + $data += $kvp; + } + $finishedModules = array_diff( + $finishedModules, $this->continueGeneratedModules + ); + } else { + // Generator and prop modules are all done. Mark it so. + $this->generatorDone = true; + } + + // Set 'continue' if any continuation data is set or if the generator + // still needs to run + if ( $data || !$this->generatorDone ) { + $data['continue'] = + ( $this->generatorDone ? '-' : join( '|', $this->generatorParams ) ) . + '||' . join( '|', $finishedModules ); + } + } + if ( $data ) { + $this->disableSizeCheck(); + $this->addValue( null, $key, $data, ApiResult::ADD_ON_TOP ); + $this->enableSizeCheck(); + } + } } diff --git a/includes/api/ApiSetNotificationTimestamp.php b/includes/api/ApiSetNotificationTimestamp.php index d5741d9a143..04be450bb6a 100644 --- a/includes/api/ApiSetNotificationTimestamp.php +++ b/includes/api/ApiSetNotificationTimestamp.php @@ -46,6 +46,8 @@ class ApiSetNotificationTimestamp extends ApiBase { $params = $this->extractRequestParams(); $this->requireMaxOneParameter( $params, 'timestamp', 'torevid', 'newerthanrevid' ); + $this->getResult()->beginContinuation( $params['continue'], array(), array() ); + $pageSet = $this->getPageSet(); if ( $params['entirewatchlist'] && $pageSet->getDataSource() !== null ) { $this->dieUsage( @@ -175,6 +177,8 @@ class ApiSetNotificationTimestamp extends ApiBase { $apiResult->setIndexedTagName( $result, 'page' ); } $apiResult->addValue( null, $this->getModuleName(), $result ); + + $apiResult->endContinuation(); } /** @@ -220,6 +224,7 @@ class ApiSetNotificationTimestamp extends ApiBase { 'newerthanrevid' => array( ApiBase::PARAM_TYPE => 'integer' ), + 'continue' => '', ); if ( $flags ) { $result += $this->getPageSet()->getFinalParams( $flags ); @@ -235,6 +240,7 @@ class ApiSetNotificationTimestamp extends ApiBase { 'torevid' => 'Revision to set the notification timestamp to (one page only)', 'newerthanrevid' => 'Revision to set the notification timestamp newer than (one page only)', 'token' => 'A token previously acquired via prop=info', + 'continue' => 'When more results are available, use this to continue', ); } diff --git a/includes/api/ApiWatch.php b/includes/api/ApiWatch.php index 6dfb1b4a836..c5aa90ef264 100644 --- a/includes/api/ApiWatch.php +++ b/includes/api/ApiWatch.php @@ -43,6 +43,9 @@ class ApiWatch extends ApiBase { } $params = $this->extractRequestParams(); + + $this->getResult()->beginContinuation( $params['continue'], array(), array() ); + $pageSet = $this->getPageSet(); // by default we use pageset to extract the page to work on. // title is still supported for backward compatibility @@ -88,6 +91,7 @@ class ApiWatch extends ApiBase { $res = $this->watchTitle( $title, $user, $params, true ); } $this->getResult()->addValue( null, $this->getModuleName(), $res ); + $this->getResult()->endContinuation(); } private function watchTitle( Title $title, User $user, array $params, @@ -180,6 +184,7 @@ class ApiWatch extends ApiBase { ApiBase::PARAM_TYPE => 'string', ApiBase::PARAM_REQUIRED => true ), + 'continue' => '', ); if ( $flags ) { $result += $this->getPageSet()->getFinalParams( $flags ); @@ -196,6 +201,7 @@ class ApiWatch extends ApiBase { 'unwatch' => 'If set the page will be unwatched rather than watched', 'uselang' => 'Language to show the message in', 'token' => 'A token previously acquired via prop=info', + 'continue' => 'When more results are available, use this to continue', ); }