wiki.techinc.nl/includes/Storage/PageEditStash.php

<?php
/**
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 */

namespace MediaWiki\Storage;

use BagOStuff;
use Content;
use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
use MediaWiki\HookContainer\HookContainer;
use MediaWiki\HookContainer\HookRunner;
use MediaWiki\Page\PageIdentity;
use MediaWiki\Page\WikiPageFactory;
use MediaWiki\Parser\ParserOutputFlags;
use MediaWiki\Revision\SlotRecord;
use MediaWiki\Storage\Hook\ParserOutputStashForEditHook;
use MediaWiki\User\UserEditTracker;
use MediaWiki\User\UserFactory;
use MediaWiki\User\UserIdentity;
use ParserOutput;
use Psr\Log\LoggerInterface;
use stdClass;
use Wikimedia\Rdbms\ILoadBalancer;
use Wikimedia\ScopedCallback;
use WikiPage;

/**
 * Manage the pre-emptive page parsing for edits to wiki pages.
 *
 * This is written to by ApiStashEdit, and consumed by ApiEditPage
 * and EditPage (via PageUpdaterFactory and DerivedPageDataUpdater).
 *
 * See also mediawiki.action.edit/stash.js.
 *
 * @since 1.34
 * @ingroup Page
 */
class PageEditStash {
	/** @var BagOStuff */
	private $cache;
	/** @var ILoadBalancer */
	private $lb;
	/** @var LoggerInterface */
	private $logger;
	/** @var StatsdDataFactoryInterface */
	private $stats;
	/** @var ParserOutputStashForEditHook */
	private $hookRunner;
	/** @var UserEditTracker */
	private $userEditTracker;
	/** @var UserFactory */
	private $userFactory;
	/** @var WikiPageFactory */
	private $wikiPageFactory;
	/** @var int */
	private $initiator;

	public const ERROR_NONE = 'stashed';
	public const ERROR_PARSE = 'error_parse';
	public const ERROR_CACHE = 'error_cache';
	public const ERROR_UNCACHEABLE = 'uncacheable';
	public const ERROR_BUSY = 'busy';

	public const PRESUME_FRESH_TTL_SEC = 30;
	public const MAX_CACHE_TTL = 300; // 5 minutes
	public const MAX_SIGNATURE_TTL = 60;

	private const MAX_CACHE_RECENT = 2;

	public const INITIATOR_USER = 1;
	public const INITIATOR_JOB_OR_CLI = 2;

	/**
	 * @param BagOStuff $cache
	 * @param ILoadBalancer $lb
	 * @param LoggerInterface $logger
	 * @param StatsdDataFactoryInterface $stats
	 * @param UserEditTracker $userEditTracker
	 * @param UserFactory $userFactory
	 * @param WikiPageFactory $wikiPageFactory
	 * @param HookContainer $hookContainer
	 * @param int $initiator Class INITIATOR__* constant
	 */
	public function __construct(
		BagOStuff $cache,
		ILoadBalancer $lb,
		LoggerInterface $logger,
		StatsdDataFactoryInterface $stats,
		UserEditTracker $userEditTracker,
		UserFactory $userFactory,
		WikiPageFactory $wikiPageFactory,
		HookContainer $hookContainer,
		$initiator
	) {
		$this->cache = $cache;
		$this->lb = $lb;
		$this->logger = $logger;
		$this->stats = $stats;
		$this->userEditTracker = $userEditTracker;
		$this->userFactory = $userFactory;
		$this->wikiPageFactory = $wikiPageFactory;
		$this->hookRunner = new HookRunner( $hookContainer );
		$this->initiator = $initiator;
	}

	/**
	 * @param PageUpdater $pageUpdater (a WikiPage instance is also supported but deprecated)
	 * @param Content $content Edit content
	 * @param UserIdentity $user
	 * @param string $summary Edit summary
	 * @return string Class ERROR_* constant
	 */
	public function parseAndCache( $pageUpdater, Content $content, UserIdentity $user, string $summary ) {
		$logger = $this->logger;

		if ( $pageUpdater instanceof WikiPage ) {
			// TODO: Trigger deprecation warning once extensions have been fixed.
			//       Or better, create PageUpdater::prepareAndStash and deprecate this method.
			$pageUpdater = $pageUpdater->newPageUpdater( $user );
		}

		$page = $pageUpdater->getPage();
		$key = $this->getStashKey( $page, $this->getContentHash( $content ), $user );
		$fname = __METHOD__;

		// Use the primary DB to allow for fast blocking locks on the "save path" where this
		// value might actually be used to complete a page edit. If the edit submission request
		// happens before this edit stash requests finishes, then the submission will block until
		// the stash request finishes parsing. For the lock acquisition below, there is not much
		// need to duplicate parsing of the same content/user/summary bundle, so try to avoid
		// blocking at all here.
		$dbw = $this->lb->getConnectionRef( DB_PRIMARY );
		if ( !$dbw->lock( $key, $fname, 0 ) ) {
			// De-duplicate requests on the same key
			return self::ERROR_BUSY;
		}
		/** @noinspection PhpUnusedLocalVariableInspection */
		$unlocker = new ScopedCallback( static function () use ( $dbw, $key, $fname ) {
			$dbw->unlock( $key, $fname );
		} );

		$cutoffTime = time() - self::PRESUME_FRESH_TTL_SEC;

		// Reuse any freshly build matching edit stash cache
		$editInfo = $this->getStashValue( $key );
		if ( $editInfo && (int)wfTimestamp( TS_UNIX, $editInfo->timestamp ) >= $cutoffTime ) {
			$alreadyCached = true;
		} else {
			$pageUpdater->setContent( SlotRecord::MAIN, $content );

			// FIXME: pass causeAction down to DerivedPageDataUpdater?!
			$update = $pageUpdater->prepareUpdate( EDIT_INTERNAL ); // applies pre-safe transform
			$output = $update->getCanonicalParserOutput(); // causes content to be parsed
			$output->setCacheTime( $update->getRevision()->getTimestamp() );

			// emulate a cache value that kind of looks like a PreparedEdit, for use below
			$editInfo = (object)[
				'pstContent' => $update->getRawContent( SlotRecord::MAIN ),
				'output'     => $output,
				'timestamp'  => $output->getCacheTime()
			];

			$alreadyCached = false;
		}

		$logContext = [ 'cachekey' => $key, 'title' => (string)$page ];

		if ( $editInfo->output ) {
			// Let extensions add ParserOutput metadata or warm other caches
			$legacyUser = $this->userFactory->newFromUserIdentity( $user );
			$legacyPage = $this->wikiPageFactory->newFromTitle( $page );
			$this->hookRunner->onParserOutputStashForEdit(
				$legacyPage, $content, $editInfo->output, $summary, $legacyUser );

			if ( $alreadyCached ) {
				$logger->debug( "Parser output for key '{cachekey}' already cached.", $logContext );

				return self::ERROR_NONE;
			}

			$code = $this->storeStashValue(
				$key,
				$editInfo->pstContent,
				$editInfo->output,
				$editInfo->timestamp,
				$user
			);

			if ( $code === true ) {
				$logger->debug( "Cached parser output for key '{cachekey}'.", $logContext );

				return self::ERROR_NONE;
			} elseif ( $code === 'uncacheable' ) {
				$logger->info(
					"Uncacheable parser output for key '{cachekey}' [{code}].",
					$logContext + [ 'code' => $code ]
				);

				return self::ERROR_UNCACHEABLE;
			} else {
				$logger->error(
					"Failed to cache parser output for key '{cachekey}'.",
					$logContext + [ 'code' => $code ]
				);

				return self::ERROR_CACHE;
			}
		}

		return self::ERROR_PARSE;
	}

	/**
	 * Check that a prepared edit is in cache and still up-to-date
	 *
	 * This method blocks if the prepared edit is already being rendered,
	 * waiting until rendering finishes before doing final validity checks.
	 *
	 * The cache is rejected if template or file changes are detected.
	 * Note that foreign template or file transclusions are not checked.
	 *
	 * This returns an object with the following fields:
	 *   - pstContent: the Content after pre-save-transform
	 *   - output: the ParserOutput instance
	 *   - timestamp: the timestamp of the parse
	 *   - edits: author edit count if they are logged in or NULL otherwise
	 *
	 * @param PageIdentity $page
	 * @param Content $content
	 * @param UserIdentity $user to get parser options from
	 * @return stdClass|false Returns edit stash object or false on cache miss
	 */
	public function checkCache( PageIdentity $page, Content $content, UserIdentity $user ) {
		$legacyUser = $this->userFactory->newFromUserIdentity( $user );
		if (
			// The context is not an HTTP POST request
			!$legacyUser->getRequest()->wasPosted() ||
			// The context is a CLI script or a job runner HTTP POST request
			$this->initiator !== self::INITIATOR_USER ||
			// The editor account is a known bot
			$legacyUser->isBot()
		) {
			// Avoid wasted queries and statsd pollution
			return false;
		}

		$logger = $this->logger;

		$key = $this->getStashKey( $page, $this->getContentHash( $content ), $user );
		$logContext = [
			'key' => $key,
			'title' => (string)$page,
			'user' => $user->getName()
		];

		$editInfo = $this->getAndWaitForStashValue( $key );
		if ( !is_object( $editInfo ) || !$editInfo->output ) {
			$this->incrStatsByContent( 'cache_misses.no_stash', $content );
			if ( $this->recentStashEntryCount( $user ) > 0 ) {
				$logger->info( "Empty cache for key '{key}' but not for user.", $logContext );
			} else {
				$logger->debug( "Empty cache for key '{key}'.", $logContext );
			}

			return false;
		}

		$age = time() - (int)wfTimestamp( TS_UNIX, $editInfo->output->getCacheTime() );
		$logContext['age'] = $age;

		$isCacheUsable = true;
		if ( $age <= self::PRESUME_FRESH_TTL_SEC ) {
			// Assume nothing changed in this time
			$this->incrStatsByContent( 'cache_hits.presumed_fresh', $content );
			$logger->debug( "Timestamp-based cache hit for key '{key}'.", $logContext );
		} elseif ( !$user->isRegistered() ) {
			$lastEdit = $this->lastEditTime( $user );
			$cacheTime = $editInfo->output->getCacheTime();
			if ( $lastEdit < $cacheTime ) {
				// Logged-out user made no local upload/template edits in the meantime
				$this->incrStatsByContent( 'cache_hits.presumed_fresh', $content );
				$logger->debug( "Edit check based cache hit for key '{key}'.", $logContext );
			} else {
				$isCacheUsable = false;
				$this->incrStatsByContent( 'cache_misses.proven_stale', $content );
				$logger->info( "Stale cache for key '{key}' due to outside edits.", $logContext );
			}
		} else {
			if ( $editInfo->edits === $this->userEditTracker->getUserEditCount( $user ) ) {
				// Logged-in user made no local upload/template edits in the meantime
				$this->incrStatsByContent( 'cache_hits.presumed_fresh', $content );
				$logger->debug( "Edit count based cache hit for key '{key}'.", $logContext );
			} else {
				$isCacheUsable = false;
				$this->incrStatsByContent( 'cache_misses.proven_stale', $content );
				$logger->info( "Stale cache for key '{key}'due to outside edits.", $logContext );
			}
		}

		if ( !$isCacheUsable ) {
			return false;
		}

		if ( $editInfo->output->getOutputFlag( ParserOutputFlags::VARY_REVISION ) ) {
			// This can be used for the initial parse, e.g. for filters or doUserEditContent(),
			// but a second parse will be triggered in doEditUpdates() no matter what
			$logger->info(
				"Cache for key '{key}' has vary-revision; post-insertion parse inevitable.",
				$logContext
			);
		} else {
			static $flagsMaybeReparse = [
				// Similar to the above if we didn't guess the ID correctly
				ParserOutputFlags::VARY_REVISION_ID,
				// Similar to the above if we didn't guess the timestamp correctly
				ParserOutputFlags::VARY_REVISION_TIMESTAMP,
				// Similar to the above if we didn't guess the content correctly
				ParserOutputFlags::VARY_REVISION_SHA1,
				// Similar to the above if we didn't guess page ID correctly
				ParserOutputFlags::VARY_PAGE_ID,
			];
			foreach ( $flagsMaybeReparse as $flag ) {
				if ( $editInfo->output->getOutputFlag( $flag ) ) {
					$logger->debug(
						"Cache for key '{key}' has $flag; post-insertion parse possible.",
						$logContext
					);
				}
			}
		}

		return $editInfo;
	}

	/**
	 * @param string $subkey
	 * @param Content $content
	 */
	private function incrStatsByContent( $subkey, Content $content ) {
		$this->stats->increment( 'editstash.' . $subkey ); // overall for b/c
		$this->stats->increment( 'editstash_by_model.' . $content->getModel() . '.' . $subkey );
	}

	/**
	 * @param string $key
	 * @return bool|stdClass
	 */
	private function getAndWaitForStashValue( $key ) {
		$editInfo = $this->getStashValue( $key );

		if ( !$editInfo ) {
			$start = microtime( true );
			// We ignore user aborts and keep parsing. Block on any prior parsing
			// so as to use its results and make use of the time spent parsing.
			// Skip this logic if there no primary connection in case this method
			// is called on an HTTP GET request for some reason.
			$dbw = $this->lb->getAnyOpenConnection( $this->lb->getWriterIndex() );
			if ( $dbw && $dbw->lock( $key, __METHOD__, 30 ) ) {
				$editInfo = $this->getStashValue( $key );
				$dbw->unlock( $key, __METHOD__ );
			}

			$timeMs = 1000 * max( 0, microtime( true ) - $start );
			$this->stats->timing( 'editstash.lock_wait_time', $timeMs );
		}

		return $editInfo;
	}

	/**
	 * @param string $textHash
	 * @return string|bool Text or false if missing
	 */
	public function fetchInputText( $textHash ) {
		$textKey = $this->cache->makeKey( 'stashedit', 'text', $textHash );

		return $this->cache->get( $textKey );
	}

	/**
	 * @param string $text
	 * @param string $textHash
	 * @return bool Success
	 */
	public function stashInputText( $text, $textHash ) {
		$textKey = $this->cache->makeKey( 'stashedit', 'text', $textHash );

		return $this->cache->set(
			$textKey,
			$text,
			self::MAX_CACHE_TTL,
			BagOStuff::WRITE_ALLOW_SEGMENTS
		);
	}

	/**
	 * @param UserIdentity $user
	 * @return string|null TS_MW timestamp or null
	 */
	private function lastEditTime( UserIdentity $user ) {
		$db = $this->lb->getConnectionRef( DB_REPLICA );

		$time = $db->newSelectQueryBuilder()
			->select( 'MAX(rc_timestamp)' )
			->from( 'recentchanges' )
			->join( 'actor', null, 'actor_id=rc_actor' )
			->where( [ 'actor_name' => $user->getName() ] )
			->caller( __METHOD__ )
			->fetchField();

		return wfTimestampOrNull( TS_MW, $time );
	}

	/**
	 * Get hash of the content, factoring in model/format
	 *
	 * @param Content $content
	 * @return string
	 */
	private function getContentHash( Content $content ) {
		return sha1( implode( "\n", [
			$content->getModel(),
			$content->getDefaultFormat(),
			$content->serialize( $content->getDefaultFormat() )
		] ) );
	}

	/**
	 * Get the temporary prepared edit stash key for a user
	 *
	 * This key can be used for caching prepared edits provided:
	 *   - a) The $user was used for PST options
	 *   - b) The parser output was made from the PST using cannonical matching options
	 *
	 * @param PageIdentity $page
	 * @param string $contentHash Result of getContentHash()
	 * @param UserIdentity $user User to get parser options from
	 * @return string
	 */
	private function getStashKey( PageIdentity $page, $contentHash, UserIdentity $user ) {
		return $this->cache->makeKey(
			'stashedit-info-v1',
			md5( "{$page->getNamespace()}\n{$page->getDBkey()}" ),
			// Account for the edit model/text
			$contentHash,
			// Account for user name related variables like signatures
			md5( "{$user->getId()}\n{$user->getName()}" )
		);
	}

	/**
	 * @param string $key
	 * @return stdClass|bool Object map (pstContent,output,outputID,timestamp,edits) or false
	 */
	private function getStashValue( $key ) {
		$stashInfo = $this->cache->get( $key );
		if ( is_object( $stashInfo ) && $stashInfo->output instanceof ParserOutput ) {
			return $stashInfo;
		}

		return false;
	}

	/**
	 * Build a value to store in memcached based on the PST content and parser output
	 *
	 * This makes a simple version of WikiPage::prepareContentForEdit() as stash info
	 *
	 * @param string $key
	 * @param Content $pstContent Pre-Save transformed content
	 * @param ParserOutput $parserOutput
	 * @param string $timestamp TS_MW
	 * @param UserIdentity $user
	 * @return string|bool True or an error code
	 */
	private function storeStashValue(
		$key,
		Content $pstContent,
		ParserOutput $parserOutput,
		$timestamp,
		UserIdentity $user
	) {
		// If an item is renewed, mind the cache TTL determined by config and parser functions.
		// Put an upper limit on the TTL to avoid extreme template/file staleness.
		$age = time() - (int)wfTimestamp( TS_UNIX, $parserOutput->getCacheTime() );
		$ttl = min( $parserOutput->getCacheExpiry() - $age, self::MAX_CACHE_TTL );
		// Avoid extremely stale user signature timestamps (T84843)
		if ( $parserOutput->getOutputFlag( ParserOutputFlags::USER_SIGNATURE ) ) {
			$ttl = min( $ttl, self::MAX_SIGNATURE_TTL );
		}

		if ( $ttl <= 0 ) {
			return 'uncacheable'; // low TTL due to a tag, magic word, or signature?
		}

		// Store what is actually needed and split the output into another key (T204742)
		$stashInfo = (object)[
			'pstContent' => $pstContent,
			'output'     => $parserOutput,
			'timestamp'  => $timestamp,
			'edits'      => $user->isRegistered() ? $this->userEditTracker->getUserEditCount( $user ) : null,
		];

		$ok = $this->cache->set( $key, $stashInfo, $ttl, BagOStuff::WRITE_ALLOW_SEGMENTS );
		if ( $ok ) {
			// These blobs can waste slots in low cardinality memcached slabs
			$this->pruneExcessStashedEntries( $user, $key );
		}

		return $ok ? true : 'store_error';
	}

	/**
	 * @param UserIdentity $user
	 * @param string $newKey
	 */
	private function pruneExcessStashedEntries( UserIdentity $user, $newKey ) {
		$key = $this->cache->makeKey( 'stash-edit-recent', sha1( $user->getName() ) );

		$keyList = $this->cache->get( $key ) ?: [];
		if ( count( $keyList ) >= self::MAX_CACHE_RECENT ) {
			$oldestKey = array_shift( $keyList );
			$this->cache->delete( $oldestKey, BagOStuff::WRITE_PRUNE_SEGMENTS );
		}

		$keyList[] = $newKey;
		$this->cache->set( $key, $keyList, 2 * self::MAX_CACHE_TTL );
	}

	/**
	 * @param UserIdentity $user
	 * @return int
	 */
	private function recentStashEntryCount( UserIdentity $user ) {
		$key = $this->cache->makeKey( 'stash-edit-recent', sha1( $user->getName() ) );

		return count( $this->cache->get( $key ) ?: [] );
	}
}