Postcard from linuxland.

* Reduced stack depth by using an internal stack in expand(), and by having some common code paths (e.g. non-subst double-brace during PST) return objects which can be expanded in that internal stack instead of the PHP stack. This is friendly to xdebug but slightly slower than the original version. Also it probably helps robustness when you don't add 7 stack levels per pair of double braces.
* Profiling indicates that expand and PPD are now good targets for porting to C. Abstracted and refactored the relevant code to allow for a drop-in replacement. A factor of 2 reduction in average-case replaceVariables() time may be possible.
* Verified with preprocessorFuzzTest.php against r29950, updated to allow better PST tests.
* Made parserTests.php respect $wgParserConf
* LST and ParserFunctions need a simultaneous update with the core due to changed interfaces. DOM objects are now wrapped rather than directly exposed.

includes/AutoLoader.php

@@ -143,6 +143,15 @@ function __autoload($className) {
 		'ParserOutput' => 'includes/ParserOutput.php',
 		'ParserOptions' => 'includes/ParserOptions.php',
 		'PatrolLog' => 'includes/PatrolLog.php',
+		'Preprocessor' => 'includes/Preprocessor.php',
+		'PPFrame' => 'includes/Preprocessor.php',
+		'PPNode' => 'includes/Preprocessor.php',
+		'Preprocessor_DOM' => 'includes/Preprocessor_DOM.php',
+		'PPFrame_DOM' => 'includes/Preprocessor_DOM.php',
+		'PPTemplateFrame_DOM' => 'includes/Preprocessor_DOM.php',
+		'PPDStack' => 'includes/Preprocessor_DOM.php',
+		'PPDStackElement' => 'includes/Preprocessor_DOM.php',
+		'PPNode_DOM' => 'includes/Preprocessor_DOM.php',
 		'ProfilerSimple' => 'includes/ProfilerSimple.php',
 		'ProfilerSimpleUDP' => 'includes/ProfilerSimpleUDP.php',
 		'Profiler' => 'includes/Profiler.php',

includes/Preprocessor.php

@@ -0,0 +1,74 @@
+<?php
+
+interface Preprocessor {
+	function __construct( $parser );
+	function newFrame();
+	function preprocessToObj( $text, $flags = 0 );
+}
+
+interface PPFrame {
+	const NO_ARGS = 1;
+	const NO_TEMPLATES = 2;
+	const STRIP_COMMENTS = 4;
+	const NO_IGNORE = 8;
+
+	const RECOVER_ORIG = 11;
+
+	/**
+	 * Create a child frame
+	 */
+	function newChild( $args = false, $title = false );
+
+	/**
+	 * Expand a document tree node
+	 */
+	function expand( $root, $flags = 0 );
+	
+	/**
+	 * Implode with flags for expand()
+	 */
+	function implodeWithFlags( $sep, $flags /*, ... */ );
+
+	/**
+	 * Implode with no flags specified
+	 */
+	function implode( $sep /*, ... */ );
+
+	/**
+	 * Makes an object that, when expand()ed, will be the same as one obtained 
+	 * with implode()
+	 */	
+	function virtualImplode( $sep /*, ... */ );
+
+	/**
+	 * Virtual implode with brackets
+	 */
+	function virtualBracketedImplode( $start, $sep, $end /*, ... */ );
+
+	/**
+	 * Returns true if there are no arguments in this frame
+	 */
+	function isEmpty();
+	
+	function getArgument( $name );
+
+	/**
+	 * Returns true if the infinite loop check is OK, false if a loop is detected
+	 */
+	function loopCheck( $title );
+}
+
+interface PPNode {
+	function getChildren();
+	function getFirstChild();
+	function getNextSibling();
+	function getChildrenOfType( $type );
+	function getLength();
+	function item( $i );
+	function getName();
+	
+	function splitArg();
+	function splitExt();
+	function splitHeading();
+}
+

maintenance/parserTests.inc

@@ -257,6 +257,7 @@ class ParserTest {
 	 * @return bool
 	 */
 	private function runTest( $desc, $input, $result, $opts ) {
+		global $wgParserConf;
 		if( $this->showProgress ) {
 			$this->showTesting( $desc );
 		}
@@ -281,7 +282,8 @@ class ParserTest {
 
 		$noxml = (bool)preg_match( '~\\b noxml \\b~x', $opts );
 
-		$parser = new Parser;
+		$class = $wgParserConf['class'];
+		$parser = new $class( $wgParserConf );
 		foreach( $this->hooks as $tag => $callback ) {
 			$parser->setHook( $tag, $callback );
 		}

maintenance/preprocessorFuzzTest.php

@@ -8,17 +8,21 @@ class PPFuzzTester {
 	var $hairs = array(
 		'[[', ']]', '{{', '}}', '{{{', '}}}', 
 		'<', '>', '<nowiki', '<gallery', '</nowiki>', '</gallery>', '<nOwIkI>', '</NoWiKi>',
-		//'<!--' , '-->',
-		//'<ref>', '</ref>', '<references/>',
+		'<!--' , '-->',
 		"\n==", "==\n",
 		'|', '=', "\n", ' ', "\t", "\x7f",
+		'~~', '~~~', '~~~~', 'subst:',
 		'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 
 		'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
+
+		// extensions
+		//'<ref>', '</ref>', '<references/>',
 	);
 	var $minLength = 0;
 	var $maxLength = 20;
 	var $maxTemplates = 5;
-	var $outputTypes = array( 'OT_HTML', 'OT_WIKI', 'OT_MSG', 'OT_PREPROCESS' );
+	//var $outputTypes = array( 'OT_HTML', 'OT_WIKI', 'OT_PREPROCESS' );
+	var $entryPoints = array( 'testSrvus', 'testPst', 'testPreprocess' );
 	static $currentTest = false;
 
 	function execute() {
@@ -71,26 +75,35 @@ class PPFuzzTester {
 		return Title::newFromText( mt_rand( 0, 1000000 ), mt_rand( 0, 10 ) );
 	}
 
+	/*
 	function pickOutputType() {
 		$count = count( $this->outputTypes );
 		return $this->outputTypes[ mt_rand( 0, $count - 1 ) ];
+	}*/
+
+	function pickEntryPoint() {
+		$count = count( $this->entryPoints );
+		return $this->entryPoints[ mt_rand( 0, $count - 1 ) ];
 	}
 }
 
 class PPFuzzTest {
-	var $templates, $mainText, $title;
+	var $templates, $mainText, $title, $entryPoint;
 
 	function __construct( $tester ) {
 		$this->parent = $tester;
 		$this->mainText = $tester->makeInputText();
 		$this->title = $tester->makeTitle();
-		$this->outputType = $tester->pickOutputType();
+		//$this->outputType = $tester->pickOutputType();
+		$this->entryPoint = $tester->pickEntryPoint();
+		$this->nickname = $tester->makeInputText();
+		$this->fancySig = (bool)mt_rand( 0, 1 );
 		$this->templates = array();
 	}
 
 	function templateHook( $title ) {
 		$titleText = $title->getPrefixedDBkey();
-				
+		
 		if ( !isset( $this->templates[$titleText] ) ) {
 			$finalTitle = $title;
 			if ( count( $this->templates ) >= $this->parent->maxTemplates ) {
@@ -116,16 +129,24 @@ class PPFuzzTest {
 	}
 
 	function execute() {
-		global $wgParser;
+		global $wgParser, $wgUser;
+
+		$wgUser = new PPFuzzUser;
+		$wgUser->mName = 'Fuzz';
+		$wgUser->mFrom = 'name';
+		$wgUser->ppfz_test = $this;
+
 		$options = new ParserOptions;
 		$options->setTemplateCallback( array( $this, 'templateHook' ) );
-		$wgParser->startExternalParse( $this->title, $options, constant( $this->outputType ) );
-		return $wgParser->srvus( $this->mainText );
+		//$wgParser->startExternalParse( $this->title, $options, constant( $this->outputType ) );
+		return call_user_func( array( $wgParser, $this->entryPoint ), $this->mainText, $this->title, $options );
 	}
 
 	function getReport() {
 		$s = "Title: " . $this->title->getPrefixedDBkey() . "\n" .
-			"Output type: {$this->outputType}\n" . 
+//			"Output type: {$this->outputType}\n" . 
+			"Entry point: {$this->entryPoint}\n" . 
+			"User: " . ( $this->fancySig ? 'fancy' : 'no-fancy' ) . ' ' . var_export( $this->nickname, true ) . "\n" . 
 			"Main text: " . var_export( $this->mainText, true ) . "\n";
 		foreach ( $this->templates as $titleText => $template ) {
 			$finalTitle = $template['finalTitle'];
@@ -139,6 +160,20 @@ class PPFuzzTest {
 	}
 }
 
+class PPFuzzUser extends User {
+	var $ppfz_test;
+
+	function getOption( $option, $defaultOverride = '' ) {
+		if ( $option === 'fancysig' ) {
+			return $this->ppfz_test->fancySig;
+		} elseif ( $option === 'nickname' ) {
+			return $this->ppfz_test->nickname;
+		} else {
+			return parent::getOption( $option, $defaultOverride );
+		}
+	}
+}
+
 ini_set( 'memory_limit', '50M' );
 if ( isset( $args[0] ) ) {
 	$testText = file_get_contents( $args[0] );