HipHop improvements:

* Added the ability to compile extensions. The build process is bootstrapped by running MediaWiki in interpreted mode. Extension setup file inclusions are slightly modified in a way that makes them register themselves for compilation. Then the same LocalSettings.php uses the compiled extension setup file when the compiled binary runs.
* Tested with Cite and ParserFunctions. The code which lets you have an extensions directory in a place other than $IP/../extensions is untested.
* Simplified WebStart.php slightly by using a custom $_SERVER variable to mark compiled mode. It will break if you don't use the supplied server.conf, but that will break a lot of things so don't do that.
* Fixed the core web entry points to include WebStart.php in compiled mode instead of interpreted.
* Made the build directory configurable. This is mostly so that I can grep the source tree without seeing loads of generated C++.
* In server.conf, added a rewrite rule allowing a /wiki/$1 article path.
* Removed server.conf log file location "/dev/stdout", breaks when you switch user
* Disable static content cache, breaks horribly when you set SourceRoot to a directory containing 7GB of files.
* Rewrote the run-server script in PHP, mostly to support the configurable build directory feature.
* Added an option to the run-server script to allow running in interpreted (hphpi) mode.

api.php

@@ -51,7 +51,11 @@ if( !function_exists( 'version_compare' ) || version_compare( phpversion(), '5.2
 }
 
 // Initialise common code.
-require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 
 wfProfileIn( 'api.php' );
 $starttime = microtime( true );

img_auth.php

@@ -26,7 +26,11 @@
  **/
 
 define( 'MW_NO_OUTPUT_COMPRESSION', 1 );
-require_once( dirname( __FILE__ ) . '/includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 wfProfileIn( 'img_auth.php' );
 require_once( dirname( __FILE__ ) . '/includes/StreamFile.php' );
 

includes/DefaultSettings.php

@@ -5313,6 +5313,12 @@ $wgUpdateRowsPerQuery = 100;
  * @{
  */
 
+/**
+ * The build directory for HipHop compilation. 
+ * Defaults to $IP/maintenance/hiphop/build.
+ */
+$wgHipHopBuildDirectory = false;
+
 /**
  * The HipHop build type. Can be either "Debug" or "Release".
  */
@@ -5324,6 +5330,31 @@ $wgHipHopBuildType = 'Debug';
  */
 $wgHipHopCompilerProcs = 'detect';
 
+/**
+ * Filesystem extensions directory. Defaults to $IP/../extensions.
+ *
+ * To compile extensions with HipHop, set $wgExtensionsDirectory correctly, 
+ * and use code like:
+ *    
+ *    require( MWInit::extensionSetupPath( 'Extension/Extension.php' ) );
+ *
+ * to include the extension setup file from LocalSettings.php. It is not 
+ * necessary to set this variable unless you use MWInit::extensionSetupPath().
+ */
+$wgExtensionsDirectory = false;
+
+/**
+ * A list of files that should be compiled into a HipHop build, in addition to 
+ * those listed in $wgAutoloadClasses. Add to this array in an extension setup 
+ * file in order to add files to the build. 
+ *
+ * The files listed here must either be either absolute paths under $IP or 
+ * under $wgExtensionsDirectory, or paths relative to the virtual source root
+ * "$IP/..", i.e. starting with "phase3" for core files, and "extensions" for 
+ * extension files.
+ */
+$wgCompiledFiles = array();
+
 /** @} */ # End of HipHop compilation }
 
 

includes/Init.php

@@ -61,12 +61,62 @@ class MWInit {
 		global $IP;
 
 		if ( defined( 'MW_COMPILED' ) ) {
-			return $file;
+			return "phase3/$file";
 		} else {
 			return "$IP/$file";
 		}
 	}
 
+	/**
+	 * The equivalent of MWInit::interpretedPath() but for files relative to the
+	 * extensions directory.
+	 */
+	static function extInterpretedPath( $file ) {
+		return getExtensionsDirectory() . '/' . $file;
+	}
+
+	/**
+	 * The equivalent of MWInit::compiledPath() but for files relative to the 
+	 * extensions directory. Any files referenced in this way must be registered
+	 * for compilation by including them in $wgCompiledFiles.
+	 */
+	static function extCompiledPath( $file ) {
+		if ( defined( 'MW_COMPILED' ) ) {
+			return "extensions/$file";
+		} else {
+			return getExtensionsDirectory() . '/' . $file;
+		}
+	}
+
+	/**
+	 * Register an extension setup file and return its path for compiled 
+	 * inclusion. Use this function in LocalSettings.php to add extensions
+	 * to the build. For example:
+	 *
+	 *    require( MWInit::extSetupPath( 'ParserFunctions/ParserFunctions.php' ) );
+	 *
+	 * @param $path The path relative to the extensions directory, as defined by 
+	 *   $wgExtensionsDirectory.
+	 */
+	static function extSetupPath( $extRel ) {
+		$baseRel = "extensions/$extRel";
+		if ( defined( 'MW_COMPILED' ) ) {
+			return $baseRel;
+		} else {
+			global $wgCompiledFiles;
+			$wgCompiledFiles[] = $baseRel;
+			return self::getExtensionsDirectory() . '/' . $extRel;
+		}
+	}
+
+	static function getExtensionsDirectory() {
+		global $wgExtensionsDirectory, $IP;
+		if ( $wgExtensionsDirectory === false ) {
+			$wgExtensionsDirectory = "$IP/../extensions";
+		}
+		return $wgExtensionsDirectory;
+	}
+
 	/**
 	 * Determine whether a class exists, using a method which works under HipHop.
 	 *

includes/WebStart.php

@@ -8,22 +8,6 @@
  * @file
  */
 
-/**
- * Detect compiled mode by looking for a function that only exists if compiled 
- * in. Note that we can't use function_exists(), because it is terribly broken 
- * under HipHop due to the "volatile" feature.
- *
- * @return bool
- */
-function wfDetectCompiledMode() {
-	try {
-		$r = new ReflectionFunction( 'wfHipHopCompilerVersion' );
-	} catch ( ReflectionException $e ) {
-		$r = false;
-	}
-	return $r !== false;
-}
-
 # Protect against register_globals
 # This must be done before any globals are set by the code
 if ( ini_get( 'register_globals' ) ) {
@@ -88,11 +72,9 @@ if ( $IP === false ) {
 	$IP = realpath( '.' );
 }
 
-if ( wfDetectCompiledMode() ) {
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
 	define( 'MW_COMPILED', 1 );
-}
-
-if ( !defined( 'MW_COMPILED' ) ) {
+} else {
 	# Get MWInit class
 	require_once( "$IP/includes/Init.php" );
 

index.php

@@ -64,7 +64,11 @@ ENDL;
 # Initialise common code.  This gives us access to GlobalFunctions, the AutoLoader, and
 # the globals $wgRequest, $wgOut, $wgUser, $wgLang and $wgContLang, amongst others; it
 # does *not* load $wgTitle
-require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 
 try {
 	wfIndexMain();

load.php

@@ -36,7 +36,12 @@ if( !function_exists( 'version_compare' ) || version_compare( phpversion(), '5.2
 	wfDie( "MediaWiki $version requires at least PHP version 5.2.3." );
 }
 
-require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
+
 wfProfileIn( 'load.php' );
 
 // URL safety checks

maintenance/doMaintenance.php

@@ -54,18 +54,10 @@ $maintenance->setup();
 $self = $maintenance->getName();
 
 // Detect compiled mode
-try {
-	$r = new ReflectionFunction( 'wfHipHopCompilerVersion' );
-} catch ( ReflectionException $e ) {
-	$r = false;
-}
-
-if ( $r ) {
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
 	define( 'MW_COMPILED', 1 );
-}
-
-# Get the MWInit class
-if ( !defined( 'MW_COMPILED' ) ) {
+} else {
+	# Get the MWInit class
 	require_once( "$IP/includes/Init.php" );
 	require_once( "$IP/includes/AutoLoader.php" );
 }
@@ -77,7 +69,7 @@ require_once( MWInit::compiledPath( 'includes/profiler/Profiler.php' ) );
 if ( !defined( 'MW_COMPILED' ) ) {
 	require_once( "$IP/includes/Defines.php" );
 }
-require_once( "$IP/includes/DefaultSettings.php" );
+require_once( MWInit::compiledPath( 'includes/DefaultSettings.php' ) );
 
 if ( defined( 'MW_CONFIG_CALLBACK' ) ) {
 	# Use a callback function to configure MediaWiki

maintenance/hiphop/make

@@ -5,13 +5,20 @@ require( dirname( __FILE__ ) . '/../Maintenance.php' );
 
 class MakeHipHop extends Maintenance {
 	function execute() {
+		global $wgHipHopBuildDirectory;
+
 		$startTime = time();
 
-		$sourceDir = realpath( dirname( __FILE__ ) );
-		$IP = realpath( "$sourceDir/../.." );
-		$buildDir = "$sourceDir/build";
+		$thisDir = realpath( dirname( __FILE__ ) );
+		$IP = realpath( "$thisDir/../.." );
+		if ( strval( $wgHipHopBuildDirectory ) !== '' ) {
+			$buildDir = $wgHipHopBuildDirectory;
+		} else {
+			$buildDir = "$thisDir/build";
+		}
+		$extensionsDir = realpath( MWInit::getExtensionsDirectory() );
 		$outDir = "$buildDir/hiphop-output";
-		$persistentDir = "$buildDir/persistent" ;
+		$persistentDir = "$buildDir/persistent";
 
 		if ( !is_dir( $buildDir ) ) {
 			mkdir( $buildDir, 0777, true );
@@ -20,6 +27,17 @@ class MakeHipHop extends Maintenance {
 			mkdir( $persistentDir, 0777, true );
 		}
 
+		if ( realpath( "$IP/../phase3" ) !== $IP
+			|| realpath( "$IP/../extensions" ) !== $extensionsDir )
+		{
+			# Set up a fake source directory with the correct layout
+			$sourceBase = "$buildDir/source";
+			$this->setupFakeSourceBase( $IP, $extensionsDir, $sourceBase );
+		} else {
+			$sourceBase = realpath( "$IP/.." );
+			unlink( "$buildDir/source" );
+		}
+
 		# With the CentOS RPMs, you just get g++44, no g++, so we have to 
 		# use the environment
 		if ( isset( $_ENV['CXX'] ) ) {
@@ -39,13 +57,8 @@ class MakeHipHop extends Maintenance {
 			"}\n"
 		);
 
-		# Generate the file list from the autoloader
-		global $wgAutoloadLocalClasses;
-		$files = array_merge(
-			array_values( $wgAutoloadLocalClasses ),
-			array_map( 'trim', file( "$sourceDir/extra-files" ) )
-		);
-		$files = array_unique( $files );
+		# Generate the file list
+		$files = $this->getFileList();
 		file_put_contents(
 			"$buildDir/file-list",
 			implode( "\n", $files ) . "\n" );
@@ -55,10 +68,10 @@ class MakeHipHop extends Maintenance {
 			'hphp' .
 			' --target=cpp' .
 			' --format=file' .
-			' --input-dir=' . wfEscapeShellArg( $IP ) .
+			' --input-dir=' . wfEscapeShellArg( $sourceBase ) .
 			' --input-list=' . wfEscapeShellArg( "$buildDir/file-list" ) .
 			' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) .
-			' -c ' . wfEscapeShellArg( "$sourceDir/compiler.conf" ) .
+			' -c ' . wfEscapeShellArg( "$thisDir/compiler.conf" ) .
 			' --parse-on-demand=false' .
 			' --program=mediawiki-hphp' .
 			' --output-dir=' . wfEscapeShellArg( $outDir ) .
@@ -181,7 +194,7 @@ class MakeHipHop extends Maintenance {
 			$elapsed -= $minutes * 60;
 		}
 		echo $elapsed . "s\n";
-		echo "The MediaWiki executable is at build/persistent/mediawiki-hphp\n";
+		echo "The MediaWiki executable is at $buildDir/persistent/mediawiki-hphp\n";
 	}
 
 	function checkVolatileClasses( $dir ) {
@@ -224,6 +237,71 @@ class MakeHipHop extends Maintenance {
 			return 1;
 		}
 	}
+
+	function setupFakeSourceBase( $phase3, $extensions, $dest ) {
+		if ( !file_exists( $dest ) ) {
+			mkdir( $dest, 0777, true );
+		}
+
+		$this->forceCreateLink( "$dest/phase3", $phase3 );
+		$this->forceCreateLink( "$dest/extensions", $extensions );
+	}
+
+	function forceCreateLink( $target, $link ) {
+		if ( file_exists( $target ) ) {
+			if ( readlink( $target ) === $link ) {
+				return;
+			}
+			unlink( $target );
+		}
+		symlink( $target, $link );
+	}
+
+	function getFileList() {
+		global $wgAutoloadClasses, $wgAutoloadLocalClasses, $wgCompiledFiles;
+		$inputFiles = array_merge(
+			array_values( $wgAutoloadClasses ),
+			array_values( $wgAutoloadLocalClasses ),
+			$wgCompiledFiles
+		);
+		$processedFiles = array();
+		foreach ( $inputFiles as $file ) {
+			if ( substr( $file, 0, 1 ) === '/' ) {
+				$processedFiles[] = $this->absoluteToRelative( $file );
+			} elseif ( preg_match( '/^extensions/', $file ) ) {
+				$processedFiles[] = $file;
+			} else {
+				$processedFiles[] = "phase3/$file";
+			}
+		}
+
+		$extraCoreFiles = array_map( 'trim', file( dirname( __FILE__ ) . '/extra-files' ) );
+		foreach ( $extraCoreFiles as $file ) {
+			if ( $file === '' ) {
+				continue;
+			}
+			$processedFiles[] = "phase3/$file";
+		}
+		return array_unique( $processedFiles );
+	}
+
+	function absoluteToRelative( $file ) {
+		global $IP;
+
+		$coreBase = realpath( $IP ) . '/';
+		$extBase = realpath( MWInit::getExtensionsDirectory() ) . '/';
+		$file = realpath( $file );
+
+		if ( substr( $file, 0, strlen( $extBase ) ) === $extBase ) {
+			return 'extensions/' . substr( $file, strlen( $extBase ) );
+		} elseif ( substr( $file, 0, strlen( $coreBase ) ) === $coreBase ) {
+			return 'phase3/' . substr( $file, strlen( $coreBase ) );
+		} else {
+			$this->error( "The following file is registered for compilation but is not in \$IP or " .
+				"\$wgExtensionsDirectory: $file \n" );
+			exit( 1 );
+		}
+	}
 }
 
 $maintClass = 'MakeHipHop';

maintenance/hiphop/run-server

@@ -1,12 +1,72 @@
-#!/bin/bash
+#!/usr/bin/hphpi -f
+<?php
 
-sourceDir=`dirname "$0"`
-sourceRoot=`readlink -f "$sourceDir/../.."`
+require( dirname( __FILE__ ) . '/../Maintenance.php' );
 
-"$sourceDir"/build/persistent/mediawiki-hphp \
-	-c "$sourceDir/server.conf" \
-	-v Server.SourceRoot="$sourceRoot" \
-	--mode=server \
-	--port=8080
+class RunHipHopServer extends Maintenance {
+	function __construct() {
+		parent::__construct();
+		$this->addOption( 'interpret', 'Run in interpreted mode' );
+	}
 
+	function execute() {
+		if ( $this->hasOption( 'interpret' ) ) {
+			$this->runInterpreted();
+		} else {
+			$this->runCompiled();
+		}
+	}
 
+	function runCompiled() {
+		global $wgHipHopBuildDirectory;
+		$thisDir = realpath( dirname( __FILE__ ) );
+		$IP = realpath( "$thisDir/../.." );
+		if ( strval( $wgHipHopBuildDirectory ) !== '' ) {
+			$buildDir = $wgHipHopBuildDirectory;
+		} else {
+			$buildDir = "$thisDir/build";
+		}
+
+		if ( file_exists( "$buildDir/source" ) ) {
+			$sourceBase = "$buildDir/source";
+		} else {
+			$sourceBase = realpath( "$IP/.." );
+		}
+
+		passthru( 
+			'cd ' . wfEscapeShellArg( $sourceBase ) . " && " .
+			'MW_INSTALL_PATH=' . wfEscapeShellArg( $IP ) . ' ' .
+			wfEscapeShellArg( 
+				"$buildDir/persistent/mediawiki-hphp",
+				'-c', "$thisDir/server.conf",
+				'-v', "Server.SourceRoot=$sourceBase",
+				'-v', "Server.IncludeSearchPaths.0=$sourceBase",
+				'--mode=server',
+				'--port=8080'
+			),
+			$ret
+		);
+		exit( $ret );
+	}
+
+	function runInterpreted() {
+		$thisDir = realpath( dirname( __FILE__ ) );
+		$IP = realpath( "$thisDir/../.." );
+		$sourceBase = realpath( "$IP/.." );
+
+		passthru(
+			wfEscapeShellArg(
+				'hphpi',
+				'-c', "$thisDir/server.conf",
+				'-v', "Server.SourceRoot=$sourceBase",
+				'-v', "Server.IncludeSearchPaths.0=$sourceBase",
+				'--mode=server',
+				'--port=8080'
+			),
+			$ret
+		);
+		exit( $ret );
+	}
+}
+$maintClass = 'RunHipHopServer';
+require_once( RUN_MAINTENANCE_IF_MAIN );

maintenance/hiphop/server.conf

@@ -1,7 +1,6 @@
 Log {
 	Level = Warning
 	UseLogFile = true
-	File = /dev/stdout
 	NativeStackTrace = true
 	InjectedStackTrace = true
 }
@@ -12,5 +11,23 @@ Debug {
 	TranslateSource = true
 }
 Server {
-	EnableStaticContentFromDisk = true
+	EnableStaticContentCache = false
+	EnableStaticContentFromDisk = false
+	AlwaysUseRelativePath = true
 }
+ServerVariables {
+	MW_COMPILED = 1
+}
+VirtualHost {
+	* {
+		ServerName = localhost
+		Pattern = .
+		RewriteRules {
+			* {
+				pattern = ^/wiki/(.*)$
+				to = /phase3/index.php?title=$1
+			}
+		}
+	}
+}
+

mw-config/index.php

@@ -9,7 +9,11 @@ define( 'MW_CONFIG_CALLBACK', 'Installer::overrideConfig' );
 define( 'MEDIAWIKI_INSTALL', true );
 
 chdir( dirname( dirname( __FILE__ ) ) );
-require( dirname( dirname( __FILE__ ) ) . '/includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require( dirname( dirname( __FILE__ ) ) . '/includes/WebStart.php' );
+}
 
 wfInstallerMain();
 

profileinfo.php

@@ -28,8 +28,12 @@
 ini_set( 'zlib.output_compression', 'off' );
 
 $wgEnableProfileInfo = $wgProfileToDatabase = false;
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 
-require_once( './includes/WebStart.php' );
 
 header( 'Content-Type: text/html; charset=utf-8' );
 

redirect.php

@@ -6,8 +6,12 @@
  *
  * @file
  */
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 
-require_once( './includes/WebStart.php' );
 global $wgArticlePath;
 
 $page = $wgRequest->getVal( 'wpDropdown' );

thumb.php

@@ -7,7 +7,11 @@
  * @ingroup Media
  */
 define( 'MW_NO_OUTPUT_COMPRESSION', 1 );
-require_once( './includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 
 $wgTrivialMimeDetection = true; //don't use fancy mime detection, just check the file extension for jpg/gif/png.
 

trackback.php

@@ -5,7 +5,11 @@
  * @ingroup SpecialPage
  */
 
-require_once( './includes/WebStart.php' );
+if ( isset( $_SERVER['MW_COMPILED'] ) ) {
+	require ( 'phase3/includes/WebStart.php' );
+} else {
+	require ( dirname( __FILE__ ) . '/includes/WebStart.php' );
+}
 
 class TrackBack {