Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

resources: Add caching for faster runs and offline use

Browse source 
Also honour any XDG variables if set (such as Wikimedia CI does)
so that gate jobs can automatically persist the cache.

Bug: T203694
Change-Id: I156af1c5097b8a87be228345fc0a0b80bb3ae338
This commit is contained in:
Timo Tijhof 2019-03-24 20:28:25 +00:00
parent dd6beafe61
commit 3540897a6f
2 changed files with 54 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -50,6 +50,7 @@ sftp-config.json
# Building & testing
npm-debug.log
node_modules/
/resources/lib/.foreign
/tests/phpunit/phpunit.phar
/tests/selenium/log
.eslintcache

61
includes/ForeignResourceManager.php
View file

@ -30,10 +30,12 @@ class ForeignResourceManager {
private $registryFile;
private $libDir;
private $tmpParentDir;
private $cacheDir;
private $infoPrinter;
private $errorPrinter;
private $verbosePrinter;
private $action;
private $registry;
/**
* @param string $registryFile Path to YAML file
@ -60,8 +62,11 @@ class ForeignResourceManager {
// Use a temporary directory under the destination directory instead
// of wfTempDir() because PHP's rename() does not work across file
// systems, as the user's /tmp and $IP may be on different filesystems.
$this->tmpParentDir = "{$this->libDir}/.tmp";
// systems, and the user's /tmp and $IP may be on different filesystems.
$this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
$cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
$this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
}
/**
@ -76,14 +81,14 @@ class ForeignResourceManager {
}
$this->action = $action;
$registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
$this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
if ( $module === 'all' ) {
$modules = $registry;
} elseif ( isset( $registry[ $module ] ) ) {
$modules = [ $module => $registry[ $module ] ];
$modules = $this->registry;
} elseif ( isset( $this->registry[ $module ] ) ) {
$modules = [ $module => $this->registry[ $module ] ];
} else {
$this->error( "Unknown module name.\n\nMust be one of:\n" .
wordwrap( implode( ', ', array_keys( $registry ) ), 80 ) .
wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
'.'
);
return false;
@ -127,8 +132,8 @@ class ForeignResourceManager {
}
}
$this->cleanUp();
$this->output( "\nDone!\n" );
$this->cleanUp();
if ( $this->hasErrors ) {
// The verify mode should check all modules/files and fail after, not during.
return false;
@ -137,7 +142,29 @@ class ForeignResourceManager {
return true;
}
private function cacheKey( $src, $integrity ) {
$key = basename( $src ) . '_' . substr( $integrity, -12 );
$key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
return rtrim( $key, '_' );
}
/** @return string|false */
private function cacheGet( $key ) {
return Wikimedia\quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
}
private function cacheSet( $key, $data ) {
wfMkdirParents( $this->cacheDir );
file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
}
private function fetch( $src, $integrity ) {
$key = $this->cacheKey( $src, $integrity );
$data = $this->cacheGet( $key );
if ( $data ) {
return $data;
}
$req = MWHttpRequest::factory( $src, [ 'method' => 'GET', 'followRedirects' => false ] );
if ( !$req->execute()->isOK() ) {
throw new Exception( "Failed to download resource at {$src}" );
@ -150,6 +177,7 @@ class ForeignResourceManager {
$actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
if ( $integrity === $actualIntegrity ) {
$this->verbose( "... passed integrity check for {$src}\n" );
$this->cacheSet( $key, $data );
} else {
if ( $this->action === 'make-sri' ) {
$this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
@ -277,6 +305,23 @@ class ForeignResourceManager {
private function cleanUp() {
wfRecursiveRemoveDir( $this->tmpParentDir );
// Prune the cache of files we don't recognise.
$knownKeys = [];
foreach ( $this->registry as $info ) {
if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
$knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
} elseif ( $info['type'] === 'multi-file' ) {
foreach ( $info['files'] as $file ) {
$knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
}
}
}
foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
unlink( $cacheFile );
}
}
}
/**