Thijs/wiki.techinc.nl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rewrite various TableCleanup scripts to subclass Maintenance instead of FiveUpgrade

Browse source
This commit is contained in:
Chad Horohoe 2009-08-24 02:14:52 +00:00
parent cb1900e608
commit f991d9cf77
5 changed files with 139 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
maintenance/cleanupCaps.php
View file

@ -24,38 +24,33 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @author Brion Vibber <brion at pobox.com>
* @ingroup maintenance
*/
$optionsWithArgs = array( 'namespace' );
require_once( dirname(__FILE__) . '/cleanupTable.inc' );
require_once( dirname(__FILE__) . '/commandLine.inc' );
require_once( 'cleanupTable.inc' );
/**
* @ingroup Maintenance
*/
class CapsCleanup extends TableCleanup {
function __construct( $dryrun = false, $namespace = 0 ) {
parent::__construct( 'page', $dryrun );
$this->namespace = intval( $namespace );
public function __construct() {
parent::__construct();
$this->mDescription = "Script to cleanup capitalization";
$this->addOption( 'namespace', 'Namespace number to run caps cleanup on', false, true );
}
function cleanup() {
global $wgCapitalLinks;
if( $wgCapitalLinks ) {
echo "\$wgCapitalLinks is on -- no need for caps links cleanup.\n";
return false;
}
public function execute() {
global $wgCapitalLinks, $wgUser;
$this->namespace = intval( $this->getOption( 'namespace', 0 ) );
$this->dryrun = $this->hasOption( 'dry-run' );
$wgUser->setName( 'Conversion script' );
if( $wgCapitalLinks )
$this->error( "\$wgCapitalLinks is on -- no need for caps links cleanup.", true );
$this->runTable( $this->targetTable,
'WHERE page_namespace=' . $this->namespace,
array( &$this, 'processPage' ) );
}
function processPage( $row ) {
protected function processPage( $row ) {
global $wgContLang;
$current = Title::makeTitle( $row->page_namespace, $row->page_title );
@ -63,27 +58,26 @@ class CapsCleanup extends TableCleanup {
$upper = $row->page_title;
$lower = $wgContLang->lcfirst( $row->page_title );
if( $upper == $lower ) {
$this->log( "\"$display\" already lowercase." );
$this->output( "\"$display\" already lowercase." );
return $this->progress( 0 );
}
$target = Title::makeTitle( $row->page_namespace, $lower );
$targetDisplay = $target->getPrefixedText();
if( $target->exists() ) {
$this->log( "\"$display\" skipped; \"$targetDisplay\" already exists" );
$this->output( "\"$display\" skipped; \"$targetDisplay\" already exists" );
return $this->progress( 0 );
}
if( $this->dryrun ) {
$this->log( "\"$display\" -> \"$targetDisplay\": DRY RUN, NOT MOVED" );
$this->output( "\"$display\" -> \"$targetDisplay\": DRY RUN, NOT MOVED" );
$ok = true;
} else {
$ok = $current->moveTo( $target, false, 'Converting page titles to lowercase' );
$this->log( "\"$display\" -> \"$targetDisplay\": $ok" );
$this->output( "\"$display\" -> \"$targetDisplay\": $ok" );
}
if( $ok === true ) {
$this->progress( 1 );
if( $row->page_namespace == $this->namespace ) {
$talk = $target->getTalkPage();
$row->page_namespace = $talk->getNamespace();
@ -95,10 +89,7 @@ class CapsCleanup extends TableCleanup {
$this->progress( 0 );
}
}
}
$wgUser->setName( 'Conversion script' );
$ns = isset( $options['namespace'] ) ? $options['namespace'] : 0;
$caps = new CapsCleanup( isset( $options['dry-run'] ), $ns );
$caps->cleanup();
$maintClass = "CapsCleanup";
require_once( DO_MAINTENANCE );

60
maintenance/cleanupImages.php
View file

@ -24,23 +24,20 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @author Brion Vibber <brion at pobox.com>
* @ingroup Maintenance
*/
require_once( dirname(__FILE__) . '/commandLine.inc' );
require_once( 'cleanupTable.inc' );
require_once( dirname(__FILE__) . '/cleanupTable.inc' );
/**
* @ingroup Maintenance
*/
class ImageCleanup extends TableCleanup {
function __construct( $dryrun = false ) {
parent::__construct( 'image', $dryrun );
protected $targetTable = 'image';
public function __construct() {
parent::__construct();
$this->mDescription = "Script to clean up broken, unparseable upload filenames";
}
function processPage( $row ) {
protected function processPage( $row ) {
global $wgContLang;
$source = $row->img_name;
@ -67,7 +64,7 @@ class ImageCleanup extends TableCleanup {
$title = Title::makeTitleSafe( NS_FILE, $cleaned );
if( is_null( $title ) ) {
$this->log( "page $source ($cleaned) is illegal." );
$this->output( "page $source ($cleaned) is illegal.\n" );
$safe = $this->buildSafeTitle( $cleaned );
if( $safe === false )
return $this->progress( 0 );
@ -77,19 +74,19 @@ class ImageCleanup extends TableCleanup {
if( $title->getDBkey() !== $source ) {
$munged = $title->getDBkey();
$this->log( "page $source ($munged) doesn't match self." );
$this->output( "page $source ($munged) doesn't match self.\n" );
$this->pokeFile( $source, $munged );
return $this->progress( 1 );
}
$this->progress( 0 );
}
function killRow( $name ) {
private function killRow( $name ) {
if( $this->dryrun ) {
$this->log( "DRY RUN: would delete bogus row '$name'" );
$this->output( "DRY RUN: would delete bogus row '$name'\n" );
} else {
$this->log( "deleting bogus row '$name'" );
$this->output( "deleting bogus row '$name'\n" );
$db = wfGetDB( DB_MASTER );
$db->delete( 'image',
array( 'img_name' => $name ),
@ -97,17 +94,17 @@ class ImageCleanup extends TableCleanup {
}
}
function filePath( $name ) {
private function filePath( $name ) {
if ( !isset( $this->repo ) ) {
$this->repo = RepoGroup::singleton()->getLocalRepo();
}
return $this->repo->getRootDirectory() . '/' . $this->repo->getHashPath( $name ) . $name;
}
function pokeFile( $orig, $new ) {
private function pokeFile( $orig, $new ) {
$path = $this->filePath( $orig );
if( !file_exists( $path ) ) {
$this->log( "missing file: $path" );
$this->output( "missing file: $path\n" );
return $this->killRow( $orig );
}
@ -117,7 +114,7 @@ class ImageCleanup extends TableCleanup {
while( $db->selectField( 'image', 'img_name', array( 'img_name' => $final ), __METHOD__ ) ||
Title::makeTitle( NS_FILE, $final )->exists() ) {
$this->log( "Rename conflicts with '$final'..." );
$this->output( "Rename conflicts with '$final'...\n" );
$version++;
$final = $this->appendTitle( $new, "_$version" );
}
@ -125,9 +122,9 @@ class ImageCleanup extends TableCleanup {
$finalPath = $this->filePath( $final );
if( $this->dryrun ) {
$this->log( "DRY RUN: would rename $path to $finalPath" );
$this->output( "DRY RUN: would rename $path to $finalPath\n" );
} else {
$this->log( "renaming $path to $finalPath" );
$this->output( "renaming $path to $finalPath\n" );
// XXX: should this use File::move()? FIXME?
$db->begin();
$db->update( 'image',
@ -153,18 +150,18 @@ class ImageCleanup extends TableCleanup {
if( rename( $path, $finalPath ) ) {
$db->commit();
} else {
$this->log( "RENAME FAILED" );
$this->error( "RENAME FAILED" );
$db->rollback();
}
}
}
function appendTitle( $name, $suffix ) {
private function appendTitle( $name, $suffix ) {
return preg_replace( '/^(.*)(\..*?)$/',
"\\1$suffix\\2", $name );
}
function buildSafeTitle( $name ) {
private function buildSafeTitle( $name ) {
global $wgLegalTitleChars;
$x = preg_replace_callback(
"/([^$wgLegalTitleChars]|~)/",
@ -173,7 +170,7 @@ class ImageCleanup extends TableCleanup {
$test = Title::makeTitleSafe( NS_FILE, $x );
if( is_null( $test ) || $test->getDBkey() !== $x ) {
$this->log( "Unable to generate safe title from '$name', got '$x'" );
$this->error( "Unable to generate safe title from '$name', got '$x'" );
return false;
}
@ -181,8 +178,5 @@ class ImageCleanup extends TableCleanup {
}
}
$wgUser->setName( 'Conversion script' );
$caps = new ImageCleanup( !isset( $options['fix'] ) );
$caps->cleanup();
$maintClass = "ImageCleanup";
require_once( DO_MAINTENANCE );

91
maintenance/cleanupTable.inc
View file

@ -1,31 +1,53 @@
<?php
require_once( 'FiveUpgrade.inc' );
/**
* Generic table cleanup class. Already subclasses maintenance
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @defgroup Wikimedia
* @ingroup Maintenance
*/
abstract class TableCleanup extends FiveUpgrade {
function __construct( $table, $dryrun = false ) {
parent::__construct();
$this->targetTable = $table;
$this->maxLag = 10; # if slaves are lagged more than 10 secs, wait
$this->dryrun = $dryrun;
require_once( dirname(__FILE__) . '/Maintenance.php' );
abstract class TableCleanup extends Maintenance {
protected $targetTable = 'page';
protected $dryrun = false;
protected $maxLag = 10; # if slaves are lagged more than 10 secs, wait
public function __construct() {
parent::__construct();
$this->addOption( 'dry-run', 'Perform a dry run' );
}
function cleanup() {
public function execute() {
global $wgUser;
$wgUser->setName( 'Conversion script' );
$this->dryrun = $this->hasOption( 'dry-run' );
if( $this->dryrun ) {
echo "Checking for bad titles...\n";
$this->output( "Checking for bad titles...\n" );
} else {
echo "Checking and fixing bad titles...\n";
$this->output( "Checking and fixing bad titles...\n" );
}
$this->runTable( $this->targetTable,
'', //'WHERE page_namespace=0',
array( $this, 'processPage' ) );
}
function init( $count, $table ) {
protected function init( $count, $table ) {
$this->processed = 0;
$this->updated = 0;
$this->count = $count;
@ -33,7 +55,7 @@ abstract class TableCleanup extends FiveUpgrade {
$this->table = $table;
}
function progress( $updated ) {
protected function progress( $updated ) {
$this->updated += $updated;
$this->processed++;
if( $this->processed % 100 != 0 ) {
@ -47,39 +69,42 @@ abstract class TableCleanup extends FiveUpgrade {
$estimatedTotalTime = $delta / $portion;
$eta = $this->startTime + $estimatedTotalTime;
printf( "%s %s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
wfWikiID(),
wfTimestamp( TS_DB, intval( $now ) ),
$portion * 100.0,
$this->table,
wfTimestamp( TS_DB, intval( $eta ) ),
$this->processed,
$this->count,
$this->processed / $delta,
$updateRate * 100.0 );
$this->output(
sprintf( "%s %s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
wfWikiID(),
wfTimestamp( TS_DB, intval( $now ) ),
$portion * 100.0,
$this->table,
wfTimestamp( TS_DB, intval( $eta ) ),
$this->processed,
$this->count,
$this->processed / $delta,
$updateRate * 100.0
)
);
flush();
}
function runTable( $table, $where, $callback ) {
$count = $this->dbw->selectField( $table, 'count(*)', '', __METHOD__ );
protected function runTable( $table, $where, $callback ) {
$dbw = wfGetDB( DB_MASTER );
$count = $dbw->selectField( $table, 'count(*)', '', __METHOD__ );
$this->init( $count, $table );
$this->log( "Processing $table..." );
$this->output( "Processing $table..." );
$tableName = $this->dbr->tableName( $table );
$tableName = $dbw->tableName( $table );
$sql = "SELECT * FROM $tableName $where";
$result = $this->dbr->query( $sql, __METHOD__ );
$result = $dbw->query( $sql, __METHOD__ );
foreach( $result as $row ) {
call_user_func( $callback, $row );
}
$this->log( "Finished $table... $this->updated of $this->processed rows updated" );
$this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
$result->free();
}
function hexChar( $matches ) {
protected function hexChar( $matches ) {
return sprintf( "\\x%02x", ord( $matches[1] ) );
}
abstract function processPage( $row );
abstract protected function processPage( $row );
}

53
maintenance/cleanupTitles.php
View file

@ -24,23 +24,19 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @author Brion Vibber <brion at pobox.com>
* @ingroup Maintenance
*/
require_once( dirname(__FILE__) . '/commandLine.inc' );
require_once( 'cleanupTable.inc' );
require_once( dirname(__FILE__) . '/cleanupTable.inc' );
/**
* @ingroup Maintenance
*/
class TitleCleanup extends TableCleanup {
function __construct( $dryrun = false ) {
parent::__construct( 'page', $dryrun );
public function __construct() {
parent::__construct();
$this->mDescription = "Script to clean up broken, unparseable titles";
}
function processPage( $row ) {
protected function processPage( $row ) {
$current = Title::makeTitle( $row->page_namespace, $row->page_title );
$display = $current->getPrefixedText();
@ -53,20 +49,20 @@ class TitleCleanup extends TableCleanup {
}
if( $row->page_namespace == NS_FILE && $this->fileExists( $row->page_title ) ) {
$this->log( "file $row->page_title needs cleanup, please run cleanupImages.php." );
$this->output( "file $row->page_title needs cleanup, please run cleanupImages.php.\n" );
return $this->progress( 0 );
} elseif( is_null( $title ) ) {
$this->log( "page $row->page_id ($display) is illegal." );
$this->output( "page $row->page_id ($display) is illegal.\n" );
$this->moveIllegalPage( $row );
return $this->progress( 1 );
} else {
$this->log( "page $row->page_id ($display) doesn't match self." );
$this->output( "page $row->page_id ($display) doesn't match self.\n" );
$this->moveInconsistentPage( $row, $title );
return $this->progress( 1 );
}
}
function fileExists( $name ) {
protected function fileExists( $name ) {
// XXX: Doesn't actually check for file existence, just presence of image record.
// This is reasonable, since cleanupImages.php only iterates over the image table.
$dbr = wfGetDB( DB_SLAVE );
@ -74,7 +70,7 @@ class TitleCleanup extends TableCleanup {
return $row !== false;
}
function moveIllegalPage( $row ) {
protected function moveIllegalPage( $row ) {
$legal = 'A-Za-z0-9_/\\\\-';
$legalized = preg_replace_callback( "!([^$legal])!",
array( &$this, 'hexChar' ),
@ -86,28 +82,28 @@ class TitleCleanup extends TableCleanup {
$title = Title::newFromText( $legalized );
if( is_null( $title ) ) {
$clean = 'Broken/id:' . $row->page_id;
$this->log( "Couldn't legalize; form '$legalized' still invalid; using '$clean'" );
$this->output( "Couldn't legalize; form '$legalized' still invalid; using '$clean'\n" );
$title = Title::newFromText( $clean );
} elseif( $title->exists() ) {
$clean = 'Broken/id:' . $row->page_id;
$this->log( "Legalized for '$legalized' exists; using '$clean'" );
$this->output( "Legalized for '$legalized' exists; using '$clean'\n" );
$title = Title::newFromText( $clean );
}
$dest = $title->getDBkey();
if( $this->dryrun ) {
$this->log( "DRY RUN: would rename $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')" );
$this->output( "DRY RUN: would rename $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')\n" );
} else {
$this->log( "renaming $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')" );
$this->output( "renaming $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')\n" );
$dbw = wfGetDB( DB_MASTER );
$dbw->update( 'page',
array( 'page_title' => $dest ),
array( 'page_id' => $row->page_id ),
'cleanupTitles::moveInconsistentPage' );
__METHOD__ );
}
}
function moveInconsistentPage( $row, $title ) {
protected function moveInconsistentPage( $row, $title ) {
if( $title->exists() || $title->getInterwiki() ) {
if( $title->getInterwiki() ) {
$prior = $title->getPrefixedDbKey();
@ -118,20 +114,20 @@ class TitleCleanup extends TableCleanup {
$verified = Title::makeTitleSafe( $row->page_namespace, $clean );
if( $verified->exists() ) {
$blah = "Broken/id:" . $row->page_id;
$this->log( "Couldn't legalize; form '$clean' exists; using '$blah'" );
$this->output( "Couldn't legalize; form '$clean' exists; using '$blah'\n" );
$verified = Title::makeTitleSafe( $row->page_namespace, $blah );
}
$title = $verified;
}
if( is_null( $title ) ) {
wfDie( "Something awry; empty title.\n" );
$this->error( "Something awry; empty title.", true );
}
$ns = $title->getNamespace();
$dest = $title->getDBkey();
if( $this->dryrun ) {
$this->log( "DRY RUN: would rename $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')" );
$this->output( "DRY RUN: would rename $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')\n" );
} else {
$this->log( "renaming $row->page_id ($row->page_namespace,'$row->page_title') to ($ns,'$dest')" );
$this->output( "renaming $row->page_id ($row->page_namespace,'$row->page_title') to ($ns,'$dest')\n" );
$dbw = wfGetDB( DB_MASTER );
$dbw->update( 'page',
array(
@ -139,15 +135,12 @@ class TitleCleanup extends TableCleanup {
'page_title' => $dest
),
array( 'page_id' => $row->page_id ),
'cleanupTitles::moveInconsistentPage' );
__METHOD__ );
$linkCache = LinkCache::singleton();
$linkCache->clear();
}
}
}
$wgUser->setName( 'Conversion script' );
$caps = new TitleCleanup( !isset( $options['fix'] ) );
$caps->cleanup();
$maintClass = "TitleCleanup";
require_once( DO_MAINTENANCE );

33
maintenance/cleanupWatchlist.php
View file

@ -24,40 +24,34 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @author Brion Vibber <brion at pobox.com>
* @ingroup Maintenance
*/
require_once( dirname(__FILE__) . '/commandLine.inc' );
require_once( 'cleanupTable.inc' );
require_once( dirname(__FILE__) . '/cleanupTable.inc' );
/**
* @ingroup Maintenance
*/
class WatchlistCleanup extends TableCleanup {
function __construct( $dryrun = false ) {
parent::__construct( 'watchlist', $dryrun );
protected $targetTable = 'watchlist';
public function __construct() {
parent::__construct();
$this->mDescription = "Script to remove broken, unparseable titles in the Watchlist";
}
function processPage( $row ) {
protected function processPage( $row ) {
$current = Title::makeTitle( $row->wl_namespace, $row->wl_title );
$display = $current->getPrefixedText();
$verified = UtfNormal::cleanUp( $display );
$title = Title::newFromText( $verified );
if( $row->wl_user == 0 || is_null( $title ) || !$title->equals( $current ) ) {
$this->log( "invalid watch by {$row->wl_user} for ({$row->wl_namespace}, \"{$row->wl_title}\")" );
$this->output( "invalid watch by {$row->wl_user} for ({$row->wl_namespace}, \"{$row->wl_title}\")\n" );
$this->removeWatch( $row );
return $this->progress( 1 );
}
$this->progress( 0 );
}
function removeWatch( $row ) {
private function removeWatch( $row ) {
if( !$this->dryrun ) {
$dbw = wfGetDB( DB_MASTER );
$dbw->delete( 'watchlist', array(
@ -65,13 +59,10 @@ class WatchlistCleanup extends TableCleanup {
'wl_namespace' => $row->wl_namespace,
'wl_title' => $row->wl_title ),
__METHOD__ );
$this->log( '- removed' );
$this->output( "- removed\n" );
}
}
}
$wgUser->setName( 'Conversion script' );
$caps = new WatchlistCleanup( !isset( $options['fix'] ) );
$caps->cleanup();
$maintClass = "WatchlistCleanup";
require_once( DO_MAINTENANCE );