2022-07-20 18:19:59 +00:00
|
|
|
<?php
|
|
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
namespace MediaWiki\Tests\Parser\Parsoid;
|
2022-07-20 18:19:59 +00:00
|
|
|
|
2022-08-12 15:02:12 +00:00
|
|
|
use Composer\Semver\Semver;
|
2022-07-20 18:19:59 +00:00
|
|
|
use Exception;
|
2022-07-26 19:41:03 +00:00
|
|
|
use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
|
|
|
|
|
use LogicException;
|
2022-08-04 21:14:47 +00:00
|
|
|
use MediaWiki\Parser\Parsoid\HTMLTransform;
|
2022-07-20 18:19:59 +00:00
|
|
|
use MediaWikiIntegrationTestCase;
|
|
|
|
|
use Wikimedia\Parsoid\Core\ClientError;
|
2022-08-04 21:14:47 +00:00
|
|
|
use Wikimedia\Parsoid\Mocks\MockPageConfig;
|
|
|
|
|
use Wikimedia\Parsoid\Parsoid;
|
2022-07-22 10:26:39 +00:00
|
|
|
use Wikimedia\Parsoid\Utils\ContentUtils;
|
2022-07-20 18:19:59 +00:00
|
|
|
|
|
|
|
|
/**
|
2022-08-04 21:14:47 +00:00
|
|
|
* @covers \MediaWiki\Parser\Parsoid\HTMLTransform
|
2022-07-20 18:19:59 +00:00
|
|
|
*/
|
2022-08-04 21:14:47 +00:00
|
|
|
class HTMLTransformTest extends MediaWikiIntegrationTestCase {
|
2022-07-20 18:19:59 +00:00
|
|
|
|
|
|
|
|
private const ORIG_BODY = '<body>Original HTML</body>';
|
|
|
|
|
private const ORIG_HTML = '<html>' . self::ORIG_BODY . '</html>';
|
|
|
|
|
private const ORIG_DATA_MW = [ 'ids' => [ 'mwAQ' => [] ] ];
|
|
|
|
|
private const ORIG_DATA_PARSOID = [ 'ids' => [ 'mwAQ' => [ 'pi' => [ [ [ 'k' => '1' ] ] ] ] ] ];
|
|
|
|
|
private const MODIFIED_DATA_MW = [ 'ids' => [ 'mwAQ' => [
|
|
|
|
|
'parts' => [ [
|
|
|
|
|
'template' => [
|
|
|
|
|
'target' => [ 'wt' => '1x', 'href' => './Template:1x' ],
|
|
|
|
|
'params' => [ '1' => [ 'wt' => 'hi' ] ],
|
|
|
|
|
'i' => 0
|
|
|
|
|
]
|
|
|
|
|
] ]
|
|
|
|
|
] ] ];
|
|
|
|
|
|
|
|
|
|
private function getOriginalData() {
|
|
|
|
|
$profileVersion = '2.4.0';
|
|
|
|
|
$htmlProfileUri = 'https://www.mediawiki.org/wiki/Specs/HTML/' . $profileVersion;
|
|
|
|
|
$dataParsoidProfileUri = 'https://www.mediawiki.org/wiki/Specs/data-parsoid/' . $profileVersion;
|
|
|
|
|
$dataMwProfileUri = 'https://www.mediawiki.org/wiki/Specs/data-mw/' . $profileVersion;
|
|
|
|
|
|
|
|
|
|
$htmlContentType = "text/html;profile=\"$htmlProfileUri\"";
|
|
|
|
|
$dataParsoidContentType = "application/json;profile=\"$dataParsoidProfileUri\"";
|
|
|
|
|
$dataMwContentType = "application/json;profile=\"$dataMwProfileUri\"";
|
|
|
|
|
|
|
|
|
|
return [
|
|
|
|
|
'revid' => 1,
|
|
|
|
|
'html' => [
|
|
|
|
|
'body' => self::ORIG_HTML,
|
|
|
|
|
'headers' => [
|
|
|
|
|
'content-type' => $htmlContentType,
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
'data-mw' => [
|
|
|
|
|
'body' => self::ORIG_DATA_MW,
|
|
|
|
|
'headers' => [
|
|
|
|
|
'content-type' => $dataMwContentType,
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
'data-parsoid' => [
|
|
|
|
|
'body' => self::ORIG_DATA_PARSOID,
|
|
|
|
|
'headers' => [
|
|
|
|
|
'content-type' => $dataParsoidContentType,
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
private function createHTMLTransformWithHTML( $html = '' ) {
|
|
|
|
|
return new HTMLTransform(
|
|
|
|
|
$html ?? self::ORIG_HTML,
|
|
|
|
|
new MockPageConfig( [], null ),
|
|
|
|
|
new Parsoid(
|
|
|
|
|
$this->getServiceContainer()->getParsoidSiteConfig(),
|
|
|
|
|
$this->getServiceContainer()->getParsoidDataAccess()
|
|
|
|
|
),
|
|
|
|
|
[]
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private function newHTMLTransform() {
|
|
|
|
|
$transform = $this->createHTMLTransformWithHTML();
|
2022-07-26 10:33:42 +00:00
|
|
|
$originalData = $this->getOriginalData();
|
2022-07-20 18:19:59 +00:00
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
// Set some options to assert on $transform object.
|
|
|
|
|
$transform->setOptions( [
|
2022-07-26 10:33:42 +00:00
|
|
|
'contentmodel' => 'wikitext',
|
|
|
|
|
'offsetType' => 'byte',
|
|
|
|
|
] );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalRevisionId( $originalData['revid'] );
|
|
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-20 18:19:59 +00:00
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
return $transform;
|
2022-07-20 18:19:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function provideBadOriginalData() {
|
|
|
|
|
$base = $this->getOriginalData();
|
|
|
|
|
|
|
|
|
|
$orig = $base;
|
|
|
|
|
unset( $orig['html']['headers']['content-type'] );
|
|
|
|
|
yield 'no html content type' => [
|
|
|
|
|
$orig,
|
|
|
|
|
new ClientError( 'Content-type of original html is missing.' )
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
$orig = $base;
|
|
|
|
|
$orig['html']['headers']['content-type'] = 'xyz';
|
|
|
|
|
yield 'bad html content type' => [
|
|
|
|
|
$orig,
|
|
|
|
|
new ClientError( 'Content-type of original html is missing.' )
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
$orig = $base;
|
|
|
|
|
unset( $orig['data-parsoid']['body']['ids'] );
|
|
|
|
|
yield 'missing ids key in data-parsoid' => [
|
|
|
|
|
$orig,
|
|
|
|
|
new ClientError( 'Invalid data-parsoid was provided.' )
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @dataProvider provideBadOriginalData
|
|
|
|
|
*
|
|
|
|
|
* @param array $original
|
|
|
|
|
* @param Exception $exception
|
|
|
|
|
*
|
|
|
|
|
* @return void
|
|
|
|
|
* @throws ClientError
|
|
|
|
|
*/
|
|
|
|
|
public function testBadOriginalData( array $original, Exception $exception ) {
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML();
|
2022-07-20 18:19:59 +00:00
|
|
|
|
|
|
|
|
$this->expectException( get_class( $exception ) );
|
|
|
|
|
$this->expectExceptionMessage( $exception->getMessage() );
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $original );
|
|
|
|
|
$transform->getOriginalSchemaVersion();
|
|
|
|
|
$transform->getOriginalPageBundle();
|
2022-07-20 18:19:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testHasOriginalHtml() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-20 18:19:59 +00:00
|
|
|
$this->assertTrue( $ctx->hasOriginalHtml() );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testGetOriginalSchemaVersion() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-20 18:19:59 +00:00
|
|
|
$this->assertSame( '2.4.0', $ctx->getOriginalSchemaVersion() );
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-22 10:26:39 +00:00
|
|
|
public function testGetSchemaVersion() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-22 10:26:39 +00:00
|
|
|
$this->assertSame( '2.4.0', $ctx->getSchemaVersion() );
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-26 19:41:03 +00:00
|
|
|
public function testHasOriginalDataParsoid() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-26 19:41:03 +00:00
|
|
|
$this->assertTrue( $ctx->hasOriginalDataParsoid() );
|
2022-07-20 18:19:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testGetOriginalPageBundle() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-20 18:19:59 +00:00
|
|
|
$this->assertSame(
|
|
|
|
|
self::ORIG_HTML,
|
|
|
|
|
$ctx->getOriginalPageBundle()->html
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testGetOriginalHtml() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-20 18:19:59 +00:00
|
|
|
$this->assertSame(
|
2022-07-26 19:41:03 +00:00
|
|
|
self::ORIG_BODY,
|
2022-07-20 18:19:59 +00:00
|
|
|
$ctx->getOriginalHtml()
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-22 10:26:39 +00:00
|
|
|
public function testGetOriginalBody() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-22 10:26:39 +00:00
|
|
|
$this->assertSame(
|
|
|
|
|
self::ORIG_BODY,
|
|
|
|
|
ContentUtils::toXML( $ctx->getOriginalBody() )
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-20 18:19:59 +00:00
|
|
|
public function testOldId() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-20 18:19:59 +00:00
|
|
|
$this->assertSame( 1, $ctx->getOriginalRevisionId() );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testGetContentModel() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-20 18:19:59 +00:00
|
|
|
$this->assertSame( 'wikitext', $ctx->getContentModel() );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testGetEnvOpts() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$ctx = $this->newHTMLTransform();
|
2022-07-26 10:33:42 +00:00
|
|
|
$this->assertSame( 'byte', $ctx->getOffsetType() );
|
2022-07-20 18:19:59 +00:00
|
|
|
}
|
2022-07-22 10:26:39 +00:00
|
|
|
|
|
|
|
|
private function getTextFromFile( string $name ): string {
|
|
|
|
|
return trim( file_get_contents( __DIR__ . "/data/Transform/$name" ) );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testDowngrade() {
|
|
|
|
|
$html = $this->getTextFromFile( 'Minimal.html' ); // Uses profile version 2.4.0
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-22 10:26:39 +00:00
|
|
|
|
|
|
|
|
$original = $this->getOriginalData();
|
|
|
|
|
|
|
|
|
|
// Specify newer profile version for original HTML
|
|
|
|
|
$original['html']['headers']['content-type'] = 'text/html;profile="https://www.mediawiki.org/wiki/Specs/HTML/999.0.0"';
|
|
|
|
|
|
|
|
|
|
// The profile version given inline in the original HTML doesn't matter, it's ignored
|
|
|
|
|
$original['html']['body'] = $html;
|
|
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $original );
|
2022-07-22 10:26:39 +00:00
|
|
|
|
2022-07-26 19:41:03 +00:00
|
|
|
// should automatically apply downgrade
|
2022-08-04 21:14:47 +00:00
|
|
|
$oldBody = $transform->getOriginalBody();
|
2022-07-22 10:26:39 +00:00
|
|
|
|
|
|
|
|
// all getters should now reflect the state after the downgrade.
|
2022-08-12 15:02:12 +00:00
|
|
|
// we expect a version >= 2.4.0 and < 3.0.0. So use ^2.4.0
|
|
|
|
|
Semver::satisfies( $transform->getOriginalSchemaVersion(), '^2.4.0' );
|
2022-08-04 21:14:47 +00:00
|
|
|
$this->assertNotSame( $html, $transform->getOriginalHtml() );
|
|
|
|
|
$this->assertNotSame( $oldBody, ContentUtils::toXML( $transform->getOriginalBody() ) );
|
2022-07-22 10:26:39 +00:00
|
|
|
}
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
public function testModifiedDataMW() {
|
|
|
|
|
$html = $this->getTextFromFile( 'Minimal-999.html' ); // Uses profile version 2.4.0
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$original = $this->getOriginalData();
|
|
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $original );
|
|
|
|
|
$transform->setModifiedDataMW( self::MODIFIED_DATA_MW );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
// should automatically apply downgrade
|
2022-08-04 21:14:47 +00:00
|
|
|
$doc = $transform->getModifiedDocument();
|
2022-07-26 19:41:03 +00:00
|
|
|
$html = ContentUtils::toXML( $doc );
|
|
|
|
|
|
|
|
|
|
// all getters should now reflect the state after the downgrade.
|
|
|
|
|
$this->assertNotSame( '"hi"', $html );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testMetrics() {
|
|
|
|
|
$html = '<html><body>xyz</body></html>'; // no schema version!
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$metrics = $this->createNoOpMock( StatsdDataFactoryInterface::class, [ 'increment' ] );
|
|
|
|
|
$metrics->expects( $this->atLeastOnce() )->method( 'increment' );
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setMetrics( $metrics );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
// getSchemaVersion should ioncrement the html2wt.original.version.notinline counter
|
|
|
|
|
// because the input HTML doesn't contain a schema version.
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->getSchemaVersion();
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testHtmlSize() {
|
2022-07-30 23:04:02 +00:00
|
|
|
$html = '<html><body>hällö</body></html>'; // use some multi-byte characters
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
2022-07-30 23:04:02 +00:00
|
|
|
// make sure it counts characters, not bytes
|
2022-08-04 21:14:47 +00:00
|
|
|
$this->assertSame( 31, $transform->getModifiedHtmlSize() );
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testSetOriginalWikitext() {
|
|
|
|
|
$html = '<html><body>xyz</body></html>'; // no schema version!
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$originalData = [
|
|
|
|
|
'wikitext' => [ 'body' => 'hi' ],
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
// mainly check that this doesn't explode.
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$this->assertFalse( $transform->hasOriginalHtml() );
|
|
|
|
|
$this->assertFalse( $transform->hasOriginalDataParsoid() );
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testSetOriginalHTML() {
|
|
|
|
|
$html = '<html><body>xyz</body></html>'; // no schema version!
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$originalData = [
|
|
|
|
|
'html' => [
|
|
|
|
|
'headers' => [
|
|
|
|
|
'content-type' => 'text/html;profile="https://www.mediawiki.org/wiki/Specs/HTML/999.0.0"',
|
|
|
|
|
],
|
|
|
|
|
'body' => 'hi',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
// mainly check that this doesn't explode.
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$this->assertTrue( $transform->hasOriginalHtml() );
|
|
|
|
|
$this->assertFalse( $transform->hasOriginalDataParsoid() );
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testSetOriginalDataTwice() {
|
|
|
|
|
$html = '<html><body>xyz</body></html>'; // no schema version!
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$originalData = $this->getOriginalData();
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$this->expectException( LogicException::class );
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testSetOriginalDataAfterGetModified() {
|
|
|
|
|
// Use HTML that contains a schema version!
|
|
|
|
|
// Otherwise, we won't trigger the right error.
|
|
|
|
|
$html = $this->getTextFromFile( 'Minimal.html' );
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML( $html );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->getModifiedDocument();
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$this->expectException( LogicException::class );
|
|
|
|
|
$this->expectExceptionMessage( 'getModifiedDocument()' );
|
|
|
|
|
|
|
|
|
|
$originalData = $this->getOriginalData();
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function testOffsetTypeMismatch() {
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform = $this->createHTMLTransformWithHTML();
|
2022-07-26 19:41:03 +00:00
|
|
|
$originalData = $this->getOriginalData();
|
|
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
// Set some options to assert on $transform.
|
|
|
|
|
$transform->setOptions( [
|
2022-07-26 19:41:03 +00:00
|
|
|
'contentmodel' => 'wikitext',
|
|
|
|
|
'offsetType' => 'byte',
|
|
|
|
|
] );
|
|
|
|
|
|
|
|
|
|
$originalData['data-parsoid']['body']['offsetType'] = 'UCS2';
|
|
|
|
|
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->setOriginalData( $originalData );
|
2022-07-26 19:41:03 +00:00
|
|
|
|
|
|
|
|
$this->expectException( ClientError::class );
|
2022-08-04 21:14:47 +00:00
|
|
|
$transform->getOriginalPageBundle();
|
2022-07-26 19:41:03 +00:00
|
|
|
}
|
2022-07-20 18:19:59 +00:00
|
|
|
}
|
