wiki.techinc.nl/tests/phpunit/includes/normal/CleanUpTest.php

<?php
/**
 * Tests for UtfNormal::cleanUp() function.
 *
 * Copyright © 2004 Brion Vibber <brion@pobox.com>
 * http://www.mediawiki.org/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 */

/**
 * Additional tests for UtfNormal::cleanUp() function, inclusion
 * regression checks for known problems.
 * Requires PHPUnit.
 *
 * @ingroup UtfNormal
 */
class CleanUpTest extends MediaWikiTestCase {
	/** @todo document */
	function testAscii() {
		$text = 'This is plain ASCII text.';
		$this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
	}

	/** @todo document */
	function testNull() {
		$text = "a \x00 null";
		$expect = "a \xef\xbf\xbd null";
		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testLatin() {
		$text = "L'\xc3\xa9cole";
		$this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
	}

	/** @todo document */
	function testLatinNormal() {
		$text = "L'e\xcc\x81cole";
		$expect = "L'\xc3\xa9cole";
		$this->assertEquals( $expect, UtfNormal::cleanUp( $text ) );
	}

	/**
	 * This test is *very* expensive!
	 * @todo document
	 */
	function XtestAllChars() {
		$rep = UTF8_REPLACEMENT;
		for( $i = 0x0; $i < UNICODE_MAX; $i++ ) {
			$char = codepointToUtf8( $i );
			$clean = UtfNormal::cleanUp( $char );
			$x = sprintf( "%04X", $i );
			if( $i % 0x1000 == 0 ) echo "U+$x\n";
			if( $i == 0x0009 ||
			    $i == 0x000a ||
			    $i == 0x000d ||
			    ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST) ||
			    ($i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) ||
			    ($i > 0xffff && $i <= UNICODE_MAX ) ) {
				if( isset( UtfNormal::$utfCanonicalComp[$char] ) || isset( UtfNormal::$utfCanonicalDecomp[$char] ) ) {
				    $comp = UtfNormal::NFC( $char );
					$this->assertEquals(
						bin2hex( $comp ),
						bin2hex( $clean ),
						"U+$x should be decomposed" );
				} else {
					$this->assertEquals(
						bin2hex( $char ),
						bin2hex( $clean ),
						"U+$x should be intact" );
				}
			} else {
				$this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
			}
		}
	}

	/** @todo document */
	function testAllBytes() {
		$this->doTestBytes( '', '' );
		$this->doTestBytes( 'x', '' );
		$this->doTestBytes( '', 'x' );
		$this->doTestBytes( 'x', 'x' );
	}

	/** @todo document */
	function doTestBytes( $head, $tail ) {
		for( $i = 0x0; $i < 256; $i++ ) {
			$char = $head . chr( $i ) . $tail;
			$clean = UtfNormal::cleanUp( $char );
			$x = sprintf( "%02X", $i );
			if( $i == 0x0009 ||
			    $i == 0x000a ||
			    $i == 0x000d ||
			    ($i > 0x001f && $i < 0x80) ) {
				$this->assertEquals(
					bin2hex( $char ),
					bin2hex( $clean ),
					"ASCII byte $x should be intact" );
				if( $char != $clean ) return;
			} else {
				$norm = $head . UTF8_REPLACEMENT . $tail;
				$this->assertEquals(
					bin2hex( $norm ),
					bin2hex( $clean ),
					"Forbidden byte $x should be rejected" );
				if( $norm != $clean ) return;
			}
		}
	}

	/** @todo document */
	function testDoubleBytes() {
		$this->doTestDoubleBytes( '', '' );
		$this->doTestDoubleBytes( 'x', '' );
		$this->doTestDoubleBytes( '', 'x' );
		$this->doTestDoubleBytes( 'x', 'x' );
	}

	/**
	 * @todo document
	 */
	function doTestDoubleBytes( $head, $tail ) {
		for( $first = 0xc0; $first < 0x100; $first+=2 ) {
			for( $second = 0x80; $second < 0x100; $second+=2 ) {
				$char = $head . chr( $first ) . chr( $second ) . $tail;
				$clean = UtfNormal::cleanUp( $char );
				$x = sprintf( "%02X,%02X", $first, $second );
				if( $first > 0xc1 &&
				    $first < 0xe0 &&
				    $second < 0xc0 ) {
				    $norm = UtfNormal::NFC( $char );
					$this->assertEquals(
						bin2hex( $norm ),
						bin2hex( $clean ),
						"Pair $x should be intact" );
				    if( $norm != $clean ) return;
				} elseif( $first > 0xfd || $second > 0xbf ) {
					# fe and ff are not legal head bytes -- expect two replacement chars
					$norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail;
					$this->assertEquals(
						bin2hex( $norm ),
						bin2hex( $clean ),
						"Forbidden pair $x should be rejected" );
					if( $norm != $clean ) return;
				} else {
					$norm = $head . UTF8_REPLACEMENT . $tail;
					$this->assertEquals(
						bin2hex( $norm ),
						bin2hex( $clean ),
						"Forbidden pair $x should be rejected" );
					if( $norm != $clean ) return;
				}
			}
		}
	}

	/** @todo document */
	function testTripleBytes() {
		$this->doTestTripleBytes( '', '' );
		$this->doTestTripleBytes( 'x', '' );
		$this->doTestTripleBytes( '', 'x' );
		$this->doTestTripleBytes( 'x', 'x' );
	}

	/** @todo document */
	function doTestTripleBytes( $head, $tail ) {
		for( $first = 0xc0; $first < 0x100; $first+=2 ) {
			for( $second = 0x80; $second < 0x100; $second+=2 ) {
				#for( $third = 0x80; $third < 0x100; $third++ ) {
				for( $third = 0x80; $third < 0x81; $third++ ) {
					$char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
					$clean = UtfNormal::cleanUp( $char );
					$x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
					if( $first >= 0xe0 &&
						$first < 0xf0 &&
						$second < 0xc0 &&
						$third < 0xc0 ) {
						if( $first == 0xe0 && $second < 0xa0 ) {
							$this->assertEquals(
								bin2hex( $head . UTF8_REPLACEMENT . $tail ),
								bin2hex( $clean ),
								"Overlong triplet $x should be rejected" );
						} elseif( $first == 0xed &&
							( chr( $first ) . chr( $second ) . chr( $third ))  >= UTF8_SURROGATE_FIRST ) {
							$this->assertEquals(
								bin2hex( $head . UTF8_REPLACEMENT . $tail ),
								bin2hex( $clean ),
								"Surrogate triplet $x should be rejected" );
						} else {
							$this->assertEquals(
								bin2hex( UtfNormal::NFC( $char ) ),
								bin2hex( $clean ),
								"Triplet $x should be intact" );
						}
					} elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
						$this->assertEquals(
							bin2hex( UtfNormal::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT . $tail ),
							bin2hex( $clean ),
							"Valid 2-byte $x + broken tail" );
					} elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
						$this->assertEquals(
							bin2hex( $head . UTF8_REPLACEMENT . UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ),
							bin2hex( $clean ),
							"Broken head + valid 2-byte $x" );
					} elseif( ( $first > 0xfd || $second > 0xfd ) &&
					            ( ( $second > 0xbf && $third > 0xbf ) ||
					              ( $second < 0xc0 && $third < 0xc0 ) ||
					              ( $second > 0xfd ) ||
					              ( $third > 0xfd ) ) ) {
						# fe and ff are not legal head bytes -- expect three replacement chars
						$this->assertEquals(
							bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
							bin2hex( $clean ),
							"Forbidden triplet $x should be rejected" );
					} elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
						$this->assertEquals(
							bin2hex( $head . UTF8_REPLACEMENT . $tail ),
							bin2hex( $clean ),
							"Forbidden triplet $x should be rejected" );
					} else {
						$this->assertEquals(
							bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
							bin2hex( $clean ),
							"Forbidden triplet $x should be rejected" );
					}
				}
			}
		}
	}

	/** @todo document */
	function testChunkRegression() {
		# Check for regression against a chunking bug
		$text   = "\x46\x55\xb8" .
		          "\xdc\x96" .
		          "\xee" .
		          "\xe7" .
		          "\x44" .
		          "\xaa" .
		          "\x2f\x25";
		$expect = "\x46\x55\xef\xbf\xbd" .
		          "\xdc\x96" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\x44" .
		          "\xef\xbf\xbd" .
		          "\x2f\x25";

		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testInterposeRegression() {
		$text   = "\x4e\x30" .
		          "\xb1" .		# bad tail
		          "\x3a" .
		          "\x92" .		# bad tail
		          "\x62\x3a" .
		          "\x84" .		# bad tail
		          "\x43" .
		          "\xc6" .		# bad head
		          "\x3f" .
		          "\x92" .		# bad tail
		          "\xad" .		# bad tail
		          "\x7d" .
		          "\xd9\x95";

		$expect = "\x4e\x30" .
		          "\xef\xbf\xbd" .
		          "\x3a" .
		          "\xef\xbf\xbd" .
		          "\x62\x3a" .
		          "\xef\xbf\xbd" .
		          "\x43" .
		          "\xef\xbf\xbd" .
		          "\x3f" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\x7d" .
		          "\xd9\x95";

		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testOverlongRegression() {
		$text   = "\x67" .
		          "\x1a" . # forbidden ascii
		          "\xea" . # bad head
		          "\xc1\xa6" . # overlong sequence
		          "\xad" . # bad tail
		          "\x1c" . # forbidden ascii
		          "\xb0" . # bad tail
		          "\x3c" .
		          "\x9e";  # bad tail
		$expect = "\x67" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\x3c" .
		          "\xef\xbf\xbd";
		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testSurrogateRegression() {
		$text   = "\xed\xb4\x96" . # surrogate 0xDD16
		          "\x83" . # bad tail
		          "\xb4" . # bad tail
		          "\xac";  # bad head
		$expect = "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd";
		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testBomRegression() {
		$text   = "\xef\xbf\xbe" . # U+FFFE, illegal char
		          "\xb2" . # bad tail
		          "\xef" . # bad head
		          "\x59";
		$expect = "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\xef\xbf\xbd" .
		          "\x59";
		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testForbiddenRegression() {
		$text   = "\xef\xbf\xbf"; # U+FFFF, illegal char
		$expect = "\xef\xbf\xbd";
		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}

	/** @todo document */
	function testHangulRegression() {
		$text = "\xed\x9c\xaf" . # Hangul char
				"\xe1\x87\x81";  # followed by another final jamo
		$expect = $text;         # Should *not* change.
		$this->assertEquals(
			bin2hex( $expect ),
			bin2hex( UtfNormal::cleanUp( $text ) ) );
	}
}