Merge "api: Add new 'raw' parameter type which avoids Unicode NFC normalization"

RELEASE-NOTES-1.36

@@ -71,6 +71,10 @@ For notes on 1.35.x and older releases, see HISTORY.
   unknown tags is numeric formatting; non-EXIF tags which are non-numeric
   should always use this method to specify the desired formatting.
 * The new 'title' type can be used to validate action API and REST API inputs.
+* The new 'raw' type can be used to validate action API inputs. It bypasses
+  the Unicode NFC normalization done on inputs of type 'string', so it more
+  suitable when the input is binary or may contain deprecated Unicode
+  sequences or characters (such as U+2001) that should be passed unmodified.
 * …
 
 === External library changes in 1.36 ===

includes/api/ApiBase.php

@@ -1042,6 +1042,7 @@ abstract class ApiBase extends ContextSource {
 		$validator = $this->getMain()->getParamValidator();
 		$value = $validator->getValue( $this, $name, $settings, [
 			'parse-limit' => $parseLimit,
+			'raw' => ( $settings[ParamValidator::PARAM_TYPE] ?? '' ) === 'raw',
 		] );
 
 		// @todo Deprecate and remove this, if possible.

includes/api/Validator/ApiParamValidator.php

@@ -61,6 +61,9 @@ class ApiParamValidator {
 			] ],
 		],
 		'password' => [ 'class' => PasswordDef::class ],
+		// Unlike 'string', the 'raw' type will not be subject to Unicode
+		// NFC normalization.
+		'raw' => [ 'class' => StringDef::class ],
 		'string' => [ 'class' => StringDef::class ],
 		'submodule' => [ 'class' => SubmoduleDef::class ],
 		'tags' => [ 'class' => TagsDef::class ],

includes/api/Validator/ApiParamValidatorCallbacks.php

@@ -39,6 +39,10 @@ class ApiParamValidatorCallbacks implements Callbacks {
 		$request = $this->apiMain->getRequest();
 		$rawValue = $request->getRawVal( $name );
 
+		if ( $options['raw'] ?? false ) {
+			// Bypass NFC normalization
+			return $rawValue;
+		}
 		if ( is_string( $rawValue ) ) {
 			// Preserve U+001F for multi-values
 			if ( substr( $rawValue, 0, 1 ) === "\x1f" ) {

resources/src/mediawiki.special.apisandbox/apisandbox.js

@@ -346,6 +346,7 @@
 					}
 					break;
 
+				case 'raw':
 				case 'text':
 					widget = new OO.ui.MultilineTextInputWidget( {
 						required: Util.apiBool( pi.required )

tests/phpunit/includes/api/Validator/ApiParamValidatorTest.php

@@ -38,7 +38,7 @@ class ApiParamValidatorTest extends ApiTestCase {
 		$this->assertSame(
 			[
 				'boolean', 'enum', 'expiry', 'integer', 'limit', 'namespace', 'NULL', 'password',
-				'string', 'submodule', 'tags', 'text', 'timestamp', 'title', 'user', 'upload',
+				'raw', 'string', 'submodule', 'tags', 'text', 'timestamp', 'title', 'user', 'upload',
 			],
 			$validator->knownTypes()
 		);
@@ -536,17 +536,21 @@ class ApiParamValidatorTest extends ApiTestCase {
 	public function testGetValue( ?string $data, $settings, $expect ) : void {
 		[ $validator, $main ] = $this->getValidator( new FauxRequest( [ 'aptest' => $data ] ) );
 		$module = $main->getModuleFromPath( 'query+allpages' );
+		$options = [
+			'parse-limit' => false,
+			'raw' => ( $settings[ParamValidator::PARAM_TYPE] ?? '' ) === 'raw',
+		];
 
 		if ( $expect instanceof ApiUsageException ) {
 			try {
-				$validator->getValue( $module, 'test', $settings, [] );
+				$validator->getValue( $module, 'test', $settings, $options );
 				$this->fail( 'Expected exception not thrown' );
 			} catch ( ApiUsageException $e ) {
 				$this->assertSame( $module->getModulePath(), $e->getModulePath() );
 				$this->assertEquals( $expect->getStatusValue(), $e->getStatusValue() );
 			}
 		} else {
-			$this->assertEquals( $expect, $validator->getValue( $module, 'test', $settings, [] ) );
+			$this->assertEquals( $expect, $validator->getValue( $module, 'test', $settings, $options ) );
 		}
 	}
 
@@ -581,6 +585,24 @@ class ApiParamValidatorTest extends ApiTestCase {
 				false,
 				true,
 			],
+			// The 'string' type will be NFC normalized (in this case,
+			// U+2001 will be converted to U+2003; see Figure 5 of
+			// of https://unicode.org/reports/tr15 for more examples).
+			'Test string (Unicode NFC)' => [
+				"\u{2001}",
+				[
+					ParamValidator::PARAM_TYPE => 'string',
+				],
+				"\u{2003}",
+			],
+			// The 'raw' type bypasses Unicode NFC normalization.
+			'Test string (raw)' => [
+				"\u{2001}",
+				[
+					ParamValidator::PARAM_TYPE => 'raw',
+				],
+				"\u{2001}",
+			],
 			'Validation failure' => [
 				'xyz',
 				[