wiki.techinc.nl/includes/media/DjVuImage.php

<?php
/**
 * DjVu image handler.
 *
 * Copyright © 2006 Brooke Vibber <bvibber@wikimedia.org>
 * https://www.mediawiki.org/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 * @ingroup Media
 */

use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use MediaWiki\Shell\Shell;
use Wikimedia\AtEase\AtEase;

/**
 * Support for detecting/validating DjVu image files and getting
 * some basic file metadata (resolution etc)
 *
 * File format docs are available in source package for DjVuLibre:
 * http://djvulibre.djvuzone.org/
 *
 * @ingroup Media
 */
class DjVuImage {

	/**
	 * Memory limit for the DjVu description software
	 */
	private const DJVUTXT_MEMORY_LIMIT = 300_000_000;

	/** @var string */
	private $mFilename;

	/**
	 * @param string $filename The DjVu file name.
	 */
	public function __construct( $filename ) {
		$this->mFilename = $filename;
	}

	/**
	 * Check if the given file is indeed a valid DjVu image file
	 * @return bool
	 */
	public function isValid() {
		$info = $this->getInfo();

		return $info !== false;
	}

	/**
	 * Return width and height
	 * @return array An array with "width" and "height" keys, or an empty array on failure.
	 */
	public function getImageSize() {
		$data = $this->getInfo();

		if ( $data !== false ) {
			return [
				'width' => $data['width'],
				'height' => $data['height']
			];
		}
		return [];
	}

	// ---------

	/**
	 * For debugging; dump the IFF chunk structure
	 */
	public function dump() {
		$file = fopen( $this->mFilename, 'rb' );
		$header = fread( $file, 12 );
		$arr = unpack( 'a4magic/a4chunk/NchunkLength', $header );
		$chunk = $arr['chunk'];
		$chunkLength = $arr['chunkLength'];
		echo "$chunk $chunkLength\n";
		$this->dumpForm( $file, $chunkLength, 1 );
		fclose( $file );
	}

	private function dumpForm( $file, $length, $indent ) {
		$start = ftell( $file );
		$secondary = fread( $file, 4 );
		echo str_repeat( ' ', $indent * 4 ) . "($secondary)\n";
		while ( ftell( $file ) - $start < $length ) {
			$chunkHeader = fread( $file, 8 );
			if ( $chunkHeader == '' ) {
				break;
			}
			$arr = unpack( 'a4chunk/NchunkLength', $chunkHeader );
			$chunk = $arr['chunk'];
			$chunkLength = $arr['chunkLength'];
			echo str_repeat( ' ', $indent * 4 ) . "$chunk $chunkLength\n";

			if ( $chunk === 'FORM' ) {
				$this->dumpForm( $file, $chunkLength, $indent + 1 );
			} else {
				fseek( $file, $chunkLength, SEEK_CUR );
				if ( $chunkLength & 1 ) {
					// Padding byte between chunks
					fseek( $file, 1, SEEK_CUR );
				}
			}
		}
	}

	private function getInfo() {
		AtEase::suppressWarnings();
		$file = fopen( $this->mFilename, 'rb' );
		AtEase::restoreWarnings();
		if ( $file === false ) {
			wfDebug( __METHOD__ . ": missing or failed file read" );

			return false;
		}

		$header = fread( $file, 16 );
		$info = false;

		if ( strlen( $header ) < 16 ) {
			wfDebug( __METHOD__ . ": too short file header" );
		} else {
			$arr = unpack( 'a4magic/a4form/NformLength/a4subtype', $header );

			$subtype = $arr['subtype'];
			if ( $arr['magic'] !== 'AT&T' ) {
				wfDebug( __METHOD__ . ": not a DjVu file" );
			} elseif ( $subtype === 'DJVU' ) {
				// Single-page document
				$info = $this->getPageInfo( $file );
			} elseif ( $subtype === 'DJVM' ) {
				// Multi-page document
				$info = $this->getMultiPageInfo( $file, $arr['formLength'] );
			} else {
				wfDebug( __METHOD__ . ": unrecognized DJVU file type '{$arr['subtype']}'" );
			}
		}
		fclose( $file );

		return $info;
	}

	private function readChunk( $file ) {
		$header = fread( $file, 8 );
		if ( strlen( $header ) < 8 ) {
			return [ false, 0 ];
		}
		$arr = unpack( 'a4chunk/Nlength', $header );

		return [ $arr['chunk'], $arr['length'] ];
	}

	private function skipChunk( $file, $chunkLength ) {
		fseek( $file, $chunkLength, SEEK_CUR );

		if ( ( $chunkLength & 1 ) && !feof( $file ) ) {
			// padding byte
			fseek( $file, 1, SEEK_CUR );
		}
	}

	private function getMultiPageInfo( $file, $formLength ) {
		// For now, we'll just look for the first page in the file
		// and report its information, hoping others are the same size.
		$start = ftell( $file );
		do {
			[ $chunk, $length ] = $this->readChunk( $file );
			if ( !$chunk ) {
				break;
			}

			if ( $chunk === 'FORM' ) {
				$subtype = fread( $file, 4 );
				if ( $subtype === 'DJVU' ) {
					wfDebug( __METHOD__ . ": found first subpage" );

					return $this->getPageInfo( $file );
				}
				$this->skipChunk( $file, $length - 4 );
			} else {
				wfDebug( __METHOD__ . ": skipping '$chunk' chunk" );
				$this->skipChunk( $file, $length );
			}
		} while ( $length != 0 && !feof( $file ) && ftell( $file ) - $start < $formLength );

		wfDebug( __METHOD__ . ": multi-page DJVU file contained no pages" );

		return false;
	}

	private function getPageInfo( $file ) {
		[ $chunk, $length ] = $this->readChunk( $file );
		if ( $chunk !== 'INFO' ) {
			wfDebug( __METHOD__ . ": expected INFO chunk, got '$chunk'" );

			return false;
		}

		if ( $length < 9 ) {
			wfDebug( __METHOD__ . ": INFO should be 9 or 10 bytes, found $length" );

			return false;
		}
		$data = fread( $file, $length );
		if ( strlen( $data ) < $length ) {
			wfDebug( __METHOD__ . ": INFO chunk cut off" );

			return false;
		}

		$arr = unpack(
			'nwidth/' .
			'nheight/' .
			'Cminor/' .
			'Cmajor/' .
			'vresolution/' .
			'Cgamma', $data );

		# Newer files have rotation info in byte 10, but we don't use it yet.

		return [
			'width' => $arr['width'],
			'height' => $arr['height'],
			'version' => "{$arr['major']}.{$arr['minor']}",
			'resolution' => $arr['resolution'],
			'gamma' => $arr['gamma'] / 10.0 ];
	}

	/**
	 * Return an array describing the DjVu image
	 * @return array|null|false
	 */
	public function retrieveMetaData() {
		$config = MediaWikiServices::getInstance()->getMainConfig();
		$djvuDump = $config->get( MainConfigNames::DjvuDump );
		$djvuTxt = $config->get( MainConfigNames::DjvuTxt );
		$djvuUseBoxedCommand = $config->get( MainConfigNames::DjvuUseBoxedCommand );
		$shell = $config->get( MainConfigNames::ShellboxShell );
		if ( !$this->isValid() ) {
			return false;
		}

		if ( $djvuTxt === null && $djvuDump === null ) {
			return [];
		}

		$txt = null;
		$dump = null;

		if ( $djvuUseBoxedCommand ) {
			$command = MediaWikiServices::getInstance()->getShellCommandFactory()
				->createBoxed( 'djvu' )
				->disableNetwork()
				->firejailDefaultSeccomp()
				->routeName( 'djvu-metadata' )
				->params( $shell, 'scripts/retrieveDjvuMetaData.sh' )
				->inputFileFromFile(
					'scripts/retrieveDjvuMetaData.sh',
					__DIR__ . '/scripts/retrieveDjvuMetaData.sh' )
				->inputFileFromFile( 'file.djvu', $this->mFilename )
				->memoryLimit( self::DJVUTXT_MEMORY_LIMIT );
			$env = [];
			if ( $djvuDump !== null ) {
				$env['DJVU_DUMP'] = $djvuDump;
				$command->outputFileToString( 'dump' );
			}
			if ( $djvuTxt !== null ) {
				$env['DJVU_TXT'] = $djvuTxt;
				$command->outputFileToString( 'txt' );
			}

			$result = $command
				->environment( $env )
				->execute();
			if ( $result->getExitCode() !== 0 ) {
				wfDebug( 'retrieveDjvuMetaData failed with exit code ' . $result->getExitCode() );
				return false;
			}
			if ( $djvuDump !== null ) {
				if ( $result->wasReceived( 'dump' ) ) {
					$dump = $result->getFileContents( 'dump' );
				} else {
					wfDebug( __METHOD__ . ": did not receive dump file" );
				}
			}

			if ( $djvuTxt !== null ) {
				if ( $result->wasReceived( 'txt' ) ) {
					$txt = $result->getFileContents( 'txt' );
				} else {
					wfDebug( __METHOD__ . ": did not receive text file" );
				}
			}
		} else { // No boxedcommand
			if ( $djvuDump !== null ) {
				# djvudump is faster than djvutoxml (now abandoned) as of version 3.5
				# https://sourceforge.net/p/djvu/bugs/71/
				$cmd = Shell::escape( $djvuDump ) . ' ' . Shell::escape( $this->mFilename );
				$dump = wfShellExec( $cmd );
			}
			if ( $djvuTxt !== null ) {
				$cmd = Shell::escape( $djvuTxt ) . ' --detail=page ' . Shell::escape( $this->mFilename );
				wfDebug( __METHOD__ . ": $cmd" );
				$retval = 0;
				$txt = wfShellExec( $cmd, $retval, [], [ 'memory' => self::DJVUTXT_MEMORY_LIMIT ] );
				if ( $retval !== 0 ) {
					$txt = null;
				}
			}
		}

		# Convert dump to array
		$json = [];
		if ( $dump !== null ) {
			$data = $this->convertDumpToJSON( $dump );
			if ( $data !== false ) {
				$json = [ 'data' => $data ];
			}
		}

		# Text layer
		if ( $txt !== null ) {
			# Strip some control characters
			# Ignore carriage returns
			$txt = preg_replace( "/\\\\013/", "", $txt );
			# Replace runs of OCR region separators with a single extra line break
			$txt = preg_replace( "/(?:\\\\(035|037))+/", "\n", $txt );

			$reg = <<<EOR
				/\(page\s[\d-]*\s[\d-]*\s[\d-]*\s[\d-]*\s*"
				((?>    # Text to match is composed of atoms of either:
					\\\\. # - any escaped character
					|     # - any character different from " and \
					[^"\\\\]+
				)*?)
				"\s*\)
				| # Or page can be empty ; in this case, djvutxt dumps ()
				\(\s*()\)/sx
EOR;
			$matches = [];
			preg_match_all( $reg, $txt, $matches );
			$json['text'] = array_map( [ $this, 'pageTextCallback' ], $matches[1] );
		} else {
			$json['text'] = [];
		}

		return $json;
	}

	private function pageTextCallback( string $match ) {
		# Get rid of invalid UTF-8
		$val = UtfNormal\Validator::cleanUp( stripcslashes( $match ) );
		return str_replace( '<27>', '', $val );
	}

	/**
	 * @param string $dump
	 * @return array|false
	 */
	private function convertDumpToJSON( $dump ) {
		if ( strval( $dump ) == '' ) {
			return false;
		}

		$dump = str_replace( "\r", '', $dump );
		$line = strtok( $dump, "\n" );
		$m = false;
		$good = false;
		$result = [];
		if ( preg_match( '/^( *)FORM:DJVU/', $line, $m ) ) {
			# Single-page
			$parsed = $this->parseFormDjvu( $line );
			if ( $parsed ) {
				$good = true;
			} else {
				return false;
			}
			$result['pages'] = [ $parsed ];
		} elseif ( preg_match( '/^( *)FORM:DJVM/', $line, $m ) ) {
			# Multi-page
			$parentLevel = strlen( $m[1] );
			# Find DIRM
			$line = strtok( "\n" );
			$result['pages'] = [];
			while ( $line !== false ) {
				$childLevel = strspn( $line, ' ' );
				if ( $childLevel <= $parentLevel ) {
					# End of chunk
					break;
				}

				if ( preg_match( '/^ *DIRM.*indirect/', $line ) ) {
					wfDebug( "Indirect multi-page DjVu document, bad for server!" );

					return false;
				}

				if ( preg_match( '/^ *FORM:DJVU/', $line ) ) {
					# Found page
					$parsed = $this->parseFormDjvu( $line );
					if ( $parsed ) {
						$good = true;
					} else {
						return false;
					}
					$result['pages'][] = $parsed;
				}
				$line = strtok( "\n" );
			}
		}
		if ( !$good ) {
			return false;
		}

		return $result;
	}

	private function parseFormDjvu( $line ) {
		$parentLevel = strspn( $line, ' ' );
		$line = strtok( "\n" );
		# Find INFO
		while ( $line !== false ) {
			$childLevel = strspn( $line, ' ' );
			if ( $childLevel <= $parentLevel ) {
				# End of chunk
				break;
			}

			if ( preg_match(
				'/^ *INFO *\[\d*] *DjVu *(\d+)x(\d+), *\w*, *(\d+) *dpi, *gamma=([0-9.-]+)/',
				$line,
				$m
			) ) {
				return [
					'height' => (int)$m[2],
					'width' => (int)$m[1],
					'dpi' => (float)$m[3],
					'gamma' => (float)$m[4],
				];
			}
			$line = strtok( "\n" );
		}

		# Not found
		return false;
	}
}