-<?php

-/**

-*

-* This file is part of the phpBB Forum Software package.

-*

-* @copyright (c) phpBB Limited <https://www.phpbb.com>

-* @license GNU General Public License, version 2 (GPL-2.0)

-*

-* For full copyright and license information, please see

-* the docs/CREDITS.txt file.

-*

-*/

-

-require_once dirname(__FILE__) . '/../../phpBB/includes/utf/utf_normalizer.php';

-

-/**

-* @group slow

-*/

-class phpbb_utf_normalizer_test extends phpbb_test_case

-{

- static private $data_dir;

-

- static public function setUpBeforeClass()

- {

- self::$data_dir = dirname(__file__) . '/../tmp';

- self::download('http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt', self::$data_dir);

- self::download('http://www.unicode.org/Public/UNIDATA/UnicodeData.txt', self::$data_dir);

- }

-

- public function test_normalizer()

- {

- $test_suite = array(

- /**

- * NFC

- * c2 == NFC(c1) == NFC(c2) == NFC(c3)

- * c4 == NFC(c4) == NFC(c5)

- */

- 'NFC' => array(

- 'c2' => array('c1', 'c2', 'c3'),

- 'c4' => array('c4', 'c5')

- ),

-

- /**

- * NFD

- * c3 == NFD(c1) == NFD(c2) == NFD(c3)

- * c5 == NFD(c4) == NFD(c5)

- */

- 'NFD' => array(

- 'c3' => array('c1', 'c2', 'c3'),

- 'c5' => array('c4', 'c5')

- ),

-

- /**

- * NFKC

- * c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)

- */

- 'NFKC' => array(

- 'c4' => array('c1', 'c2', 'c3', 'c4', 'c5')

- ),

-

- /**

- * NFKD

- * c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)

- */

- 'NFKD' => array(

- 'c5' => array('c1', 'c2', 'c3', 'c4', 'c5')

- )

- );

-

- $tested_chars = array();

-

- $fp = fopen(self::$data_dir . '/NormalizationTest.txt', 'rb');

- while (!feof($fp))

- {

- $line = fgets($fp);

-

- if ($line[0] == '@')

- {

- continue;

- }

-

- if (!strpos(' 0123456789ABCDEF', $line[0]))

- {

- continue;

- }

-

- list($c1, $c2, $c3, $c4, $c5) = explode(';', $line);

-

- if (!strpos($c1, ' '))

- {

- /**

- * We are currently testing a single character, we add it to the list of

- * characters we have processed so that we can exclude it when testing

- * for invariants

- */

- $tested_chars[$c1] = 1;

- }

-

- foreach ($test_suite as $form => $serie)

- {

- foreach ($serie as $expected => $tests)

- {

- $hex_expected = ${$expected};

- $utf_expected = $this->hexseq_to_utf($hex_expected);

-

- foreach ($tests as $test)

- {

- $utf_result = $utf_expected;

- call_user_func_array(array('utf_normalizer', $form), array(&$utf_result));

-

- $hex_result = $this->utf_to_hexseq($utf_result);

- $this->assertEquals($utf_expected, $utf_result, "$expected == $form($test) ($hex_expected != $hex_result)");

- }

- }

- }

- }

- fclose($fp);

-

- return $tested_chars;

- }

-

- /**

- * @depends test_normalizer

- */

- public function test_invariants(array $tested_chars)

- {

- $fp = fopen(self::$data_dir . '/UnicodeData.txt', 'rb');

-

- while (!feof($fp))

- {

- $line = fgets($fp, 1024);

-

- if (!$pos = strpos($line, ';'))

- {

- continue;

- }

-

- $hex_tested = $hex_expected = substr($line, 0, $pos);

-

- if (isset($tested_chars[$hex_tested]))

- {

- continue;

- }

-

- $utf_expected = $this->hex_to_utf($hex_expected);

-

- if ($utf_expected >= UTF8_SURROGATE_FIRST

- && $utf_expected <= UTF8_SURROGATE_LAST)

- {

- /**

- * Surrogates are illegal on their own, we expect the normalizer

- * to return a replacement char

- */

- $utf_expected = UTF8_REPLACEMENT;

- $hex_expected = $this->utf_to_hexseq($utf_expected);

- }

-

- foreach (array('nfc', 'nfkc', 'nfd', 'nfkd') as $form)

- {

- $utf_result = $utf_expected;

- call_user_func_array(array('utf_normalizer', $form), array(&$utf_result));

- $hex_result = $this->utf_to_hexseq($utf_result);

-

- $this->assertEquals($utf_expected, $utf_result, "$hex_expected == $form($hex_tested) ($hex_expected != $hex_result)");

- }

- }

- fclose($fp);

- }

-

- /**

- * Convert a UTF string to a sequence of codepoints in hexadecimal

- *

- * @param string $utf UTF string

- * @return integer Unicode codepoints in hex

- */

- protected function utf_to_hexseq($str)

- {

- $pos = 0;

- $len = strlen($str);

- $ret = array();

-

- while ($pos < $len)

- {

- $c = $str[$pos];

- switch ($c & "\xF0")

- {

- case "\xC0":

- case "\xD0":

- $utf_char = substr($str, $pos, 2);

- $pos += 2;

- break;

-

- case "\xE0":

- $utf_char = substr($str, $pos, 3);

- $pos += 3;

- break;

-

- case "\xF0":

- $utf_char = substr($str, $pos, 4);

- $pos += 4;

- break;

-

- default:

- $utf_char = $c;

- ++$pos;

- }

-

- $hex = dechex($this->utf_to_cp($utf_char));

-

- if (!isset($hex[3]))

- {

- $hex = substr('000' . $hex, -4);

- }

-

- $ret[] = $hex;

- }

-

- return strtr(implode(' ', $ret), 'abcdef', 'ABCDEF');

- }

-

- /**

- * Convert a UTF-8 char to its codepoint

- *

- * @param string $utf_char UTF-8 char

- * @return integer Unicode codepoint

- */

- protected function utf_to_cp($utf_char)

- {

- switch (strlen($utf_char))

- {

- case 1:

- return ord($utf_char);

-

- case 2:

- return ((ord($utf_char[0]) & 0x1F) << 6) | (ord($utf_char[1]) & 0x3F);

-

- case 3:

- return ((ord($utf_char[0]) & 0x0F) << 12) | ((ord($utf_char[1]) & 0x3F) << 6) | (ord($utf_char[2]) & 0x3F);

-

- case 4:

- return ((ord($utf_char[0]) & 0x07) << 18) | ((ord($utf_char[1]) & 0x3F) << 12) | ((ord($utf_char[2]) & 0x3F) << 6) | (ord($utf_char[3]) & 0x3F);

-

- default:

- throw new RuntimeException('UTF-8 chars can only be 1-4 bytes long');

- }

- }

-

- /**

- * Return a UTF string formed from a sequence of codepoints in hexadecimal

- *

- * @param string $seq Sequence of codepoints, separated with a space

- * @return string UTF-8 string

- */

- protected function hexseq_to_utf($seq)

- {

- return implode('', array_map(array($this, 'hex_to_utf'), explode(' ', $seq)));

- }

-

- /**

- * Convert a codepoint in hexadecimal to a UTF-8 char

- *

- * @param string $hex Codepoint, in hexadecimal

- * @return string UTF-8 char

- */

- protected function hex_to_utf($hex)

- {

- return $this->cp_to_utf(hexdec($hex));

- }

-

- /**

- * Convert a codepoint to a UTF-8 char

- *

- * @param integer $cp Unicode codepoint

- * @return string UTF-8 string

- */

- protected function cp_to_utf($cp)

- {

- if ($cp > 0xFFFF)

- {

- return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));

- }

- else if ($cp > 0x7FF)

- {

- return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));

- }

- else if ($cp > 0x7F)

- {

- return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));

- }

- else

- {

- return chr($cp);

- }

- }

-

- // chunked download helper

- static protected function download($url, $to)

- {

- $target = $to . '/' . basename($url);

-

- if (file_exists($target))

- {

- return;

- }

-

- if (!$fpr = fopen($url, 'rb'))

- {

- echo "Failed to download $url\n";

- return;

- }

-

- if (!$fpw = fopen($target, 'wb'))

- {

- echo "Failed to open $target for writing\n";

- return;

- }

-

- $chunk = 32768;

-

- while (!feof($fpr))

- {

- fwrite($fpw, fread($fpr, $chunk));

- }

- fclose($fpr);

- fclose($fpw);

- }

-}