diff options
| author | Meik Sievertsen <acydburn@phpbb.com> | 2009-10-04 18:13:59 +0000 |
|---|---|---|
| committer | Meik Sievertsen <acydburn@phpbb.com> | 2009-10-04 18:13:59 +0000 |
| commit | bf8ac19eaa8d74f9dfd6d597190f5664e7339382 (patch) | |
| tree | c3ad876736748e36cb9176a0248cc43badfc1d9a /phpBB/develop/generate_utf_tables.php | |
| parent | 3215bbf88864139dc8c7e9ac5773b1ea8a7e96c1 (diff) | |
| download | forums-bf8ac19eaa8d74f9dfd6d597190f5664e7339382.tar forums-bf8ac19eaa8d74f9dfd6d597190f5664e7339382.tar.gz forums-bf8ac19eaa8d74f9dfd6d597190f5664e7339382.tar.bz2 forums-bf8ac19eaa8d74f9dfd6d597190f5664e7339382.tar.xz forums-bf8ac19eaa8d74f9dfd6d597190f5664e7339382.zip | |
Move trunk/phpBB to old_trunk/phpBB
git-svn-id: file:///svn/phpbb/trunk@10210 89ea8834-ac86-4346-8a33-228a782c2dd0
Diffstat (limited to 'phpBB/develop/generate_utf_tables.php')
| -rw-r--r-- | phpBB/develop/generate_utf_tables.php | 570 |
1 files changed, 0 insertions, 570 deletions
diff --git a/phpBB/develop/generate_utf_tables.php b/phpBB/develop/generate_utf_tables.php deleted file mode 100644 index fcf6395b24..0000000000 --- a/phpBB/develop/generate_utf_tables.php +++ /dev/null @@ -1,570 +0,0 @@ -<?php -/** -* -* @package phpBB3 -* @version $Id$ -* @copyright (c) 2005 phpBB Group -* @license http://opensource.org/licenses/gpl-license.php GNU Public License -* -*/ - -if (php_sapi_name() != 'cli') -{ - die("This program must be run from the command line.\n"); -} - -// -// Security message: -// -// This script is potentially dangerous. -// Remove or comment the next line (die(".... ) to enable this script. -// Do NOT FORGET to either remove this script or disable it after you have used it. -// -die("Please read the first lines of this script for instructions on how to enable it"); - -set_time_limit(0); - -define('IN_PHPBB', true); -define('PHPBB_ROOT_PATH', './../'); -define('PHP_EXT', substr(strrchr(__FILE__, '.'), 1)); - -echo "Checking for required files\n"; -download('http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt'); -download('http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt'); -download('http://www.unicode.org/Public/UNIDATA/UnicodeData.txt'); -echo "\n"; - -require_once(PHPBB_ROOT_PATH . 'includes/utf/utf_normalizer.' . PHP_EXT); -$file_contents = array(); - -/** -* Generate some Hangul/Jamo stuff -*/ -echo "\nGenerating Hangul and Jamo tables\n"; -for ($i = 0; $i < UNICODE_HANGUL_LCOUNT; ++$i) -{ - $utf_char = cp_to_utf(UNICODE_HANGUL_LBASE + $i); - $file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_VCOUNT * UNICODE_HANGUL_TCOUNT + UNICODE_HANGUL_SBASE; - $file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_L; -} - -for ($i = 0; $i < UNICODE_HANGUL_VCOUNT; ++$i) -{ - $utf_char = cp_to_utf(UNICODE_HANGUL_VBASE + $i); - $file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_TCOUNT; - $file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_V; -} - -for ($i = 0; $i < UNICODE_HANGUL_TCOUNT; ++$i) -{ - $utf_char = cp_to_utf(UNICODE_HANGUL_TBASE + $i); - $file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i; - $file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_T; -} - -/** -* Load the CompositionExclusions table -*/ -echo "Loading CompositionExclusion\n"; -$fp = fopen('CompositionExclusions.txt', 'rt'); - -$exclude = array(); -while (!feof($fp)) -{ - $line = fgets($fp, 1024); - - if (!strpos(' 0123456789ABCDEFabcdef', $line[0])) - { - continue; - } - - $cp = strtok($line, ' '); - - if ($pos = strpos($cp, '..')) - { - $start = hexdec(substr($cp, 0, $pos)); - $end = hexdec(substr($cp, $pos + 2)); - - for ($i = $start; $i < $end; ++$i) - { - $exclude[$i] = 1; - } - } - else - { - $exclude[hexdec($cp)] = 1; - } -} -fclose($fp); - -/** -* Load QuickCheck tables -*/ -echo "Generating QuickCheck tables\n"; -$fp = fopen('DerivedNormalizationProps.txt', 'rt'); - -while (!feof($fp)) -{ - $line = fgets($fp, 1024); - - if (!strpos(' 0123456789ABCDEFabcdef', $line[0])) - { - continue; - } - - $p = array_map('trim', explode(';', strtok($line, '#'))); - - /** - * Capture only NFC_QC, NFKC_QC - */ - if (!preg_match('#^NFK?C_QC$#', $p[1])) - { - continue; - } - - if ($pos = strpos($p[0], '..')) - { - $start = hexdec(substr($p[0], 0, $pos)); - $end = hexdec(substr($p[0], $pos + 2)); - } - else - { - $start = $end = hexdec($p[0]); - } - - if ($start >= UTF8_HANGUL_FIRST && $end <= UTF8_HANGUL_LAST) - { - /** - * We do not store Hangul syllables in the array - */ - continue; - } - - if ($p[2] == 'M') - { - $val = UNICODE_QC_MAYBE; - } - else - { - $val = UNICODE_QC_NO; - } - - if ($p[1] == 'NFKC_QC') - { - $file = 'utf_nfkc_qc'; - } - else - { - $file = 'utf_nfc_qc'; - } - - for ($i = $start; $i <= $end; ++$i) - { - /** - * The vars have the same name as the file: $utf_nfc_qc is in utf_nfc_qc.php - */ - $file_contents[$file][$file][cp_to_utf($i)] = $val; - } -} -fclose($fp); - -/** -* Do mappings -*/ -echo "Loading Unicode decomposition mappings\n"; -$fp = fopen(PHPBB_ROOT_PATH . 'develop/UnicodeData.txt', 'rt'); - -$map = array(); -while (!feof($fp)) -{ - $p = explode(';', fgets($fp, 1024)); - $cp = hexdec($p[0]); - - if (!empty($p[3])) - { - /** - * Store combining class > 0 - */ - $file_contents['utf_normalizer_common']['utf_combining_class'][cp_to_utf($cp)] = (int) $p[3]; - } - - if (!isset($p[5]) || !preg_match_all('#[0-9A-F]+#', strip_tags($p[5]), $m)) - { - continue; - } - - if (strpos($p[5], '>')) - { - $map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0])); - } - else - { - $map['NFD'][$cp] = $map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0])); - } -} -fclose($fp); - -/** -* Build the canonical composition table -*/ -echo "Generating the Canonical Composition table\n"; -foreach ($map['NFD'] as $cp => $decomp_seq) -{ - if (!strpos($decomp_seq, ' ') || isset($exclude[$cp])) - { - /** - * Singletons are excluded from canonical composition - */ - continue; - } - - $utf_seq = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq))); - - if (!isset($file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq])) - { - $file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq] = cp_to_utf($cp); - } -} - -/** -* Decompose the NF[K]D mappings recursively and prepare the file contents -*/ -echo "Generating the Canonical and Compatibility Decomposition tables\n\n"; -foreach ($map as $type => $decomp_map) -{ - foreach ($decomp_map as $cp => $decomp_seq) - { - $decomp_map[$cp] = decompose($decomp_map, $decomp_seq); - } - unset($decomp_seq); - - if ($type == 'NFKD') - { - $file = 'utf_compatibility_decomp'; - $var = 'utf_compatibility_decomp'; - } - else - { - $file = 'utf_canonical_decomp'; - $var = 'utf_canonical_decomp'; - } - - /** - * Generate the corresponding file - */ - foreach ($decomp_map as $cp => $decomp_seq) - { - $file_contents[$file][$var][cp_to_utf($cp)] = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq))); - } -} - -/** -* Generate and/or alter the files -*/ -foreach ($file_contents as $file => $contents) -{ - /** - * Generate a new file - */ - echo "Writing to $file." . PHP_EXT . "\n"; - - if (!$fp = fopen(PHPBB_ROOT_PATH . 'includes/utf/data/' . $file . '.' . PHP_EXT, 'wb')) - { - trigger_error('Cannot open ' . $file . ' for write'); - } - - fwrite($fp, '<?php'); - foreach ($contents as $var => $val) - { - fwrite($fp, "\n\$GLOBALS[" . my_var_export($var) . ']=' . my_var_export($val) . ";"); - } - fclose($fp); -} - -echo "\n*** UTF-8 normalization tables done\n\n"; - -/** -* Now we'll generate the files needed by the search indexer -*/ -echo "Generating search indexer tables\n"; - -$fp = fopen(PHPBB_ROOT_PATH . 'develop/UnicodeData.txt', 'rt'); - -$map = array(); -while ($line = fgets($fp, 1024)) -{ - /** - * The current line is split, $m[0] hold the codepoint in hexadecimal and - * all other fields numbered as in http://www.unicode.org/Public/UNIDATA/UCD.html#UnicodeData.txt - */ - $m = explode(';', $line); - - /** - * @var integer $cp Current char codepoint - * @var string $utf_char UTF-8 representation of current char - */ - $cp = hexdec($m[0]); - $utf_char = cp_to_utf($cp); - - /** - * $m[2] holds the "General Category" of the character - * @link http://www.unicode.org/Public/UNIDATA/UCD.html#General_Category_Values - */ - switch ($m[2][0]) - { - case 'L': - /** - * We allow all letters and map them to their lowercased counterpart on the fly - */ - $map_to_hex = (isset($m[13][0])) ? $m[13] : $m[0]; - - if (preg_match('#^LATIN.*(?:LETTER|LIGATURE) ([A-Z]{2}(?![A-Z]))$#', $m[1], $capture)) - { - /** - * Special hack for some latin ligatures. Using the name of a character - * is bad practice, but for now it works well enough. - * - * @todo Note that ligatures with combining marks such as U+01E2 are - * not supported at this time - */ - $map[$cp] = strtolower($capture[1]); - } - else if (isset($m[13][0])) - { - /** - * If the letter has a lowercased form, use it - */ - $map[$cp] = hex_to_utf($m[13]); - } - else - { - /** - * In all other cases, map the letter to itself - */ - $map[$cp] = $utf_char; - } - break; - - case 'M': - /** - * We allow all marks, they are mapped to themselves - */ - $map[$cp] = $utf_char; - break; - - case 'N': - /** - * We allow all numbers, but we map them to their numeric value whenever - * possible. The numeric value (field #8) is in ASCII already - * - * @todo Note that fractions such as U+00BD will be converted to something - * like "1/2", with a slash. However, "1/2" entered in ASCII is converted - * to "1 2". This will have to be fixed. - */ - $map[$cp] = (isset($m[8][0])) ? $m[8] : $utf_char; - break; - - default: - /** - * Everything else is ignored, skip to the next line - */ - continue 2; - } -} -fclose($fp); - -/** -* Add some cheating -*/ -$cheats = array( - '00DF' => 'ss', # German sharp S - '00C5' => 'ae', # Capital A with diaeresis - '00E4' => 'ae', # Small A with diaeresis - '00D6' => 'oe', # Capital O with diaeresis - '00F6' => 'oe', # Small O with diaeresis - '00DC' => 'ue', # Capital U with diaeresis - '00FC' => 'ue', # Small U with diaeresis -); - -/** -* Add our "cheat replacements" to the map -*/ -foreach ($cheats as $hex => $map_to) -{ - $map[hexdec($hex)] = $map_to; -} - -/** -* Split the map into smaller blocks -*/ -$file_contents = array(); -foreach ($map as $cp => $map_to) -{ - $file_contents[$cp >> 11][cp_to_utf($cp)] = $map_to; -} -unset($map); - -foreach ($file_contents as $idx => $contents) -{ - echo "Writing to search_indexer_$idx." . PHP_EXT . "\n"; - $fp = fopen(PHPBB_ROOT_PATH . 'includes/utf/data/search_indexer_' . $idx . '.' . PHP_EXT, 'wb'); - fwrite($fp, '<?php return ' . my_var_export($contents) . ';'); - fclose($fp); -} -echo "\n*** Search indexer tables done\n\n"; - - -die("\nAll done!\n"); - - -//////////////////////////////////////////////////////////////////////////////// -// Internal functions // -//////////////////////////////////////////////////////////////////////////////// - -/** -* Decompose a sequence recusively -* -* @param array $decomp_map Decomposition mapping, passed by reference -* @param string $decomp_seq Decomposition sequence as decimal codepoints separated with a space -* @return string Decomposition sequence, fully decomposed -*/ -function decompose(&$decomp_map, $decomp_seq) -{ - $ret = array(); - foreach (explode(' ', $decomp_seq) as $cp) - { - if (isset($decomp_map[$cp])) - { - $ret[] = decompose($decomp_map, $decomp_map[$cp]); - } - else - { - $ret[] = $cp; - } - } - - return implode(' ', $ret); -} - - -/** -* Return a parsable string representation of a variable -* -* This is function is limited to array/strings/integers -* -* @param mixed $var Variable -* @return string PHP code representing the variable -*/ -function my_var_export($var) -{ - if (is_array($var)) - { - $lines = array(); - - foreach ($var as $k => $v) - { - $lines[] = my_var_export($k) . '=>' . my_var_export($v); - } - - return 'array(' . implode(',', $lines) . ')'; - } - else if (is_string($var)) - { - return "'" . str_replace(array('\\', "'"), array('\\\\', "\\'"), $var) . "'"; - } - else - { - return $var; - } -} - -/** -* Download a file to the develop/ dir -* -* @param string $url URL of the file to download -* @return void -*/ -function download($url) -{ - if (file_exists(PHPBB_ROOT_PATH . 'develop/' . basename($url))) - { - return; - } - - echo 'Downloading from ', $url, ' '; - - if (!$fpr = fopen($url, 'rb')) - { - die("Can't download from $url\nPlease download it yourself and put it in the develop/ dir, kthxbai"); - } - - if (!$fpw = fopen(PHPBB_ROOT_PATH . 'develop/' . basename($url), 'wb')) - { - die("Can't open develop/" . basename($url) . " for output... please check your permissions or something"); - } - - $i = 0; - $chunk = 32768; - $done = ''; - - while (!feof($fpr)) - { - $i += fwrite($fpw, fread($fpr, $chunk)); - echo str_repeat("\x08", strlen($done)); - - $done = ($i >> 10) . ' KiB'; - echo $done; - } - fclose($fpr); - fclose($fpw); - - echo "\n"; -} - -/** -* Convert a codepoint in hexadecimal to a UTF-8 char -* -* @param string $hex Codepoint, in hexadecimal -* @return string UTF-8 char -*/ -function hex_to_utf($hex) -{ - return cp_to_utf(hexdec($hex)); -} - -/** -* Return a UTF string formed from a sequence of codepoints in hexadecimal -* -* @param string $seq Sequence of codepoints, separated with a space -* @return string UTF-8 string -*/ -function hexseq_to_utf($seq) -{ - return implode('', array_map('hex_to_utf', explode(' ', $seq))); -} - -/** -* Convert a codepoint to a UTF-8 char -* -* @param integer $cp Unicode codepoint -* @return string UTF-8 string -*/ -function cp_to_utf($cp) -{ - if ($cp > 0xFFFF) - { - return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F)); - } - else if ($cp > 0x7FF) - { - return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F)); - } - else if ($cp > 0x7F) - { - return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F)); - } - else - { - return chr($cp); - } -}
\ No newline at end of file |