diff options
Diffstat (limited to 'phpBB/develop/generate_utf_tables.php')
| -rw-r--r-- | phpBB/develop/generate_utf_tables.php | 290 |
1 files changed, 1 insertions, 289 deletions
diff --git a/phpBB/develop/generate_utf_tables.php b/phpBB/develop/generate_utf_tables.php index 16a449679b..888c07676d 100644 --- a/phpBB/develop/generate_utf_tables.php +++ b/phpBB/develop/generate_utf_tables.php @@ -32,262 +32,11 @@ $phpbb_root_path = '../'; $phpEx = substr(strrchr(__FILE__, '.'), 1); echo "Checking for required files\n"; -download('http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt'); -download('http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt'); download('http://www.unicode.org/Public/UNIDATA/UnicodeData.txt'); echo "\n"; -require_once($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx); -$file_contents = array(); - -/** -* Generate some Hangul/Jamo stuff -*/ -echo "\nGenerating Hangul and Jamo tables\n"; -for ($i = 0; $i < UNICODE_HANGUL_LCOUNT; ++$i) -{ - $utf_char = cp_to_utf(UNICODE_HANGUL_LBASE + $i); - $file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_VCOUNT * UNICODE_HANGUL_TCOUNT + UNICODE_HANGUL_SBASE; - $file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_L; -} - -for ($i = 0; $i < UNICODE_HANGUL_VCOUNT; ++$i) -{ - $utf_char = cp_to_utf(UNICODE_HANGUL_VBASE + $i); - $file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_TCOUNT; - $file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_V; -} - -for ($i = 0; $i < UNICODE_HANGUL_TCOUNT; ++$i) -{ - $utf_char = cp_to_utf(UNICODE_HANGUL_TBASE + $i); - $file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i; - $file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_T; -} - -/** -* Load the CompositionExclusions table -*/ -echo "Loading CompositionExclusion\n"; -$fp = fopen('CompositionExclusions.txt', 'rt'); - -$exclude = array(); -while (!feof($fp)) -{ - $line = fgets($fp, 1024); - - if (!strpos(' 0123456789ABCDEFabcdef', $line[0])) - { - continue; - } - - $cp = strtok($line, ' '); - - if ($pos = strpos($cp, '..')) - { - $start = hexdec(substr($cp, 0, $pos)); - $end = hexdec(substr($cp, $pos + 2)); - - for ($i = $start; $i < $end; ++$i) - { - $exclude[$i] = 1; - } - } - else - { - $exclude[hexdec($cp)] = 1; - } -} -fclose($fp); - -/** -* Load QuickCheck tables -*/ -echo "Generating QuickCheck tables\n"; -$fp = fopen('DerivedNormalizationProps.txt', 'rt'); - -while (!feof($fp)) -{ - $line = fgets($fp, 1024); - - if (!strpos(' 0123456789ABCDEFabcdef', $line[0])) - { - continue; - } - - $p = array_map('trim', explode(';', strtok($line, '#'))); - - /** - * Capture only NFC_QC, NFKC_QC - */ - if (!preg_match('#^NFK?C_QC$#', $p[1])) - { - continue; - } - - if ($pos = strpos($p[0], '..')) - { - $start = hexdec(substr($p[0], 0, $pos)); - $end = hexdec(substr($p[0], $pos + 2)); - } - else - { - $start = $end = hexdec($p[0]); - } - - if ($start >= UTF8_HANGUL_FIRST && $end <= UTF8_HANGUL_LAST) - { - /** - * We do not store Hangul syllables in the array - */ - continue; - } - - if ($p[2] == 'M') - { - $val = UNICODE_QC_MAYBE; - } - else - { - $val = UNICODE_QC_NO; - } - - if ($p[1] == 'NFKC_QC') - { - $file = 'utf_nfkc_qc'; - } - else - { - $file = 'utf_nfc_qc'; - } - - for ($i = $start; $i <= $end; ++$i) - { - /** - * The vars have the same name as the file: $utf_nfc_qc is in utf_nfc_qc.php - */ - $file_contents[$file][$file][cp_to_utf($i)] = $val; - } -} -fclose($fp); - /** -* Do mappings -*/ -echo "Loading Unicode decomposition mappings\n"; -$fp = fopen($phpbb_root_path . 'develop/UnicodeData.txt', 'rt'); - -$map = array(); -while (!feof($fp)) -{ - $p = explode(';', fgets($fp, 1024)); - $cp = hexdec($p[0]); - - if (!empty($p[3])) - { - /** - * Store combining class > 0 - */ - $file_contents['utf_normalizer_common']['utf_combining_class'][cp_to_utf($cp)] = (int) $p[3]; - } - - if (!isset($p[5]) || !preg_match_all('#[0-9A-F]+#', strip_tags($p[5]), $m)) - { - continue; - } - - if (strpos($p[5], '>')) - { - $map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0])); - } - else - { - $map['NFD'][$cp] = $map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0])); - } -} -fclose($fp); - -/** -* Build the canonical composition table -*/ -echo "Generating the Canonical Composition table\n"; -foreach ($map['NFD'] as $cp => $decomp_seq) -{ - if (!strpos($decomp_seq, ' ') || isset($exclude[$cp])) - { - /** - * Singletons are excluded from canonical composition - */ - continue; - } - - $utf_seq = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq))); - - if (!isset($file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq])) - { - $file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq] = cp_to_utf($cp); - } -} - -/** -* Decompose the NF[K]D mappings recursively and prepare the file contents -*/ -echo "Generating the Canonical and Compatibility Decomposition tables\n\n"; -foreach ($map as $type => $decomp_map) -{ - foreach ($decomp_map as $cp => $decomp_seq) - { - $decomp_map[$cp] = decompose($decomp_map, $decomp_seq); - } - unset($decomp_seq); - - if ($type == 'NFKD') - { - $file = 'utf_compatibility_decomp'; - $var = 'utf_compatibility_decomp'; - } - else - { - $file = 'utf_canonical_decomp'; - $var = 'utf_canonical_decomp'; - } - - /** - * Generate the corresponding file - */ - foreach ($decomp_map as $cp => $decomp_seq) - { - $file_contents[$file][$var][cp_to_utf($cp)] = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq))); - } -} - -/** -* Generate and/or alter the files -*/ -foreach ($file_contents as $file => $contents) -{ - /** - * Generate a new file - */ - echo "Writing to $file.$phpEx\n"; - - if (!$fp = fopen($phpbb_root_path . 'includes/utf/data/' . $file . '.' . $phpEx, 'wb')) - { - trigger_error('Cannot open ' . $file . ' for write'); - } - - fwrite($fp, '<?php'); - foreach ($contents as $var => $val) - { - fwrite($fp, "\n\$GLOBALS[" . my_var_export($var) . ']=' . my_var_export($val) . ";"); - } - fclose($fp); -} - -echo "\n*** UTF-8 normalization tables done\n\n"; - -/** -* Now we'll generate the files needed by the search indexer +* Generate the files needed by the search indexer */ echo "Generating search indexer tables\n"; @@ -425,32 +174,6 @@ die("\nAll done!\n"); //////////////////////////////////////////////////////////////////////////////// /** -* Decompose a sequence recusively -* -* @param array $decomp_map Decomposition mapping, passed by reference -* @param string $decomp_seq Decomposition sequence as decimal codepoints separated with a space -* @return string Decomposition sequence, fully decomposed -*/ -function decompose(&$decomp_map, $decomp_seq) -{ - $ret = array(); - foreach (explode(' ', $decomp_seq) as $cp) - { - if (isset($decomp_map[$cp])) - { - $ret[] = decompose($decomp_map, $decomp_map[$cp]); - } - else - { - $ret[] = $cp; - } - } - - return implode(' ', $ret); -} - - -/** * Return a parsable string representation of a variable * * This is function is limited to array/strings/integers @@ -538,17 +261,6 @@ function hex_to_utf($hex) } /** -* Return a UTF string formed from a sequence of codepoints in hexadecimal -* -* @param string $seq Sequence of codepoints, separated with a space -* @return string UTF-8 string -*/ -function hexseq_to_utf($seq) -{ - return implode('', array_map('hex_to_utf', explode(' ', $seq))); -} - -/** * Convert a codepoint to a UTF-8 char * * @param integer $cp Unicode codepoint |