diff options
Diffstat (limited to 'phpBB/includes/utf/utf_tools.php')
| -rw-r--r-- | phpBB/includes/utf/utf_tools.php | 863 |
1 files changed, 715 insertions, 148 deletions
diff --git a/phpBB/includes/utf/utf_tools.php b/phpBB/includes/utf/utf_tools.php index 0a1409041b..0c855be7c3 100644 --- a/phpBB/includes/utf/utf_tools.php +++ b/phpBB/includes/utf/utf_tools.php @@ -338,7 +338,7 @@ else */ function utf8_strtolower($string) { - static $UTF8_UPPER_TO_LOWER = array( + static $utf8_upper_to_lower = array( "\xC3\x80" => "\xC3\xA0", "\xC3\x81" => "\xC3\xA1", "\xC3\x82" => "\xC3\xA2", "\xC3\x83" => "\xC3\xA3", "\xC3\x84" => "\xC3\xA4", "\xC3\x85" => "\xC3\xA5", "\xC3\x86" => "\xC3\xA6", "\xC3\x87" => "\xC3\xA7", "\xC3\x88" => "\xC3\xA8", "\xC3\x89" => "\xC3\xA9", @@ -387,7 +387,7 @@ else "\xE1\xBA\x82" => "\xE1\xBA\x83", "\xE1\xBA\x84" => "\xE1\xBA\x85", "\xE1\xBB\xB2" => "\xE1\xBB\xB3" ); - return strtr(strtolower($string), $UTF8_UPPER_TO_LOWER); + return strtr(strtolower($string), $utf8_upper_to_lower); } /** @@ -403,7 +403,7 @@ else */ function utf8_strtoupper($string) { - static $UTF8_LOWER_TO_UPPER = array( + static $utf8_lower_to_upper = array( "\xC3\xA0" => "\xC3\x80", "\xC3\xA1" => "\xC3\x81", "\xC3\xA2" => "\xC3\x82", "\xC3\xA3" => "\xC3\x83", "\xC3\xA4" => "\xC3\x84", "\xC3\xA5" => "\xC3\x85", "\xC3\xA6" => "\xC3\x86", "\xC3\xA7" => "\xC3\x87", "\xC3\xA8" => "\xC3\x88", "\xC3\xA9" => "\xC3\x89", @@ -452,7 +452,7 @@ else "\xE1\xBA\x83" => "\xE1\xBA\x82", "\xE1\xBA\x85" => "\xE1\xBA\x84", "\xE1\xBB\xB3" => "\xE1\xBB\xB2" ); - return strtr(strtoupper($string), $UTF8_LOWER_TO_UPPER); + return strtr(strtoupper($string), $utf8_lower_to_upper); } /** @@ -1028,8 +1028,7 @@ function utf8_decode_ncr_callback($m) } /** -* Takes an array of ints representing the Unicode characters and returns -* a UTF-8 string. +* Case folds a unicode string as per Unicode 5.0, section 3.13 * * @param string $text text to be case folded * @param string $option determines how we will fold the cases @@ -1058,15 +1057,705 @@ function utf8_case_fold($text, $option = 'full') $uniarray['s'] = include($phpbb_root_path . 'includes/utf/data/case_fold_s.' . $phpEx); } + // common is always replaced $text = strtr($text, $uniarray['c']); + if ($option === 'full') { + // full replaces a character with multiple characters $text = strtr($text, $uniarray['f']); } else { + // simple replaces a character with another character $text = strtr($text, $uniarray['s']); } + + return $text; +} + +/** +* Takes the input and does a "special" case fold. It does minor normalization +* and returns NFKC compatable text +* +* @param string $text text to be case folded +* @param string $option determines how we will fold the cases +* @return string case folded text +*/ +function utf8_case_fold_nfkc($text, $option = 'full') +{ + static $fc_nfkc_closure = array( + "\xCD\xBA" => "\x20\xCE\xB9", + "\xCF\x92" => "\xCF\x85", + "\xCF\x93" => "\xCF\x8D", + "\xCF\x94" => "\xCF\x8B", + "\xCF\xB2" => "\xCF\x83", + "\xCF\xB9" => "\xCF\x83", + "\xE1\xB4\xAC" => "\x61", + "\xE1\xB4\xAD" => "\xC3\xA6", + "\xE1\xB4\xAE" => "\x62", + "\xE1\xB4\xB0" => "\x64", + "\xE1\xB4\xB1" => "\x65", + "\xE1\xB4\xB2" => "\xC7\x9D", + "\xE1\xB4\xB3" => "\x67", + "\xE1\xB4\xB4" => "\x68", + "\xE1\xB4\xB5" => "\x69", + "\xE1\xB4\xB6" => "\x6A", + "\xE1\xB4\xB7" => "\x6B", + "\xE1\xB4\xB8" => "\x6C", + "\xE1\xB4\xB9" => "\x6D", + "\xE1\xB4\xBA" => "\x6E", + "\xE1\xB4\xBC" => "\x6F", + "\xE1\xB4\xBD" => "\xC8\xA3", + "\xE1\xB4\xBE" => "\x70", + "\xE1\xB4\xBF" => "\x72", + "\xE1\xB5\x80" => "\x74", + "\xE1\xB5\x81" => "\x75", + "\xE1\xB5\x82" => "\x77", + "\xE2\x82\xA8" => "\x72\x73", + "\xE2\x84\x82" => "\x63", + "\xE2\x84\x83" => "\xC2\xB0\x63", + "\xE2\x84\x87" => "\xC9\x9B", + "\xE2\x84\x89" => "\xC2\xB0\x66", + "\xE2\x84\x8B" => "\x68", + "\xE2\x84\x8C" => "\x68", + "\xE2\x84\x8D" => "\x68", + "\xE2\x84\x90" => "\x69", + "\xE2\x84\x91" => "\x69", + "\xE2\x84\x92" => "\x6C", + "\xE2\x84\x95" => "\x6E", + "\xE2\x84\x96" => "\x6E\x6F", + "\xE2\x84\x99" => "\x70", + "\xE2\x84\x9A" => "\x71", + "\xE2\x84\x9B" => "\x72", + "\xE2\x84\x9C" => "\x72", + "\xE2\x84\x9D" => "\x72", + "\xE2\x84\xA0" => "\x73\x6D", + "\xE2\x84\xA1" => "\x74\x65\x6C", + "\xE2\x84\xA2" => "\x74\x6D", + "\xE2\x84\xA4" => "\x7A", + "\xE2\x84\xA8" => "\x7A", + "\xE2\x84\xAC" => "\x62", + "\xE2\x84\xAD" => "\x63", + "\xE2\x84\xB0" => "\x65", + "\xE2\x84\xB1" => "\x66", + "\xE2\x84\xB3" => "\x6D", + "\xE2\x84\xBB" => "\x66\x61\x78", + "\xE2\x84\xBE" => "\xCE\xB3", + "\xE2\x84\xBF" => "\xCF\x80", + "\xE2\x85\x85" => "\x64", + "\xE3\x89\x90" => "\x70\x74\x65", + "\xE3\x8B\x8C" => "\x68\x67", + "\xE3\x8B\x8E" => "\x65\x76", + "\xE3\x8B\x8F" => "\x6C\x74\x64", + "\xE3\x8D\xB1" => "\x68\x70\x61", + "\xE3\x8D\xB3" => "\x61\x75", + "\xE3\x8D\xB5" => "\x6F\x76", + "\xE3\x8D\xBA" => "\x69\x75", + "\xE3\x8E\x80" => "\x70\x61", + "\xE3\x8E\x81" => "\x6E\x61", + "\xE3\x8E\x82" => "\xCE\xBC\x61", + "\xE3\x8E\x83" => "\x6D\x61", + "\xE3\x8E\x84" => "\x6B\x61", + "\xE3\x8E\x85" => "\x6B\x62", + "\xE3\x8E\x86" => "\x6D\x62", + "\xE3\x8E\x87" => "\x67\x62", + "\xE3\x8E\x8A" => "\x70\x66", + "\xE3\x8E\x8B" => "\x6E\x66", + "\xE3\x8E\x8C" => "\xCE\xBC\x66", + "\xE3\x8E\x90" => "\x68\x7A", + "\xE3\x8E\x91" => "\x6B\x68\x7A", + "\xE3\x8E\x92" => "\x6D\x68\x7A", + "\xE3\x8E\x93" => "\x67\x68\x7A", + "\xE3\x8E\x94" => "\x74\x68\x7A", + "\xE3\x8E\xA9" => "\x70\x61", + "\xE3\x8E\xAA" => "\x6B\x70\x61", + "\xE3\x8E\xAB" => "\x6D\x70\x61", + "\xE3\x8E\xAC" => "\x67\x70\x61", + "\xE3\x8E\xB4" => "\x70\x76", + "\xE3\x8E\xB5" => "\x6E\x76", + "\xE3\x8E\xB6" => "\xCE\xBC\x76", + "\xE3\x8E\xB7" => "\x6D\x76", + "\xE3\x8E\xB8" => "\x6B\x76", + "\xE3\x8E\xB9" => "\x6D\x76", + "\xE3\x8E\xBA" => "\x70\x77", + "\xE3\x8E\xBB" => "\x6E\x77", + "\xE3\x8E\xBC" => "\xCE\xBC\x77", + "\xE3\x8E\xBD" => "\x6D\x77", + "\xE3\x8E\xBE" => "\x6B\x77", + "\xE3\x8E\xBF" => "\x6D\x77", + "\xE3\x8F\x80" => "\x6B\xCF\x89", + "\xE3\x8F\x81" => "\x6D\xCF\x89", + "\xE3\x8F\x83" => "\x62\x71", + "\xE3\x8F\x86" => "\x63\xE2\x88\x95\x6B\x67", + "\xE3\x8F\x87" => "\x63\x6F\x2E", + "\xE3\x8F\x88" => "\x64\x62", + "\xE3\x8F\x89" => "\x67\x79", + "\xE3\x8F\x8B" => "\x68\x70", + "\xE3\x8F\x8D" => "\x6B\x6B", + "\xE3\x8F\x8E" => "\x6B\x6D", + "\xE3\x8F\x97" => "\x70\x68", + "\xE3\x8F\x99" => "\x70\x70\x6D", + "\xE3\x8F\x9A" => "\x70\x72", + "\xE3\x8F\x9C" => "\x73\x76", + "\xE3\x8F\x9D" => "\x77\x62", + "\xE3\x8F\x9E" => "\x76\xE2\x88\x95\x6D", + "\xE3\x8F\x9F" => "\x61\xE2\x88\x95\x6D", + "\xF0\x9D\x90\x80" => "\x61", + "\xF0\x9D\x90\x81" => "\x62", + "\xF0\x9D\x90\x82" => "\x63", + "\xF0\x9D\x90\x83" => "\x64", + "\xF0\x9D\x90\x84" => "\x65", + "\xF0\x9D\x90\x85" => "\x66", + "\xF0\x9D\x90\x86" => "\x67", + "\xF0\x9D\x90\x87" => "\x68", + "\xF0\x9D\x90\x88" => "\x69", + "\xF0\x9D\x90\x89" => "\x6A", + "\xF0\x9D\x90\x8A" => "\x6B", + "\xF0\x9D\x90\x8B" => "\x6C", + "\xF0\x9D\x90\x8C" => "\x6D", + "\xF0\x9D\x90\x8D" => "\x6E", + "\xF0\x9D\x90\x8E" => "\x6F", + "\xF0\x9D\x90\x8F" => "\x70", + "\xF0\x9D\x90\x90" => "\x71", + "\xF0\x9D\x90\x91" => "\x72", + "\xF0\x9D\x90\x92" => "\x73", + "\xF0\x9D\x90\x93" => "\x74", + "\xF0\x9D\x90\x94" => "\x75", + "\xF0\x9D\x90\x95" => "\x76", + "\xF0\x9D\x90\x96" => "\x77", + "\xF0\x9D\x90\x97" => "\x78", + "\xF0\x9D\x90\x98" => "\x79", + "\xF0\x9D\x90\x99" => "\x7A", + "\xF0\x9D\x90\xB4" => "\x61", + "\xF0\x9D\x90\xB5" => "\x62", + "\xF0\x9D\x90\xB6" => "\x63", + "\xF0\x9D\x90\xB7" => "\x64", + "\xF0\x9D\x90\xB8" => "\x65", + "\xF0\x9D\x90\xB9" => "\x66", + "\xF0\x9D\x90\xBA" => "\x67", + "\xF0\x9D\x90\xBB" => "\x68", + "\xF0\x9D\x90\xBC" => "\x69", + "\xF0\x9D\x90\xBD" => "\x6A", + "\xF0\x9D\x90\xBE" => "\x6B", + "\xF0\x9D\x90\xBF" => "\x6C", + "\xF0\x9D\x91\x80" => "\x6D", + "\xF0\x9D\x91\x81" => "\x6E", + "\xF0\x9D\x91\x82" => "\x6F", + "\xF0\x9D\x91\x83" => "\x70", + "\xF0\x9D\x91\x84" => "\x71", + "\xF0\x9D\x91\x85" => "\x72", + "\xF0\x9D\x91\x86" => "\x73", + "\xF0\x9D\x91\x87" => "\x74", + "\xF0\x9D\x91\x88" => "\x75", + "\xF0\x9D\x91\x89" => "\x76", + "\xF0\x9D\x91\x8A" => "\x77", + "\xF0\x9D\x91\x8B" => "\x78", + "\xF0\x9D\x91\x8C" => "\x79", + "\xF0\x9D\x91\x8D" => "\x7A", + "\xF0\x9D\x91\xA8" => "\x61", + "\xF0\x9D\x91\xA9" => "\x62", + "\xF0\x9D\x91\xAA" => "\x63", + "\xF0\x9D\x91\xAB" => "\x64", + "\xF0\x9D\x91\xAC" => "\x65", + "\xF0\x9D\x91\xAD" => "\x66", + "\xF0\x9D\x91\xAE" => "\x67", + "\xF0\x9D\x91\xAF" => "\x68", + "\xF0\x9D\x91\xB0" => "\x69", + "\xF0\x9D\x91\xB1" => "\x6A", + "\xF0\x9D\x91\xB2" => "\x6B", + "\xF0\x9D\x91\xB3" => "\x6C", + "\xF0\x9D\x91\xB4" => "\x6D", + "\xF0\x9D\x91\xB5" => "\x6E", + "\xF0\x9D\x91\xB6" => "\x6F", + "\xF0\x9D\x91\xB7" => "\x70", + "\xF0\x9D\x91\xB8" => "\x71", + "\xF0\x9D\x91\xB9" => "\x72", + "\xF0\x9D\x91\xBA" => "\x73", + "\xF0\x9D\x91\xBB" => "\x74", + "\xF0\x9D\x91\xBC" => "\x75", + "\xF0\x9D\x91\xBD" => "\x76", + "\xF0\x9D\x91\xBE" => "\x77", + "\xF0\x9D\x91\xBF" => "\x78", + "\xF0\x9D\x92\x80" => "\x79", + "\xF0\x9D\x92\x81" => "\x7A", + "\xF0\x9D\x92\x9C" => "\x61", + "\xF0\x9D\x92\x9E" => "\x63", + "\xF0\x9D\x92\x9F" => "\x64", + "\xF0\x9D\x92\xA2" => "\x67", + "\xF0\x9D\x92\xA5" => "\x6A", + "\xF0\x9D\x92\xA6" => "\x6B", + "\xF0\x9D\x92\xA9" => "\x6E", + "\xF0\x9D\x92\xAA" => "\x6F", + "\xF0\x9D\x92\xAB" => "\x70", + "\xF0\x9D\x92\xAC" => "\x71", + "\xF0\x9D\x92\xAE" => "\x73", + "\xF0\x9D\x92\xAF" => "\x74", + "\xF0\x9D\x92\xB0" => "\x75", + "\xF0\x9D\x92\xB1" => "\x76", + "\xF0\x9D\x92\xB2" => "\x77", + "\xF0\x9D\x92\xB3" => "\x78", + "\xF0\x9D\x92\xB4" => "\x79", + "\xF0\x9D\x92\xB5" => "\x7A", + "\xF0\x9D\x93\x90" => "\x61", + "\xF0\x9D\x93\x91" => "\x62", + "\xF0\x9D\x93\x92" => "\x63", + "\xF0\x9D\x93\x93" => "\x64", + "\xF0\x9D\x93\x94" => "\x65", + "\xF0\x9D\x93\x95" => "\x66", + "\xF0\x9D\x93\x96" => "\x67", + "\xF0\x9D\x93\x97" => "\x68", + "\xF0\x9D\x93\x98" => "\x69", + "\xF0\x9D\x93\x99" => "\x6A", + "\xF0\x9D\x93\x9A" => "\x6B", + "\xF0\x9D\x93\x9B" => "\x6C", + "\xF0\x9D\x93\x9C" => "\x6D", + "\xF0\x9D\x93\x9D" => "\x6E", + "\xF0\x9D\x93\x9E" => "\x6F", + "\xF0\x9D\x93\x9F" => "\x70", + "\xF0\x9D\x93\xA0" => "\x71", + "\xF0\x9D\x93\xA1" => "\x72", + "\xF0\x9D\x93\xA2" => "\x73", + "\xF0\x9D\x93\xA3" => "\x74", + "\xF0\x9D\x93\xA4" => "\x75", + "\xF0\x9D\x93\xA5" => "\x76", + "\xF0\x9D\x93\xA6" => "\x77", + "\xF0\x9D\x93\xA7" => "\x78", + "\xF0\x9D\x93\xA8" => "\x79", + "\xF0\x9D\x93\xA9" => "\x7A", + "\xF0\x9D\x94\x84" => "\x61", + "\xF0\x9D\x94\x85" => "\x62", + "\xF0\x9D\x94\x87" => "\x64", + "\xF0\x9D\x94\x88" => "\x65", + "\xF0\x9D\x94\x89" => "\x66", + "\xF0\x9D\x94\x8A" => "\x67", + "\xF0\x9D\x94\x8D" => "\x6A", + "\xF0\x9D\x94\x8E" => "\x6B", + "\xF0\x9D\x94\x8F" => "\x6C", + "\xF0\x9D\x94\x90" => "\x6D", + "\xF0\x9D\x94\x91" => "\x6E", + "\xF0\x9D\x94\x92" => "\x6F", + "\xF0\x9D\x94\x93" => "\x70", + "\xF0\x9D\x94\x94" => "\x71", + "\xF0\x9D\x94\x96" => "\x73", + "\xF0\x9D\x94\x97" => "\x74", + "\xF0\x9D\x94\x98" => "\x75", + "\xF0\x9D\x94\x99" => "\x76", + "\xF0\x9D\x94\x9A" => "\x77", + "\xF0\x9D\x94\x9B" => "\x78", + "\xF0\x9D\x94\x9C" => "\x79", + "\xF0\x9D\x94\xB8" => "\x61", + "\xF0\x9D\x94\xB9" => "\x62", + "\xF0\x9D\x94\xBB" => "\x64", + "\xF0\x9D\x94\xBC" => "\x65", + "\xF0\x9D\x94\xBD" => "\x66", + "\xF0\x9D\x94\xBE" => "\x67", + "\xF0\x9D\x95\x80" => "\x69", + "\xF0\x9D\x95\x81" => "\x6A", + "\xF0\x9D\x95\x82" => "\x6B", + "\xF0\x9D\x95\x83" => "\x6C", + "\xF0\x9D\x95\x84" => "\x6D", + "\xF0\x9D\x95\x86" => "\x6F", + "\xF0\x9D\x95\x8A" => "\x73", + "\xF0\x9D\x95\x8B" => "\x74", + "\xF0\x9D\x95\x8C" => "\x75", + "\xF0\x9D\x95\x8D" => "\x76", + "\xF0\x9D\x95\x8E" => "\x77", + "\xF0\x9D\x95\x8F" => "\x78", + "\xF0\x9D\x95\x90" => "\x79", + "\xF0\x9D\x95\xAC" => "\x61", + "\xF0\x9D\x95\xAD" => "\x62", + "\xF0\x9D\x95\xAE" => "\x63", + "\xF0\x9D\x95\xAF" => "\x64", + "\xF0\x9D\x95\xB0" => "\x65", + "\xF0\x9D\x95\xB1" => "\x66", + "\xF0\x9D\x95\xB2" => "\x67", + "\xF0\x9D\x95\xB3" => "\x68", + "\xF0\x9D\x95\xB4" => "\x69", + "\xF0\x9D\x95\xB5" => "\x6A", + "\xF0\x9D\x95\xB6" => "\x6B", + "\xF0\x9D\x95\xB7" => "\x6C", + "\xF0\x9D\x95\xB8" => "\x6D", + "\xF0\x9D\x95\xB9" => "\x6E", + "\xF0\x9D\x95\xBA" => "\x6F", + "\xF0\x9D\x95\xBB" => "\x70", + "\xF0\x9D\x95\xBC" => "\x71", + "\xF0\x9D\x95\xBD" => "\x72", + "\xF0\x9D\x95\xBE" => "\x73", + "\xF0\x9D\x95\xBF" => "\x74", + "\xF0\x9D\x96\x80" => "\x75", + "\xF0\x9D\x96\x81" => "\x76", + "\xF0\x9D\x96\x82" => "\x77", + "\xF0\x9D\x96\x83" => "\x78", + "\xF0\x9D\x96\x84" => "\x79", + "\xF0\x9D\x96\x85" => "\x7A", + "\xF0\x9D\x96\xA0" => "\x61", + "\xF0\x9D\x96\xA1" => "\x62", + "\xF0\x9D\x96\xA2" => "\x63", + "\xF0\x9D\x96\xA3" => "\x64", + "\xF0\x9D\x96\xA4" => "\x65", + "\xF0\x9D\x96\xA5" => "\x66", + "\xF0\x9D\x96\xA6" => "\x67", + "\xF0\x9D\x96\xA7" => "\x68", + "\xF0\x9D\x96\xA8" => "\x69", + "\xF0\x9D\x96\xA9" => "\x6A", + "\xF0\x9D\x96\xAA" => "\x6B", + "\xF0\x9D\x96\xAB" => "\x6C", + "\xF0\x9D\x96\xAC" => "\x6D", + "\xF0\x9D\x96\xAD" => "\x6E", + "\xF0\x9D\x96\xAE" => "\x6F", + "\xF0\x9D\x96\xAF" => "\x70", + "\xF0\x9D\x96\xB0" => "\x71", + "\xF0\x9D\x96\xB1" => "\x72", + "\xF0\x9D\x96\xB2" => "\x73", + "\xF0\x9D\x96\xB3" => "\x74", + "\xF0\x9D\x96\xB4" => "\x75", + "\xF0\x9D\x96\xB5" => "\x76", + "\xF0\x9D\x96\xB6" => "\x77", + "\xF0\x9D\x96\xB7" => "\x78", + "\xF0\x9D\x96\xB8" => "\x79", + "\xF0\x9D\x96\xB9" => "\x7A", + "\xF0\x9D\x97\x94" => "\x61", + "\xF0\x9D\x97\x95" => "\x62", + "\xF0\x9D\x97\x96" => "\x63", + "\xF0\x9D\x97\x97" => "\x64", + "\xF0\x9D\x97\x98" => "\x65", + "\xF0\x9D\x97\x99" => "\x66", + "\xF0\x9D\x97\x9A" => "\x67", + "\xF0\x9D\x97\x9B" => "\x68", + "\xF0\x9D\x97\x9C" => "\x69", + "\xF0\x9D\x97\x9D" => "\x6A", + "\xF0\x9D\x97\x9E" => "\x6B", + "\xF0\x9D\x97\x9F" => "\x6C", + "\xF0\x9D\x97\xA0" => "\x6D", + "\xF0\x9D\x97\xA1" => "\x6E", + "\xF0\x9D\x97\xA2" => "\x6F", + "\xF0\x9D\x97\xA3" => "\x70", + "\xF0\x9D\x97\xA4" => "\x71", + "\xF0\x9D\x97\xA5" => "\x72", + "\xF0\x9D\x97\xA6" => "\x73", + "\xF0\x9D\x97\xA7" => "\x74", + "\xF0\x9D\x97\xA8" => "\x75", + "\xF0\x9D\x97\xA9" => "\x76", + "\xF0\x9D\x97\xAA" => "\x77", + "\xF0\x9D\x97\xAB" => "\x78", + "\xF0\x9D\x97\xAC" => "\x79", + "\xF0\x9D\x97\xAD" => "\x7A", + "\xF0\x9D\x98\x88" => "\x61", + "\xF0\x9D\x98\x89" => "\x62", + "\xF0\x9D\x98\x8A" => "\x63", + "\xF0\x9D\x98\x8B" => "\x64", + "\xF0\x9D\x98\x8C" => "\x65", + "\xF0\x9D\x98\x8D" => "\x66", + "\xF0\x9D\x98\x8E" => "\x67", + "\xF0\x9D\x98\x8F" => "\x68", + "\xF0\x9D\x98\x90" => "\x69", + "\xF0\x9D\x98\x91" => "\x6A", + "\xF0\x9D\x98\x92" => "\x6B", + "\xF0\x9D\x98\x93" => "\x6C", + "\xF0\x9D\x98\x94" => "\x6D", + "\xF0\x9D\x98\x95" => "\x6E", + "\xF0\x9D\x98\x96" => "\x6F", + "\xF0\x9D\x98\x97" => "\x70", + "\xF0\x9D\x98\x98" => "\x71", + "\xF0\x9D\x98\x99" => "\x72", + "\xF0\x9D\x98\x9A" => "\x73", + "\xF0\x9D\x98\x9B" => "\x74", + "\xF0\x9D\x98\x9C" => "\x75", + "\xF0\x9D\x98\x9D" => "\x76", + "\xF0\x9D\x98\x9E" => "\x77", + "\xF0\x9D\x98\x9F" => "\x78", + "\xF0\x9D\x98\xA0" => "\x79", + "\xF0\x9D\x98\xA1" => "\x7A", + "\xF0\x9D\x98\xBC" => "\x61", + "\xF0\x9D\x98\xBD" => "\x62", + "\xF0\x9D\x98\xBE" => "\x63", + "\xF0\x9D\x98\xBF" => "\x64", + "\xF0\x9D\x99\x80" => "\x65", + "\xF0\x9D\x99\x81" => "\x66", + "\xF0\x9D\x99\x82" => "\x67", + "\xF0\x9D\x99\x83" => "\x68", + "\xF0\x9D\x99\x84" => "\x69", + "\xF0\x9D\x99\x85" => "\x6A", + "\xF0\x9D\x99\x86" => "\x6B", + "\xF0\x9D\x99\x87" => "\x6C", + "\xF0\x9D\x99\x88" => "\x6D", + "\xF0\x9D\x99\x89" => "\x6E", + "\xF0\x9D\x99\x8A" => "\x6F", + "\xF0\x9D\x99\x8B" => "\x70", + "\xF0\x9D\x99\x8C" => "\x71", + "\xF0\x9D\x99\x8D" => "\x72", + "\xF0\x9D\x99\x8E" => "\x73", + "\xF0\x9D\x99\x8F" => "\x74", + "\xF0\x9D\x99\x90" => "\x75", + "\xF0\x9D\x99\x91" => "\x76", + "\xF0\x9D\x99\x92" => "\x77", + "\xF0\x9D\x99\x93" => "\x78", + "\xF0\x9D\x99\x94" => "\x79", + "\xF0\x9D\x99\x95" => "\x7A", + "\xF0\x9D\x99\xB0" => "\x61", + "\xF0\x9D\x99\xB1" => "\x62", + "\xF0\x9D\x99\xB2" => "\x63", + "\xF0\x9D\x99\xB3" => "\x64", + "\xF0\x9D\x99\xB4" => "\x65", + "\xF0\x9D\x99\xB5" => "\x66", + "\xF0\x9D\x99\xB6" => "\x67", + "\xF0\x9D\x99\xB7" => "\x68", + "\xF0\x9D\x99\xB8" => "\x69", + "\xF0\x9D\x99\xB9" => "\x6A", + "\xF0\x9D\x99\xBA" => "\x6B", + "\xF0\x9D\x99\xBB" => "\x6C", + "\xF0\x9D\x99\xBC" => "\x6D", + "\xF0\x9D\x99\xBD" => "\x6E", + "\xF0\x9D\x99\xBE" => "\x6F", + "\xF0\x9D\x99\xBF" => "\x70", + "\xF0\x9D\x9A\x80" => "\x71", + "\xF0\x9D\x9A\x81" => "\x72", + "\xF0\x9D\x9A\x82" => "\x73", + "\xF0\x9D\x9A\x83" => "\x74", + "\xF0\x9D\x9A\x84" => "\x75", + "\xF0\x9D\x9A\x85" => "\x76", + "\xF0\x9D\x9A\x86" => "\x77", + "\xF0\x9D\x9A\x87" => "\x78", + "\xF0\x9D\x9A\x88" => "\x79", + "\xF0\x9D\x9A\x89" => "\x7A", + "\xF0\x9D\x9A\xA8" => "\xCE\xB1", + "\xF0\x9D\x9A\xA9" => "\xCE\xB2", + "\xF0\x9D\x9A\xAA" => "\xCE\xB3", + "\xF0\x9D\x9A\xAB" => "\xCE\xB4", + "\xF0\x9D\x9A\xAC" => "\xCE\xB5", + "\xF0\x9D\x9A\xAD" => "\xCE\xB6", + "\xF0\x9D\x9A\xAE" => "\xCE\xB7", + "\xF0\x9D\x9A\xAF" => "\xCE\xB8", + "\xF0\x9D\x9A\xB0" => "\xCE\xB9", + "\xF0\x9D\x9A\xB1" => "\xCE\xBA", + "\xF0\x9D\x9A\xB2" => "\xCE\xBB", + "\xF0\x9D\x9A\xB3" => "\xCE\xBC", + "\xF0\x9D\x9A\xB4" => "\xCE\xBD", + "\xF0\x9D\x9A\xB5" => "\xCE\xBE", + "\xF0\x9D\x9A\xB6" => "\xCE\xBF", + "\xF0\x9D\x9A\xB7" => "\xCF\x80", + "\xF0\x9D\x9A\xB8" => "\xCF\x81", + "\xF0\x9D\x9A\xB9" => "\xCE\xB8", + "\xF0\x9D\x9A\xBA" => "\xCF\x83", + "\xF0\x9D\x9A\xBB" => "\xCF\x84", + "\xF0\x9D\x9A\xBC" => "\xCF\x85", + "\xF0\x9D\x9A\xBD" => "\xCF\x86", + "\xF0\x9D\x9A\xBE" => "\xCF\x87", + "\xF0\x9D\x9A\xBF" => "\xCF\x88", + "\xF0\x9D\x9B\x80" => "\xCF\x89", + "\xF0\x9D\x9B\x93" => "\xCF\x83", + "\xF0\x9D\x9B\xA2" => "\xCE\xB1", + "\xF0\x9D\x9B\xA3" => "\xCE\xB2", + "\xF0\x9D\x9B\xA4" => "\xCE\xB3", + "\xF0\x9D\x9B\xA5" => "\xCE\xB4", + "\xF0\x9D\x9B\xA6" => "\xCE\xB5", + "\xF0\x9D\x9B\xA7" => "\xCE\xB6", + "\xF0\x9D\x9B\xA8" => "\xCE\xB7", + "\xF0\x9D\x9B\xA9" => "\xCE\xB8", + "\xF0\x9D\x9B\xAA" => "\xCE\xB9", + "\xF0\x9D\x9B\xAB" => "\xCE\xBA", + "\xF0\x9D\x9B\xAC" => "\xCE\xBB", + "\xF0\x9D\x9B\xAD" => "\xCE\xBC", + "\xF0\x9D\x9B\xAE" => "\xCE\xBD", + "\xF0\x9D\x9B\xAF" => "\xCE\xBE", + "\xF0\x9D\x9B\xB0" => "\xCE\xBF", + "\xF0\x9D\x9B\xB1" => "\xCF\x80", + "\xF0\x9D\x9B\xB2" => "\xCF\x81", + "\xF0\x9D\x9B\xB3" => "\xCE\xB8", + "\xF0\x9D\x9B\xB4" => "\xCF\x83", + "\xF0\x9D\x9B\xB5" => "\xCF\x84", + "\xF0\x9D\x9B\xB6" => "\xCF\x85", + "\xF0\x9D\x9B\xB7" => "\xCF\x86", + "\xF0\x9D\x9B\xB8" => "\xCF\x87", + "\xF0\x9D\x9B\xB9" => "\xCF\x88", + "\xF0\x9D\x9B\xBA" => "\xCF\x89", + "\xF0\x9D\x9C\x8D" => "\xCF\x83", + "\xF0\x9D\x9C\x9C" => "\xCE\xB1", + "\xF0\x9D\x9C\x9D" => "\xCE\xB2", + "\xF0\x9D\x9C\x9E" => "\xCE\xB3", + "\xF0\x9D\x9C\x9F" => "\xCE\xB4", + "\xF0\x9D\x9C\xA0" => "\xCE\xB5", + "\xF0\x9D\x9C\xA1" => "\xCE\xB6", + "\xF0\x9D\x9C\xA2" => "\xCE\xB7", + "\xF0\x9D\x9C\xA3" => "\xCE\xB8", + "\xF0\x9D\x9C\xA4" => "\xCE\xB9", + "\xF0\x9D\x9C\xA5" => "\xCE\xBA", + "\xF0\x9D\x9C\xA6" => "\xCE\xBB", + "\xF0\x9D\x9C\xA7" => "\xCE\xBC", + "\xF0\x9D\x9C\xA8" => "\xCE\xBD", + "\xF0\x9D\x9C\xA9" => "\xCE\xBE", + "\xF0\x9D\x9C\xAA" => "\xCE\xBF", + "\xF0\x9D\x9C\xAB" => "\xCF\x80", + "\xF0\x9D\x9C\xAC" => "\xCF\x81", + "\xF0\x9D\x9C\xAD" => "\xCE\xB8", + "\xF0\x9D\x9C\xAE" => "\xCF\x83", + "\xF0\x9D\x9C\xAF" => "\xCF\x84", + "\xF0\x9D\x9C\xB0" => "\xCF\x85", + "\xF0\x9D\x9C\xB1" => "\xCF\x86", + "\xF0\x9D\x9C\xB2" => "\xCF\x87", + "\xF0\x9D\x9C\xB3" => "\xCF\x88", + "\xF0\x9D\x9C\xB4" => "\xCF\x89", + "\xF0\x9D\x9D\x87" => "\xCF\x83", + "\xF0\x9D\x9D\x96" => "\xCE\xB1", + "\xF0\x9D\x9D\x97" => "\xCE\xB2", + "\xF0\x9D\x9D\x98" => "\xCE\xB3", + "\xF0\x9D\x9D\x99" => "\xCE\xB4", + "\xF0\x9D\x9D\x9A" => "\xCE\xB5", + "\xF0\x9D\x9D\x9B" => "\xCE\xB6", + "\xF0\x9D\x9D\x9C" => "\xCE\xB7", + "\xF0\x9D\x9D\x9D" => "\xCE\xB8", + "\xF0\x9D\x9D\x9E" => "\xCE\xB9", + "\xF0\x9D\x9D\x9F" => "\xCE\xBA", + "\xF0\x9D\x9D\xA0" => "\xCE\xBB", + "\xF0\x9D\x9D\xA1" => "\xCE\xBC", + "\xF0\x9D\x9D\xA2" => "\xCE\xBD", + "\xF0\x9D\x9D\xA3" => "\xCE\xBE", + "\xF0\x9D\x9D\xA4" => "\xCE\xBF", + "\xF0\x9D\x9D\xA5" => "\xCF\x80", + "\xF0\x9D\x9D\xA6" => "\xCF\x81", + "\xF0\x9D\x9D\xA7" => "\xCE\xB8", + "\xF0\x9D\x9D\xA8" => "\xCF\x83", + "\xF0\x9D\x9D\xA9" => "\xCF\x84", + "\xF0\x9D\x9D\xAA" => "\xCF\x85", + "\xF0\x9D\x9D\xAB" => "\xCF\x86", + "\xF0\x9D\x9D\xAC" => "\xCF\x87", + "\xF0\x9D\x9D\xAD" => "\xCF\x88", + "\xF0\x9D\x9D\xAE" => "\xCF\x89", + "\xF0\x9D\x9E\x81" => "\xCF\x83", + "\xF0\x9D\x9E\x90" => "\xCE\xB1", + "\xF0\x9D\x9E\x91" => "\xCE\xB2", + "\xF0\x9D\x9E\x92" => "\xCE\xB3", + "\xF0\x9D\x9E\x93" => "\xCE\xB4", + "\xF0\x9D\x9E\x94" => "\xCE\xB5", + "\xF0\x9D\x9E\x95" => "\xCE\xB6", + "\xF0\x9D\x9E\x96" => "\xCE\xB7", + "\xF0\x9D\x9E\x97" => "\xCE\xB8", + "\xF0\x9D\x9E\x98" => "\xCE\xB9", + "\xF0\x9D\x9E\x99" => "\xCE\xBA", + "\xF0\x9D\x9E\x9A" => "\xCE\xBB", + "\xF0\x9D\x9E\x9B" => "\xCE\xBC", + "\xF0\x9D\x9E\x9C" => "\xCE\xBD", + "\xF0\x9D\x9E\x9D" => "\xCE\xBE", + "\xF0\x9D\x9E\x9E" => "\xCE\xBF", + "\xF0\x9D\x9E\x9F" => "\xCF\x80", + "\xF0\x9D\x9E\xA0" => "\xCF\x81", + "\xF0\x9D\x9E\xA1" => "\xCE\xB8", + "\xF0\x9D\x9E\xA2" => "\xCF\x83", + "\xF0\x9D\x9E\xA3" => "\xCF\x84", + "\xF0\x9D\x9E\xA4" => "\xCF\x85", + "\xF0\x9D\x9E\xA5" => "\xCF\x86", + "\xF0\x9D\x9E\xA6" => "\xCF\x87", + "\xF0\x9D\x9E\xA7" => "\xCF\x88", + "\xF0\x9D\x9E\xA8" => "\xCF\x89", + "\xF0\x9D\x9E\xBB" => "\xCF\x83", + "\xF0\x9D\x9F\x8A" => "\xCF\x9D", + ); + global $phpbb_root_path, $phpEx; + + // do the case fold + $text = utf8_case_fold($text, $option); + + if (!class_exists('utf_normalizer')) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx); + } + + // convert to NFKC + utf_normalizer::nfkc($text); + + // FC_NFKC_Closure, http://www.unicode.org/Public/5.0.0/ucd/DerivedNormalizationProps.txt + $text = strtr($text, $fc_nfkc_closure); + + return $text; +} + +/** +* Assume the input is NFC: +* Takes the input and does a "special" case fold. It does minor normalization as well. +* +* @param string $text text to be case folded +* @param string $option determines how we will fold the cases +* @return string case folded text +*/ +function utf8_case_fold_nfc($text, $option = 'full') +{ + static $uniarray = array(); + static $ypogegrammeni = array( + "\xCD\xBA" => "\x20\xCD\x85", + "\xE1\xBE\x80" => "\xE1\xBC\x80\xCD\x85", + "\xE1\xBE\x81" => "\xE1\xBC\x81\xCD\x85", + "\xE1\xBE\x82" => "\xE1\xBC\x82\xCD\x85", + "\xE1\xBE\x83" => "\xE1\xBC\x83\xCD\x85", + "\xE1\xBE\x84" => "\xE1\xBC\x84\xCD\x85", + "\xE1\xBE\x85" => "\xE1\xBC\x85\xCD\x85", + "\xE1\xBE\x86" => "\xE1\xBC\x86\xCD\x85", + "\xE1\xBE\x87" => "\xE1\xBC\x87\xCD\x85", + "\xE1\xBE\x88" => "\xE1\xBC\x88\xCD\x85", + "\xE1\xBE\x89" => "\xE1\xBC\x89\xCD\x85", + "\xE1\xBE\x8A" => "\xE1\xBC\x8A\xCD\x85", + "\xE1\xBE\x8B" => "\xE1\xBC\x8B\xCD\x85", + "\xE1\xBE\x8C" => "\xE1\xBC\x8C\xCD\x85", + "\xE1\xBE\x8D" => "\xE1\xBC\x8D\xCD\x85", + "\xE1\xBE\x8E" => "\xE1\xBC\x8E\xCD\x85", + "\xE1\xBE\x8F" => "\xE1\xBC\x8F\xCD\x85", + "\xE1\xBE\x90" => "\xE1\xBC\xA0\xCD\x85", + "\xE1\xBE\x91" => "\xE1\xBC\xA1\xCD\x85", + "\xE1\xBE\x92" => "\xE1\xBC\xA2\xCD\x85", + "\xE1\xBE\x93" => "\xE1\xBC\xA3\xCD\x85", + "\xE1\xBE\x94" => "\xE1\xBC\xA4\xCD\x85", + "\xE1\xBE\x95" => "\xE1\xBC\xA5\xCD\x85", + "\xE1\xBE\x96" => "\xE1\xBC\xA6\xCD\x85", + "\xE1\xBE\x97" => "\xE1\xBC\xA7\xCD\x85", + "\xE1\xBE\x98" => "\xE1\xBC\xA8\xCD\x85", + "\xE1\xBE\x99" => "\xE1\xBC\xA9\xCD\x85", + "\xE1\xBE\x9A" => "\xE1\xBC\xAA\xCD\x85", + "\xE1\xBE\x9B" => "\xE1\xBC\xAB\xCD\x85", + "\xE1\xBE\x9C" => "\xE1\xBC\xAC\xCD\x85", + "\xE1\xBE\x9D" => "\xE1\xBC\xAD\xCD\x85", + "\xE1\xBE\x9E" => "\xE1\xBC\xAE\xCD\x85", + "\xE1\xBE\x9F" => "\xE1\xBC\xAF\xCD\x85", + "\xE1\xBE\xA0" => "\xE1\xBD\xA0\xCD\x85", + "\xE1\xBE\xA1" => "\xE1\xBD\xA1\xCD\x85", + "\xE1\xBE\xA2" => "\xE1\xBD\xA2\xCD\x85", + "\xE1\xBE\xA3" => "\xE1\xBD\xA3\xCD\x85", + "\xE1\xBE\xA4" => "\xE1\xBD\xA4\xCD\x85", + "\xE1\xBE\xA5" => "\xE1\xBD\xA5\xCD\x85", + "\xE1\xBE\xA6" => "\xE1\xBD\xA6\xCD\x85", + "\xE1\xBE\xA7" => "\xE1\xBD\xA7\xCD\x85", + "\xE1\xBE\xA8" => "\xE1\xBD\xA8\xCD\x85", + "\xE1\xBE\xA9" => "\xE1\xBD\xA9\xCD\x85", + "\xE1\xBE\xAA" => "\xE1\xBD\xAA\xCD\x85", + "\xE1\xBE\xAB" => "\xE1\xBD\xAB\xCD\x85", + "\xE1\xBE\xAC" => "\xE1\xBD\xAC\xCD\x85", + "\xE1\xBE\xAD" => "\xE1\xBD\xAD\xCD\x85", + "\xE1\xBE\xAE" => "\xE1\xBD\xAE\xCD\x85", + "\xE1\xBE\xAF" => "\xE1\xBD\xAF\xCD\x85", + "\xE1\xBE\xB2" => "\xE1\xBD\xB0\xCD\x85", + "\xE1\xBE\xB3" => "\xCE\xB1\xCD\x85", + "\xE1\xBE\xB4" => "\xCE\xAC\xCD\x85", + "\xE1\xBE\xB7" => "\xE1\xBE\xB6\xCD\x85", + "\xE1\xBE\xBC" => "\xCE\x91\xCD\x85", + "\xE1\xBF\x82" => "\xE1\xBD\xB4\xCD\x85", + "\xE1\xBF\x83" => "\xCE\xB7\xCD\x85", + "\xE1\xBF\x84" => "\xCE\xAE\xCD\x85", + "\xE1\xBF\x87" => "\xE1\xBF\x86\xCD\x85", + "\xE1\xBF\x8C" => "\xCE\x97\xCD\x85", + "\xE1\xBF\xB2" => "\xE1\xBD\xBC\xCD\x85", + "\xE1\xBF\xB3" => "\xCF\x89\xCD\x85", + "\xE1\xBF\xB4" => "\xCF\x8E\xCD\x85", + "\xE1\xBF\xB7" => "\xE1\xBF\xB6\xCD\x85", + "\xE1\xBF\xBC" => "\xCE\xA9\xCD\x85", + ); + global $phpbb_root_path, $phpEx; + + // perform a small trick, avoid further normalization on composed points that contain U+0345 in their decomposition + $text = strtr($text, $ypogegrammeni); + + // do the case fold + $text = utf8_case_fold($text, $option); + return $text; } @@ -1098,7 +1787,17 @@ function utf8_normalize_nfc($strings) { foreach ($strings as $key => $string) { - utf_normalizer::nfc($strings[$key]); + if (is_array($string)) + { + foreach ($string as $_key => $_string) + { + utf_normalizer::nfc($strings[$key][$_key]); + } + } + else + { + utf_normalizer::nfc($strings[$key]); + } } } @@ -1120,154 +1819,22 @@ function utf8_normalize_nfc($strings) */ function utf8_clean_string($text) { - $text = utf8_case_fold($text); - - if (!class_exists('utf_normalizer')) + global $phpbb_root_path, $phpEx; + + static $homographs = array(); + if (empty($homographs)) { - global $phpbb_root_path, $phpEx; - include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx); + $homographs = include($phpbb_root_path . 'includes/utf/data/confusables.' . $phpEx); } - utf_normalizer::nfc($text); - - static $homographs = array( - "\xC2\xA1" => "\x69", // EXCLAMATION MARK, INVERTED => LATIN SMALL LETTER I - "\xC2\xAD" => '', // HYPHEN, SOFT => empty string - "\xC4\x90" => "\xC3\x90", // LATIN CAPITAL LETTER D WITH STROKE => LATIN CAPITAL LETTER ETH - "\xC7\x83" => "\x21", // LATIN LETTER RETROFLEX CLICK => EXCLAMATION MARK - "\xC9\x85" => "\xCE\x9B", // LATIN CAPITAL LETTER TURNED V => GREEK CAPITAL LETTER LAMDA - "\xC9\x99" => "\xC7\x9D", // LATIN SMALL LETTER SCHWA => LATIN SMALL LETTER TURNED E - "\xCA\x99" => "\xD0\xB2", // LATIN LETTER SMALL CAPITAL B => CYRILLIC SMALL LETTER VE - "\xCA\x9C" => "\xD0\xBD", // LATIN LETTER SMALL CAPITAL H => CYRILLIC SMALL LETTER EN - "\xCE\x91" => "\x41", // GREEK CAPITAL LETTER ALPHA => LATIN CAPITAL LETTER A - "\xCE\x92" => "\x42", // GREEK CAPITAL LETTER BETA => LATIN CAPITAL LETTER B - "\xCE\x95" => "\x45", // GREEK CAPITAL LETTER EPSILON => LATIN CAPITAL LETTER E - "\xCE\x96" => "\x5A", // GREEK CAPITAL LETTER ZETA => LATIN CAPITAL LETTER Z - "\xCE\x97" => "\x48", // GREEK CAPITAL LETTER ETA => LATIN CAPITAL LETTER H - "\xCE\x99" => "\x49", // GREEK CAPITAL LETTER IOTA => LATIN CAPITAL LETTER I - "\xCE\x9A" => "\x4B", // GREEK CAPITAL LETTER KAPPA => LATIN CAPITAL LETTER K - "\xCE\x9C" => "\x4D", // GREEK CAPITAL LETTER MU => LATIN CAPITAL LETTER M - "\xCE\x9D" => "\x4E", // GREEK CAPITAL LETTER NU => LATIN CAPITAL LETTER N - "\xCE\x9F" => "\x4F", // GREEK CAPITAL LETTER OMICRON => LATIN CAPITAL LETTER O - "\xCE\xA1" => "\x50", // GREEK CAPITAL LETTER RHO => LATIN CAPITAL LETTER P - "\xCE\xA3" => "\xC6\xA9", // GREEK CAPITAL LETTER SIGMA => LATIN CAPITAL LETTER ESH - "\xCE\xA4" => "\x54", // GREEK CAPITAL LETTER TAU => LATIN CAPITAL LETTER T - "\xCE\xA5" => "\x59", // GREEK CAPITAL LETTER UPSILON => LATIN CAPITAL LETTER Y - "\xCE\xA7" => "\x58", // GREEK CAPITAL LETTER CHI => LATIN CAPITAL LETTER X - "\xCE\xB1" => "\x61", // GREEK SMALL LETTER ALPHA => LATIN SMALL LETTER A - "\xCE\xB5" => "\xC9\x9B", // GREEK SMALL LETTER EPSILON => LATIN SMALL LETTER OPEN E - "\xCE\xB9" => "\xC9\xA9", // GREEK SMALL LETTER IOTA => LATIN SMALL LETTER IOTA - "\xCE\xBF" => "\x6F", // GREEK SMALL LETTER OMICRON => LATIN SMALL LETTER O - "\xCF\xB3" => "\x6A", // GREEK LETTER YOT => LATIN SMALL LETTER J - "\xD0\x85" => "\x53", // CYRILLIC CAPITAL LETTER DZE => LATIN CAPITAL LETTER S - "\xD0\x88" => "\x4A", // CYRILLIC CAPITAL LETTER JE => LATIN CAPITAL LETTER J - "\xD0\x91" => "\xC6\x82", // CYRILLIC CAPITAL LETTER BE => LATIN CAPITAL LETTER B WITH TOPBAR - "\xD0\x93" => "\xCE\x93", // CYRILLIC CAPITAL LETTER GHE => GREEK CAPITAL LETTER GAMMA - "\xD0\x9F" => "\xCE\xA0", // CYRILLIC CAPITAL LETTER PE => GREEK CAPITAL LETTER PI - "\xD0\xA1" => "\x43", // CYRILLIC CAPITAL LETTER ES => LATIN CAPITAL LETTER C - "\xD0\xB0" => "\x61", // CYRILLIC SMALL LETTER A => LATIN SMALL LETTER A - "\xD0\xB5" => "\x65", // CYRILLIC SMALL LETTER IE => LATIN SMALL LETTER E - "\xD0\xBA" => "\xC4\xB8", // CYRILLIC SMALL LETTER KA => LATIN SMALL LETTER KRA - "\xD0\xBE" => "\x6F", // CYRILLIC SMALL LETTER O => LATIN SMALL LETTER O - "\xD1\x80" => "\x70", // CYRILLIC SMALL LETTER ER => LATIN SMALL LETTER P - "\xD1\x81" => "\x63", // CYRILLIC SMALL LETTER ES => LATIN SMALL LETTER C - "\xD1\x83" => "\x79", // CYRILLIC SMALL LETTER U => LATIN SMALL LETTER Y - "\xD1\x85" => "\x78", // CYRILLIC SMALL LETTER HA => LATIN SMALL LETTER X - "\xD1\x95" => "\x73", // CYRILLIC SMALL LETTER DZE => LATIN SMALL LETTER S - "\xD1\x96" => "\x69", // CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I => LATIN SMALL LETTER I - "\xD1\x98" => "\x6A", // CYRILLIC SMALL LETTER JE => LATIN SMALL LETTER J - "\xD2\xBB" => "\x68", // CYRILLIC SMALL LETTER SHHA => LATIN SMALL LETTER H - "\xD3\x8F" => "\xC9\xAA", // CYRILLIC SMALL LETTER PALOCHKA => LATIN LETTER SMALL CAPITAL I - "\xD3\x94" => "\xC3\x86", // CYRILLIC CAPITAL LIGATURE A IE => LATIN CAPITAL LETTER AE - "\xD3\x95" => "\xC3\xA6", // CYRILLIC SMALL LIGATURE A IE => LATIN SMALL LETTER AE - "\xD3\x98" => "\xC6\x8E", // CYRILLIC CAPITAL LETTER SCHWA => LATIN CAPITAL LETTER REVERSED E - "\xD3\x99" => "\xC7\x9D", // CYRILLIC SMALL LETTER SCHWA => LATIN SMALL LETTER TURNED E - "\xD3\xA1" => "\xCA\x92", // CYRILLIC SMALL LETTER ABKHASIAN DZE => LATIN SMALL LETTER EZH - "\xD3\xA8" => "\xC6\x9F", // CYRILLIC CAPITAL LETTER BARRED O => LATIN CAPITAL LETTER O WITH MIDDLE TILDE - "\xD3\xA9" => "\xC9\xB5", // CYRILLIC SMALL LETTER BARRED O => LATIN SMALL LETTER BARRED O - "\xD4\x81" => "\x64", // CYRILLIC SMALL LETTER KOMI DE => LATIN SMALL LETTER D - "\xE1\x81\x80" => "\xE1\x80\x9D", // MYANMAR DIGIT ZERO => MYANMAR LETTER WA - "\xE1\x9E\xA3" => "\xE1\x9E\xA2", // KHMER INDEPENDENT VOWEL QAQ => KHMER LETTER QA - "\xE1\xA1\x95" => "\xE1\xA0\xB5", // MONGOLIAN LETTER TODO YA => MONGOLIAN LETTER JA - "\xE1\xA7\x90" => "\xE1\xA6\x9E", // NEW TAI LUE DIGIT ZERO => NEW TAI LUE LETTER LOW VA - "\xE1\xAD\x92" => "\xE1\xAC\x8D", // BALINESE DIGIT TWO => BALINESE LETTER LA LENGA - "\xE1\xAD\x93" => "\xE1\xAC\x91", // BALINESE DIGIT THREE => BALINESE LETTER OKARA - "\xE1\xAD\x98" => "\xE1\xAC\xA8", // BALINESE DIGIT EIGHT => BALINESE LETTER PA KAPAL - "\xE1\xAD\x9C" => "\xE1\xAD\x90", // BALINESE WINDU => BALINESE DIGIT ZERO - "\xE1\xB4\x8D" => "\xD0\xBC", // LATIN LETTER SMALL CAPITAL M => CYRILLIC SMALL LETTER EM - "\xE1\xB4\x9B" => "\xD1\x82", // LATIN LETTER SMALL CAPITAL T => CYRILLIC SMALL LETTER TE - "\xE1\xB4\xA6" => "\xD0\xB3", // GREEK LETTER SMALL CAPITAL GAMMA => CYRILLIC SMALL LETTER GHE - "\xE1\xB4\xA8" => "\xD0\xBF", // GREEK LETTER SMALL CAPITAL PI => CYRILLIC SMALL LETTER PE - "\xE1\xB4\xA9" => "\xE1\xB4\x98", // GREEK LETTER SMALL CAPITAL RHO => LATIN LETTER SMALL CAPITAL P - "\xE1\xB4\xAB" => "\xD0\xBB", // CYRILLIC LETTER SMALL CAPITAL EL => CYRILLIC SMALL LETTER EL - "\xE2\x8D\xB3" => "\xC9\xA9", // APL FUNCTIONAL SYMBOL IOTA => LATIN SMALL LETTER IOTA - "\xE2\x8D\xB4" => "\xCF\x81", // APL FUNCTIONAL SYMBOL RHO => GREEK SMALL LETTER RHO - "\xE2\x8D\xB5" => "\xCF\x89", // APL FUNCcTIONAL SYMBOL OMEGA => GREEK SMALL LETTER OMEGA - "\xE2\x8D\xBA" => "\xCE\xB1", // APL FUNCTIONAL SYMBOL ALPHA => GREEK SMALL LETTER ALPHA - "\xE2\xB1\xA7" => "\xD2\xA2", // LATIN CAPITAL LETTER H WITH DESCENDER => CYRILLIC CAPITAL LETTER EN WITH DESCENDER - "\xE2\xB1\xA9" => "\xD2\x9A", // LATIN CAPITAL LETTER K WITH DESCENDER => CYRILLIC CAPITAL LETTER KA WITH DESCENDER - "\xF0\x90\x8F\x91" => "\xF0\x90\x8E\x82", // OLD PERSIAN NUMBER ONE => UGARITIC LETTER GAMLA - "\xF0\x90\x8F\x93" => "\xF0\x90\x8E\x93", // OLD PERSIAN NUMBER TEN => UGARITIC LETTER AIN - "\xF0\x90\x92\xA0" => "\xF0\x90\x92\x86", // OSMANYA DIGIT ZERO => OSMANYA LETTER DEEL - "\xF0\x92\x80\xB8" => "\xF0\x90\x8E\x9A", // CUNEIFORM SIGN ASH => UGARITIC LETTER TO - - "\xC2\xA0" => "\x20", // NO-BREAK SPACE - "\xE1\x9A\x80" => "\x20", // OGHAM SPACE MARK - "\xE2\x80\x80" => "\x20", // EN QUAD - "\xE2\x80\x81" => "\x20", // EM QUAD - "\xE2\x80\x82" => "\x20", // EN SPACE - "\xE2\x80\x83" => "\x20", // EM SPACE - "\xE2\x80\x84" => "\x20", // THREE-PER-EM SPACE - "\xE2\x80\x85" => "\x20", // FOUR-PER-EM SPACE - "\xE2\x80\x86" => "\x20", // SIX-PER-EM SPACE - "\xE2\x80\x87" => "\x20", // FIGURE SPACE - "\xE2\x80\x88" => "\x20", // PUNCTUATION SPACE - "\xE2\x80\x89" => "\x20", // THIN SPACE - "\xE2\x80\x8A" => "\x20", // HAIR SPACE - "\xE2\x80\xAF" => "\x20", // NARROW NO-BREAK SPACE - "\xE2\x81\x9F" => "\x20", // MEDIUM MATHEMATICAL SPACE - "\xE3\x80\x80" => "\x20", // IDEOGRAPHIC SPACE - - "\xDB\x9D" => '', // ARABIC END OF AYAH - "\xDC\x8F" => '', // SYRIAC ABBREVIATION MARK - "\xE1\xA0\x86" => '', // MONGOLIAN TODO SOFT HYPHEN - "\xE1\xA0\x8E" => '', // MONGOLIAN VOWEL SEPARATOR - "\xE2\x80\x8B" => '', // ZERO WIDTH SPACE - "\xE2\x80\x8C" => '', // ZERO WIDTH NON-JOINER - "\xE2\x80\x8D" => '', // ZERO WIDTH JOINER - "\xE2\x80\xA8" => '', // LINE SEPARATOR - "\xE2\x80\xA9" => '', // PARAGRAPH SEPARATOR - "\xE2\x81\xA0" => '', // WORD JOINER - "\xE2\x81\xA1" => '', // FUNCTION APPLICATION - "\xE2\x81\xA2" => '', // INVISIBLE TIMES - "\xE2\x81\xA3" => '', // INVISIBLE SEPARATOR - "\xE2\x81\xAA" => '', // [CONTROL CHARACTERS] - "\xE2\x81\xAB" => '', // [CONTROL CHARACTERS] - "\xE2\x81\xAC" => '', // [CONTROL CHARACTERS] - "\xE2\x81\xAD" => '', // [CONTROL CHARACTERS] - "\xE2\x81\xAE" => '', // [CONTROL CHARACTERS] - "\xE2\x81\xAF" => '', // [CONTROL CHARACTERS] - "\xEF\xBB\xBF" => '', // ZERO WIDTH NO-BREAK SPACE - "\xEF\xBF\xB9" => '', // [CONTROL CHARACTERS] - "\xEF\xBF\xBA" => '', // [CONTROL CHARACTERS] - "\xEF\xBF\xBB" => '', // [CONTROL CHARACTERS] - "\xEF\xBF\xBC" => '', // [CONTROL CHARACTERS] - "\xF0\x9D\x85\xB3" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xB4" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xB5" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xB6" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xB7" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xB8" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xB9" => '', // [MUSICAL CONTROL CHARACTERS] - "\xF0\x9D\x85\xBA" => '', // [MUSICAL CONTROL CHARACTERS] - ); - + $text = utf8_case_fold_nfkc($text); $text = strtr($text, $homographs); - // Other control characters $text = preg_replace('#(?:[\x00-\x1F\x7F]+|(?:\xC2[\x80-\x9F])+)#', '', $text); - return $text; + // we can use trim here as all the other space characters should have been turned + // into normal ASCII spaces by now + return trim($text); } /** |