aboutsummaryrefslogtreecommitdiffstats
path: root/phpBB/includes/utf/utf_tools.php
diff options
context:
space:
mode:
authorNils Adermann <naderman@naderman.de>2007-07-15 20:53:27 +0000
committerNils Adermann <naderman@naderman.de>2007-07-15 20:53:27 +0000
commit909e195a9b54f38294f217ee8e10b17a62876756 (patch)
tree2d7999ce793f784c8a0915c98bf957b322c8952e /phpBB/includes/utf/utf_tools.php
parentf27fa04b8c7f6d668e9cc651afaa10334f5d7067 (diff)
downloadforums-909e195a9b54f38294f217ee8e10b17a62876756.tar
forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.gz
forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.bz2
forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.xz
forums-909e195a9b54f38294f217ee8e10b17a62876756.zip
- search result extract shouldn't end in the middle of a multibyte character [Bug #11863]
- missing localisation for an imageset shouldn't create lots of "imageset refreshed" log messages [Bug #12027] - explain that themes which need parsing cannot be stored on the filesystem [Bug #11134] - normalize usernames (we really need to make sure we normalize everything) - improved utf8_clean_string, more complete list of homographs and NFKC normalization, also the resulting string is now trimmed - corrected searching subforums explanation [Bug #12209] git-svn-id: file:///svn/phpbb/trunk@7890 89ea8834-ac86-4346-8a33-228a782c2dd0
Diffstat (limited to 'phpBB/includes/utf/utf_tools.php')
-rw-r--r--phpBB/includes/utf/utf_tools.php150
1 files changed, 9 insertions, 141 deletions
diff --git a/phpBB/includes/utf/utf_tools.php b/phpBB/includes/utf/utf_tools.php
index d39220b613..cb13d370a4 100644
--- a/phpBB/includes/utf/utf_tools.php
+++ b/phpBB/includes/utf/utf_tools.php
@@ -1848,154 +1848,22 @@ function utf8_normalize_nfc($strings)
*/
function utf8_clean_string($text)
{
- $text = utf8_case_fold($text);
-
- if (!class_exists('utf_normalizer'))
+ global $phpbb_root_path, $phpEx;
+
+ static $homographs = array();
+ if (empty($homographs))
{
- global $phpbb_root_path, $phpEx;
- include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
+ $homographs = include($phpbb_root_path . 'includes/utf/data/confusables.' . $phpEx);
}
- utf_normalizer::nfc($text);
-
- static $homographs = array(
- "\xC2\xA1" => "\x69", // EXCLAMATION MARK, INVERTED => LATIN SMALL LETTER I
- "\xC2\xAD" => '', // HYPHEN, SOFT => empty string
- "\xC4\x90" => "\xC3\x90", // LATIN CAPITAL LETTER D WITH STROKE => LATIN CAPITAL LETTER ETH
- "\xC7\x83" => "\x21", // LATIN LETTER RETROFLEX CLICK => EXCLAMATION MARK
- "\xC9\x85" => "\xCE\x9B", // LATIN CAPITAL LETTER TURNED V => GREEK CAPITAL LETTER LAMDA
- "\xC9\x99" => "\xC7\x9D", // LATIN SMALL LETTER SCHWA => LATIN SMALL LETTER TURNED E
- "\xCA\x99" => "\xD0\xB2", // LATIN LETTER SMALL CAPITAL B => CYRILLIC SMALL LETTER VE
- "\xCA\x9C" => "\xD0\xBD", // LATIN LETTER SMALL CAPITAL H => CYRILLIC SMALL LETTER EN
- "\xCE\x91" => "\x41", // GREEK CAPITAL LETTER ALPHA => LATIN CAPITAL LETTER A
- "\xCE\x92" => "\x42", // GREEK CAPITAL LETTER BETA => LATIN CAPITAL LETTER B
- "\xCE\x95" => "\x45", // GREEK CAPITAL LETTER EPSILON => LATIN CAPITAL LETTER E
- "\xCE\x96" => "\x5A", // GREEK CAPITAL LETTER ZETA => LATIN CAPITAL LETTER Z
- "\xCE\x97" => "\x48", // GREEK CAPITAL LETTER ETA => LATIN CAPITAL LETTER H
- "\xCE\x99" => "\x49", // GREEK CAPITAL LETTER IOTA => LATIN CAPITAL LETTER I
- "\xCE\x9A" => "\x4B", // GREEK CAPITAL LETTER KAPPA => LATIN CAPITAL LETTER K
- "\xCE\x9C" => "\x4D", // GREEK CAPITAL LETTER MU => LATIN CAPITAL LETTER M
- "\xCE\x9D" => "\x4E", // GREEK CAPITAL LETTER NU => LATIN CAPITAL LETTER N
- "\xCE\x9F" => "\x4F", // GREEK CAPITAL LETTER OMICRON => LATIN CAPITAL LETTER O
- "\xCE\xA1" => "\x50", // GREEK CAPITAL LETTER RHO => LATIN CAPITAL LETTER P
- "\xCE\xA3" => "\xC6\xA9", // GREEK CAPITAL LETTER SIGMA => LATIN CAPITAL LETTER ESH
- "\xCE\xA4" => "\x54", // GREEK CAPITAL LETTER TAU => LATIN CAPITAL LETTER T
- "\xCE\xA5" => "\x59", // GREEK CAPITAL LETTER UPSILON => LATIN CAPITAL LETTER Y
- "\xCE\xA7" => "\x58", // GREEK CAPITAL LETTER CHI => LATIN CAPITAL LETTER X
- "\xCE\xB1" => "\x61", // GREEK SMALL LETTER ALPHA => LATIN SMALL LETTER A
- "\xCE\xB5" => "\xC9\x9B", // GREEK SMALL LETTER EPSILON => LATIN SMALL LETTER OPEN E
- "\xCE\xB9" => "\xC9\xA9", // GREEK SMALL LETTER IOTA => LATIN SMALL LETTER IOTA
- "\xCE\xBF" => "\x6F", // GREEK SMALL LETTER OMICRON => LATIN SMALL LETTER O
- "\xCF\xB3" => "\x6A", // GREEK LETTER YOT => LATIN SMALL LETTER J
- "\xD0\x85" => "\x53", // CYRILLIC CAPITAL LETTER DZE => LATIN CAPITAL LETTER S
- "\xD0\x88" => "\x4A", // CYRILLIC CAPITAL LETTER JE => LATIN CAPITAL LETTER J
- "\xD0\x91" => "\xC6\x82", // CYRILLIC CAPITAL LETTER BE => LATIN CAPITAL LETTER B WITH TOPBAR
- "\xD0\x93" => "\xCE\x93", // CYRILLIC CAPITAL LETTER GHE => GREEK CAPITAL LETTER GAMMA
- "\xD0\x9F" => "\xCE\xA0", // CYRILLIC CAPITAL LETTER PE => GREEK CAPITAL LETTER PI
- "\xD0\xA1" => "\x43", // CYRILLIC CAPITAL LETTER ES => LATIN CAPITAL LETTER C
- "\xD0\xB0" => "\x61", // CYRILLIC SMALL LETTER A => LATIN SMALL LETTER A
- "\xD0\xB5" => "\x65", // CYRILLIC SMALL LETTER IE => LATIN SMALL LETTER E
- "\xD0\xBA" => "\xC4\xB8", // CYRILLIC SMALL LETTER KA => LATIN SMALL LETTER KRA
- "\xD0\xBE" => "\x6F", // CYRILLIC SMALL LETTER O => LATIN SMALL LETTER O
- "\xD1\x80" => "\x70", // CYRILLIC SMALL LETTER ER => LATIN SMALL LETTER P
- "\xD1\x81" => "\x63", // CYRILLIC SMALL LETTER ES => LATIN SMALL LETTER C
- "\xD1\x83" => "\x79", // CYRILLIC SMALL LETTER U => LATIN SMALL LETTER Y
- "\xD1\x85" => "\x78", // CYRILLIC SMALL LETTER HA => LATIN SMALL LETTER X
- "\xD1\x95" => "\x73", // CYRILLIC SMALL LETTER DZE => LATIN SMALL LETTER S
- "\xD1\x96" => "\x69", // CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I => LATIN SMALL LETTER I
- "\xD1\x98" => "\x6A", // CYRILLIC SMALL LETTER JE => LATIN SMALL LETTER J
- "\xD2\xBB" => "\x68", // CYRILLIC SMALL LETTER SHHA => LATIN SMALL LETTER H
- "\xD3\x8F" => "\xC9\xAA", // CYRILLIC SMALL LETTER PALOCHKA => LATIN LETTER SMALL CAPITAL I
- "\xD3\x94" => "\xC3\x86", // CYRILLIC CAPITAL LIGATURE A IE => LATIN CAPITAL LETTER AE
- "\xD3\x95" => "\xC3\xA6", // CYRILLIC SMALL LIGATURE A IE => LATIN SMALL LETTER AE
- "\xD3\x98" => "\xC6\x8E", // CYRILLIC CAPITAL LETTER SCHWA => LATIN CAPITAL LETTER REVERSED E
- "\xD3\x99" => "\xC7\x9D", // CYRILLIC SMALL LETTER SCHWA => LATIN SMALL LETTER TURNED E
- "\xD3\xA1" => "\xCA\x92", // CYRILLIC SMALL LETTER ABKHASIAN DZE => LATIN SMALL LETTER EZH
- "\xD3\xA8" => "\xC6\x9F", // CYRILLIC CAPITAL LETTER BARRED O => LATIN CAPITAL LETTER O WITH MIDDLE TILDE
- "\xD3\xA9" => "\xC9\xB5", // CYRILLIC SMALL LETTER BARRED O => LATIN SMALL LETTER BARRED O
- "\xD4\x81" => "\x64", // CYRILLIC SMALL LETTER KOMI DE => LATIN SMALL LETTER D
- "\xE1\x81\x80" => "\xE1\x80\x9D", // MYANMAR DIGIT ZERO => MYANMAR LETTER WA
- "\xE1\x9E\xA3" => "\xE1\x9E\xA2", // KHMER INDEPENDENT VOWEL QAQ => KHMER LETTER QA
- "\xE1\xA1\x95" => "\xE1\xA0\xB5", // MONGOLIAN LETTER TODO YA => MONGOLIAN LETTER JA
- "\xE1\xA7\x90" => "\xE1\xA6\x9E", // NEW TAI LUE DIGIT ZERO => NEW TAI LUE LETTER LOW VA
- "\xE1\xAD\x92" => "\xE1\xAC\x8D", // BALINESE DIGIT TWO => BALINESE LETTER LA LENGA
- "\xE1\xAD\x93" => "\xE1\xAC\x91", // BALINESE DIGIT THREE => BALINESE LETTER OKARA
- "\xE1\xAD\x98" => "\xE1\xAC\xA8", // BALINESE DIGIT EIGHT => BALINESE LETTER PA KAPAL
- "\xE1\xAD\x9C" => "\xE1\xAD\x90", // BALINESE WINDU => BALINESE DIGIT ZERO
- "\xE1\xB4\x8D" => "\xD0\xBC", // LATIN LETTER SMALL CAPITAL M => CYRILLIC SMALL LETTER EM
- "\xE1\xB4\x9B" => "\xD1\x82", // LATIN LETTER SMALL CAPITAL T => CYRILLIC SMALL LETTER TE
- "\xE1\xB4\xA6" => "\xD0\xB3", // GREEK LETTER SMALL CAPITAL GAMMA => CYRILLIC SMALL LETTER GHE
- "\xE1\xB4\xA8" => "\xD0\xBF", // GREEK LETTER SMALL CAPITAL PI => CYRILLIC SMALL LETTER PE
- "\xE1\xB4\xA9" => "\xE1\xB4\x98", // GREEK LETTER SMALL CAPITAL RHO => LATIN LETTER SMALL CAPITAL P
- "\xE1\xB4\xAB" => "\xD0\xBB", // CYRILLIC LETTER SMALL CAPITAL EL => CYRILLIC SMALL LETTER EL
- "\xE2\x8D\xB3" => "\xC9\xA9", // APL FUNCTIONAL SYMBOL IOTA => LATIN SMALL LETTER IOTA
- "\xE2\x8D\xB4" => "\xCF\x81", // APL FUNCTIONAL SYMBOL RHO => GREEK SMALL LETTER RHO
- "\xE2\x8D\xB5" => "\xCF\x89", // APL FUNCTIONAL SYMBOL OMEGA => GREEK SMALL LETTER OMEGA
- "\xE2\x8D\xBA" => "\xCE\xB1", // APL FUNCTIONAL SYMBOL ALPHA => GREEK SMALL LETTER ALPHA
- "\xE2\xB1\xA7" => "\xD2\xA2", // LATIN CAPITAL LETTER H WITH DESCENDER => CYRILLIC CAPITAL LETTER EN WITH DESCENDER
- "\xE2\xB1\xA9" => "\xD2\x9A", // LATIN CAPITAL LETTER K WITH DESCENDER => CYRILLIC CAPITAL LETTER KA WITH DESCENDER
- "\xF0\x90\x8F\x91" => "\xF0\x90\x8E\x82", // OLD PERSIAN NUMBER ONE => UGARITIC LETTER GAMLA
- "\xF0\x90\x8F\x93" => "\xF0\x90\x8E\x93", // OLD PERSIAN NUMBER TEN => UGARITIC LETTER AIN
- "\xF0\x90\x92\xA0" => "\xF0\x90\x92\x86", // OSMANYA DIGIT ZERO => OSMANYA LETTER DEEL
- "\xF0\x92\x80\xB8" => "\xF0\x90\x8E\x9A", // CUNEIFORM SIGN ASH => UGARITIC LETTER TO
-
- "\xC2\xA0" => "\x20", // NO-BREAK SPACE
- "\xE1\x9A\x80" => "\x20", // OGHAM SPACE MARK
- "\xE2\x80\x80" => "\x20", // EN QUAD
- "\xE2\x80\x81" => "\x20", // EM QUAD
- "\xE2\x80\x82" => "\x20", // EN SPACE
- "\xE2\x80\x83" => "\x20", // EM SPACE
- "\xE2\x80\x84" => "\x20", // THREE-PER-EM SPACE
- "\xE2\x80\x85" => "\x20", // FOUR-PER-EM SPACE
- "\xE2\x80\x86" => "\x20", // SIX-PER-EM SPACE
- "\xE2\x80\x87" => "\x20", // FIGURE SPACE
- "\xE2\x80\x88" => "\x20", // PUNCTUATION SPACE
- "\xE2\x80\x89" => "\x20", // THIN SPACE
- "\xE2\x80\x8A" => "\x20", // HAIR SPACE
- "\xE2\x80\xAF" => "\x20", // NARROW NO-BREAK SPACE
- "\xE2\x81\x9F" => "\x20", // MEDIUM MATHEMATICAL SPACE
- "\xE3\x80\x80" => "\x20", // IDEOGRAPHIC SPACE
-
- "\xDB\x9D" => '', // ARABIC END OF AYAH
- "\xDC\x8F" => '', // SYRIAC ABBREVIATION MARK
- "\xE1\xA0\x86" => '', // MONGOLIAN TODO SOFT HYPHEN
- "\xE1\xA0\x8E" => '', // MONGOLIAN VOWEL SEPARATOR
- "\xE2\x80\x8B" => '', // ZERO WIDTH SPACE
- "\xE2\x80\x8C" => '', // ZERO WIDTH NON-JOINER
- "\xE2\x80\x8D" => '', // ZERO WIDTH JOINER
- "\xE2\x80\xA8" => '', // LINE SEPARATOR
- "\xE2\x80\xA9" => '', // PARAGRAPH SEPARATOR
- "\xE2\x81\xA0" => '', // WORD JOINER
- "\xE2\x81\xA1" => '', // FUNCTION APPLICATION
- "\xE2\x81\xA2" => '', // INVISIBLE TIMES
- "\xE2\x81\xA3" => '', // INVISIBLE SEPARATOR
- "\xE2\x81\xAA" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAB" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAC" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAD" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAE" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAF" => '', // [CONTROL CHARACTERS]
- "\xEF\xBB\xBF" => '', // ZERO WIDTH NO-BREAK SPACE
- "\xEF\xBF\xB9" => '', // [CONTROL CHARACTERS]
- "\xEF\xBF\xBA" => '', // [CONTROL CHARACTERS]
- "\xEF\xBF\xBB" => '', // [CONTROL CHARACTERS]
- "\xEF\xBF\xBC" => '', // [CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB3" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB4" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB5" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB6" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB7" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB8" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB9" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xBA" => '', // [MUSICAL CONTROL CHARACTERS]
- );
-
+ $text = utf8_case_fold_nfkc($text);
$text = strtr($text, $homographs);
-
// Other control characters
$text = preg_replace('#(?:[\x00-\x1F\x7F]+|(?:\xC2[\x80-\x9F])+)#', '', $text);
- return $text;
+ // we can use trim here as all the other space characters should have been turned
+ // into normal ASCII spaces by now
+ return trim($text);
}
/**
generated by cgit v1.2.1 (git 2.21.0) at 2024-11-23 07:36:08 +0000