- search result extract shouldn't end in the middle of a multibyte character [Bug #11863]

- missing localisation for an imageset shouldn't create lots of "imageset refreshed" log messages [Bug #12027] - explain that themes which need parsing cannot be stored on the filesystem [Bug #11134] - normalize usernames (we really need to make sure we normalize everything) - improved utf8_clean_string, more complete list of homographs and NFKC normalization, also the resulting string is now trimmed - corrected searching subforums explanation [Bug #12209] git-svn-id: file:///svn/phpbb/trunk@7890 89ea8834-ac86-4346-8a33-228a782c2dd0
author: Nils Adermann <naderman@naderman.de> 2007-07-15 20:53:27 +0000
committer: Nils Adermann <naderman@naderman.de> 2007-07-15 20:53:27 +0000
commit: 909e195a9b54f38294f217ee8e10b17a62876756 (patch)
tree: 2d7999ce793f784c8a0915c98bf957b322c8952e /phpBB/develop
parent: f27fa04b8c7f6d668e9cc651afaa10334f5d7067 (diff)
download: forums-909e195a9b54f38294f217ee8e10b17a62876756.tar
forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.gz
forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.bz2
forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.xz
forums-909e195a9b54f38294f217ee8e10b17a62876756.zip
2 files changed, 331 insertions, 0 deletions
diff --git a/phpBB/develop/generate_utf_confusables.php b/phpBB/develop/generate_utf_confusables.php
new file mode 100644
index 0000000000..c4ffd21fef
--- /dev/null
+++ b/phpBB/develop/generate_utf_confusables.php
@@ -0,0 +1,211 @@
+<?php
+/** 
+*
+* @package phpBB3
+* @version $Id$
+* @copyright (c) 2005 phpBB Group 
+* @license http://opensource.org/licenses/gpl-license.php GNU Public License 
+*
+*/
+
+if (php_sapi_name() != 'cli')
+{
+	die("This program must be run from the command line.\n");
+}
+
+//
+// Security message:
+//
+// This script is potentially dangerous.
+// Remove or comment the next line (die(".... ) to enable this script.
+// Do NOT FORGET to either remove this script or disable it after you have used it.
+//
+die("Please read the first lines of this script for instructions on how to enable it");
+
+set_time_limit(0);
+
+define('IN_PHPBB', true);
+$phpbb_root_path = '../';
+$phpEx = substr(strrchr(__FILE__, '.'), 1);
+
+echo "Checking for required files\n";
+download('http://unicode.org/reports/tr39/data/confusables.txt');
+echo "\n";
+
+
+/**
+* Load the CaseFolding table
+*/
+echo "Loading confusables\n";
+$unidata = file_get_contents('confusables.txt');
+
+
+function utf8_chr($cp)
+{
+    if ($cp > 0xFFFF)
+    {
+        return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
+    }
+    else if ($cp > 0x7FF)
+    {
+        return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
+    }
+    else if ($cp > 0x7F)
+    {
+        return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
+    }
+    else
+    {
+        return chr($cp);
+    }
+}
+
+preg_match_all('/^([0-9A-F]+) ;\s((?:[0-9A-F]+ )*);/im', $unidata, $array, PREG_SET_ORDER);
+
+// some that we defined ourselves
+$uniarray = array(
+		"\xC2\xA1"			=>	"\x69",	// EXCLAMATION MARK, INVERTED => LATIN SMALL LETTER I
+		"\xC7\x83"			=>	"\x21",	// LATIN LETTER RETROFLEX CLICK => EXCLAMATION MARK
+		"\xCE\xB1"			=>	"\x61",	// GREEK SMALL LETTER ALPHA => LATIN SMALL LETTER A
+		"\xE1\x9A\x80"		=>	"\x20",	// OGHAM SPACE MARK
+
+		"\xC2\xAD"			=>	'',		// HYPHEN, SOFT => empty string
+		"\xDB\x9D"			=>	'',		// ARABIC END OF AYAH
+		"\xDC\x8F"			=>	'',		// SYRIAC ABBREVIATION MARK
+		"\xE1\xA0\x86"		=>	'',		// MONGOLIAN TODO SOFT HYPHEN
+		"\xE1\xA0\x8E"		=>	'',		// MONGOLIAN VOWEL SEPARATOR
+		"\xE2\x80\x8B"		=>	'',		// ZERO WIDTH SPACE
+		"\xE2\x80\x8C"		=>	'',		// ZERO WIDTH NON-JOINER
+		"\xE2\x80\x8D"		=>	'',		// ZERO WIDTH JOINER
+		"\xE2\x80\xA8"		=>	'',		// LINE SEPARATOR
+		"\xE2\x80\xA9"		=>	'',		// PARAGRAPH SEPARATOR
+		"\xE2\x81\xA0"		=>	'',		// WORD JOINER
+		"\xE2\x81\xA1"		=>	'',		// FUNCTION APPLICATION
+		"\xE2\x81\xA2"		=>	'',		// INVISIBLE TIMES
+		"\xE2\x81\xA3"		=>	'',		// INVISIBLE SEPARATOR
+		"\xE2\x81\xAA"		=>	'',		// [CONTROL CHARACTERS]
+		"\xE2\x81\xAB"		=>	'',		// [CONTROL CHARACTERS]
+		"\xE2\x81\xAC"		=>	'',		// [CONTROL CHARACTERS]
+		"\xE2\x81\xAD"		=>	'',		// [CONTROL CHARACTERS]
+		"\xE2\x81\xAE"		=>	'',		// [CONTROL CHARACTERS]
+		"\xE2\x81\xAF"		=>	'',		// [CONTROL CHARACTERS]
+		"\xEF\xBB\xBF"		=>	'',		// ZERO WIDTH NO-BREAK SPACE
+		"\xEF\xBF\xB9"		=>	'',		// [CONTROL CHARACTERS]
+		"\xEF\xBF\xBA"		=>	'',		// [CONTROL CHARACTERS]
+		"\xEF\xBF\xBB"		=>	'',		// [CONTROL CHARACTERS]
+		"\xEF\xBF\xBC"		=>	'',		// [CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB3"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB4"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB5"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB6"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB7"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB8"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xB9"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+		"\xF0\x9D\x85\xBA"	=>	'',		// [MUSICAL CONTROL CHARACTERS]
+);
+
+$copy = $uniarray;
+
+foreach ($array as $value)
+{
+	if (isset($copy[utf8_chr(hexdec((string)$value[1]))]))
+	{
+		$num = '';
+		$string = utf8_chr(hexdec((string)$value[1]));
+		for ($i = 0; $i < strlen($string); $i++)
+		{
+			$num .= '\x' . str_pad(base_convert(ord($string[$i]), 10, 16), 2, '0', STR_PAD_LEFT);
+		}
+		echo $num . "\n";
+		if ($uniarray[$string] != implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2]))))))
+		{
+			echo "  --> $string\n";
+			echo "  --> " . implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2]))))) . "\n";
+		}
+	}
+    $uniarray[utf8_chr(hexdec((string)$value[1]))] = implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2])))));
+}
+
+echo "Writing to confusables.$phpEx\n";
+
+$fp = fopen($phpbb_root_path . 'includes/utf/data/confusables.' . $phpEx, 'wb');
+fwrite($fp, '<?php return ' . my_var_export($uniarray) . ';');
+fclose($fp);
+
+/**
+* Return a parsable string representation of a variable
+*
+* This is function is limited to array/strings/integers
+*
+* @param	mixed	$var		Variable
+* @return	string				PHP code representing the variable
+*/
+function my_var_export($var)
+{
+	if (is_array($var))
+	{
+		$lines = array();
+
+		foreach ($var as $k => $v)
+		{
+			$lines[] = my_var_export($k) . '=>' . my_var_export($v);
+		}
+
+		return 'array(' . implode(',', $lines) . ')';
+	}
+	else if (is_string($var))
+	{
+		return "'" . str_replace(array('\\', "'"), array('\\\\', "\\'"), $var) . "'";
+	}
+	else
+	{
+		return $var;
+	}
+}
+
+/**
+* Download a file to the develop/ dir
+*
+* @param	string	$url		URL of the file to download
+* @return	void
+*/
+function download($url)
+{
+	global $phpbb_root_path;
+
+	if (file_exists($phpbb_root_path . 'develop/' . basename($url)))
+	{
+		return;
+	}
+
+	echo 'Downloading from ', $url, ' ';
+
+	if (!$fpr = fopen($url, 'rb'))
+	{
+		die("Can't download from $url\nPlease download it yourself and put it in the develop/ dir, kthxbai");
+	}
+
+	if (!$fpw = fopen($phpbb_root_path . 'develop/' . basename($url), 'wb'))
+	{
+		die("Can't open develop/" . basename($url) . " for output... please check your permissions or something");
+	}
+
+	$i = 0;
+	$chunk = 32768;
+	$done = '';
+
+	while (!feof($fpr))
+	{
+		$i += fwrite($fpw, fread($fpr, $chunk));
+		echo str_repeat("\x08", strlen($done));
+
+		$done = ($i >> 10) . ' KiB';
+		echo $done;
+	}
+	fclose($fpr);
+	fclose($fpw);
+
+	echo "\n";
+}
+
+?>
+\ No newline at end of file
diff --git a/phpBB/develop/unicode_testing.php b/phpBB/develop/unicode_testing.php
new file mode 100644
index 0000000000..25a13d1325
--- /dev/null
+++ b/phpBB/develop/unicode_testing.php
@@ -0,0 +1,120 @@
+<?php
+//
+// This file provides some useful functions for debugging the unicode/UTF-8 library
+// It requires utf_tools.php to be loaded
+//
+die("Please read the first lines of this script for instructions on how to enable it");
+
+if (!headers_sent())
+{
+	header('Content-type: text/html; charset=UTF-8');
+}
+
+/**
+ * Converts unicode escape sequences (\u0123) into UTF-8 characters
+ *
+ * @param	string	A unicode sequence
+ * @return	string	UTF-8 representation of the given unicode sequence
+ */
+function unicode_to_utf8($string)
+{
+	$utf8 = '';
+	$chars = array();
+	for ($i = 0; $i < strlen($string); $i++)
+	{
+		if (isset($string[$i + 5]) && substr($string, $i, 2) == '\\u' && ctype_xdigit(substr($string, $i + 2, 4)))
+		{
+			$utf8 .= utf8_from_unicode(array(base_convert(substr($string, $i + 2, 4), 16, 10)));
+			$i += 5;
+		}
+		else
+		{
+			$utf8 .= $string[$i];
+		}
+	}
+	return $utf8;
+}
+
+/**
+ * Takes an array of ints representing the Unicode characters and returns
+ * a UTF-8 string.
+ *
+ * @param array $array array of unicode code points representing a string
+ * @return string UTF-8 character string
+ */
+function utf8_from_unicode($array)
+{
+	$str = '';
+	foreach ($array as $value)
+	{
+		$str .= utf8_chr($value);
+	}
+	return $str;
+}
+
+/**
+* Converts a UTF-8 string to unicode code points
+*
+* @param	string	$text		UTF-8 string
+* @return	string				Unicode code points
+*/
+function utf8_to_unicode($text)
+{
+	return preg_replace_callback(
+		'#[\\xC2-\\xF4][\\x80-\\xBF]?[\\x80-\\xBF]?[\\x80-\\xBF]#',
+		'utf8_to_unicode_callback',
+		preg_replace_callback(
+			'#[\\x00-\\x7f]#',
+			'utf8_to_unicode_callback',
+			$text
+		)
+	);
+}
+
+/**
+* Takes a UTF-8 char and replaces it with its unicode escape sequence. Attention, $m is an array
+*
+* @param	array	$m			0-based numerically indexed array passed by preg_replace_callback()
+* @return	string				A unicode escape sequence
+*/
+function utf8_to_unicode_callback($m)
+{
+	return '\u' . str_pad(base_convert(utf8_ord($m[0]), 10, 16), 4, '0', STR_PAD_LEFT) . '';
+}
+
+/**
+* A wrapper function for the normalizer which takes care of including the class if required and modifies the passed strings
+* to be in NFKC
+*
+* @param	mixed	$strings	a string or an array of strings to normalize
+* @return	mixed				the normalized content, preserving array keys if array given.
+*/
+function utf8_normalize_nfkc($strings)
+{
+	if (empty($strings))
+	{
+		return $strings;
+	}
+
+	if (!class_exists('utf_normalizer'))
+	{
+		global $phpbb_root_path, $phpEx;
+		include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
+	}
+
+	if (!is_array($strings))
+	{
+		utf_normalizer::nfkc($strings);
+	}
+	else if (is_array($strings))
+	{
+		foreach ($strings as $key => $string)
+		{
+			utf_normalizer::nfkc($strings[$key]);
+		}
+	}
+
+	return $strings;
+}
+
+?>
+\ No newline at end of file
author	Nils Adermann <naderman@naderman.de>	2007-07-15 20:53:27 +0000
committer	Nils Adermann <naderman@naderman.de>	2007-07-15 20:53:27 +0000
commit	909e195a9b54f38294f217ee8e10b17a62876756 (patch)
tree	2d7999ce793f784c8a0915c98bf957b322c8952e /phpBB/develop
parent	f27fa04b8c7f6d668e9cc651afaa10334f5d7067 (diff)
download	forums-909e195a9b54f38294f217ee8e10b17a62876756.tar forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.gz forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.bz2 forums-909e195a9b54f38294f217ee8e10b17a62876756.tar.xz forums-909e195a9b54f38294f217ee8e10b17a62876756.zip