* The ultimate convenience function! Clean up invalid UTF-8 sequences,

* and convert to Normal Form C, canonical composition.

*

* @return string The same string, all shiny and cleaned-up

*/

{

// The string below is the list of all autorized characters, sorted by frequency in latin text

$pos = strspn($str, "\x20\x65\x69\x61\x73\x6E\x74\x72\x6F\x6C\x75\x64\x5D\x5B\x63\x6D\x70\x27\x0A\x67\x7C\x68\x76\x2E\x66\x62\x2C\x3A\x3D\x2D\x71\x31\x30\x43\x32\x2A\x79\x78\x29\x28\x4C\x39\x41\x53\x2F\x50\x22\x45\x6A\x4D\x49\x6B\x33\x3E\x35\x54\x3C\x44\x34\x7D\x42\x7B\x38\x46\x77\x52\x36\x37\x55\x47\x4E\x3B\x4A\x7A\x56\x23\x48\x4F\x57\x5F\x26\x21\x4B\x3F\x58\x51\x25\x59\x5C\x09\x5A\x2B\x7E\x5E\x24\x40\x60\x7F\x0D");

if ($pos == $len)

{

// ASCII strings with no special chars return immediately

}

// Note: we do not check for $GLOBALS['utf_canonical_decomp']. It is assumed they are always loaded together

// Replace any byte in the range 0x00..0x1F, except for \r, \n and \t

// We replace those characters with a 0xFF byte, which is illegal in UTF-8 and will in turn be replaced with a UTF replacement char

);

}

/**

* Validate and normalize a UTF string to NFC

*

* @return string The string, validated and in normal form

*/

{

$pos = strspn($str, UTF8_ASCII_RANGE);

$len = strlen($str);

if ($pos == $len)

{

// ASCII strings return immediately

}

if (!isset($GLOBALS['utf_nfc_qc']))

include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx);

}

}

/**

* Validate and normalize a UTF string to NFKC

*

* @return string The string, validated and in normal form

*/

{

$pos = strspn($str, UTF8_ASCII_RANGE);

$len = strlen($str);

if ($pos == $len)

{

// ASCII strings return immediately

}

if (!isset($GLOBALS['utf_nfkc_qc']))

include($phpbb_root_path . 'includes/utf/data/utf_canonical_comp.' . $phpEx);

}

}

/**

* Validate and normalize a UTF string to NFD

*

* @return string The string, validated and in normal form

*/

{

$pos = strspn($str, UTF8_ASCII_RANGE);

$len = strlen($str);

if ($pos == $len)

{

// ASCII strings return immediately

}

if (!isset($GLOBALS['utf_canonical_decomp']))

include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx);

}

}

/**

* Validate and normalize a UTF string to NFKD

*

* @return string The string, validated and in normal form

*/

{

$pos = strspn($str, UTF8_ASCII_RANGE);

$len = strlen($str);

if ($pos == $len)

{

// ASCII strings return immediately

}

if (!isset($GLOBALS['utf_compatibility_decomp']))

include($phpbb_root_path . 'includes/utf/data/utf_compatibility_decomp.' . $phpEx);

}

}

/**

* Recompose a UTF string

*

*

* @access private

*/

$tmp = '';

$i = $tmp_pos = $last_cc = 0;

// UTF char length array

// This array is used to determine the length of a UTF character.

// has been encoded in a five- or six- byte sequence

if ($utf_char[0] >= "\xF8")

{

{

$trailing_bytes = 4;

}

{

$trailing_bytes = 0;

}

/**

* Decompose a UTF string

*

*

* @access private

*/

function decompose($str, $pos, $len, &$decomp_map)

{

// Load some commonly-used tables

if (!isset($utf_combining_class))

ksort($utf_sort);

}

{

$tmp .= implode('', $utf_chars);

}

// LIndex can only range from 0 to 18, therefore it cannot influence the first two bytes of the L Jamo, which allows us to hardcode them (based on LBase).

//

// The same goes for VIndex, but for TIndex there's a catch: the value of the third byte could exceed 0xBF and we would have to increment the second byte

{

{

$utf_char = "\xE1\x84\x00\xE1\x85\x00\xE1\x86\x00";

}

else

{

$utf_char = "\xE1\x84\x00\xE1\x85\x00\xE1\x87\x00";

}

}

else

}

return $tmp;

}

else if ($tmp_pos)

{

* @license http://opensource.org/licenses/gpl-license.php GNU Public License

*

* @todo make sure the replacements are called correctly

*/

/**

/**

* Implementation of PHP's native utf8_decode for people without XML support

*

* @return string ISO-8859-1 encoded data

*/

function utf8_decode($str)

return false;

}

}

}

else

function utf8_strrpos($str, $needle, $offset = null)

{

// offset for mb_strrpos was added in 5.2.0

{

// Emulate behaviour of strrpos rather than raising warning

if (empty($str))

return false;

}

}

else

{

$str = mb_substr($str, $offset);

{

return $pos + $offset;

}

*/

function utf8_strpos($str, $needle, $offset = null)

{

{

return mb_strpos($str, $needle);

}

* UTF-8 aware alternative to substr

* @ignore

*/

{

{

return mb_substr($str, $offset);

}

* Find position of last occurrence of a char in a string

*

* @author Harry Fuecks

* @return mixed integer position or FALSE on failure

*/

function utf8_strrpos($str, $needle, $offset = null)

* Find position of first occurrence of a string

*

* @author Harry Fuecks

* @return mixed integer position or FALSE on failure

*/

function utf8_strpos($str, $needle, $offset = null)

* necessary. It isn't necessary for +ve offsets and no specified length

*

* @author Chris Smith<chris@jalakai.co.uk>

* @return mixed string or FALSE if failure

*/

function utf8_substr($str, $offset, $length = NULL)

* Convert a string to an array

*

* @author Harry Fuecks

* @return string characters in string reverses

*/

function utf8_str_split($str, $split_len = 1)

* Find length of initial segment not matching mask

*

* @author Harry Fuecks

*/

function utf8_strspn($str, $mask, $start = null, $length = null)

{

/**

* Converts an NCR to a UTF-8 char

*

*/

function utf8_chr($cp)

{

* Convert Numeric Character References to UTF-8 chars

*

* Notes:

*

* @param string $text String to convert, encoded in UTF-8 (no normal form required)

* @return string UTF-8 string where NCRs have been replaced with the actual chars

* Takes an array of ints representing the Unicode characters and returns

* a UTF-8 string.

*

*/

function utf8_case_fold($text, $option = 'full')

{

* A wrapper function for the normalizer which takes care of including the class if required and modifies the passed strings

* to be in NFC (Normalization Form Composition).

*

*/

function utf8_normalize_nfc($strings)

{

{

}

}

}

/**

* functions used here you need to rebuild/update the username_clean column in the users table. And all other

* columns that store a clean string otherwise you will break this functionality.

*

*/

function utf8_clean_string($text)

{

include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);

}

static $homographs = array(

// cyrllic