1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
|
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/
/**
* @ignore
*/
if (!defined('IN_PHPBB'))
{
exit;
}
/**
* A wrapper function for the normalizer which takes care of including the class if required and modifies the passed strings
* to be in NFC (Normalization Form Composition).
*
* @param mixed $strings a string or an array of strings to normalize
* @return mixed the normalized content, preserving array keys if array given.
*/
function utf8_new_normalize_nfc($strings)
{
if (empty($strings))
{
return $strings;
}
if (!is_array($strings))
{
utf_new_normalizer::nfc($strings);
}
else if (is_array($strings))
{
foreach ($strings as $key => $string)
{
if (is_array($string))
{
foreach ($string as $_key => $_string)
{
utf_new_normalizer::nfc($strings[$key][$_key]);
}
}
else
{
utf_new_normalizer::nfc($strings[$key]);
}
}
}
return $strings;
}
class utf_new_normalizer
{
/**
* Validate, cleanup and normalize a string
*
* The ultimate convenience function! Clean up invalid UTF-8 sequences,
* and convert to Normal Form C, canonical composition.
*
* @param string &$str The dirty string
* @return string The same string, all shiny and cleaned-up
*/
function cleanup(&$str)
{
// The string below is the list of all autorized characters, sorted by frequency in latin text
$pos = strspn($str, "\x20\x65\x69\x61\x73\x6E\x74\x72\x6F\x6C\x75\x64\x5D\x5B\x63\x6D\x70\x27\x0A\x67\x7C\x68\x76\x2E\x66\x62\x2C\x3A\x3D\x2D\x71\x31\x30\x43\x32\x2A\x79\x78\x29\x28\x4C\x39\x41\x53\x2F\x50\x22\x45\x6A\x4D\x49\x6B\x33\x3E\x35\x54\x3C\x44\x34\x7D\x42\x7B\x38\x46\x77\x52\x36\x37\x55\x47\x4E\x3B\x4A\x7A\x56\x23\x48\x4F\x57\x5F\x26\x21\x4B\x3F\x58\x51\x25\x59\x5C\x09\x5A\x2B\x7E\x5E\x24\x40\x60\x7F\x0D");
$len = strlen($str);
if ($pos == $len)
{
// ASCII strings with no special chars return immediately
return;
}
// Note: we do not check for $GLOBALS['utf_canonical_decomp']. It is assumed they are always loaded together
if (!isset($GLOBALS['utf_nfc_qc']))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx);
}
if (!isset($GLOBALS['utf_canonical_decomp']))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx);
}
// Replace any byte in the range 0x00..0x1F, except for \r, \n and \t
// We replace those characters with a 0xFF byte, which is illegal in UTF-8 and will in turn be replaced with a UTF replacement char
$str = strtr(
$str,
"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F",
"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
);
$str = utf_new_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_canonical_decomp']);
}
/**
* Validate and normalize a UTF string to NFC
*
* @param string &$str Unchecked UTF string
* @return string The string, validated and in normal form
*/
function nfc(&$str)
{
$pos = strspn($str, UTF8_ASCII_RANGE);
$len = strlen($str);
if ($pos == $len)
{
// ASCII strings return immediately
return;
}
if (!isset($GLOBALS['utf_nfc_qc']))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx);
}
if (!isset($GLOBALS['utf_canonical_decomp']))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx);
}
$str = utf_new_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_canonical_decomp']);
}
/**
* Validate and normalize a UTF string to NFKC
*
* @param string &$str Unchecked UTF string
* @return string The string, validated and in normal form
*/
function nfkc(&$str)
{
$pos = strspn($str, UTF8_ASCII_RANGE);
$len = strlen($str);
if ($pos == $len)
{
// ASCII strings return immediately
return;
}
if (!isset($GLOBALS['utf_nfkc_qc']))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_nfkc_qc.' . $phpEx);
}
if (!isset($GLOBALS['utf_compatibility_decomp']))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_compatibility_decomp.' . $phpEx);
}
$str = utf_new_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfkc_qc'], $GLOBALS['utf_compatibility_decomp']);
}
/**
* Recompose a UTF string
*
* @param string $str Unchecked UTF string
* @param integer $pos Position of the first UTF char (in bytes)
* @param integer $len Length of the string (in bytes)
* @param array &$qc Quick-check array, passed by reference but never modified
* @param array &$decomp_map Decomposition mapping, passed by reference but never modified
* @return string The string, validated and recomposed
*
* @access private
*/
function recompose($str, $pos, $len, &$qc, &$decomp_map)
{
global $utf_canonical_comp;
// Load the canonical composition table
if (!isset($utf_canonical_comp))
{
global $phpbb_root_path, $phpEx;
include($phpbb_root_path . 'includes/utf/data/utf_canonical_comp.' . $phpEx);
}
return utf_normalizer::recompose($str, $pos, $len, $qc, $decomp_map);
}
}
|
