12 files changed, 144 insertions, 177 deletions
diff --git a/phpBB/includes/acp/acp_search.php b/phpBB/includes/acp/acp_search.php
index b3caae028e..2ff1b778bb 100644
--- a/phpBB/includes/acp/acp_search.php
+++ b/phpBB/includes/acp/acp_search.php
@@ -320,7 +320,7 @@ class acp_search
 				}
 				else
 				{
-					$sql = 'SELECT post_id, post_subject, post_text, post_encoding, poster_id, forum_id
+					$sql = 'SELECT post_id, post_subject, post_text, poster_id, forum_id
 						FROM ' . POSTS_TABLE . '
 						WHERE post_id >= ' . (int) ($post_counter + 1) . '
 							AND post_id < ' . (int) ($post_counter + $this->batch_size);
@@ -328,7 +328,7 @@ class acp_search
 
 					while ($row = $db->sql_fetchrow($result))
 					{
-						$this->search->index('post', $row['post_id'], $row['post_text'], $row['post_subject'], $row['post_encoding'], $row['poster_id'], $row['forum_id']);
+						$this->search->index('post', $row['post_id'], $row['post_text'], $row['post_subject'], $row['poster_id'], $row['forum_id']);
 					}
 					$db->sql_freeresult($result);
 
diff --git a/phpBB/includes/db/firebird.php b/phpBB/includes/db/firebird.php
index 3446513e19..44bbcb4eb3 100644
--- a/phpBB/includes/db/firebird.php
+++ b/phpBB/includes/db/firebird.php
@@ -355,7 +355,7 @@ class dbal_firebird extends dbal
 	*/
 	function sql_escape($msg)
 	{
-		return (@ini_get('magic_quotes_sybase') || strtolower(@ini_get('magic_quotes_sybase')) == 'on') ? str_replace('\\\'', '\'', addslashes($msg)) : str_replace('\'', '\'\'', stripslashes($msg));
+		return (@ini_get('magic_quotes_sybase') == 1 || strtolower(@ini_get('magic_quotes_sybase')) == 'on') ? str_replace('\\\'', '\'', addslashes($msg)) : str_replace('\'', '\'\'', stripslashes($msg));
 	}
 
 	/**
diff --git a/phpBB/includes/functions.php b/phpBB/includes/functions.php
index 4aa4f4a2c0..d7a594eeeb 100644
--- a/phpBB/includes/functions.php
+++ b/phpBB/includes/functions.php
@@ -24,14 +24,26 @@ function set_var(&$result, $var, $type, $multibyte = false)
 
 	if ($type == 'string')
 	{
-		$result = trim(htmlspecialchars(str_replace(array("\r\n", "\r"), array("\n", "\n"), $result)));
-		$result = (STRIP) ? stripslashes($result) : $result;
+		$result = trim(htmlspecialchars(str_replace(array("\r\n", "\r"), array("\n", "\n"), $result), ENT_QUOTES, 'UTF-8'));
 
-		// Check for possible multibyte characters to save a preg_replace call if nothing is in there...
-		if ($multibyte && strpos($result, '&amp;#') !== false)
+		if (!empty($result))
 		{
-			$result = preg_replace('#&amp;(\#[0-9]+;)#', '&\1', $result);
+			// Make sure multibyte characters are wellformed
+			if ($multibyte)
+			{
+				if (!preg_match('/^./u', $result))
+				{
+					$result = '';
+				}
+			}
+			else
+			{
+				// no multibyte, allow only ASCII (0-127)
+				$result = preg_replace('/[\x80-\xFF]/', '?', $result);
+			}
 		}
+
+		$result = (STRIP) ? stripslashes($result) : $result;
 	}
 }
 
@@ -2039,7 +2051,7 @@ function get_context($text, $words, $length = 400)
 		// find the starting indizes of all words
 		foreach ($words as $word)
 		{
-			if (preg_match('#(?: |^)(' . str_replace('\*', '\w*?', preg_quote($word, '#')) . ')(?: |$)#i', $text, $match))
+			if (preg_match('#(?:[^\w]|^)(' . str_replace('\*', '\w*?', preg_quote($word, '#')) . ')(?:[^\w]|$)#i', $text, $match))
 			{
 				$pos = strpos($text, $match[1]);
 				if ($pos !== false)
@@ -2363,6 +2375,12 @@ function parse_inline_attachments(&$text, &$attachments, &$update_count, $forum_
 {
 	global $config, $user;
 
+	if (!function_exists('display_attachments'))
+	{
+		global $phpbb_root_path, $phpEx;
+		include_once("{$phpbb_root_path}includes/functions_display.$phpEx");
+	}
+
 	$attachments = display_attachments($forum_id, NULL, $attachments, $update_count, false, true);
 	$tpl_size = sizeof($attachments);
 
@@ -3143,7 +3161,7 @@ function page_header($page_title = '', $display_online_list = true)
 		'S_USER_BROWSER'		=> (isset($user->data['session_browser'])) ? $user->data['session_browser'] : $user->lang['UNKNOWN_BROWSER'],
 		'S_USERNAME'			=> $user->data['username'],
 		'S_CONTENT_DIRECTION'	=> $user->lang['DIRECTION'],
-		'S_CONTENT_ENCODING'	=> $user->lang['ENCODING'],
+		'S_CONTENT_ENCODING'	=> 'UTF-8',
 		'S_CONTENT_DIR_LEFT'	=> $user->lang['LEFT'],
 		'S_CONTENT_DIR_RIGHT'	=> $user->lang['RIGHT'],
 		'S_TIMEZONE'			=> ($user->data['user_dst'] || ($user->data['user_id'] == ANONYMOUS && $config['board_dst'])) ? sprintf($user->lang['ALL_TIMES'], $user->lang['tz'][$tz], $user->lang['tz']['dst']) : sprintf($user->lang['ALL_TIMES'], $user->lang['tz'][$tz], ''),
@@ -3164,7 +3182,7 @@ function page_header($page_title = '', $display_online_list = true)
 		'T_ICONS_PATH'			=> "{$phpbb_root_path}{$config['icons_path']}/",
 		'T_RANKS_PATH'			=> "{$phpbb_root_path}{$config['ranks_path']}/",
 		'T_UPLOAD_PATH'			=> "{$phpbb_root_path}{$config['upload_path']}/",
-		'T_STYLESHEET_LINK'		=> (!$user->theme['theme_storedb']) ? "{$phpbb_root_path}styles/" . $user->theme['theme_path'] . '/theme/stylesheet.css' : "{$phpbb_root_path}style.$phpEx?sid=$user->session_id&amp;id=" . $user->theme['style_id'],
+		'T_STYLESHEET_LINK'		=> (!$user->theme['theme_storedb']) ? "{$phpbb_root_path}styles/" . $user->theme['theme_path'] . '/theme/stylesheet.css' : "{$phpbb_root_path}style.$phpEx?sid=$user->session_id&amp;id=" . $user->theme['style_id'] . '&amp;lang=' . $user->data['user_lang'],
 		'T_STYLESHEET_NAME'		=> $user->theme['theme_name'],
 		'T_THEME_DATA'			=> (!$user->theme['theme_storedb']) ? '' : $user->theme['theme_data'],
 
@@ -3173,7 +3191,7 @@ function page_header($page_title = '', $display_online_list = true)
 
 	if ($config['send_encoding'])
 	{
-		header('Content-type: text/html; charset=' . $user->lang['ENCODING']);
+		header('Content-type: text/html; charset=UTF-8');
 	}
 	header('Cache-Control: private, no-cache="set-cookie"');
 	header('Expires: 0');
diff --git a/phpBB/includes/functions_jabber.php b/phpBB/includes/functions_jabber.php
index 96d2e686e4..9efe17dc6b 100644
--- a/phpBB/includes/functions_jabber.php
+++ b/phpBB/includes/functions_jabber.php
@@ -25,7 +25,6 @@
 */
 class jabber
 {
-	var $encoding;
 	var $server;
 	var $port;
 	var $username;
@@ -68,8 +67,6 @@ class jabber
 		$this->packet_queue			= $this->subscription_queue = array();
 		$this->iq_sleep_timer		= $this->delay_disconnect = 1;
 
-		$this->encoding				= 'UTF-8';
-
 		$this->returned_keep_alive	= true;
 		$this->txnid				= 0;
 
@@ -454,7 +451,6 @@ class jabber
 			}
 
 			$this->_array_xmlspecialchars($content);
-			$this->_array_conv_utf8($content);
 
 			$xml = "<message to='$to' type='$type' id='$id'>\n";
 
@@ -795,69 +791,6 @@ class jabber
 		$string = str_replace(array('&', '>', '<', '"', '\''), array('&amp;', '&gt;', '&lt;', '&quot;', '&apos;'), $string);
 	}
 
-	/**
-	* Recursively converts all elements in an array to UTF-8 from the encoding stored in {@link encoding the encoding attribute}.
-	* @access private
-	*/
-	function _array_conv_utf8(&$array)
-	{
-		// no need to do anything if the encoding already is UTF-8
-		if (strtoupper($this->encoding) == 'UTF-8')
-		{
-			return true;
-		}
-
-		if (is_array($array))
-		{
-			foreach ($array as $k => $v)
-			{
-				if (is_array($v))
-				{
-					$this->_array_conv_utf8($array[$k]);
-				}
-				else
-				{
-					$this->_conv_utf8($array[$k]);
-				}
-			}
-		}
-	}
-
-	/**
-	* Converts a string to utf8 encoding.
-	*
-	* @param string $string has to have the same encoding as {@link encoding the encoding attribute} is set to.
-	*
-	* @return boolean True on success, false on failure.
-	*
-	* @access private
-	*/
-	function _conv_utf8(&$string)
-	{
-		// no need to do anything if the encoding already is UTF-8
-		if (strtoupper($this->encoding) == 'UTF-8')
-		{
-			return true;
-		}
-
-		// first try iconv then mb_convert_encoding and as a last fall back try recode_string
-		if (function_exists('iconv') && (($string = iconv($this->encoding, 'UTF-8', $string)) !== false))
-		{
-			return true;
-		}
-		elseif (function_exists('mb_convert_encoding') && (($string = mb_convert_encoding($string, 'UTF-8', $this->encoding)) !== false))
-		{
-			return true;
-		}
-		elseif (function_exists('recode_string') && (($string = recode_string($this->encoding . '..UTF-8', $string)) !== false))
-		{
-			return true;
-		}
-
-		// if everything fails we will just have to live with what we have, good luck!
-		return false;
-	}
-
 	// ======================================================================
 	// <message/> parsers
 	// ======================================================================
diff --git a/phpBB/includes/functions_messenger.php b/phpBB/includes/functions_messenger.php
index d8dfb2837e..d5d9c8dd5e 100644
--- a/phpBB/includes/functions_messenger.php
+++ b/phpBB/includes/functions_messenger.php
@@ -14,7 +14,7 @@
 */
 class messenger
 {
-	var $vars, $msg, $extra_headers, $replyto, $from, $subject, $encoding;
+	var $vars, $msg, $extra_headers, $replyto, $from, $subject;
 	var $addresses = array();
 
 	var $mail_priority = MAIL_NORMAL_PRIORITY;
@@ -45,7 +45,7 @@ class messenger
 	function reset()
 	{
 		$this->addresses = array();
-		$this->vars = $this->msg = $this->extra_headers = $this->replyto = $this->from = $this->encoding = '';
+		$this->vars = $this->msg = $this->extra_headers = $this->replyto = $this->from = '';
 		$this->mail_priority = MAIL_NORMAL_PRIORITY;
 	}
 
@@ -221,16 +221,6 @@ class messenger
 			$this->subject = (($this->subject != '') ? $this->subject : $user->lang['NO_SUBJECT']);
 		}
 
-		if (preg_match('#^(Charset:(.*?))$#m', $this->msg, $match))
-		{
-			$this->encoding = (trim($match[2]) != '') ? trim($match[2]) : trim($user->lang['ENCODING']);
-			$drop_header .= '[\r\n]*?' . preg_quote($match[1], '#');
-		}
-		else
-		{
-			$this->encoding = trim($user->lang['ENCODING']);
-		}
-
 		if ($drop_header)
 		{
 			$this->msg = trim(preg_replace('#' . $drop_header . '#s', '', $this->msg));
@@ -324,7 +314,7 @@ class messenger
 
 			foreach ($address_ary as $which_ary)
 			{
-				$$type .= (($$type != '') ? ', ' : '') . (($which_ary['name'] != '') ?  '"' . mail_encode($which_ary['name'], $this->encoding) . '" <' . $which_ary['email'] . '>' : $which_ary['email']);
+				$$type .= (($$type != '') ? ', ' : '') . (($which_ary['name'] != '') ?  '"' . mail_encode($which_ary['name']) . '" <' . $which_ary['email'] . '>' : $which_ary['email']);
 			}
 		}
 
@@ -348,7 +338,7 @@ class messenger
 		$headers .= "MIME-Version: 1.0\n";
 		$headers .= 'Message-ID: <' . md5(unique_id(time())) . "@" . $config['server_name'] . ">\n";
 		$headers .= 'Date: ' . gmdate('D, d M Y H:i:s T', time()) . "\n";
-		$headers .= "Content-type: text/plain; charset={$this->encoding}\n";
+		$headers .= "Content-type: text/plain; charset=UTF-8\n";
 		$headers .= "Content-transfer-encoding: 8bit\n";
 		$headers .= "X-Priority: {$this->mail_priority}\n";
 		$headers .= 'X-MSMail-Priority: ' . (($this->mail_priority == MAIL_LOW_PRIORITY) ? 'Low' : (($this->mail_priority == MAIL_NORMAL_PRIORITY) ? 'Normal' : 'High')) . "\n";
@@ -357,7 +347,7 @@ class messenger
 		$headers .= "X-phpBB-Origin: phpbb://" . str_replace(array('http://', 'https://'), array('', ''), generate_board_url()) . "\n";
 		$headers .= ($this->extra_headers != '') ? $this->extra_headers : '';
 
-		// Send message ... removed $this->encode() from subject for time being
+		// Send message ...
 		if (!$use_queue)
 		{
 			$mail_to = ($to == '') ? 'Undisclosed-Recipient:;' : $to;
@@ -365,7 +355,7 @@ class messenger
 
 			if ($config['smtp_delivery'])
 			{
-				$result = smtpmail($this->addresses, $this->subject, wordwrap($this->msg), $err_msg, $this->encoding, $headers);
+				$result = smtpmail($this->addresses, $this->subject, wordwrap($this->msg), $err_msg, $headers);
 			}
 			else
 			{
@@ -387,7 +377,6 @@ class messenger
 				'addresses'		=> $this->addresses,
 				'subject'		=> $this->subject,
 				'msg'			=> $this->msg,
-				'encoding'		=> $this->encoding,
 				'headers'		=> $headers)
 			);
 		}
@@ -435,7 +424,6 @@ class messenger
 			$this->jabber->username = $config['jab_username'];
 			$this->jabber->password = $config['jab_password'];
 			$this->jabber->resource = ($config['jab_resource']) ? $config['jab_resource'] : '';
-			$this->jabber->encoding = $this->encoding;
 
 			if (!$this->jabber->connect())
 			{
@@ -607,7 +595,7 @@ class queue
 						$err_msg = '';
 						$to = (!$to) ? 'Undisclosed-Recipient:;' : $to;
 
-						$result = ($config['smtp_delivery']) ? smtpmail($addresses, $subject, wordwrap($msg), $err_msg, $encoding, $headers) : @$config['email_function_name']($to, $subject, implode("\n", preg_split("/\r?\n/", wordwrap($msg))), $headers);
+						$result = ($config['smtp_delivery']) ? smtpmail($addresses, $subject, wordwrap($msg), $err_msg, $headers) : @$config['email_function_name']($to, $subject, implode("\n", preg_split("/\r?\n/", wordwrap($msg))), $headers);
 
 						if (!$result)
 						{
@@ -739,7 +727,7 @@ class queue
 /**
 * Replacement or substitute for PHP's mail command
 */
-function smtpmail($addresses, $subject, $message, &$err_msg, $encoding, $headers = '')
+function smtpmail($addresses, $subject, $message, &$err_msg, $headers = '')
 {
 	global $config, $user;
 
@@ -794,7 +782,7 @@ function smtpmail($addresses, $subject, $message, &$err_msg, $encoding, $headers
 	{
 		foreach ($addresses['to'] as $which_ary)
 		{
-			$mail_to[] = ($which_ary['name'] != '') ? mail_encode(trim($which_ary['name']), $encoding) . ' <' . trim($which_ary['email']) . '>' : '<' . trim($which_ary['email']) . '>';
+			$mail_to[] = ($which_ary['name'] != '') ? mail_encode(trim($which_ary['name'])) . ' <' . trim($which_ary['email']) . '>' : '<' . trim($which_ary['email']) . '>';
 			$mail_rcpt['to'][] = '<' . trim($which_ary['email']) . '>';
 		}
 	}
@@ -811,7 +799,7 @@ function smtpmail($addresses, $subject, $message, &$err_msg, $encoding, $headers
 	{
 		foreach ($addresses['cc'] as $which_ary)
 		{
-			$mail_cc[] = ($which_ary['name'] != '') ? mail_encode(trim($which_ary['name']), $encoding) . ' <' . trim($which_ary['email']) . '>' : '<' . trim($which_ary['email']) . '>';
+			$mail_cc[] = ($which_ary['name'] != '') ? mail_encode(trim($which_ary['name'])) . ' <' . trim($which_ary['email']) . '>' : '<' . trim($which_ary['email']) . '>';
 			$mail_rcpt['cc'][] = '<' . trim($which_ary['email']) . '>';
 		}
 	}
@@ -1363,12 +1351,12 @@ class smtp_class
 }
 
 /**
-* Encodes the given string for proper display for this encoding ... nabbed 
+* Encodes the given string for proper display in UTF-8 ... nabbed 
 * from php.net and modified. There is an alternative encoding method which 
 * may produce less output but it's questionable as to its worth in this 
-* scenario IMO
+* scenario.
 */
-function mail_encode($str, $encoding)
+function mail_encode($str)
 {
 	if ($encoding == '')
 	{
@@ -1377,7 +1365,7 @@ function mail_encode($str, $encoding)
 
 	// define start delimimter, end delimiter and spacer
 	$end = "?=";
-	$start = "=?$encoding?B?";
+	$start = "=?UTF-8?B?";
 	$spacer = "$end\r\n $start";
 
 	// determine length of encoded text within chunks and ensure length is even
diff --git a/phpBB/includes/functions_posting.php b/phpBB/includes/functions_posting.php
index 4a46dc1062..1518f1d1e5 100644
--- a/phpBB/includes/functions_posting.php
+++ b/phpBB/includes/functions_posting.php
@@ -1414,7 +1414,6 @@ function submit_post($mode, $subject, $username, $topic_type, &$poll, &$data, $u
 				'post_subject'		=> $subject,
 				'post_text'			=> $data['message'],
 				'post_checksum'		=> $data['message_md5'],
-				'post_encoding'		=> $user->lang['ENCODING'],
 				'post_attachment'	=> (sizeof($data['attachment_data'])) ? 1 : 0,
 				'bbcode_bitfield'	=> $data['bbcode_bitfield'],
 				'bbcode_uid'		=> $data['bbcode_uid'],
@@ -1468,7 +1467,6 @@ function submit_post($mode, $subject, $username, $topic_type, &$poll, &$data, $u
 				'post_edit_reason'	=> $data['post_edit_reason'],
 				'post_edit_user'	=> (int) $data['post_edit_user'],
 				'post_checksum'		=> $data['message_md5'],
-				'post_encoding'		=> $user->lang['ENCODING'],
 				'post_attachment'	=> (sizeof($data['attachment_data'])) ? 1 : 0,
 				'bbcode_bitfield'	=> $data['bbcode_bitfield'],
 				'bbcode_uid'		=> $data['bbcode_uid'],
@@ -1911,7 +1909,7 @@ function submit_post($mode, $subject, $username, $topic_type, &$poll, &$data, $u
 			trigger_error($error);
 		}
 
-		$search->index($mode, $data['post_id'], $data['message'], $subject, $user->lang['ENCODING'], $poster_id, ($topic_type == POST_GLOBAL) ? 0 : $data['forum_id']);
+		$search->index($mode, $data['post_id'], $data['message'], $subject, $poster_id, ($topic_type == POST_GLOBAL) ? 0 : $data['forum_id']);
 	}
 
 	$db->sql_transaction('commit');
diff --git a/phpBB/includes/functions_privmsgs.php b/phpBB/includes/functions_privmsgs.php
index 94b6368493..1f53dc4511 100644
--- a/phpBB/includes/functions_privmsgs.php
+++ b/phpBB/includes/functions_privmsgs.php
@@ -1323,7 +1323,6 @@ function submit_pm($mode, $subject, &$data, $update_message, $put_in_outbox = tr
 				'enable_sig'		=> $data['enable_sig'],
 				'message_subject'	=> $subject,
 				'message_text'		=> $data['message'],
-				'message_encoding'	=> $user->lang['ENCODING'],
 				'message_attachment'=> (sizeof($data['attachment_data'])) ? 1 : 0,
 				'bbcode_bitfield'	=> $data['bbcode_bitfield'],
 				'bbcode_uid'		=> $data['bbcode_uid'],
@@ -1342,7 +1341,6 @@ function submit_pm($mode, $subject, &$data, $update_message, $put_in_outbox = tr
 				'enable_sig'		=> $data['enable_sig'],
 				'message_subject'	=> $subject,
 				'message_text'		=> $data['message'],
-				'message_encoding'	=> $user->lang['ENCODING'],
 				'message_attachment'=> (sizeof($data['attachment_data'])) ? 1 : 0,
 				'bbcode_bitfield'	=> $data['bbcode_bitfield'],
 				'bbcode_uid'		=> $data['bbcode_uid']
diff --git a/phpBB/includes/mcp/mcp_main.php b/phpBB/includes/mcp/mcp_main.php
index fe1047db79..2d298a4108 100644
--- a/phpBB/includes/mcp/mcp_main.php
+++ b/phpBB/includes/mcp/mcp_main.php
@@ -893,7 +893,6 @@ function mcp_fork_topic($topic_ids)
 					'post_edit_reason'	=> (string) $row['post_edit_reason'],
 					'post_edit_user'	=> (int) $row['post_edit_user'],
 					'post_checksum'		=> (string) $row['post_checksum'],
-					'post_encoding'		=> (string) $row['post_encoding'],
 					'post_attachment'	=> (int) $row['post_attachment'],
 					'bbcode_bitfield'	=> $row['bbcode_bitfield'],
 					'bbcode_uid'		=> (string) $row['bbcode_uid'],
diff --git a/phpBB/includes/search/fulltext_mysql.php b/phpBB/includes/search/fulltext_mysql.php
index 4a9a57df34..f4bf0d9eaf 100644
--- a/phpBB/includes/search/fulltext_mysql.php
+++ b/phpBB/includes/search/fulltext_mysql.php
@@ -561,7 +561,7 @@ class fulltext_mysql extends search_backend
 	*
 	* @param string $mode contains the post mode: edit, post, reply, quote ...
 	*/
-	function index($mode, $post_id, &$message, &$subject, $encoding, $poster_id, $forum_id)
+	function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
 	{
 		global $db;
 
diff --git a/phpBB/includes/search/fulltext_native.php b/phpBB/includes/search/fulltext_native.php
index 90da8abcd1..1dcb599718 100755
--- a/phpBB/includes/search/fulltext_native.php
+++ b/phpBB/includes/search/fulltext_native.php
@@ -56,10 +56,6 @@ class fulltext_native extends search_backend
 		{
 			include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
 		}
-		if (!function_exists('utf8_strlen'))
-		{
-			include($phpbb_root_path . 'includes/utf/utf_tools.' . $phpEx);
-		}
 
 
 		$error = false;
@@ -86,47 +82,89 @@ class fulltext_native extends search_backend
 	{
 		global $db, $config, $user;
 
-		// Clean up the query search
-		$match = array(
-			// Replace multiple spaces with a single space
-			'#  +#',
+		$keywords = trim($this->cleanup($keywords, '+-|()*'));
 
-			// Strip spaces after: +-|(
-			'#([+\\-|(]) #',
+		// allow word|word|word without brackets
+		if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false))
+		{
+			$keywords = '(' . $keywords . ')';
+		}
 
-			// Strip spaces before: |*)
-			'# ([|*)])#',
+		$open_bracket = $space = false;
+		for ($i = 0, $n = strlen($keywords); $i < $n; $i++)
+		{
+			if ($open_bracket !== false)
+			{
+				switch ($keywords[$i])
+				{
+					case ')':
+						if ($open_bracket + 1 == $i)
+						{
+							$keywords[$i - 1] = '|';
+							$keywords[$i] = '|';
+						}
+						$open_bracket = false;
+					break;
+					case '(':
+						$keywords[$i] = '|';
+					break;
+					case '+':
+					case '-':
+					case ' ':
+						$keywords[$i] = '|';
+					break;
+				}
+			}
+			else
+			{
+				switch ($keywords[$i])
+				{
+					case ')':
+						$keywords[$i] = ' ';
+					break;
+					case '(':
+						$open_bracket = $i;
+					break;
+					case '|':
+						$keywords[$i] = ' ';
+					break;
+					case '-':
+					case '+':
+						$space = $keywords[$i];
+					break;
+					case ' ':
+						if ($space !== false)
+						{
+							$keywords[$i] = $space;
+						}
+					break;
+					default:
+						$space = false;
+				}
+			}
+		}
 
-			// Make word|word|word work without brackets
-			'#^[^()]*\\|[^()]*$#',
+		if ($open_bracket)
+		{
+			$keywords .= ')';
+		}
 
-			// Remove nested brackets
-			'#(\\([^()]*)(?=\\()#',
-			'#\\)([^()]*)(?=\\))#',
+		$match = array(
+			'#  +#',
+			'#\|\|+#',
+			'#(\+|\-)(?:\+|\-)+#',
+			'#\(\|#',
+			'#\|\)#',
 		);
-
 		$replace = array(
 			' ',
+			'|',
 			'$1',
-			'$1',
-			'($0)',
-			'$1)',
-			'$1',
+			'(',
+			')',
 		);
 
-		$keywords = trim(preg_replace($match, $replace, $this->cleanup($keywords, '+-|()*', $user->lang['ENCODING'])));
-
-		// remove some useless bracket combinations which might be created by the previous regexps
-		$keywords = str_replace(array('()', ')|('), array('', '|'), $keywords);
-
-		$keywords = preg_replace_callback(
-			'#\((?:(?:[^)]*?) )*?[^)]*?\)#',
-			create_function(
-				'$matches',
-				'return str_replace(" ", "|", $matches[0]);'
-			),
-			$keywords
-		);
+		$keywords = preg_replace($match, $replace, $keywords);
 
 		// $keywords input format: each word seperated by a space, words in a bracket are not seperated
 
@@ -143,7 +181,7 @@ class fulltext_native extends search_backend
 		}
 
 		// set the search_query which is shown to the user
-		$this->search_query = utf8_encode_ncr($keywords, ENT_QUOTES);
+		$this->search_query = $keywords;
 
 		$exact_words = array();
 		preg_match_all('#([^\\s+\\-|*()]+)(?:$|[\\s+\\-|()])#', $keywords, $exact_words);
@@ -224,6 +262,11 @@ class fulltext_native extends search_backend
 				$mode = 'must_contain';
 			}
 
+			if (empty($word))
+			{
+				continue;
+			}
+
 			// if this is an array of words then retrieve an id for each
 			if (is_array($word))
 			{
@@ -255,7 +298,7 @@ class fulltext_native extends search_backend
 				// throw an error if we shall not ignore unexistant words
 				else if (!$ignore_no_id)
 				{
-					trigger_error(sprintf($user->lang['WORDS_IN_NO_POST'], utf8_encode_ncr(implode(', ', $word))));
+					trigger_error(sprintf($user->lang['WORDS_IN_NO_POST'], implode(', ', $word)));
 				}
 			}
 			// else we only need one id
@@ -273,7 +316,7 @@ class fulltext_native extends search_backend
 			// throw an error if we shall not ignore unexistant words
 			else if (!$ignore_no_id)
 			{
-				trigger_error(sprintf($user->lang['WORD_IN_NO_POST'], utf8_encode_ncr($word)));
+				trigger_error(sprintf($user->lang['WORD_IN_NO_POST'], $word));
 			}
 		}
 
@@ -558,9 +601,16 @@ class fulltext_native extends search_backend
 				case 'mysql':
 				case 'mysql4':
 				case 'mysqli':
-					$sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];
-					$is_mysql = true;
-				break;
+					// 3.x does not support SQL_CALC_FOUND_ROWS
+					if (SQL_LAYER != 'mysql' || $db->sql_server_info[6] != '3')
+					{
+						$sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];
+						$is_mysql = true;
+
+						// that's everything for MySQL >= 4.0
+						break;
+					}
+					// no break for MySQL 3.x
 
 				case 'sqlite':
 					$sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id';
@@ -871,7 +921,7 @@ class fulltext_native extends search_backend
 	*
 	* NOTE: duplicates are NOT removed from the return array
 	*
-	* @param	string	$text	Text to split, encoded in user's encoding
+	* @param	string	$text	Text to split, encoded in UTF-8
 	* @return	array			Array of UTF-8 words
 	*
 	* @access	private
@@ -899,7 +949,7 @@ class fulltext_native extends search_backend
 		/**
 		* Clean up the string, remove HTML tags, remove BBCodes
 		*/
-		$word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1, $user->lang['ENCODING']), ' ');
+		$word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1), ' ');
 
 		while (isset($word[0]))
 		{
@@ -952,13 +1002,12 @@ class fulltext_native extends search_backend
 	* @param	int		$post_id	The id of the post which is modified/created
 	* @param	string	$message	New or updated post content
 	* @param	string	$subject	New or updated post subject
-	* @param	string	$encoding	The post content's encoding
 	* @param	int		$poster_id	Post author's user id
 	* @param	int		$forum_id	The id of the forum in which the post is located
 	*
 	* @access	public
 	*/
-	function index($mode, $post_id, &$message, &$subject, $encoding, $poster_id, $forum_id)
+	function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
 	{
 		global $config, $db, $user;
 
@@ -1104,22 +1153,6 @@ class fulltext_native extends search_backend
 	}
 
 	/**
-	* Used by index() to sort strings by string length, longest first
-	*/
-	function strlencmp($a, $b)
-	{
-		$len_a = strlen($a);
-		$len_b = strlen($b);
-
-		if ($len_a == $len_b)
-		{
-			return 0;
-		}
-
-		return ($len_a > $len_b) ? -1 : 1;
-	}
-
-	/**
 	* Removes entries from the wordmatch table for the specified post_ids
 	*/
 	function index_remove($post_ids, $author_ids, $forum_ids)
@@ -1278,7 +1311,7 @@ class fulltext_native extends search_backend
 	* @param	string	$encoding		Text encoding
 	* @return	string					Cleaned up text, only alphanumeric chars are left
 	*/
-	function cleanup($text, $allowed_chars = null, $encoding = 'iso-8859-1')
+	function cleanup($text, $allowed_chars = null, $encoding = 'utf-8')
 	{
 		global $phpbb_root_path, $phpEx;
 		static $conv = array(), $conv_loaded = array();
@@ -1537,7 +1570,7 @@ class fulltext_native extends search_backend
 		// These are fields required in the config table
 		return array(
 			'tpl'		=> $tpl,
-			'config'	=> array('fulltext_native_load_upd' => 'bool', 'fulltext_native_min_chars' => 'integer:0:252', 'fulltext_native_max_chars' => 'integer:0:252')
+			'config'	=> array('fulltext_native_load_upd' => 'bool', 'fulltext_native_min_chars' => 'integer:0:252', 'fulltext_native_max_chars' => 'integer:0:255')
 		);
 	}
 }
diff --git a/phpBB/includes/ucp/ucp_prefs.php b/phpBB/includes/ucp/ucp_prefs.php
index ff8df44148..60ed5dc360 100644
--- a/phpBB/includes/ucp/ucp_prefs.php
+++ b/phpBB/includes/ucp/ucp_prefs.php
@@ -61,7 +61,7 @@ class ucp_prefs
 
 					$var_ary = array(
 						'dateformat'	=> array('string', false, 3, 30),
-						'lang'			=> array('match', false, '#^[a-z_\-]{2,}$#i'),
+						'lang'			=> array('match', false, '#^[a-z0-9_\-]{2,}$#i'),
 						'tz'			=> array('num', false, -14, 14),
 					);
 
diff --git a/phpBB/includes/utf/utf_tools.php b/phpBB/includes/utf/utf_tools.php
index 2f7c8de69a..a4de83ab0f 100644
--- a/phpBB/includes/utf/utf_tools.php
+++ b/phpBB/includes/utf/utf_tools.php
@@ -741,7 +741,7 @@ function utf8_recode($string, $encoding)
 */
 function utf8_encode_ncr($text)
 {
-	return preg_replace_callback('#[\\xC2-\\xF4][\\x80-\\xBF]+#', 'utf8_encode_ncr_callback', $text);
+	return preg_replace_callback('#[\\xC2-\\xF4][\\x80-\\xBF]?[\\x80-\\xBF]?[\\x80-\\xBF]+#', 'utf8_encode_ncr_callback', $text);
 }
 
 /**