From 4f0b736f14e6d51dcf19e93897fe5386023369e7 Mon Sep 17 00:00:00 2001 From: rxu Date: Tue, 24 Oct 2017 20:35:25 +0700 Subject: [ticket/15419] Fix Sphinx UTF8 search in delta index PHPBB3-15419 --- phpBB/phpbb/search/fulltext_sphinx.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index 59c3d55076..a20e3ad4b5 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -304,7 +304,7 @@ class fulltext_sphinx array('sql_attr_string', 'post_subject'), ), 'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array( - array('sql_query_pre', ''), + array('sql_query_pre', 'SET NAMES \'utf8\''), array('sql_query_range', ''), array('sql_range_step', ''), array('sql_query', 'SELECT @@ -324,6 +324,7 @@ class fulltext_sphinx WHERE p.topic_id = t.topic_id AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), + array('sql_query_post_index', ''), ), 'index index_phpbb_' . $this->id . '_main' => array( array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), -- cgit v1.2.1 From f9b9d659a1638389e6250110ce54d18e9d9bd59c Mon Sep 17 00:00:00 2001 From: rxu Date: Sat, 4 Nov 2017 19:14:04 +0700 Subject: [ticket/15224] Fix some MySQL fulltext index searching errors PHPBB3-15224 --- phpBB/phpbb/search/fulltext_mysql.php | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_mysql.php b/phpBB/phpbb/search/fulltext_mysql.php index da1aad1c3a..c8df1951e3 100644 --- a/phpBB/phpbb/search/fulltext_mysql.php +++ b/phpBB/phpbb/search/fulltext_mysql.php @@ -591,6 +591,7 @@ class fulltext_mysql extends \phpbb\search\base WHERE MATCH ($sql_match) AGAINST ('" . $this->db->sql_escape(htmlspecialchars_decode($this->search_query)) . "' IN BOOLEAN MODE) $sql_where_options ORDER BY $sql_sort"; + $this->db->sql_return_on_error(true); $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); while ($row = $this->db->sql_fetchrow($result)) @@ -602,7 +603,7 @@ class fulltext_mysql extends \phpbb\search\base $id_ary = array_unique($id_ary); // if the total result count is not cached yet, retrieve it from the db - if (!$result_count) + if (!$result_count && count($id_ary)) { $sql_found_rows = 'SELECT FOUND_ROWS() as result_count'; $result = $this->db->sql_query($sql_found_rows); @@ -1004,6 +1005,11 @@ class fulltext_mysql extends \phpbb\search\base } } + if (!isset($this->stats['post_text'])) + { + $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ADD FULLTEXT post_text (post_text)'); + } + $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); return false; @@ -1039,6 +1045,11 @@ class fulltext_mysql extends \phpbb\search\base $alter[] = 'DROP INDEX post_content'; } + if (isset($this->stats['post_text'])) + { + $alter[] = 'DROP INDEX post_text'; + } + if (sizeof($alter)) { $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter)); @@ -1059,7 +1070,7 @@ class fulltext_mysql extends \phpbb\search\base $this->get_stats(); } - return isset($this->stats['post_subject']) && isset($this->stats['post_content']); + return isset($this->stats['post_subject']) && isset($this->stats['post_content']) && isset($this->stats['post_text']); } /** @@ -1103,6 +1114,10 @@ class fulltext_mysql extends \phpbb\search\base { $this->stats['post_subject'] = $row; } + else if ($row['Key_name'] == 'post_text') + { + $this->stats['post_text'] = $row; + } else if ($row['Key_name'] == 'post_content') { $this->stats['post_content'] = $row; -- cgit v1.2.1 From f8fbe3793680af1dae2db2829cfc84068831c52f Mon Sep 17 00:00:00 2001 From: rxu Date: Wed, 28 Jun 2017 00:58:03 +0700 Subject: [ticket/14972] replace all occurrences of sizeof() with the count() PHPBB3-14972 --- phpBB/phpbb/search/base.php | 10 +++--- phpBB/phpbb/search/fulltext_mysql.php | 22 ++++++------- phpBB/phpbb/search/fulltext_native.php | 48 ++++++++++++++-------------- phpBB/phpbb/search/fulltext_postgres.php | 14 ++++---- phpBB/phpbb/search/fulltext_sphinx.php | 10 +++--- phpBB/phpbb/search/sphinx/config.php | 4 +-- phpBB/phpbb/search/sphinx/config_section.php | 6 ++-- 7 files changed, 57 insertions(+), 57 deletions(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/base.php b/phpBB/phpbb/search/base.php index 56de973b65..e7d0774b6c 100644 --- a/phpBB/phpbb/search/base.php +++ b/phpBB/phpbb/search/base.php @@ -133,7 +133,7 @@ class base { global $cache, $config, $db, $user; - $length = min(sizeof($id_ary), $config['search_block_size']); + $length = min(count($id_ary), $config['search_block_size']); // nothing to cache so exit if (!$length) @@ -148,7 +148,7 @@ class base if (!($store = $cache->get('_search_results_' . $search_key))) { // add the current keywords to the recent searches in the cache which are listed on the search page - if (!empty($keywords) || sizeof($author_ary)) + if (!empty($keywords) || count($author_ary)) { $sql = 'SELECT search_time FROM ' . SEARCH_RESULTS_TABLE . ' @@ -201,7 +201,7 @@ class base $store += $store_ids; // if the cache is too big - if (sizeof($store) - 2 > 20 * $config['search_block_size']) + if (count($store) - 2 > 20 * $config['search_block_size']) { // remove everything in front of two blocks in front of the current start index for ($i = 0, $n = $id_range[0] - 2 * $config['search_block_size']; $i < $n; $i++) @@ -243,7 +243,7 @@ class base global $db, $cache, $config; // clear all searches that searched for the specified words - if (sizeof($words)) + if (count($words)) { $sql_where = ''; foreach ($words as $word) @@ -264,7 +264,7 @@ class base } // clear all searches that searched for the specified authors - if (is_array($authors) && sizeof($authors)) + if (is_array($authors) && count($authors)) { $sql_where = ''; foreach ($authors as $author) diff --git a/phpBB/phpbb/search/fulltext_mysql.php b/phpBB/phpbb/search/fulltext_mysql.php index c8df1951e3..51c5fe8b76 100644 --- a/phpBB/phpbb/search/fulltext_mysql.php +++ b/phpBB/phpbb/search/fulltext_mysql.php @@ -232,9 +232,9 @@ class fulltext_mysql extends \phpbb\search\base $this->split_words = $matches[1]; // We limit the number of allowed keywords to minimize load on the database - if ($this->config['max_num_search_keywords'] && sizeof($this->split_words) > $this->config['max_num_search_keywords']) + if ($this->config['max_num_search_keywords'] && count($this->split_words) > $this->config['max_num_search_keywords']) { - trigger_error($this->user->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], sizeof($this->split_words))); + trigger_error($this->user->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], count($this->split_words))); } // to allow phrase search, we need to concatenate quoted words @@ -361,7 +361,7 @@ class fulltext_mysql extends \phpbb\search\base // remove too short or too long words $text = array_values($text); - for ($i = 0, $n = sizeof($text); $i < $n; $i++) + for ($i = 0, $n = count($text); $i < $n; $i++) { $text[$i] = trim($text[$i]); if (utf8_strlen($text[$i]) < $this->config['fulltext_mysql_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_mysql_max_word_len']) @@ -563,12 +563,12 @@ class fulltext_mysql extends \phpbb\search\base $sql_select = ($type == 'posts') ? $sql_select . 'p.post_id' : 'DISTINCT ' . $sql_select . 't.topic_id'; $sql_from = ($join_topic) ? TOPICS_TABLE . ' t, ' : ''; $field = ($type == 'posts') ? 'post_id' : 'topic_id'; - if (sizeof($author_ary) && $author_name) + if (count($author_ary) && $author_name) { // first one matches post of registered users, second one guests and deleted users $sql_author = ' AND (' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; } - else if (sizeof($author_ary)) + else if (count($author_ary)) { $sql_author = ' AND ' . $this->db->sql_in_set('p.poster_id', $author_ary); } @@ -580,7 +580,7 @@ class fulltext_mysql extends \phpbb\search\base $sql_where_options = $sql_sort_join; $sql_where_options .= ($topic_id) ? ' AND p.topic_id = ' . $topic_id : ''; $sql_where_options .= ($join_topic) ? ' AND t.topic_id = p.topic_id' : ''; - $sql_where_options .= (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_where_options .= (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_where_options .= ' AND ' . $post_visibility; $sql_where_options .= $sql_author; $sql_where_options .= ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; @@ -660,7 +660,7 @@ class fulltext_mysql extends \phpbb\search\base public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { // No author? No posts - if (!sizeof($author_ary)) + if (!count($author_ary)) { return 0; } @@ -737,7 +737,7 @@ class fulltext_mysql extends \phpbb\search\base { $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); } - $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; @@ -890,7 +890,7 @@ class fulltext_mysql extends \phpbb\search\base $id_ary = array_unique($id_ary); } - if (sizeof($id_ary)) + if (count($id_ary)) { $this->save_ids($search_key, '', $author_ary, $result_count, $id_ary, $start, $sort_dir); $id_ary = array_slice($id_ary, 0, $per_page); @@ -997,7 +997,7 @@ class fulltext_mysql extends \phpbb\search\base $alter_list[] = $alter_entry; } - if (sizeof($alter_list)) + if (count($alter_list)) { foreach ($alter_list as $alter) { @@ -1050,7 +1050,7 @@ class fulltext_mysql extends \phpbb\search\base $alter[] = 'DROP INDEX post_text'; } - if (sizeof($alter)) + if (count($alter)) { $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter)); } diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 73dcfce9a5..bd222488a0 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -285,7 +285,7 @@ class fulltext_native extends \phpbb\search\base ); $keywords = preg_replace($match, $replace, $keywords); - $num_keywords = sizeof(explode(' ', $keywords)); + $num_keywords = count(explode(' ', $keywords)); // We limit the number of allowed keywords to minimize load on the database if ($this->config['max_num_search_keywords'] && $num_keywords > $this->config['max_num_search_keywords']) @@ -301,7 +301,7 @@ class fulltext_native extends \phpbb\search\base $words = array(); preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words); - if (sizeof($words[1])) + if (count($words[1])) { $keywords = '(' . implode('|', $words[1]) . ')'; } @@ -316,7 +316,7 @@ class fulltext_native extends \phpbb\search\base $common_ids = $words = array(); - if (sizeof($exact_words)) + if (count($exact_words)) { $sql = 'SELECT word_id, word_text, word_common FROM ' . SEARCH_WORDLIST_TABLE . ' @@ -426,10 +426,10 @@ class fulltext_native extends \phpbb\search\base } } } - if (sizeof($id_words)) + if (count($id_words)) { sort($id_words); - if (sizeof($id_words) > 1) + if (count($id_words) > 1) { $this->{$mode . '_ids'}[] = $id_words; } @@ -440,7 +440,7 @@ class fulltext_native extends \phpbb\search\base } } // throw an error if we shall not ignore unexistant words - else if (!$ignore_no_id && sizeof($non_common_words)) + else if (!$ignore_no_id && count($non_common_words)) { trigger_error(sprintf($this->user->lang['WORDS_IN_NO_POST'], implode($this->user->lang['COMMA_SEPARATOR'], $non_common_words))); } @@ -480,7 +480,7 @@ class fulltext_native extends \phpbb\search\base } // Return true if all words are not common words - if (sizeof($exact_words) - sizeof($this->common_words) > 0) + if (count($exact_words) - count($this->common_words) > 0) { return true; } @@ -716,7 +716,7 @@ class fulltext_native extends \phpbb\search\base } } - if (sizeof($this->must_not_contain_ids)) + if (count($this->must_not_contain_ids)) { $sql_array['LEFT_JOIN'][] = array( 'FROM' => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num), @@ -826,7 +826,7 @@ class fulltext_native extends \phpbb\search\base $sql_where[] = 'p.topic_id = ' . $topic_id; } - if (sizeof($author_ary)) + if (count($author_ary)) { if ($author_name) { @@ -840,7 +840,7 @@ class fulltext_native extends \phpbb\search\base $sql_where[] = $sql_author; } - if (sizeof($ex_fid_ary)) + if (count($ex_fid_ary)) { $sql_where[] = $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true); } @@ -1010,7 +1010,7 @@ class fulltext_native extends \phpbb\search\base public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { // No author? No posts - if (!sizeof($author_ary)) + if (!count($author_ary)) { return 0; } @@ -1082,7 +1082,7 @@ class fulltext_native extends \phpbb\search\base { $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); } - $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; @@ -1289,7 +1289,7 @@ class fulltext_native extends \phpbb\search\base $this->db->sql_freeresult($result); } - if (sizeof($id_ary)) + if (count($id_ary)) { $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir); $id_ary = array_slice($id_ary, 0, $per_page); @@ -1443,7 +1443,7 @@ class fulltext_native extends \phpbb\search\base // individual arrays of added and removed words for text and title. What // we need to do now is add the new words (if they don't already exist) // and then add (or remove) matches between the words and this post - if (sizeof($unique_add_words)) + if (count($unique_add_words)) { $sql = 'SELECT word_id, word_text FROM ' . SEARCH_WORDLIST_TABLE . ' @@ -1459,7 +1459,7 @@ class fulltext_native extends \phpbb\search\base $new_words = array_diff($unique_add_words, array_keys($word_ids)); $this->db->sql_transaction('begin'); - if (sizeof($new_words)) + if (count($new_words)) { $sql_ary = array(); @@ -1483,7 +1483,7 @@ class fulltext_native extends \phpbb\search\base { $title_match = ($word_in == 'title') ? 1 : 0; - if (sizeof($word_ary)) + if (count($word_ary)) { $sql_in = array(); foreach ($word_ary as $word) @@ -1512,7 +1512,7 @@ class fulltext_native extends \phpbb\search\base { $title_match = ($word_in == 'title') ? 1 : 0; - if (sizeof($word_ary)) + if (count($word_ary)) { $sql = 'INSERT INTO ' . SEARCH_WORDMATCH_TABLE . ' (post_id, word_id, title_match) SELECT ' . (int) $post_id . ', word_id, ' . (int) $title_match . ' @@ -1543,7 +1543,7 @@ class fulltext_native extends \phpbb\search\base */ public function index_remove($post_ids, $author_ids, $forum_ids) { - if (sizeof($post_ids)) + if (count($post_ids)) { $sql = 'SELECT w.word_id, w.word_text, m.title_match FROM ' . SEARCH_WORDMATCH_TABLE . ' m, ' . SEARCH_WORDLIST_TABLE . ' w @@ -1566,7 +1566,7 @@ class fulltext_native extends \phpbb\search\base } $this->db->sql_freeresult($result); - if (sizeof($title_word_ids)) + if (count($title_word_ids)) { $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . ' SET word_count = word_count - 1 @@ -1575,7 +1575,7 @@ class fulltext_native extends \phpbb\search\base $this->db->sql_query($sql); } - if (sizeof($message_word_ids)) + if (count($message_word_ids)) { $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . ' SET word_count = word_count - 1 @@ -1630,7 +1630,7 @@ class fulltext_native extends \phpbb\search\base } $this->db->sql_freeresult($result); - if (sizeof($sql_in)) + if (count($sql_in)) { // Flag the words $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . ' @@ -1650,7 +1650,7 @@ class fulltext_native extends \phpbb\search\base unset($sql_in); } - if (sizeof($destroy_cache_words)) + if (count($destroy_cache_words)) { // destroy cached search results containing any of the words that are now common or were removed $this->destroy_cache(array_unique($destroy_cache_words)); @@ -1685,7 +1685,7 @@ class fulltext_native extends \phpbb\search\base */ public function index_created() { - if (!sizeof($this->stats)) + if (!count($this->stats)) { $this->get_stats(); } @@ -1698,7 +1698,7 @@ class fulltext_native extends \phpbb\search\base */ public function index_stats() { - if (!sizeof($this->stats)) + if (!count($this->stats)) { $this->get_stats(); } diff --git a/phpBB/phpbb/search/fulltext_postgres.php b/phpBB/phpbb/search/fulltext_postgres.php index 8dbc7212a1..4fe7e3b88d 100644 --- a/phpBB/phpbb/search/fulltext_postgres.php +++ b/phpBB/phpbb/search/fulltext_postgres.php @@ -294,7 +294,7 @@ class fulltext_postgres extends \phpbb\search\base // remove too short or too long words $text = array_values($text); - for ($i = 0, $n = sizeof($text); $i < $n; $i++) + for ($i = 0, $n = count($text); $i < $n; $i++) { $text[$i] = trim($text[$i]); if (utf8_strlen($text[$i]) < $this->config['fulltext_postgres_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_postgres_max_word_len']) @@ -502,12 +502,12 @@ class fulltext_postgres extends \phpbb\search\base $sql_from = ($join_topic) ? TOPICS_TABLE . ' t, ' : ''; $field = ($type == 'posts') ? 'post_id' : 'topic_id'; - if (sizeof($author_ary) && $author_name) + if (count($author_ary) && $author_name) { // first one matches post of registered users, second one guests and deleted users $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; } - else if (sizeof($author_ary)) + else if (count($author_ary)) { $sql_author = ' AND ' . $this->db->sql_in_set('p.poster_id', $author_ary); } @@ -519,7 +519,7 @@ class fulltext_postgres extends \phpbb\search\base $sql_where_options = $sql_sort_join; $sql_where_options .= ($topic_id) ? ' AND p.topic_id = ' . $topic_id : ''; $sql_where_options .= ($join_topic) ? ' AND t.topic_id = p.topic_id' : ''; - $sql_where_options .= (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_where_options .= (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_where_options .= ' AND ' . $post_visibility; $sql_where_options .= $sql_author; $sql_where_options .= ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; @@ -609,7 +609,7 @@ class fulltext_postgres extends \phpbb\search\base public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { // No author? No posts - if (!sizeof($author_ary)) + if (!count($author_ary)) { return 0; } @@ -686,7 +686,7 @@ class fulltext_postgres extends \phpbb\search\base { $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); } - $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; @@ -861,7 +861,7 @@ class fulltext_postgres extends \phpbb\search\base $id_ary = array_unique($id_ary); } - if (sizeof($id_ary)) + if (count($id_ary)) { $this->save_ids($search_key, '', $author_ary, $result_count, $id_ary, $start, $sort_dir); $id_ary = array_slice($id_ary, 0, $per_page); diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index a20e3ad4b5..54d32ca371 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -483,7 +483,7 @@ class fulltext_sphinx global $user, $phpbb_log; // No keywords? No posts. - if (!strlen($this->search_query) && !sizeof($author_ary)) + if (!strlen($this->search_query) && !count($author_ary)) { return false; } @@ -623,7 +623,7 @@ class fulltext_sphinx break; } - if (sizeof($author_ary)) + if (count($author_ary)) { $this->sphinx->SetFilter('poster_id', $author_ary); } @@ -633,14 +633,14 @@ class fulltext_sphinx // but at least it will also cause the same for normal users. $this->sphinx->SetFilter('post_visibility', array(ITEM_APPROVED)); - if (sizeof($ex_fid_ary)) + if (count($ex_fid_ary)) { // All forums that a user is allowed to access $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true)))); // All forums that the user wants to and can search in $search_forums = array_diff($fid_ary, $ex_fid_ary); - if (sizeof($search_forums)) + if (count($search_forums)) { $this->sphinx->SetFilter('forum_id', $search_forums); } @@ -790,7 +790,7 @@ class fulltext_sphinx } $this->db->sql_freeresult($result); - if (sizeof($post_updates)) + if (count($post_updates)) { $this->sphinx->UpdateAttributes($this->indexes, array('topic_last_post_time'), $post_updates); } diff --git a/phpBB/phpbb/search/sphinx/config.php b/phpBB/phpbb/search/sphinx/config.php index 675649b460..3205574b45 100644 --- a/phpBB/phpbb/search/sphinx/config.php +++ b/phpBB/phpbb/search/sphinx/config.php @@ -46,7 +46,7 @@ class config */ function get_section_by_name($name) { - for ($i = 0, $size = sizeof($this->sections); $i < $size; $i++) + for ($i = 0, $size = count($this->sections); $i < $size; $i++) { // Make sure this is really a section object and not a comment if (($this->sections[$i] instanceof \phpbb\search\sphinx\config_section) && $this->sections[$i]->get_name() == $name) @@ -67,7 +67,7 @@ class config function add_section($name) { $this->sections[] = new \phpbb\search\sphinx\config_section($name, ''); - return $this->sections[sizeof($this->sections) - 1]; + return $this->sections[count($this->sections) - 1]; } /** diff --git a/phpBB/phpbb/search/sphinx/config_section.php b/phpBB/phpbb/search/sphinx/config_section.php index 14ab3a752c..2fc8b2da17 100644 --- a/phpBB/phpbb/search/sphinx/config_section.php +++ b/phpBB/phpbb/search/sphinx/config_section.php @@ -87,7 +87,7 @@ class config_section */ function get_variable_by_name($name) { - for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) + for ($i = 0, $size = count($this->variables); $i < $size; $i++) { // Make sure this is a variable object and not a comment if (($this->variables[$i] instanceof \phpbb\search\sphinx\config_variable) && $this->variables[$i]->get_name() == $name) @@ -106,7 +106,7 @@ class config_section */ function delete_variables_by_name($name) { - for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) + for ($i = 0, $size = count($this->variables); $i < $size; $i++) { // Make sure this is a variable object and not a comment if (($this->variables[$i] instanceof \phpbb\search\sphinx\config_variable) && $this->variables[$i]->get_name() == $name) @@ -129,7 +129,7 @@ class config_section function create_variable($name, $value) { $this->variables[] = new \phpbb\search\sphinx\config_variable($name, $value, ''); - return $this->variables[sizeof($this->variables) - 1]; + return $this->variables[count($this->variables) - 1]; } /** -- cgit v1.2.1 From d05c7fd34288375683c5b0467930748b1457ac75 Mon Sep 17 00:00:00 2001 From: Marc Alexander Date: Sat, 6 Jan 2018 16:06:59 +0100 Subject: [ticket/15496] Add sort key to SELECT in fulltext_postgres Postgres requires that the column being ordered also appears in the SELECT part of the query. PHPBB3-15496 --- phpBB/phpbb/search/fulltext_postgres.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_postgres.php b/phpBB/phpbb/search/fulltext_postgres.php index 8dbc7212a1..30aeb35ee0 100644 --- a/phpBB/phpbb/search/fulltext_postgres.php +++ b/phpBB/phpbb/search/fulltext_postgres.php @@ -498,7 +498,7 @@ class fulltext_postgres extends \phpbb\search\base ); extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_keywords_main_query_before', compact($vars))); - $sql_select = ($type == 'posts') ? 'p.post_id' : 'DISTINCT t.topic_id'; + $sql_select = ($type == 'posts') ? 'p.post_id' : 'DISTINCT t.topic_id, ' . $sort_by_sql[$sort_key]; $sql_from = ($join_topic) ? TOPICS_TABLE . ' t, ' : ''; $field = ($type == 'posts') ? 'post_id' : 'topic_id'; -- cgit v1.2.1 From 013f9819b00bb941faee9c0578bd6fb2f3378a72 Mon Sep 17 00:00:00 2001 From: kasimi Date: Sun, 28 Jan 2018 16:14:29 +0100 Subject: [ticket/15537] Add core.search_(native|mysql|postgres|sphinx)_index_before PHPBB3-15537 --- phpBB/phpbb/search/fulltext_mysql.php | 28 ++++++++++++++++++++++++++++ phpBB/phpbb/search/fulltext_native.php | 29 +++++++++++++++++++++++++++++ phpBB/phpbb/search/fulltext_postgres.php | 28 ++++++++++++++++++++++++++++ phpBB/phpbb/search/fulltext_sphinx.php | 22 ++++++++++++++++++++++ 4 files changed, 107 insertions(+) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_mysql.php b/phpBB/phpbb/search/fulltext_mysql.php index 51c5fe8b76..cc8180ec69 100644 --- a/phpBB/phpbb/search/fulltext_mysql.php +++ b/phpBB/phpbb/search/fulltext_mysql.php @@ -918,6 +918,34 @@ class fulltext_mysql extends \phpbb\search\base $words = array_unique(array_merge($split_text, $split_title)); + /** + * Event to modify method arguments and words before the MySQL search index is updated + * + * @event core.search_mysql_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @var array words List of words added to the index + * @var array split_text Array of words from the message + * @var array split_title Array of words from the title + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + 'words', + 'split_text', + 'split_title', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_index_before', compact($vars))); + unset($split_text); unset($split_title); diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index bd222488a0..76d86526af 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -1433,6 +1433,35 @@ class fulltext_native extends \phpbb\search\base $words['del']['post'] = array(); $words['del']['title'] = array(); } + + /** + * Event to modify method arguments and words before the native search index is updated + * + * @event core.search_native_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @var array words Grouped lists of words added to or remove from the index + * @var array split_text Array of words from the message + * @var array split_title Array of words from the title + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + 'words', + 'split_text', + 'split_title', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_index_before', compact($vars))); + unset($split_text); unset($split_title); diff --git a/phpBB/phpbb/search/fulltext_postgres.php b/phpBB/phpbb/search/fulltext_postgres.php index 6443342057..ac76f2f87a 100644 --- a/phpBB/phpbb/search/fulltext_postgres.php +++ b/phpBB/phpbb/search/fulltext_postgres.php @@ -889,6 +889,34 @@ class fulltext_postgres extends \phpbb\search\base $words = array_unique(array_merge($split_text, $split_title)); + /** + * Event to modify method arguments and words before the PostgreSQL search index is updated + * + * @event core.search_postgres_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @var array words Array of words added to the index + * @var array split_text Array of words from the message + * @var array split_title Array of words from the title + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + 'words', + 'split_text', + 'split_title', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_index_before', compact($vars))); + unset($split_text); unset($split_title); diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index 54d32ca371..e2eeb5f7f3 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -758,6 +758,28 @@ class fulltext_sphinx */ public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id) { + /** + * Event to modify method arguments before the Sphinx search index is updated + * + * @event core.search_sphinx_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_index_before', compact($vars))); + if ($mode == 'edit') { $this->sphinx->UpdateAttributes($this->indexes, array('forum_id', 'poster_id'), array((int) $post_id => array((int) $forum_id, (int) $poster_id))); -- cgit v1.2.1 From 9f7f3d14634c2995714800e93cf99272d3d531f0 Mon Sep 17 00:00:00 2001 From: kasimi Date: Sun, 28 Jan 2018 18:02:22 +0100 Subject: [ticket/15537] Add cur_words argument to core.search_native_index_before PHPBB3-15537 --- phpBB/phpbb/search/fulltext_native.php | 3 +++ 1 file changed, 3 insertions(+) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 76d86526af..eb972a257a 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -1447,6 +1447,8 @@ class fulltext_native extends \phpbb\search\base * @var array words Grouped lists of words added to or remove from the index * @var array split_text Array of words from the message * @var array split_title Array of words from the title + * @var array cur_words Array of words currently in the index for comparing to new words + * when mode is edit. Empty for other modes. * @since 3.2.3-RC1 */ $vars = array( @@ -1459,6 +1461,7 @@ class fulltext_native extends \phpbb\search\base 'words', 'split_text', 'split_title', + 'cur_words', ); extract($this->phpbb_dispatcher->trigger_event('core.search_native_index_before', compact($vars))); -- cgit v1.2.1 From a6dc32b381dea235c979d089fc7897dd7e213011 Mon Sep 17 00:00:00 2001 From: kasimi Date: Fri, 16 Feb 2018 21:31:36 +0100 Subject: [ticket/15561] Add events for adding columns to search index PHPBB3-15561 --- phpBB/phpbb/search/fulltext_mysql.php | 57 ++++++++++++++++++++++++++++---- phpBB/phpbb/search/fulltext_native.php | 35 ++++++++++++++++---- phpBB/phpbb/search/fulltext_postgres.php | 54 +++++++++++++++++++++++++++--- 3 files changed, 129 insertions(+), 17 deletions(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_mysql.php b/phpBB/phpbb/search/fulltext_mysql.php index 51c5fe8b76..0d07f5fbbf 100644 --- a/phpBB/phpbb/search/fulltext_mysql.php +++ b/phpBB/phpbb/search/fulltext_mysql.php @@ -997,17 +997,37 @@ class fulltext_mysql extends \phpbb\search\base $alter_list[] = $alter_entry; } - if (count($alter_list)) + $sql_queries = []; + + foreach ($alter_list as $alter) { - foreach ($alter_list as $alter) - { - $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter)); - } + $sql_queries[] = 'ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter); } if (!isset($this->stats['post_text'])) { - $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ADD FULLTEXT post_text (post_text)'); + $sql_queries[] = 'ALTER TABLE ' . POSTS_TABLE . ' ADD FULLTEXT post_text (post_text)'; + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the MySQL search index is created + * + * @event core.search_mysql_create_index_before + * @var array sql_queries Array with queries for creating the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_create_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); @@ -1050,9 +1070,32 @@ class fulltext_mysql extends \phpbb\search\base $alter[] = 'DROP INDEX post_text'; } + $sql_queries = []; + if (count($alter)) { - $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter)); + $sql_queries[] = 'ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter); + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the MySQL search index is deleted + * + * @event core.search_mysql_delete_index_before + * @var array sql_queries Array with queries for deleting the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_delete_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index bd222488a0..3d97896be6 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -1664,20 +1664,43 @@ class fulltext_native extends \phpbb\search\base */ public function delete_index($acp_module, $u_action) { + $sql_queries = []; + switch ($this->db->get_sql_layer()) { case 'sqlite3': - $this->db->sql_query('DELETE FROM ' . SEARCH_WORDLIST_TABLE); - $this->db->sql_query('DELETE FROM ' . SEARCH_WORDMATCH_TABLE); - $this->db->sql_query('DELETE FROM ' . SEARCH_RESULTS_TABLE); + $sql_queries[] = 'DELETE FROM ' . SEARCH_WORDLIST_TABLE; + $sql_queries[] = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE; + $sql_queries[] = 'DELETE FROM ' . SEARCH_RESULTS_TABLE; break; default: - $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_WORDLIST_TABLE); - $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_WORDMATCH_TABLE); - $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); + $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_WORDLIST_TABLE; + $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_WORDMATCH_TABLE; + $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE; break; } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the native search index is deleted + * + * @event core.search_native_delete_index_before + * @var array sql_queries Array with queries for deleting the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_delete_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } } /** diff --git a/phpBB/phpbb/search/fulltext_postgres.php b/phpBB/phpbb/search/fulltext_postgres.php index 6443342057..e81e828f26 100644 --- a/phpBB/phpbb/search/fulltext_postgres.php +++ b/phpBB/phpbb/search/fulltext_postgres.php @@ -935,14 +935,37 @@ class fulltext_postgres extends \phpbb\search\base $this->get_stats(); } + $sql_queries = []; + if (!isset($this->stats['post_subject'])) { - $this->db->sql_query("CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_subject ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_subject))"); + $sql_queries[] = "CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_subject ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_subject))"; } if (!isset($this->stats['post_content'])) { - $this->db->sql_query("CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_content ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_text || ' ' || post_subject))"); + $sql_queries[] = "CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_content ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_text || ' ' || post_subject))"; + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the Postgres search index is created + * + * @event core.search_postgres_create_index_before + * @var array sql_queries Array with queries for creating the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_create_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); @@ -968,14 +991,37 @@ class fulltext_postgres extends \phpbb\search\base $this->get_stats(); } + $sql_queries = []; + if (isset($this->stats['post_subject'])) { - $this->db->sql_query('DROP INDEX ' . $this->stats['post_subject']['relname']); + $sql_queries[] = 'DROP INDEX ' . $this->stats['post_subject']['relname']; } if (isset($this->stats['post_content'])) { - $this->db->sql_query('DROP INDEX ' . $this->stats['post_content']['relname']); + $sql_queries[] = 'DROP INDEX ' . $this->stats['post_content']['relname']; + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the Postgres search index is created + * + * @event core.search_postgres_delete_index_before + * @var array sql_queries Array with queries for deleting the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_delete_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); -- cgit v1.2.1 From 1f00f48e717d1a9e82ef627c16353595ddb8ae61 Mon Sep 17 00:00:00 2001 From: Zoddo Date: Mon, 23 Apr 2018 10:49:53 +0200 Subject: [ticket/13128] Remove deprecated Sphinx options These configuration options are deprecated: * sql_query_info (since sphinx 2.2.2; used only by the "search" command which was removed; wiki need to be updated accordingly) * max_matches (since sphinx 2.2.3; should be set in the sphinx query) * charset_type (since sphinx 2.2.2; sphinx now supports only utf-8) * compat_sphinxql_magics (since sphinx 2.2.1) PHPBB3-13128 --- phpBB/phpbb/search/fulltext_sphinx.php | 4 ---- 1 file changed, 4 deletions(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index e2eeb5f7f3..d780442f89 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -292,7 +292,6 @@ class fulltext_sphinx AND p.post_id >= $start AND p.post_id <= $end'), array('sql_query_post', ''), array('sql_query_post_index', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'), - array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'), array('sql_attr_uint', 'forum_id'), array('sql_attr_uint', 'topic_id'), array('sql_attr_uint', 'poster_id'), @@ -333,7 +332,6 @@ class fulltext_sphinx array('morphology', 'none'), array('stopwords', ''), array('min_word_len', '2'), - array('charset_type', 'utf-8'), array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), array('min_prefix_len', '0'), array('min_infix_len', '0'), @@ -346,14 +344,12 @@ class fulltext_sphinx array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'), ), 'searchd' => array( - array('compat_sphinxql_magics' , '0'), array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')), array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), array('read_timeout', '5'), array('max_children', '30'), array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'), - array('max_matches', (string) SPHINX_MAX_MATCHES), array('binlog_path', $this->config['fulltext_sphinx_data_path']), ), ); -- cgit v1.2.1 From 4ac3c545e1b95fb2abfdd6f4343865e7646c978e Mon Sep 17 00:00:00 2001 From: Marc Alexander Date: Fri, 14 Sep 2018 15:02:07 +0200 Subject: [ticket/15770] Make sure sphinx limits are cast to int PHPBB3-15770 --- phpBB/phpbb/search/fulltext_sphinx.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index e2eeb5f7f3..878c98bf72 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -648,7 +648,7 @@ class fulltext_sphinx $this->sphinx->SetFilter('deleted', array(0)); - $this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES); + $this->sphinx->SetLimits((int) $start, (int) $per_page, SPHINX_MAX_MATCHES); $result = $this->sphinx->Query($search_query_prefix . $this->sphinx->EscapeString(str_replace('"', '"', $this->search_query)), $this->indexes); // Could be connection to localhost:9312 failed (errno=111, -- cgit v1.2.1 From 56060caa4c44620929b6e17fe4622343750ad302 Mon Sep 17 00:00:00 2001 From: Derky Date: Thu, 14 Mar 2019 21:46:02 +0100 Subject: [ticket/security/235] Apply wildcard char count patch SECURITY-235 --- phpBB/phpbb/search/fulltext_native.php | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 4172e2cc4f..9a6d62f9d8 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -190,7 +190,7 @@ class fulltext_native extends \phpbb\search\base */ public function split_keywords($keywords, $terms) { - $tokens = '+-|()*'; + $tokens = '+-|()* '; $keywords = trim($this->cleanup($keywords, $tokens)); @@ -224,12 +224,10 @@ class fulltext_native extends \phpbb\search\base $keywords[$i] = '|'; break; case '*': - if ($i === 0 || ($keywords[$i - 1] !== '*' && strcspn($keywords[$i - 1], $tokens) === 0)) + // $i can never be 0 here since $open_bracket is initialised to false + if (strpos($tokens, $keywords[$i - 1]) !== false && ($i + 1 === $n || strpos($tokens, $keywords[$i + 1]) !== false)) { - if ($i === $n - 1 || ($keywords[$i + 1] !== '*' && strcspn($keywords[$i + 1], $tokens) === 0)) - { - $keywords = substr($keywords, 0, $i) . substr($keywords, $i + 1); - } + $keywords[$i] = '|'; } break; } @@ -264,7 +262,7 @@ class fulltext_native extends \phpbb\search\base } } - if ($open_bracket) + if ($open_bracket !== false) { $keywords .= ')'; } @@ -409,8 +407,16 @@ class fulltext_native extends \phpbb\search\base { if (strpos($word_part, '*') !== false) { - $id_words[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word_part)) . '\''; - $non_common_words[] = $word_part; + $len = utf8_strlen(str_replace('*', '', $word_part)); + if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) + { + $id_words[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word_part)) . '\''; + $non_common_words[] = $word_part; + } + else + { + $this->common_words[] = $word_part; + } } else if (isset($words[$word_part])) { -- cgit v1.2.1 From fd195fba210c8625e968ef5553e61864747c8d44 Mon Sep 17 00:00:00 2001 From: Derky Date: Thu, 25 Apr 2019 21:51:04 +0200 Subject: [ticket/security/235] Remove non trailing wildcards from search keywords Database indexes are only used if wildcards are used at the end. SECURITY-235 --- phpBB/phpbb/search/fulltext_native.php | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 9a6d62f9d8..478fe5616d 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -305,6 +305,11 @@ class fulltext_native extends \phpbb\search\base } } + // Remove non trailing wildcards from each word to prevent a full table scan (it's now using the database index) + $match = '#\*(?!$)\b#'; + $replace = '$1'; + $keywords = preg_replace($match, $replace, $keywords); + // set the search_query which is shown to the user $this->search_query = $keywords; -- cgit v1.2.1 From 8a73eb5f0ff912454e6479539f972081e54baa1c Mon Sep 17 00:00:00 2001 From: Derky Date: Fri, 26 Apr 2019 00:52:43 +0200 Subject: [ticket/security/235] Use whitespace instead of word boundary regex to remove wildcards This fixes removing the wildcard in the following search query: *.test SECURITY-235 --- phpBB/phpbb/search/fulltext_native.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 478fe5616d..1925623b80 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -306,7 +306,7 @@ class fulltext_native extends \phpbb\search\base } // Remove non trailing wildcards from each word to prevent a full table scan (it's now using the database index) - $match = '#\*(?!$)\b#'; + $match = '#\*(?!$|\s)#'; $replace = '$1'; $keywords = preg_replace($match, $replace, $keywords); -- cgit v1.2.1 From da9910850a168f73c6b8dd8407a01f47d27ca1d8 Mon Sep 17 00:00:00 2001 From: Derky Date: Fri, 26 Apr 2019 00:56:48 +0200 Subject: [ticket/security/235] Only allow one wildcard in the search query to limit the database load SECURITY-235 --- phpBB/phpbb/search/fulltext_native.php | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 1925623b80..c83de75eed 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -310,6 +310,15 @@ class fulltext_native extends \phpbb\search\base $replace = '$1'; $keywords = preg_replace($match, $replace, $keywords); + // Only allow one wildcard in the search query to limit the database load + $match = '#\*#'; + $replace = '$1'; + $count_wildcards = substr_count($keywords, '*'); + + // Reverse the string to remove all wildcards except the first one + $keywords = strrev(preg_replace($match, $replace, strrev($keywords), $count_wildcards - 1)); + unset($count_wildcards); + // set the search_query which is shown to the user $this->search_query = $keywords; -- cgit v1.2.1 From e33e5727413543c74b3ede43ad437bb261c72839 Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Mon, 8 Jul 2019 05:46:41 +0000 Subject: [ticket/16096] Use InnoDB fulltext limits for InnoDB tables The max and min search length for the MySQL database vary based on the engine for the underlying table. For MyISAM tables, the variables are ft_max_word_len and ft_min_word_len, but for InnoDB tables the variables are innodb_ft_max_token_size and innodb_ft_min_token_size. Take the posts table type into account when setting the max and min search length. PHPBB3-16096 --- phpBB/phpbb/search/fulltext_mysql.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'phpBB/phpbb/search') diff --git a/phpBB/phpbb/search/fulltext_mysql.php b/phpBB/phpbb/search/fulltext_mysql.php index 137ed7433d..1105d0892f 100644 --- a/phpBB/phpbb/search/fulltext_mysql.php +++ b/phpBB/phpbb/search/fulltext_mysql.php @@ -188,7 +188,7 @@ class fulltext_mysql extends \phpbb\search\base } $sql = 'SHOW VARIABLES - LIKE \'ft\_%\''; + LIKE \'%ft\_%\''; $result = $this->db->sql_query($sql); $mysql_info = array(); @@ -198,8 +198,16 @@ class fulltext_mysql extends \phpbb\search\base } $this->db->sql_freeresult($result); - $this->config->set('fulltext_mysql_max_word_len', $mysql_info['ft_max_word_len']); - $this->config->set('fulltext_mysql_min_word_len', $mysql_info['ft_min_word_len']); + if ($engine === 'MyISAM') + { + $this->config->set('fulltext_mysql_max_word_len', $mysql_info['ft_max_word_len']); + $this->config->set('fulltext_mysql_min_word_len', $mysql_info['ft_min_word_len']); + } + else if ($engine === 'InnoDB') + { + $this->config->set('fulltext_mysql_max_word_len', $mysql_info['innodb_ft_max_token_size']); + $this->config->set('fulltext_mysql_min_word_len', $mysql_info['innodb_ft_min_token_size']); + } return false; } -- cgit v1.2.1