diff options
Diffstat (limited to 'phpBB/phpbb/search')
| -rw-r--r-- | phpBB/phpbb/search/base.php | 58 | ||||
| -rw-r--r-- | phpBB/phpbb/search/fulltext_mysql.php | 352 | ||||
| -rw-r--r-- | phpBB/phpbb/search/fulltext_native.php | 400 | ||||
| -rw-r--r-- | phpBB/phpbb/search/fulltext_postgres.php | 288 | ||||
| -rw-r--r-- | phpBB/phpbb/search/fulltext_sphinx.php | 139 | ||||
| -rw-r--r-- | phpBB/phpbb/search/sphinx/config.php | 4 | ||||
| -rw-r--r-- | phpBB/phpbb/search/sphinx/config_section.php | 6 |
7 files changed, 1022 insertions, 225 deletions
diff --git a/phpBB/phpbb/search/base.php b/phpBB/phpbb/search/base.php index 30781975d8..e7d0774b6c 100644 --- a/phpBB/phpbb/search/base.php +++ b/phpBB/phpbb/search/base.php @@ -37,52 +37,6 @@ class base } /** - * Retrieves a language dependend list of words that should be ignored by the search - */ - function get_ignore_words() - { - if (!sizeof($this->ignore_words)) - { - global $user, $phpEx; - - $words = array(); - - if (file_exists("{$user->lang_path}{$user->lang_name}/search_ignore_words.$phpEx")) - { - // include the file containing ignore words - include("{$user->lang_path}{$user->lang_name}/search_ignore_words.$phpEx"); - } - - $this->ignore_words = $words; - unset($words); - } - } - - /** - * Stores a list of synonyms that should be replaced in $this->match_synonym and $this->replace_synonym and caches them - */ - function get_synonyms() - { - if (!sizeof($this->match_synonym)) - { - global $user, $phpEx; - - $synonyms = array(); - - if (file_exists("{$user->lang_path}{$user->lang_name}/search_synonyms.$phpEx")) - { - // include the file containing synonyms - include("{$user->lang_path}{$user->lang_name}/search_synonyms.$phpEx"); - } - - $this->match_synonym = array_keys($synonyms); - $this->replace_synonym = array_values($synonyms); - - unset($synonyms); - } - } - - /** * Retrieves cached search results * * @param string $search_key an md5 string generated from all the passed search options to identify the results @@ -179,7 +133,7 @@ class base { global $cache, $config, $db, $user; - $length = min(sizeof($id_ary), $config['search_block_size']); + $length = min(count($id_ary), $config['search_block_size']); // nothing to cache so exit if (!$length) @@ -194,7 +148,7 @@ class base if (!($store = $cache->get('_search_results_' . $search_key))) { // add the current keywords to the recent searches in the cache which are listed on the search page - if (!empty($keywords) || sizeof($author_ary)) + if (!empty($keywords) || count($author_ary)) { $sql = 'SELECT search_time FROM ' . SEARCH_RESULTS_TABLE . ' @@ -247,7 +201,7 @@ class base $store += $store_ids; // if the cache is too big - if (sizeof($store) - 2 > 20 * $config['search_block_size']) + if (count($store) - 2 > 20 * $config['search_block_size']) { // remove everything in front of two blocks in front of the current start index for ($i = 0, $n = $id_range[0] - 2 * $config['search_block_size']; $i < $n; $i++) @@ -289,7 +243,7 @@ class base global $db, $cache, $config; // clear all searches that searched for the specified words - if (sizeof($words)) + if (count($words)) { $sql_where = ''; foreach ($words as $word) @@ -310,7 +264,7 @@ class base } // clear all searches that searched for the specified authors - if (is_array($authors) && sizeof($authors)) + if (is_array($authors) && count($authors)) { $sql_where = ''; foreach ($authors as $author) @@ -332,7 +286,7 @@ class base $sql = 'DELETE FROM ' . SEARCH_RESULTS_TABLE . ' - WHERE search_time < ' . (time() - $config['search_store_results']); + WHERE search_time < ' . (time() - (int) $config['search_store_results']); $db->sql_query($sql); } } diff --git a/phpBB/phpbb/search/fulltext_mysql.php b/phpBB/phpbb/search/fulltext_mysql.php index 1a0aba096f..137ed7433d 100644 --- a/phpBB/phpbb/search/fulltext_mysql.php +++ b/phpBB/phpbb/search/fulltext_mysql.php @@ -43,6 +43,12 @@ class fulltext_mysql extends \phpbb\search\base protected $db; /** + * phpBB event dispatcher object + * @var \phpbb\event\dispatcher_interface + */ + protected $phpbb_dispatcher; + + /** * User object * @var \phpbb\user */ @@ -79,11 +85,13 @@ class fulltext_mysql extends \phpbb\search\base * @param \phpbb\config\config $config Config object * @param \phpbb\db\driver\driver_interface Database object * @param \phpbb\user $user User object + * @param \phpbb\event\dispatcher_interface $phpbb_dispatcher Event dispatcher object */ - public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user) + public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher) { $this->config = $config; $this->db = $db; + $this->phpbb_dispatcher = $phpbb_dispatcher; $this->user = $user; $this->word_length = array('min' => $this->config['fulltext_mysql_min_word_len'], 'max' => $this->config['fulltext_mysql_max_word_len']); @@ -169,8 +177,10 @@ class fulltext_mysql extends \phpbb\search\base $engine === 'MyISAM' || // FULLTEXT is supported on InnoDB since MySQL 5.6.4 according to // http://dev.mysql.com/doc/refman/5.6/en/innodb-storage-engine.html + // We also require https://bugs.mysql.com/bug.php?id=67004 to be + // fixed for proper overall operation. Hence we require 5.6.8. $engine === 'InnoDB' && - phpbb_version_compare($this->db->sql_server_info(true), '5.6.4', '>='); + phpbb_version_compare($this->db->sql_server_info(true), '5.6.8', '>='); if (!$fulltext_supported) { @@ -188,8 +198,8 @@ class fulltext_mysql extends \phpbb\search\base } $this->db->sql_freeresult($result); - set_config('fulltext_mysql_max_word_len', $mysql_info['ft_max_word_len']); - set_config('fulltext_mysql_min_word_len', $mysql_info['ft_min_word_len']); + $this->config->set('fulltext_mysql_max_word_len', $mysql_info['ft_max_word_len']); + $this->config->set('fulltext_mysql_min_word_len', $mysql_info['ft_min_word_len']); return false; } @@ -222,9 +232,9 @@ class fulltext_mysql extends \phpbb\search\base $this->split_words = $matches[1]; // We limit the number of allowed keywords to minimize load on the database - if ($this->config['max_num_search_keywords'] && sizeof($this->split_words) > $this->config['max_num_search_keywords']) + if ($this->config['max_num_search_keywords'] && count($this->split_words) > $this->config['max_num_search_keywords']) { - trigger_error($this->user->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], sizeof($this->split_words))); + trigger_error($this->user->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], count($this->split_words))); } // to allow phrase search, we need to concatenate quoted words @@ -262,6 +272,27 @@ class fulltext_mysql extends \phpbb\search\base foreach ($this->split_words as $i => $word) { + // Check for not allowed search queries for InnoDB. + // We assume similar restrictions for MyISAM, which is usually even + // slower but not as restrictive as InnoDB. + // InnoDB full-text search does not support the use of a leading + // plus sign with wildcard ('+*'), a plus and minus sign + // combination ('+-'), or leading a plus and minus sign combination. + // InnoDB full-text search only supports leading plus or minus signs. + // For example, InnoDB supports '+apple' but does not support 'apple+'. + // Specifying a trailing plus or minus sign causes InnoDB to report + // a syntax error. InnoDB full-text search does not support the use + // of multiple operators on a single search word, as in this example: + // '++apple'. Use of multiple operators on a single search word + // returns a syntax error to standard out. + // Also, ensure that the wildcard character is only used at the + // end of the line as it's intended by MySQL. + if (preg_match('#^(\+[+-]|\+\*|.+[+-]$|.+\*(?!$))#', $word)) + { + unset($this->split_words[$i]); + continue; + } + $clean_word = preg_replace('#^[+\-|"]#', '', $word); // check word length @@ -330,7 +361,7 @@ class fulltext_mysql extends \phpbb\search\base // remove too short or too long words $text = array_values($text); - for ($i = 0, $n = sizeof($text); $i < $n; $i++) + for ($i = 0, $n = count($text); $i < $n; $i++) { $text[$i] = trim($text[$i]); if (utf8_strlen($text[$i]) < $this->config['fulltext_mysql_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_mysql_max_word_len']) @@ -371,7 +402,7 @@ class fulltext_mysql extends \phpbb\search\base } // generate a search_key from all the options to identify the results - $search_key = md5(implode('#', array( + $search_key_array = array( implode(', ', $this->split_words), $type, $fields, @@ -382,7 +413,39 @@ class fulltext_mysql extends \phpbb\search\base implode(',', $ex_fid_ary), $post_visibility, implode(',', $author_ary) - ))); + ); + + /** + * Allow changing the search_key for cached results + * + * @event core.search_mysql_by_keyword_modify_search_key + * @var array search_key_array Array with search parameters to generate the search_key + * @var string type Searching type ('posts', 'topics') + * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all') + * @var string terms Searching terms ('all', 'any') + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @since 3.1.7-RC1 + */ + $vars = array( + 'search_key_array', + 'type', + 'fields', + 'terms', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_by_keyword_modify_search_key', compact($vars))); + + $search_key = md5(implode('#', $search_key_array)); if ($start < 0) { @@ -447,16 +510,65 @@ class fulltext_mysql extends \phpbb\search\base break; } + $search_query = $this->search_query; + + /** + * Allow changing the query used to search for posts using fulltext_mysql + * + * @event core.search_mysql_keywords_main_query_before + * @var string search_query The parsed keywords used for this search + * @var int result_count The previous result count for the format of the query. + * Set to 0 to force a re-count + * @var bool join_topic Weather or not TOPICS_TABLE should be CROSS JOIN'ED + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name An extra username to search on (!empty(author_ary) must be true, to be relevant) + * @var array ex_fid_ary Which forums not to search on + * @var int topic_id Limit the search to this topic_id only + * @var string sql_sort_table Extra tables to include in the SQL query. + * Used in conjunction with sql_sort_join + * @var string sql_sort_join SQL conditions to join all the tables used together. + * Used in conjunction with sql_sort_table + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sql_match Which columns to do the search on. + * @var string sql_match_where Extra conditions to use to properly filter the matching process + * @var string sort_by_sql The possible predefined sort types + * @var string sort_key The sort type used from the possible sort types + * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used + * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir + * @var int start How many posts to skip in the search results (used for pagination) + * @since 3.1.5-RC1 + */ + $vars = array( + 'search_query', + 'result_count', + 'join_topic', + 'author_ary', + 'author_name', + 'ex_fid_ary', + 'topic_id', + 'sql_sort_table', + 'sql_sort_join', + 'sort_days', + 'sql_match', + 'sql_match_where', + 'sort_by_sql', + 'sort_key', + 'sort_dir', + 'sql_sort', + 'start', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_keywords_main_query_before', compact($vars))); + $sql_select = (!$result_count) ? 'SQL_CALC_FOUND_ROWS ' : ''; $sql_select = ($type == 'posts') ? $sql_select . 'p.post_id' : 'DISTINCT ' . $sql_select . 't.topic_id'; $sql_from = ($join_topic) ? TOPICS_TABLE . ' t, ' : ''; $field = ($type == 'posts') ? 'post_id' : 'topic_id'; - if (sizeof($author_ary) && $author_name) + if (count($author_ary) && $author_name) { // first one matches post of registered users, second one guests and deleted users $sql_author = ' AND (' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; } - else if (sizeof($author_ary)) + else if (count($author_ary)) { $sql_author = ' AND ' . $this->db->sql_in_set('p.poster_id', $author_ary); } @@ -468,7 +580,7 @@ class fulltext_mysql extends \phpbb\search\base $sql_where_options = $sql_sort_join; $sql_where_options .= ($topic_id) ? ' AND p.topic_id = ' . $topic_id : ''; $sql_where_options .= ($join_topic) ? ' AND t.topic_id = p.topic_id' : ''; - $sql_where_options .= (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_where_options .= (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_where_options .= ' AND ' . $post_visibility; $sql_where_options .= $sql_author; $sql_where_options .= ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; @@ -479,6 +591,7 @@ class fulltext_mysql extends \phpbb\search\base WHERE MATCH ($sql_match) AGAINST ('" . $this->db->sql_escape(htmlspecialchars_decode($this->search_query)) . "' IN BOOLEAN MODE) $sql_where_options ORDER BY $sql_sort"; + $this->db->sql_return_on_error(true); $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); while ($row = $this->db->sql_fetchrow($result)) @@ -490,7 +603,7 @@ class fulltext_mysql extends \phpbb\search\base $id_ary = array_unique($id_ary); // if the total result count is not cached yet, retrieve it from the db - if (!$result_count) + if (!$result_count && count($id_ary)) { $sql_found_rows = 'SELECT FOUND_ROWS() as result_count'; $result = $this->db->sql_query($sql_found_rows); @@ -547,13 +660,13 @@ class fulltext_mysql extends \phpbb\search\base public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { // No author? No posts - if (!sizeof($author_ary)) + if (!count($author_ary)) { return 0; } // generate a search_key from all the options to identify the results - $search_key = md5(implode('#', array( + $search_key_array = array( '', $type, ($firstpost_only) ? 'firstpost' : '', @@ -566,7 +679,39 @@ class fulltext_mysql extends \phpbb\search\base $post_visibility, implode(',', $author_ary), $author_name, - ))); + ); + + /** + * Allow changing the search_key for cached results + * + * @event core.search_mysql_by_author_modify_search_key + * @var array search_key_array Array with search parameters to generate the search_key + * @var string type Searching type ('posts', 'topics') + * @var boolean firstpost_only Flag indicating if only topic starting posts are considered + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name The username to search on + * @since 3.1.7-RC1 + */ + $vars = array( + 'search_key_array', + 'type', + 'firstpost_only', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + 'author_name', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_by_author_modify_search_key', compact($vars))); + + $search_key = md5(implode('#', $search_key_array)); if ($start < 0) { @@ -592,7 +737,7 @@ class fulltext_mysql extends \phpbb\search\base { $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); } - $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; @@ -620,6 +765,59 @@ class fulltext_mysql extends \phpbb\search\base $m_approve_fid_sql = ' AND ' . $post_visibility; + /** + * Allow changing the query used to search for posts by author in fulltext_mysql + * + * @event core.search_mysql_author_query_before + * @var int result_count The previous result count for the format of the query. + * Set to 0 to force a re-count + * @var string sql_sort_table CROSS JOIN'ed table to allow doing the sort chosen + * @var string sql_sort_join Condition to define how to join the CROSS JOIN'ed table specifyed in sql_sort_table + * @var string type Either "posts" or "topics" specifying the type of search being made + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name An extra username to search on + * @var string sql_author SQL WHERE condition for the post author ids + * @var int topic_id Limit the search to this topic_id only + * @var string sql_topic_id SQL of topic_id + * @var string sort_by_sql The possible predefined sort types + * @var string sort_key The sort type used from the possible sort types + * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used + * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir + * @var string sort_days Time, in days, that the oldest post showing can have + * @var string sql_time The SQL to search on the time specifyed by sort_days + * @var bool firstpost_only Wether or not to search only on the first post of the topics + * @var string sql_firstpost The SQL with the conditions to join the tables when using firstpost_only + * @var array ex_fid_ary Forum ids that must not be searched on + * @var array sql_fora SQL query for ex_fid_ary + * @var string m_approve_fid_sql WHERE clause condition on post_visibility restrictions + * @var int start How many posts to skip in the search results (used for pagination) + * @since 3.1.5-RC1 + */ + $vars = array( + 'result_count', + 'sql_sort_table', + 'sql_sort_join', + 'type', + 'author_ary', + 'author_name', + 'sql_author', + 'topic_id', + 'sql_topic_id', + 'sort_by_sql', + 'sort_key', + 'sort_dir', + 'sql_sort', + 'sort_days', + 'sql_time', + 'firstpost_only', + 'sql_firstpost', + 'ex_fid_ary', + 'sql_fora', + 'm_approve_fid_sql', + 'start', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_author_query_before', compact($vars))); + // If the cache was completely empty count the results $calc_results = ($result_count) ? '' : 'SQL_CALC_FOUND_ROWS '; @@ -692,7 +890,7 @@ class fulltext_mysql extends \phpbb\search\base $id_ary = array_unique($id_ary); } - if (sizeof($id_ary)) + if (count($id_ary)) { $this->save_ids($search_key, '', $author_ary, $result_count, $id_ary, $start, $sort_dir); $id_ary = array_slice($id_ary, 0, $per_page); @@ -720,6 +918,34 @@ class fulltext_mysql extends \phpbb\search\base $words = array_unique(array_merge($split_text, $split_title)); + /** + * Event to modify method arguments and words before the MySQL search index is updated + * + * @event core.search_mysql_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @var array words List of words added to the index + * @var array split_text Array of words from the message + * @var array split_title Array of words from the title + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + 'words', + 'split_text', + 'split_title', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_index_before', compact($vars))); + unset($split_text); unset($split_title); @@ -745,7 +971,7 @@ class fulltext_mysql extends \phpbb\search\base // destroy too old cached search results $this->destroy_cache(array()); - set_config('search_last_gc', time(), true); + $this->config->set('search_last_gc', time(), false); } /** @@ -766,38 +992,70 @@ class fulltext_mysql extends \phpbb\search\base $this->get_stats(); } - $alter = array(); + $alter_list = array(); if (!isset($this->stats['post_subject'])) { + $alter_entry = array(); if ($this->db->get_sql_layer() == 'mysqli' || version_compare($this->db->sql_server_info(true), '4.1.3', '>=')) { - $alter[] = 'MODIFY post_subject varchar(255) COLLATE utf8_unicode_ci DEFAULT \'\' NOT NULL'; + $alter_entry[] = 'MODIFY post_subject varchar(255) COLLATE utf8_unicode_ci DEFAULT \'\' NOT NULL'; } else { - $alter[] = 'MODIFY post_subject text NOT NULL'; + $alter_entry[] = 'MODIFY post_subject text NOT NULL'; } - $alter[] = 'ADD FULLTEXT (post_subject)'; + $alter_entry[] = 'ADD FULLTEXT (post_subject)'; + $alter_list[] = $alter_entry; } if (!isset($this->stats['post_content'])) { + $alter_entry = array(); if ($this->db->get_sql_layer() == 'mysqli' || version_compare($this->db->sql_server_info(true), '4.1.3', '>=')) { - $alter[] = 'MODIFY post_text mediumtext COLLATE utf8_unicode_ci NOT NULL'; + $alter_entry[] = 'MODIFY post_text mediumtext COLLATE utf8_unicode_ci NOT NULL'; } else { - $alter[] = 'MODIFY post_text mediumtext NOT NULL'; + $alter_entry[] = 'MODIFY post_text mediumtext NOT NULL'; } - $alter[] = 'ADD FULLTEXT post_content (post_text, post_subject)'; + $alter_entry[] = 'ADD FULLTEXT post_content (post_text, post_subject)'; + $alter_list[] = $alter_entry; } - if (sizeof($alter)) + $sql_queries = []; + + foreach ($alter_list as $alter) + { + $sql_queries[] = 'ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter); + } + + if (!isset($this->stats['post_text'])) + { + $sql_queries[] = 'ALTER TABLE ' . POSTS_TABLE . ' ADD FULLTEXT post_text (post_text)'; + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the MySQL search index is created + * + * @event core.search_mysql_create_index_before + * @var array sql_queries Array with queries for creating the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_create_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) { - $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter)); + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); @@ -835,9 +1093,37 @@ class fulltext_mysql extends \phpbb\search\base $alter[] = 'DROP INDEX post_content'; } - if (sizeof($alter)) + if (isset($this->stats['post_text'])) { - $this->db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter)); + $alter[] = 'DROP INDEX post_text'; + } + + $sql_queries = []; + + if (count($alter)) + { + $sql_queries[] = 'ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter); + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the MySQL search index is deleted + * + * @event core.search_mysql_delete_index_before + * @var array sql_queries Array with queries for deleting the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_mysql_delete_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); @@ -855,7 +1141,7 @@ class fulltext_mysql extends \phpbb\search\base $this->get_stats(); } - return isset($this->stats['post_subject']) && isset($this->stats['post_content']); + return isset($this->stats['post_subject']) && isset($this->stats['post_content']) && isset($this->stats['post_text']); } /** @@ -899,6 +1185,10 @@ class fulltext_mysql extends \phpbb\search\base { $this->stats['post_subject'] = $row; } + else if ($row['Key_name'] == 'post_text') + { + $this->stats['post_text'] = $row; + } else if ($row['Key_name'] == 'post_content') { $this->stats['post_content'] = $row; diff --git a/phpBB/phpbb/search/fulltext_native.php b/phpBB/phpbb/search/fulltext_native.php index 93ea46ca60..4172e2cc4f 100644 --- a/phpBB/phpbb/search/fulltext_native.php +++ b/phpBB/phpbb/search/fulltext_native.php @@ -18,6 +18,13 @@ namespace phpbb\search; */ class fulltext_native extends \phpbb\search\base { + const UTF8_HANGUL_FIRST = "\xEA\xB0\x80"; + const UTF8_HANGUL_LAST = "\xED\x9E\xA3"; + const UTF8_CJK_FIRST = "\xE4\xB8\x80"; + const UTF8_CJK_LAST = "\xE9\xBE\xBB"; + const UTF8_CJK_B_FIRST = "\xF0\xA0\x80\x80"; + const UTF8_CJK_B_LAST = "\xF0\xAA\x9B\x96"; + /** * Associative array holding index stats * @var array @@ -57,7 +64,7 @@ class fulltext_native extends \phpbb\search\base protected $must_not_contain_ids = array(); /** - * Post ids of posts containing atleast one word that needs to be excluded + * Post ids of posts containing at least one word that needs to be excluded * @var array */ protected $must_exclude_one_ids = array(); @@ -87,33 +94,37 @@ class fulltext_native extends \phpbb\search\base protected $db; /** + * phpBB event dispatcher object + * @var \phpbb\event\dispatcher_interface + */ + protected $phpbb_dispatcher; + + /** * User object * @var \phpbb\user */ protected $user; /** - * Initialises the fulltext_native search backend with min/max word length and makes sure the UTF-8 normalizer is loaded + * Initialises the fulltext_native search backend with min/max word length * * @param boolean|string &$error is passed by reference and should either be set to false on success or an error message on failure + * @param \phpbb\event\dispatcher_interface $phpbb_dispatcher Event dispatcher object */ - public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user) + public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher) { $this->phpbb_root_path = $phpbb_root_path; $this->php_ext = $phpEx; $this->config = $config; $this->db = $db; + $this->phpbb_dispatcher = $phpbb_dispatcher; $this->user = $user; - $this->word_length = array('min' => $this->config['fulltext_native_min_chars'], 'max' => $this->config['fulltext_native_max_chars']); + $this->word_length = array('min' => (int) $this->config['fulltext_native_min_chars'], 'max' => (int) $this->config['fulltext_native_max_chars']); /** * Load the UTF tools */ - if (!class_exists('utf_normalizer')) - { - include($this->phpbb_root_path . 'includes/utf/utf_normalizer.' . $this->php_ext); - } if (!function_exists('utf8_decode_ncr')) { include($this->phpbb_root_path . 'includes/utf/utf_tools.' . $this->php_ext); @@ -274,7 +285,7 @@ class fulltext_native extends \phpbb\search\base ); $keywords = preg_replace($match, $replace, $keywords); - $num_keywords = sizeof(explode(' ', $keywords)); + $num_keywords = count(explode(' ', $keywords)); // We limit the number of allowed keywords to minimize load on the database if ($this->config['max_num_search_keywords'] && $num_keywords > $this->config['max_num_search_keywords']) @@ -290,7 +301,7 @@ class fulltext_native extends \phpbb\search\base $words = array(); preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words); - if (sizeof($words[1])) + if (count($words[1])) { $keywords = '(' . implode('|', $words[1]) . ')'; } @@ -305,7 +316,7 @@ class fulltext_native extends \phpbb\search\base $common_ids = $words = array(); - if (sizeof($exact_words)) + if (count($exact_words)) { $sql = 'SELECT word_id, word_text, word_common FROM ' . SEARCH_WORDLIST_TABLE . ' @@ -341,9 +352,6 @@ class fulltext_native extends \phpbb\search\base $this->must_not_contain_ids = array(); $this->must_exclude_one_ids = array(); - $mode = ''; - $ignore_no_id = true; - foreach ($query as $word) { if (empty($word)) @@ -418,10 +426,10 @@ class fulltext_native extends \phpbb\search\base } } } - if (sizeof($id_words)) + if (count($id_words)) { sort($id_words); - if (sizeof($id_words) > 1) + if (count($id_words) > 1) { $this->{$mode . '_ids'}[] = $id_words; } @@ -432,7 +440,7 @@ class fulltext_native extends \phpbb\search\base } } // throw an error if we shall not ignore unexistant words - else if (!$ignore_no_id && sizeof($non_common_words)) + else if (!$ignore_no_id && count($non_common_words)) { trigger_error(sprintf($this->user->lang['WORDS_IN_NO_POST'], implode($this->user->lang['COMMA_SEPARATOR'], $non_common_words))); } @@ -472,7 +480,7 @@ class fulltext_native extends \phpbb\search\base } // Return true if all words are not common words - if (sizeof($exact_words) - sizeof($this->common_words) > 0) + if (count($exact_words) - count($this->common_words) > 0) { return true; } @@ -522,7 +530,7 @@ class fulltext_native extends \phpbb\search\base sort($must_exclude_one_ids); // generate a search_key from all the options to identify the results - $search_key = md5(implode('#', array( + $search_key_array = array( serialize($must_contain_ids), serialize($must_not_contain_ids), serialize($must_exclude_one_ids), @@ -536,7 +544,45 @@ class fulltext_native extends \phpbb\search\base $post_visibility, implode(',', $author_ary), $author_name, - ))); + ); + + /** + * Allow changing the search_key for cached results + * + * @event core.search_native_by_keyword_modify_search_key + * @var array search_key_array Array with search parameters to generate the search_key + * @var array must_contain_ids Array with post ids of posts containing words that are to be included + * @var array must_not_contain_ids Array with post ids of posts containing words that should not be included + * @var array must_exclude_one_ids Array with post ids of posts containing at least one word that needs to be excluded + * @var string type Searching type ('posts', 'topics') + * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all') + * @var string terms Searching terms ('all', 'any') + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @since 3.1.7-RC1 + */ + $vars = array( + 'search_key_array', + 'must_contain_ids', + 'must_not_contain_ids', + 'must_exclude_one_ids', + 'type', + 'fields', + 'terms', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_keyword_modify_search_key', compact($vars))); + + $search_key = md5(implode('#', $search_key_array)); // try reading the results from cache $total_results = 0; @@ -548,7 +594,6 @@ class fulltext_native extends \phpbb\search\base $id_ary = array(); $sql_where = array(); - $group_by = false; $m_num = 0; $w_num = 0; @@ -671,7 +716,7 @@ class fulltext_native extends \phpbb\search\base } } - if (sizeof($this->must_not_contain_ids)) + if (count($this->must_not_contain_ids)) { $sql_array['LEFT_JOIN'][] = array( 'FROM' => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num), @@ -712,12 +757,76 @@ class fulltext_native extends \phpbb\search\base $sql_where[] = $post_visibility; + $search_query = $this->search_query; + $must_exclude_one_ids = $this->must_exclude_one_ids; + $must_not_contain_ids = $this->must_not_contain_ids; + $must_contain_ids = $this->must_contain_ids; + + /** + * Allow changing the query used for counting for posts using fulltext_native + * + * @event core.search_native_keywords_count_query_before + * @var string search_query The parsed keywords used for this search + * @var array must_not_contain_ids Ids that cannot be taken into account for the results + * @var array must_exclude_one_ids Ids that cannot be on the results + * @var array must_contain_ids Ids that must be on the results + * @var int total_results The previous result count for the format of the query + * Set to 0 to force a re-count + * @var array sql_array The data on how to search in the DB at this point + * @var bool left_join_topics Whether or not TOPICS_TABLE should be CROSS JOIN'ED + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name An extra username to search on (!empty(author_ary) must be true, to be relevant) + * @var array ex_fid_ary Which forums not to search on + * @var int topic_id Limit the search to this topic_id only + * @var string sql_sort_table Extra tables to include in the SQL query. + * Used in conjunction with sql_sort_join + * @var string sql_sort_join SQL conditions to join all the tables used together. + * Used in conjunction with sql_sort_table + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sql_where An array of the current WHERE clause conditions + * @var string sql_match Which columns to do the search on + * @var string sql_match_where Extra conditions to use to properly filter the matching process + * @var bool group_by Whether or not the SQL query requires a GROUP BY for the elements in the SELECT clause + * @var string sort_by_sql The possible predefined sort types + * @var string sort_key The sort type used from the possible sort types + * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used + * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir + * @var int start How many posts to skip in the search results (used for pagination) + * @since 3.1.5-RC1 + */ + $vars = array( + 'search_query', + 'must_not_contain_ids', + 'must_exclude_one_ids', + 'must_contain_ids', + 'total_results', + 'sql_array', + 'left_join_topics', + 'author_ary', + 'author_name', + 'ex_fid_ary', + 'topic_id', + 'sql_sort_table', + 'sql_sort_join', + 'sort_days', + 'sql_where', + 'sql_match', + 'sql_match_where', + 'group_by', + 'sort_by_sql', + 'sort_key', + 'sort_dir', + 'sql_sort', + 'start', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_keywords_count_query_before', compact($vars))); + if ($topic_id) { $sql_where[] = 'p.topic_id = ' . $topic_id; } - if (sizeof($author_ary)) + if (count($author_ary)) { if ($author_name) { @@ -731,7 +840,7 @@ class fulltext_native extends \phpbb\search\base $sql_where[] = $sql_author; } - if (sizeof($ex_fid_ary)) + if (count($ex_fid_ary)) { $sql_where[] = $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true); } @@ -769,7 +878,6 @@ class fulltext_native extends \phpbb\search\base break; - case 'sqlite': case 'sqlite3': $sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id'; $sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results @@ -823,6 +931,13 @@ class fulltext_native extends \phpbb\search\base ); } + // if using mysql and the total result count is not calculated yet, get it from the db + if (!$total_results && $is_mysql) + { + // Also count rows for the query as if there was not LIMIT. Add SQL_CALC_FOUND_ROWS to SQL + $sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT']; + } + $sql_array['WHERE'] = implode(' AND ', $sql_where); $sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : ''; $sql_array['ORDER_BY'] = $sql_sort; @@ -838,19 +953,9 @@ class fulltext_native extends \phpbb\search\base } $this->db->sql_freeresult($result); - // if we use mysql and the total result count is not cached yet, retrieve it from the db if (!$total_results && $is_mysql) { - // Count rows for the executed queries. Replace $select within $sql with SQL_CALC_FOUND_ROWS, and run it - $sql_array_copy = $sql_array; - $sql_array_copy['SELECT'] = 'SQL_CALC_FOUND_ROWS p.post_id '; - - $sql_calc = $this->db->sql_build_query('SELECT', $sql_array_copy); - unset($sql_array_copy); - - $this->db->sql_query($sql_calc); - $this->db->sql_freeresult($result); - + // Get the number of results as calculated by MySQL $sql_count = 'SELECT FOUND_ROWS() as total_results'; $result = $this->db->sql_query($sql_count); $total_results = (int) $this->db->sql_fetchfield('total_results'); @@ -905,13 +1010,13 @@ class fulltext_native extends \phpbb\search\base public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { // No author? No posts - if (!sizeof($author_ary)) + if (!count($author_ary)) { return 0; } // generate a search_key from all the options to identify the results - $search_key = md5(implode('#', array( + $search_key_array = array( '', $type, ($firstpost_only) ? 'firstpost' : '', @@ -924,7 +1029,39 @@ class fulltext_native extends \phpbb\search\base $post_visibility, implode(',', $author_ary), $author_name, - ))); + ); + + /** + * Allow changing the search_key for cached results + * + * @event core.search_native_by_author_modify_search_key + * @var array search_key_array Array with search parameters to generate the search_key + * @var string type Searching type ('posts', 'topics') + * @var boolean firstpost_only Flag indicating if only topic starting posts are considered + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name The username to search on + * @since 3.1.7-RC1 + */ + $vars = array( + 'search_key_array', + 'type', + 'firstpost_only', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + 'author_name', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_author_modify_search_key', compact($vars))); + + $search_key = md5(implode('#', $search_key_array)); // try reading the results from cache $total_results = 0; @@ -945,7 +1082,7 @@ class fulltext_native extends \phpbb\search\base { $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); } - $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; @@ -975,6 +1112,53 @@ class fulltext_native extends \phpbb\search\base $select = ($type == 'posts') ? 'p.post_id' : 't.topic_id'; $is_mysql = false; + /** + * Allow changing the query used to search for posts by author in fulltext_native + * + * @event core.search_native_author_count_query_before + * @var int total_results The previous result count for the format of the query. + * Set to 0 to force a re-count + * @var string type The type of search being made + * @var string select SQL SELECT clause for what to get + * @var string sql_sort_table CROSS JOIN'ed table to allow doing the sort chosen + * @var string sql_sort_join Condition to define how to join the CROSS JOIN'ed table specifyed in sql_sort_table + * @var array sql_author SQL WHERE condition for the post author ids + * @var int topic_id Limit the search to this topic_id only + * @var string sort_by_sql The possible predefined sort types + * @var string sort_key The sort type used from the possible sort types + * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used + * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir + * @var string sort_days Time, in days, that the oldest post showing can have + * @var string sql_time The SQL to search on the time specifyed by sort_days + * @var bool firstpost_only Wether or not to search only on the first post of the topics + * @var string sql_firstpost The SQL used in the WHERE claused to filter by firstpost. + * @var array ex_fid_ary Forum ids that must not be searched on + * @var array sql_fora SQL query for ex_fid_ary + * @var int start How many posts to skip in the search results (used for pagination) + * @since 3.1.5-RC1 + */ + $vars = array( + 'total_results', + 'type', + 'select', + 'sql_sort_table', + 'sql_sort_join', + 'sql_author', + 'topic_id', + 'sort_by_sql', + 'sort_key', + 'sort_dir', + 'sql_sort', + 'sort_days', + 'sql_time', + 'firstpost_only', + 'sql_firstpost', + 'ex_fid_ary', + 'sql_fora', + 'start', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_author_count_query_before', compact($vars))); + // If the cache was completely empty count the results if (!$total_results) { @@ -1000,7 +1184,7 @@ class fulltext_native extends \phpbb\search\base } else { - if ($this->db->get_sql_layer() == 'sqlite' || $this->db->get_sql_layer() == 'sqlite3') + if ($this->db->get_sql_layer() == 'sqlite3') { $sql = 'SELECT COUNT(topic_id) as total_results FROM (SELECT DISTINCT t.topic_id'; @@ -1017,7 +1201,7 @@ class fulltext_native extends \phpbb\search\base $post_visibility $sql_fora AND t.topic_id = p.topic_id - $sql_time" . (($this->db->get_sql_layer() == 'sqlite' || $this->db->get_sql_layer() == 'sqlite3') ? ')' : ''); + $sql_time" . ($this->db->get_sql_layer() == 'sqlite3' ? ')' : ''); } $result = $this->db->sql_query($sql); @@ -1076,9 +1260,9 @@ class fulltext_native extends \phpbb\search\base if (!$total_results && $is_mysql) { // Count rows for the executed queries. Replace $select within $sql with SQL_CALC_FOUND_ROWS, and run it. - $sql_calc = str_replace('SELECT ' . $select, 'SELECT DISTINCT SQL_CALC_FOUND_ROWS p.post_id', $sql); + $sql_calc = str_replace('SELECT ' . $select, 'SELECT SQL_CALC_FOUND_ROWS ' . $select, $sql); - $this->db->sql_query($sql_calc); + $result = $this->db->sql_query($sql_calc); $this->db->sql_freeresult($result); $sql_count = 'SELECT FOUND_ROWS() as total_results'; @@ -1105,7 +1289,7 @@ class fulltext_native extends \phpbb\search\base $this->db->sql_freeresult($result); } - if (sizeof($id_ary)) + if (count($id_ary)) { $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir); $id_ary = array_slice($id_ary, 0, $per_page); @@ -1139,7 +1323,6 @@ class fulltext_native extends \phpbb\search\base $match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#'; $min = $this->word_length['min']; - $max = $this->word_length['max']; $isset_min = $min - 1; @@ -1175,9 +1358,9 @@ class fulltext_native extends \phpbb\search\base * Note: this could be optimized. If the codepoint is lower than Hangul's range * we know that it will also be lower than CJK ranges */ - if ((strncmp($word, UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, UTF8_HANGUL_LAST, 3) > 0) - && (strncmp($word, UTF8_CJK_FIRST, 3) < 0 || strncmp($word, UTF8_CJK_LAST, 3) > 0) - && (strncmp($word, UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, UTF8_CJK_B_LAST, 4) > 0)) + if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0) + && (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0) + && (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0)) { $word = strtok(' '); continue; @@ -1250,6 +1433,38 @@ class fulltext_native extends \phpbb\search\base $words['del']['post'] = array(); $words['del']['title'] = array(); } + + /** + * Event to modify method arguments and words before the native search index is updated + * + * @event core.search_native_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @var array words Grouped lists of words added to or remove from the index + * @var array split_text Array of words from the message + * @var array split_title Array of words from the title + * @var array cur_words Array of words currently in the index for comparing to new words + * when mode is edit. Empty for other modes. + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + 'words', + 'split_text', + 'split_title', + 'cur_words', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_index_before', compact($vars))); + unset($split_text); unset($split_title); @@ -1260,7 +1475,7 @@ class fulltext_native extends \phpbb\search\base // individual arrays of added and removed words for text and title. What // we need to do now is add the new words (if they don't already exist) // and then add (or remove) matches between the words and this post - if (sizeof($unique_add_words)) + if (count($unique_add_words)) { $sql = 'SELECT word_id, word_text FROM ' . SEARCH_WORDLIST_TABLE . ' @@ -1276,7 +1491,7 @@ class fulltext_native extends \phpbb\search\base $new_words = array_diff($unique_add_words, array_keys($word_ids)); $this->db->sql_transaction('begin'); - if (sizeof($new_words)) + if (count($new_words)) { $sql_ary = array(); @@ -1300,7 +1515,7 @@ class fulltext_native extends \phpbb\search\base { $title_match = ($word_in == 'title') ? 1 : 0; - if (sizeof($word_ary)) + if (count($word_ary)) { $sql_in = array(); foreach ($word_ary as $word) @@ -1329,7 +1544,7 @@ class fulltext_native extends \phpbb\search\base { $title_match = ($word_in == 'title') ? 1 : 0; - if (sizeof($word_ary)) + if (count($word_ary)) { $sql = 'INSERT INTO ' . SEARCH_WORDMATCH_TABLE . ' (post_id, word_id, title_match) SELECT ' . (int) $post_id . ', word_id, ' . (int) $title_match . ' @@ -1360,7 +1575,7 @@ class fulltext_native extends \phpbb\search\base */ public function index_remove($post_ids, $author_ids, $forum_ids) { - if (sizeof($post_ids)) + if (count($post_ids)) { $sql = 'SELECT w.word_id, w.word_text, m.title_match FROM ' . SEARCH_WORDMATCH_TABLE . ' m, ' . SEARCH_WORDLIST_TABLE . ' w @@ -1383,7 +1598,7 @@ class fulltext_native extends \phpbb\search\base } $this->db->sql_freeresult($result); - if (sizeof($title_word_ids)) + if (count($title_word_ids)) { $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . ' SET word_count = word_count - 1 @@ -1392,7 +1607,7 @@ class fulltext_native extends \phpbb\search\base $this->db->sql_query($sql); } - if (sizeof($message_word_ids)) + if (count($message_word_ids)) { $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . ' SET word_count = word_count - 1 @@ -1422,7 +1637,7 @@ class fulltext_native extends \phpbb\search\base // carry on ... it's okay ... I know when I'm not wanted boo hoo if (!$this->config['fulltext_native_load_upd']) { - set_config('search_last_gc', time(), true); + $this->config->set('search_last_gc', time(), false); return; } @@ -1447,7 +1662,7 @@ class fulltext_native extends \phpbb\search\base } $this->db->sql_freeresult($result); - if (sizeof($sql_in)) + if (count($sql_in)) { // Flag the words $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . ' @@ -1457,7 +1672,7 @@ class fulltext_native extends \phpbb\search\base // by setting search_last_gc to the new time here we make sure that if a user reloads because the // following query takes too long, he won't run into it again - set_config('search_last_gc', time(), true); + $this->config->set('search_last_gc', time(), false); // Delete the matches $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . ' @@ -1467,13 +1682,13 @@ class fulltext_native extends \phpbb\search\base unset($sql_in); } - if (sizeof($destroy_cache_words)) + if (count($destroy_cache_words)) { // destroy cached search results containing any of the words that are now common or were removed $this->destroy_cache(array_unique($destroy_cache_words)); } - set_config('search_last_gc', time(), true); + $this->config->set('search_last_gc', time(), false); } /** @@ -1481,21 +1696,43 @@ class fulltext_native extends \phpbb\search\base */ public function delete_index($acp_module, $u_action) { + $sql_queries = []; + switch ($this->db->get_sql_layer()) { - case 'sqlite': case 'sqlite3': - $this->db->sql_query('DELETE FROM ' . SEARCH_WORDLIST_TABLE); - $this->db->sql_query('DELETE FROM ' . SEARCH_WORDMATCH_TABLE); - $this->db->sql_query('DELETE FROM ' . SEARCH_RESULTS_TABLE); + $sql_queries[] = 'DELETE FROM ' . SEARCH_WORDLIST_TABLE; + $sql_queries[] = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE; + $sql_queries[] = 'DELETE FROM ' . SEARCH_RESULTS_TABLE; break; default: - $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_WORDLIST_TABLE); - $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_WORDMATCH_TABLE); - $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); + $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_WORDLIST_TABLE; + $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_WORDMATCH_TABLE; + $sql_queries[] = 'TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE; break; } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the native search index is deleted + * + * @event core.search_native_delete_index_before + * @var array sql_queries Array with queries for deleting the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_native_delete_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } } /** @@ -1503,7 +1740,7 @@ class fulltext_native extends \phpbb\search\base */ public function index_created() { - if (!sizeof($this->stats)) + if (!count($this->stats)) { $this->get_stats(); } @@ -1516,7 +1753,7 @@ class fulltext_native extends \phpbb\search\base */ public function index_stats() { - if (!sizeof($this->stats)) + if (!count($this->stats)) { $this->get_stats(); } @@ -1544,13 +1781,11 @@ class fulltext_native extends \phpbb\search\base * @param string $allowed_chars String of special chars to allow * @param string $encoding Text encoding * @return string Cleaned up text, only alphanumeric chars are left - * - * @todo \normalizer::cleanup being able to be used? */ protected function cleanup($text, $allowed_chars = null, $encoding = 'utf-8') { static $conv = array(), $conv_loaded = array(); - $words = $allow = array(); + $allow = array(); // Convert the text to UTF-8 $encoding = strtolower($encoding); @@ -1572,12 +1807,9 @@ class fulltext_native extends \phpbb\search\base $text = htmlspecialchars_decode(utf8_decode_ncr($text), ENT_QUOTES); /** - * Load the UTF-8 normalizer - * - * If we use it more widely, an instance of that class should be held in a - * a global variable instead + * Normalize to NFC */ - \utf_normalizer::nfc($text); + $text = \Normalizer::normalize($text); /** * The first thing we do is: @@ -1670,9 +1902,9 @@ class fulltext_native extends \phpbb\search\base $utf_char = substr($text, $pos, $utf_len); $pos += $utf_len; - if (($utf_char >= UTF8_HANGUL_FIRST && $utf_char <= UTF8_HANGUL_LAST) - || ($utf_char >= UTF8_CJK_FIRST && $utf_char <= UTF8_CJK_LAST) - || ($utf_char >= UTF8_CJK_B_FIRST && $utf_char <= UTF8_CJK_B_LAST)) + if (($utf_char >= self::UTF8_HANGUL_FIRST && $utf_char <= self::UTF8_HANGUL_LAST) + || ($utf_char >= self::UTF8_CJK_FIRST && $utf_char <= self::UTF8_CJK_LAST) + || ($utf_char >= self::UTF8_CJK_B_FIRST && $utf_char <= self::UTF8_CJK_B_LAST)) { /** * All characters within these ranges are valid @@ -1789,15 +2021,15 @@ class fulltext_native extends \phpbb\search\base </dl> <dl> <dt><label for="fulltext_native_min_chars">' . $this->user->lang['MIN_SEARCH_CHARS'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['MIN_SEARCH_CHARS_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_native_min_chars" type="number" size="3" maxlength="3" min="0" max="255" name="config[fulltext_native_min_chars]" value="' . (int) $this->config['fulltext_native_min_chars'] . '" /></dd> + <dd><input id="fulltext_native_min_chars" type="number" min="0" max="255" name="config[fulltext_native_min_chars]" value="' . (int) $this->config['fulltext_native_min_chars'] . '" /></dd> </dl> <dl> <dt><label for="fulltext_native_max_chars">' . $this->user->lang['MAX_SEARCH_CHARS'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['MAX_SEARCH_CHARS_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_native_max_chars" type="number" size="3" maxlength="3" min="0" max="255" name="config[fulltext_native_max_chars]" value="' . (int) $this->config['fulltext_native_max_chars'] . '" /></dd> + <dd><input id="fulltext_native_max_chars" type="number" min="0" max="255" name="config[fulltext_native_max_chars]" value="' . (int) $this->config['fulltext_native_max_chars'] . '" /></dd> </dl> <dl> <dt><label for="fulltext_native_common_thres">' . $this->user->lang['COMMON_WORD_THRESHOLD'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['COMMON_WORD_THRESHOLD_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_native_common_thres" type="text" size="3" maxlength="3" name="config[fulltext_native_common_thres]" value="' . (double) $this->config['fulltext_native_common_thres'] . '" /> %</dd> + <dd><input id="fulltext_native_common_thres" type="text" name="config[fulltext_native_common_thres]" value="' . (double) $this->config['fulltext_native_common_thres'] . '" /> %</dd> </dl> '; diff --git a/phpBB/phpbb/search/fulltext_postgres.php b/phpBB/phpbb/search/fulltext_postgres.php index b6af371d13..2f387e791e 100644 --- a/phpBB/phpbb/search/fulltext_postgres.php +++ b/phpBB/phpbb/search/fulltext_postgres.php @@ -56,6 +56,12 @@ class fulltext_postgres extends \phpbb\search\base protected $db; /** + * phpBB event dispatcher object + * @var \phpbb\event\dispatcher_interface + */ + protected $phpbb_dispatcher; + + /** * User object * @var \phpbb\user */ @@ -92,11 +98,13 @@ class fulltext_postgres extends \phpbb\search\base * @param \phpbb\config\config $config Config object * @param \phpbb\db\driver\driver_interface Database object * @param \phpbb\user $user User object + * @param \phpbb\event\dispatcher_interface $phpbb_dispatcher Event dispatcher object */ - public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user) + public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher) { $this->config = $config; $this->db = $db; + $this->phpbb_dispatcher = $phpbb_dispatcher; $this->user = $user; $this->word_length = array('min' => $this->config['fulltext_postgres_min_word_len'], 'max' => $this->config['fulltext_postgres_max_word_len']); @@ -286,7 +294,7 @@ class fulltext_postgres extends \phpbb\search\base // remove too short or too long words $text = array_values($text); - for ($i = 0, $n = sizeof($text); $i < $n; $i++) + for ($i = 0, $n = count($text); $i < $n; $i++) { $text[$i] = trim($text[$i]); if (utf8_strlen($text[$i]) < $this->config['fulltext_postgres_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_postgres_max_word_len']) @@ -333,7 +341,7 @@ class fulltext_postgres extends \phpbb\search\base } // generate a search_key from all the options to identify the results - $search_key = md5(implode('#', array( + $search_key_array = array( implode(', ', $this->split_words), $type, $fields, @@ -344,7 +352,39 @@ class fulltext_postgres extends \phpbb\search\base implode(',', $ex_fid_ary), $post_visibility, implode(',', $author_ary) - ))); + ); + + /** + * Allow changing the search_key for cached results + * + * @event core.search_postgres_by_keyword_modify_search_key + * @var array search_key_array Array with search parameters to generate the search_key + * @var string type Searching type ('posts', 'topics') + * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all') + * @var string terms Searching terms ('all', 'any') + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @since 3.1.7-RC1 + */ + $vars = array( + 'search_key_array', + 'type', + 'fields', + 'terms', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_by_keyword_modify_search_key', compact($vars))); + + $search_key = md5(implode('#', $search_key_array)); if ($start < 0) { @@ -409,17 +449,65 @@ class fulltext_postgres extends \phpbb\search\base break; } - $sql_select = ($type == 'posts') ? 'p.post_id' : 'DISTINCT t.topic_id'; + $tsearch_query = $this->tsearch_query; + + /** + * Allow changing the query used to search for posts using fulltext_postgres + * + * @event core.search_postgres_keywords_main_query_before + * @var string tsearch_query The parsed keywords used for this search + * @var int result_count The previous result count for the format of the query. + * Set to 0 to force a re-count + * @var bool join_topic Weather or not TOPICS_TABLE should be CROSS JOIN'ED + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name An extra username to search on (!empty(author_ary) must be true, to be relevant) + * @var array ex_fid_ary Which forums not to search on + * @var int topic_id Limit the search to this topic_id only + * @var string sql_sort_table Extra tables to include in the SQL query. + * Used in conjunction with sql_sort_join + * @var string sql_sort_join SQL conditions to join all the tables used together. + * Used in conjunction with sql_sort_table + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sql_match Which columns to do the search on. + * @var string sql_match_where Extra conditions to use to properly filter the matching process + * @var string sort_by_sql The possible predefined sort types + * @var string sort_key The sort type used from the possible sort types + * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used + * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir + * @var int start How many posts to skip in the search results (used for pagination) + * @since 3.1.5-RC1 + */ + $vars = array( + 'tsearch_query', + 'result_count', + 'join_topic', + 'author_ary', + 'author_name', + 'ex_fid_ary', + 'topic_id', + 'sql_sort_table', + 'sql_sort_join', + 'sort_days', + 'sql_match', + 'sql_match_where', + 'sort_by_sql', + 'sort_key', + 'sort_dir', + 'sql_sort', + 'start', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_keywords_main_query_before', compact($vars))); + + $sql_select = ($type == 'posts') ? 'p.post_id' : 'DISTINCT t.topic_id, ' . $sort_by_sql[$sort_key]; $sql_from = ($join_topic) ? TOPICS_TABLE . ' t, ' : ''; $field = ($type == 'posts') ? 'post_id' : 'topic_id'; - $sql_author = (sizeof($author_ary) == 1) ? ' = ' . $author_ary[0] : 'IN (' . implode(', ', $author_ary) . ')'; - if (sizeof($author_ary) && $author_name) + if (count($author_ary) && $author_name) { // first one matches post of registered users, second one guests and deleted users $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; } - else if (sizeof($author_ary)) + else if (count($author_ary)) { $sql_author = ' AND ' . $this->db->sql_in_set('p.poster_id', $author_ary); } @@ -431,13 +519,12 @@ class fulltext_postgres extends \phpbb\search\base $sql_where_options = $sql_sort_join; $sql_where_options .= ($topic_id) ? ' AND p.topic_id = ' . $topic_id : ''; $sql_where_options .= ($join_topic) ? ' AND t.topic_id = p.topic_id' : ''; - $sql_where_options .= (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_where_options .= (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_where_options .= ' AND ' . $post_visibility; $sql_where_options .= $sql_author; $sql_where_options .= ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_where_options .= $sql_match_where; - $tmp_sql_match = array(); $sql_match = str_replace(',', " || ' ' ||", $sql_match); $tmp_sql_match = "to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', " . $sql_match . ") @@ to_tsquery ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', '" . $this->db->sql_escape($this->tsearch_query) . "')"; @@ -522,13 +609,13 @@ class fulltext_postgres extends \phpbb\search\base public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { // No author? No posts - if (!sizeof($author_ary)) + if (!count($author_ary)) { return 0; } // generate a search_key from all the options to identify the results - $search_key = md5(implode('#', array( + $search_key_array = array( '', $type, ($firstpost_only) ? 'firstpost' : '', @@ -541,7 +628,39 @@ class fulltext_postgres extends \phpbb\search\base $post_visibility, implode(',', $author_ary), $author_name, - ))); + ); + + /** + * Allow changing the search_key for cached results + * + * @event core.search_postgres_by_author_modify_search_key + * @var array search_key_array Array with search parameters to generate the search_key + * @var string type Searching type ('posts', 'topics') + * @var boolean firstpost_only Flag indicating if only topic starting posts are considered + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name The username to search on + * @since 3.1.7-RC1 + */ + $vars = array( + 'search_key_array', + 'type', + 'firstpost_only', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + 'author_name', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_by_author_modify_search_key', compact($vars))); + + $search_key = md5(implode('#', $search_key_array)); if ($start < 0) { @@ -567,7 +686,7 @@ class fulltext_postgres extends \phpbb\search\base { $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); } - $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; @@ -595,6 +714,55 @@ class fulltext_postgres extends \phpbb\search\base $m_approve_fid_sql = ' AND ' . $post_visibility; + /** + * Allow changing the query used to search for posts by author in fulltext_postgres + * + * @event core.search_postgres_author_count_query_before + * @var int result_count The previous result count for the format of the query. + * Set to 0 to force a re-count + * @var string sql_sort_table CROSS JOIN'ed table to allow doing the sort chosen + * @var string sql_sort_join Condition to define how to join the CROSS JOIN'ed table specifyed in sql_sort_table + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name An extra username to search on + * @var string sql_author SQL WHERE condition for the post author ids + * @var int topic_id Limit the search to this topic_id only + * @var string sql_topic_id SQL of topic_id + * @var string sort_by_sql The possible predefined sort types + * @var string sort_key The sort type used from the possible sort types + * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used + * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir + * @var string sort_days Time, in days, that the oldest post showing can have + * @var string sql_time The SQL to search on the time specifyed by sort_days + * @var bool firstpost_only Wether or not to search only on the first post of the topics + * @var array ex_fid_ary Forum ids that must not be searched on + * @var array sql_fora SQL query for ex_fid_ary + * @var string m_approve_fid_sql WHERE clause condition on post_visibility restrictions + * @var int start How many posts to skip in the search results (used for pagination) + * @since 3.1.5-RC1 + */ + $vars = array( + 'result_count', + 'sql_sort_table', + 'sql_sort_join', + 'author_ary', + 'author_name', + 'sql_author', + 'topic_id', + 'sql_topic_id', + 'sort_by_sql', + 'sort_key', + 'sort_dir', + 'sql_sort', + 'sort_days', + 'sql_time', + 'firstpost_only', + 'ex_fid_ary', + 'sql_fora', + 'm_approve_fid_sql', + 'start', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_author_count_query_before', compact($vars))); + // Build the query for really selecting the post_ids if ($type == 'posts') { @@ -668,7 +836,7 @@ class fulltext_postgres extends \phpbb\search\base GROUP BY t.topic_id, $sort_by_sql[$sort_key]"; } - $result = $this->db->sql_query($sql_count); + $this->db->sql_query($sql_count); $result_count = (int) $this->db->sql_fetchfield('result_count'); if (!$result_count) @@ -693,7 +861,7 @@ class fulltext_postgres extends \phpbb\search\base $id_ary = array_unique($id_ary); } - if (sizeof($id_ary)) + if (count($id_ary)) { $this->save_ids($search_key, '', $author_ary, $result_count, $id_ary, $start, $sort_dir); $id_ary = array_slice($id_ary, 0, $per_page); @@ -721,6 +889,34 @@ class fulltext_postgres extends \phpbb\search\base $words = array_unique(array_merge($split_text, $split_title)); + /** + * Event to modify method arguments and words before the PostgreSQL search index is updated + * + * @event core.search_postgres_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @var array words Array of words added to the index + * @var array split_text Array of words from the message + * @var array split_title Array of words from the title + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + 'words', + 'split_text', + 'split_title', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_index_before', compact($vars))); + unset($split_text); unset($split_title); @@ -746,7 +942,7 @@ class fulltext_postgres extends \phpbb\search\base // destroy too old cached search results $this->destroy_cache(array()); - set_config('search_last_gc', time(), true); + $this->config->set('search_last_gc', time(), false); } /** @@ -767,14 +963,37 @@ class fulltext_postgres extends \phpbb\search\base $this->get_stats(); } + $sql_queries = []; + if (!isset($this->stats['post_subject'])) { - $this->db->sql_query("CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_subject ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_subject))"); + $sql_queries[] = "CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_subject ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_subject))"; } if (!isset($this->stats['post_content'])) { - $this->db->sql_query("CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_content ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_text || ' ' || post_subject))"); + $sql_queries[] = "CREATE INDEX " . POSTS_TABLE . "_" . $this->config['fulltext_postgres_ts_name'] . "_post_content ON " . POSTS_TABLE . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', post_text || ' ' || post_subject))"; + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the Postgres search index is created + * + * @event core.search_postgres_create_index_before + * @var array sql_queries Array with queries for creating the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_create_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); @@ -800,14 +1019,37 @@ class fulltext_postgres extends \phpbb\search\base $this->get_stats(); } + $sql_queries = []; + if (isset($this->stats['post_subject'])) { - $this->db->sql_query('DROP INDEX ' . $this->stats['post_subject']['relname']); + $sql_queries[] = 'DROP INDEX ' . $this->stats['post_subject']['relname']; } if (isset($this->stats['post_content'])) { - $this->db->sql_query('DROP INDEX ' . $this->stats['post_content']['relname']); + $sql_queries[] = 'DROP INDEX ' . $this->stats['post_content']['relname']; + } + + $stats = $this->stats; + + /** + * Event to modify SQL queries before the Postgres search index is created + * + * @event core.search_postgres_delete_index_before + * @var array sql_queries Array with queries for deleting the search index + * @var array stats Array with statistics of the current index (read only) + * @since 3.2.3-RC1 + */ + $vars = array( + 'sql_queries', + 'stats', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_postgres_delete_index_before', compact($vars))); + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); } $this->db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE); @@ -919,11 +1161,11 @@ class fulltext_postgres extends \phpbb\search\base </dl> <dl> <dt><label for="fulltext_postgres_min_word_len">' . $this->user->lang['FULLTEXT_POSTGRES_MIN_WORD_LEN'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_POSTGRES_MIN_WORD_LEN_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_postgres_min_word_len" type="number" size="3" maxlength="3" min="0" max="255" name="config[fulltext_postgres_min_word_len]" value="' . (int) $this->config['fulltext_postgres_min_word_len'] . '" /></dd> + <dd><input id="fulltext_postgres_min_word_len" type="number" min="0" max="255" name="config[fulltext_postgres_min_word_len]" value="' . (int) $this->config['fulltext_postgres_min_word_len'] . '" /></dd> </dl> <dl> <dt><label for="fulltext_postgres_max_word_len">' . $this->user->lang['FULLTEXT_POSTGRES_MAX_WORD_LEN'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_postgres_max_word_len" type="number" size="3" maxlength="3" min="0" max="255" name="config[fulltext_postgres_max_word_len]" value="' . (int) $this->config['fulltext_postgres_max_word_len'] . '" /></dd> + <dd><input id="fulltext_postgres_max_word_len" type="number" min="0" max="255" name="config[fulltext_postgres_max_word_len]" value="' . (int) $this->config['fulltext_postgres_max_word_len'] . '" /></dd> </dl> '; diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index eb53ca6d40..d780442f89 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -85,7 +85,7 @@ class fulltext_sphinx /** * Database Tools object - * @var \phpbb\db\tools + * @var \phpbb\db\tools\tools_interface */ protected $db_tools; @@ -96,6 +96,12 @@ class fulltext_sphinx protected $dbtype; /** + * phpBB event dispatcher object + * @var \phpbb\event\dispatcher_interface + */ + protected $phpbb_dispatcher; + + /** * User object * @var \phpbb\user */ @@ -125,22 +131,25 @@ class fulltext_sphinx * @param \phpbb\config\config $config Config object * @param \phpbb\db\driver\driver_interface Database object * @param \phpbb\user $user User object + * @param \phpbb\event\dispatcher_interface $phpbb_dispatcher Event dispatcher object */ - public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user) + public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher) { $this->phpbb_root_path = $phpbb_root_path; $this->php_ext = $phpEx; $this->config = $config; + $this->phpbb_dispatcher = $phpbb_dispatcher; $this->user = $user; $this->db = $db; $this->auth = $auth; - // Initialize \phpbb\db\tools object - $this->db_tools = new \phpbb\db\tools($this->db); + // Initialize \phpbb\db\tools\tools object + global $phpbb_container; // TODO inject into object + $this->db_tools = $phpbb_container->get('dbal.tools'); - if(!$this->config['fulltext_sphinx_id']) + if (!$this->config['fulltext_sphinx_id']) { - set_config('fulltext_sphinx_id', unique_id()); + $this->config->set('fulltext_sphinx_id', unique_id()); } $this->id = $this->config['fulltext_sphinx_id']; $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main'; @@ -211,7 +220,7 @@ class fulltext_sphinx } // Move delta to main index each hour - set_config('search_gc', 3600); + $this->config->set('search_gc', 3600); return false; } @@ -283,7 +292,6 @@ class fulltext_sphinx AND p.post_id >= $start AND p.post_id <= $end'), array('sql_query_post', ''), array('sql_query_post_index', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'), - array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'), array('sql_attr_uint', 'forum_id'), array('sql_attr_uint', 'topic_id'), array('sql_attr_uint', 'poster_id'), @@ -295,7 +303,7 @@ class fulltext_sphinx array('sql_attr_string', 'post_subject'), ), 'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array( - array('sql_query_pre', ''), + array('sql_query_pre', 'SET NAMES \'utf8\''), array('sql_query_range', ''), array('sql_range_step', ''), array('sql_query', 'SELECT @@ -315,6 +323,7 @@ class fulltext_sphinx WHERE p.topic_id = t.topic_id AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), + array('sql_query_post_index', ''), ), 'index index_phpbb_' . $this->id . '_main' => array( array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), @@ -323,7 +332,6 @@ class fulltext_sphinx array('morphology', 'none'), array('stopwords', ''), array('min_word_len', '2'), - array('charset_type', 'utf-8'), array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), array('min_prefix_len', '0'), array('min_infix_len', '0'), @@ -336,20 +344,35 @@ class fulltext_sphinx array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'), ), 'searchd' => array( - array('compat_sphinxql_magics' , '0'), array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')), array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), array('read_timeout', '5'), array('max_children', '30'), array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'), - array('max_matches', (string) SPHINX_MAX_MATCHES), array('binlog_path', $this->config['fulltext_sphinx_data_path']), ), ); $non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true); $delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true); + + /** + * Allow adding/changing the Sphinx configuration data + * + * @event core.search_sphinx_modify_config_data + * @var array config_data Array with the Sphinx configuration data + * @var array non_unique Array with the Sphinx non-unique variables to delete + * @var array delete Array with the Sphinx variables to delete + * @since 3.1.7-RC1 + */ + $vars = array( + 'config_data', + 'non_unique', + 'delete', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_modify_config_data', compact($vars))); + foreach ($config_data as $section_name => $section_data) { $section = $config_object->get_section_by_name($section_name); @@ -378,7 +401,7 @@ class fulltext_sphinx $variable = $section->get_variable_by_name($key); if (!$variable) { - $variable = $section->create_variable($key, $value); + $section->create_variable($key, $value); } else { @@ -387,7 +410,7 @@ class fulltext_sphinx } else { - $variable = $section->create_variable($key, $value); + $section->create_variable($key, $value); } } } @@ -411,7 +434,6 @@ class fulltext_sphinx $match = array('#\sand\s#i', '#\sor\s#i', '#\snot\s#i', '#\+#', '#-#', '#\|#', '#@#'); $replace = array(' & ', ' | ', ' - ', ' +', ' -', ' |', ''); - $replacements = 0; $keywords = preg_replace($match, $replace, $keywords); $this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED); } @@ -454,16 +476,16 @@ class fulltext_sphinx */ public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page) { + global $user, $phpbb_log; + // No keywords? No posts. - if (!strlen($this->search_query) && !sizeof($author_ary)) + if (!strlen($this->search_query) && !count($author_ary)) { return false; } $id_ary = array(); - $join_topic = ($type != 'posts'); - // Sorting if ($type == 'topics') @@ -523,6 +545,41 @@ class fulltext_sphinx $this->sphinx->SetFilter('topic_id', array($topic_id)); } + /** + * Allow modifying the Sphinx search options + * + * @event core.search_sphinx_keywords_modify_options + * @var string type Searching type ('posts', 'topics') + * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all') + * @var string terms Searching terms ('all', 'any') + * @var int sort_days Time, in days, of the oldest possible post to list + * @var string sort_key The sort type used from the possible sort types + * @var int topic_id Limit the search to this topic_id only + * @var array ex_fid_ary Which forums not to search on + * @var string post_visibility Post visibility data + * @var array author_ary Array of user_id containing the users to filter the results to + * @var string author_name The username to search on + * @var object sphinx The Sphinx searchd client object + * @since 3.1.7-RC1 + */ + $sphinx = $this->sphinx; + $vars = array( + 'type', + 'fields', + 'terms', + 'sort_days', + 'sort_key', + 'topic_id', + 'ex_fid_ary', + 'post_visibility', + 'author_ary', + 'author_name', + 'sphinx', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_keywords_modify_options', compact($vars))); + $this->sphinx = $sphinx; + unset($sphinx); + $search_query_prefix = ''; switch ($fields) @@ -562,7 +619,7 @@ class fulltext_sphinx break; } - if (sizeof($author_ary)) + if (count($author_ary)) { $this->sphinx->SetFilter('poster_id', $author_ary); } @@ -572,14 +629,14 @@ class fulltext_sphinx // but at least it will also cause the same for normal users. $this->sphinx->SetFilter('post_visibility', array(ITEM_APPROVED)); - if (sizeof($ex_fid_ary)) + if (count($ex_fid_ary)) { // All forums that a user is allowed to access $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true)))); // All forums that the user wants to and can search in $search_forums = array_diff($fid_ary, $ex_fid_ary); - if (sizeof($search_forums)) + if (count($search_forums)) { $this->sphinx->SetFilter('forum_id', $search_forums); } @@ -588,7 +645,7 @@ class fulltext_sphinx $this->sphinx->SetFilter('deleted', array(0)); $this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES); - $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); + $result = $this->sphinx->Query($search_query_prefix . $this->sphinx->EscapeString(str_replace('"', '"', $this->search_query)), $this->indexes); // Could be connection to localhost:9312 failed (errno=111, // msg=Connection refused) during rotate, retry if so @@ -596,12 +653,12 @@ class fulltext_sphinx while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--) { usleep(SPHINX_CONNECT_WAIT_TIME); - $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); + $result = $this->sphinx->Query($search_query_prefix . $this->sphinx->EscapeString(str_replace('"', '"', $this->search_query)), $this->indexes); } if ($this->sphinx->GetLastError()) { - add_log('critical', 'LOG_SPHINX_ERROR', $this->sphinx->GetLastError()); + $phpbb_log->add('critical', $user->data['user_id'], $user->ip, 'LOG_SPHINX_ERROR', false, array($this->sphinx->GetLastError())); if ($this->auth->acl_get('a_')) { trigger_error($this->user->lang('SPHINX_SEARCH_FAILED', $this->sphinx->GetLastError())); @@ -619,7 +676,7 @@ class fulltext_sphinx $start = floor(($result_count - 1) / $per_page) * $per_page; $this->sphinx->SetLimits((int) $start, (int) $per_page, SPHINX_MAX_MATCHES); - $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); + $result = $this->sphinx->Query($search_query_prefix . $this->sphinx->EscapeString(str_replace('"', '"', $this->search_query)), $this->indexes); // Could be connection to localhost:9312 failed (errno=111, // msg=Connection refused) during rotate, retry if so @@ -627,7 +684,7 @@ class fulltext_sphinx while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--) { usleep(SPHINX_CONNECT_WAIT_TIME); - $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); + $result = $this->sphinx->Query($search_query_prefix . $this->sphinx->EscapeString(str_replace('"', '"', $this->search_query)), $this->indexes); } } @@ -697,6 +754,28 @@ class fulltext_sphinx */ public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id) { + /** + * Event to modify method arguments before the Sphinx search index is updated + * + * @event core.search_sphinx_index_before + * @var string mode Contains the post mode: edit, post, reply, quote + * @var int post_id The id of the post which is modified/created + * @var string message New or updated post content + * @var string subject New or updated post subject + * @var int poster_id Post author's user id + * @var int forum_id The id of the forum in which the post is located + * @since 3.2.3-RC1 + */ + $vars = array( + 'mode', + 'post_id', + 'message', + 'subject', + 'poster_id', + 'forum_id', + ); + extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_index_before', compact($vars))); + if ($mode == 'edit') { $this->sphinx->UpdateAttributes($this->indexes, array('forum_id', 'poster_id'), array((int) $post_id => array((int) $forum_id, (int) $poster_id))); @@ -729,7 +808,7 @@ class fulltext_sphinx } $this->db->sql_freeresult($result); - if (sizeof($post_updates)) + if (count($post_updates)) { $this->sphinx->UpdateAttributes($this->indexes, array('topic_last_post_time'), $post_updates); } @@ -755,7 +834,7 @@ class fulltext_sphinx */ public function tidy($create = false) { - set_config('search_last_gc', time(), true); + $this->config->set('search_last_gc', time(), false); } /** @@ -892,11 +971,11 @@ class fulltext_sphinx </dl> <dl> <dt><label for="fulltext_sphinx_port">' . $this->user->lang['FULLTEXT_SPHINX_PORT'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_sphinx_port" type="number" size="4" maxlength="10" name="config[fulltext_sphinx_port]" value="' . $this->config['fulltext_sphinx_port'] . '" /></dd> + <dd><input id="fulltext_sphinx_port" type="number" min="0" max="9999999999" name="config[fulltext_sphinx_port]" value="' . $this->config['fulltext_sphinx_port'] . '" /></dd> </dl> <dl> <dt><label for="fulltext_sphinx_indexer_mem_limit">' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '</span></dt> - <dd><input id="fulltext_sphinx_indexer_mem_limit" type="number" size="4" maxlength="10" name="config[fulltext_sphinx_indexer_mem_limit]" value="' . $this->config['fulltext_sphinx_indexer_mem_limit'] . '" /> ' . $this->user->lang['MIB'] . '</dd> + <dd><input id="fulltext_sphinx_indexer_mem_limit" type="number" min="0" max="9999999999" name="config[fulltext_sphinx_indexer_mem_limit]" value="' . $this->config['fulltext_sphinx_indexer_mem_limit'] . '" /> ' . $this->user->lang['MIB'] . '</dd> </dl> <dl> <dt><label for="fulltext_sphinx_config_file">' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN'] . '</span></dt> diff --git a/phpBB/phpbb/search/sphinx/config.php b/phpBB/phpbb/search/sphinx/config.php index 675649b460..3205574b45 100644 --- a/phpBB/phpbb/search/sphinx/config.php +++ b/phpBB/phpbb/search/sphinx/config.php @@ -46,7 +46,7 @@ class config */ function get_section_by_name($name) { - for ($i = 0, $size = sizeof($this->sections); $i < $size; $i++) + for ($i = 0, $size = count($this->sections); $i < $size; $i++) { // Make sure this is really a section object and not a comment if (($this->sections[$i] instanceof \phpbb\search\sphinx\config_section) && $this->sections[$i]->get_name() == $name) @@ -67,7 +67,7 @@ class config function add_section($name) { $this->sections[] = new \phpbb\search\sphinx\config_section($name, ''); - return $this->sections[sizeof($this->sections) - 1]; + return $this->sections[count($this->sections) - 1]; } /** diff --git a/phpBB/phpbb/search/sphinx/config_section.php b/phpBB/phpbb/search/sphinx/config_section.php index 14ab3a752c..2fc8b2da17 100644 --- a/phpBB/phpbb/search/sphinx/config_section.php +++ b/phpBB/phpbb/search/sphinx/config_section.php @@ -87,7 +87,7 @@ class config_section */ function get_variable_by_name($name) { - for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) + for ($i = 0, $size = count($this->variables); $i < $size; $i++) { // Make sure this is a variable object and not a comment if (($this->variables[$i] instanceof \phpbb\search\sphinx\config_variable) && $this->variables[$i]->get_name() == $name) @@ -106,7 +106,7 @@ class config_section */ function delete_variables_by_name($name) { - for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) + for ($i = 0, $size = count($this->variables); $i < $size; $i++) { // Make sure this is a variable object and not a comment if (($this->variables[$i] instanceof \phpbb\search\sphinx\config_variable) && $this->variables[$i]->get_name() == $name) @@ -129,7 +129,7 @@ class config_section function create_variable($name, $value) { $this->variables[] = new \phpbb\search\sphinx\config_variable($name, $value, ''); - return $this->variables[sizeof($this->variables) - 1]; + return $this->variables[count($this->variables) - 1]; } /** |
