diff options
| author | Paul S. Owen <psotfx@users.sourceforge.net> | 2002-01-14 14:17:18 +0000 |
|---|---|---|
| committer | Paul S. Owen <psotfx@users.sourceforge.net> | 2002-01-14 14:17:18 +0000 |
| commit | 09f4ad7012d43c5dfb9e40f5e15830cbfad2803c (patch) | |
| tree | f51a2dc4490b3bea2256d26dda24b9d5d3d31ddf /phpBB | |
| parent | 8e642d5d60aca68bee95ac78b208b8c5d6c09c08 (diff) | |
| download | forums-09f4ad7012d43c5dfb9e40f5e15830cbfad2803c.tar forums-09f4ad7012d43c5dfb9e40f5e15830cbfad2803c.tar.gz forums-09f4ad7012d43c5dfb9e40f5e15830cbfad2803c.tar.bz2 forums-09f4ad7012d43c5dfb9e40f5e15830cbfad2803c.tar.xz forums-09f4ad7012d43c5dfb9e40f5e15830cbfad2803c.zip | |
Simplified routines somewhat and introduced title searching
git-svn-id: file:///svn/phpbb/trunk@1868 89ea8834-ac86-4346-8a33-228a782c2dd0
Diffstat (limited to 'phpBB')
| -rw-r--r-- | phpBB/includes/search.php | 433 |
1 files changed, 153 insertions, 280 deletions
diff --git a/phpBB/includes/search.php b/phpBB/includes/search.php index 4bd555ce36..ce6fb935c3 100644 --- a/phpBB/includes/search.php +++ b/phpBB/includes/search.php @@ -19,57 +19,56 @@ * ***************************************************************************/ -function clean_words($entry, &$stopword_list, &$synonym_list) +function clean_words($mode, $entry, &$stopword_list, &$synonym_list) { // Weird, $init_match doesn't work with static when double quotes (") are used... - static $init_match = array('^', '$', '&', '(', ')', '<', '>', '`', "'", '|', ',', '@', '_', '?', '%'); - static $init_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", "", " ", " ", " ", " ", " ", " "); - - static $later_match = array("-", "~", "+", ".", "[", "]", "{", "}", ":", "\\", "/", "=", "#", "\"", ";", "*", "!"); - static $later_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", " ", " " , " ", " ", " ", " ", " ", " ", " "); - - static $sgml_match = array(" ", "ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ"); - static $sgml_replace = array(" ", "s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y"); + static $drop_char_match = array('^', '$', '&', '(', ')', '<', '>', '`', "'", '|', ',', '@', '_', '?', '%', '-', '~', '+', '.', '[', ']', '{', '}', ':', '\\', '/', '=', '#', '\'', ';', '*', '!'); + static $drop_char_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", "", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " " , " ", " ", " ", " ", " ", " ", " "); static $accent_match = array("ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ"); static $accent_replace = array("s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y"); $entry = " " . strip_tags(strtolower($entry)) . " "; - $entry = str_replace($sgml_match, $sgml_match, $entry); - $entry = str_replace($accent_match, $accent_replace, $entry); + for($i = 0; $i < count($accent_match); $i++) + { + $entry = str_replace($accent_match[$i], $accent_replace[$i], $entry); + } + + if( $mode == "post" ) + { + // HTML entities like + $entry = preg_replace("/\b&[a-z]+;\b/is", " ", $entry); + // Replace line endings by a space + $entry = preg_replace("/[\n\r]/is", " ", $entry); + // Remove URL's + $entry = preg_replace("/\b[a-z0-9]+:\/\/[a-z0-9\.\-]+(\/[a-z0-9\?\.%_\-\+=&\/]+)?/si", " ", $entry); + // Quickly remove BBcode. + $entry = preg_replace("/\[img:[a-z0-9]{10,}\].*?\[\/img:[a-z0-9]{10,}\]/is", " ", $entry); + $entry = preg_replace("/\[\/?url(=.*?)?\]/si", " ", $entry); + $entry = preg_replace("/\[\/?[a-z\*=\+\-]+(\:?[0-9a-z]+)?:[a-z0-9]{10,}(\:[a-z0-9]+)?=?.*?\]/si", " ", $entry); + } + else if( $mode == "search" ) + { + $entry = str_replace("+", " and ", $entry); + $entry = str_replace("-", " not ", $entry); + } - // Replace line endings by a space - $entry = preg_replace("/[\n\r]/is", " ", $entry); - // Remove URL's - $entry = preg_replace("/\b[a-z0-9]+:\/\/[a-z0-9\.\-]+(\/[a-z0-9\?\.%_\-\+=&\/]+)?/si", " ", $entry); + // Replace numbers on their own + $entry = preg_replace("/\b[0-9]+\b/si", " ", $entry); + // // Filter out strange characters like ^, $, &, change "it's" to "its" - // str_replace with arrays is buggy in some PHP versions so traverse the arrays manually ;( - for($i = 0; $i < count($init_match); $i++) + // + for($i = 0; $i < count($drop_char_match); $i++) { - $entry = str_replace($init_match[$i], $init_replace[$i], $entry); + $entry = str_replace($drop_char_match[$i], $drop_char_replace[$i], $entry); } - // Quickly remove BBcode. - $entry = preg_replace("/\[code:[0-9]+:[0-9a-z]{10,}\].*?\[\/code:[0-9]+:[0-9a-z]{10,}\]/is", " ", $entry); - $entry = preg_replace("/\[img\].*?\[\/img\]/is", " ", $entry); - $entry = preg_replace("/\[\/?[a-z\*=\+\-]+[0-9a-z]?(\:[a-z0-9]+)?:[a-z0-9]{10,}(\:[a-z0-9]+)?=?.*?\]/si", " ", $entry); - // URLs - $entry = preg_replace("/\[\/?[a-z\*]+[=\+\-]?[0-9a-z]+?:[a-z0-9]{10,}[=.*?]?\]/si", " ", $entry); - $entry = preg_replace("/\[\/?url(=.*?)?\]/si", " ", $entry); - // Numbers - $entry = preg_replace("/\b[0-9]+\b/si", " ", $entry); - // HTML entities like &1234; - $entry = preg_replace("/\b&[a-z]+;\b/is", " ", $entry); - // 'words' that consist of <2 or >50 characters are removed. - $entry = preg_replace("/\b[a-z0-9]{1,2}?\b/si", " ", $entry); - $entry = preg_replace("/\b[a-z0-9]{50,}?\b/si", " ", $entry); - - // Remove some more strange characters - for($i = 0; $i < count($later_match); $i++) + if( $mode == "post" ) { - $entry = str_replace($later_match[$i], $later_replace[$i], $entry); + // 'words' that consist of <=2 or >=50 characters are removed. + $entry = preg_replace("/\b([a-z0-9]{1,2}|[a-z0-9]{50,})\b/si", " ", $entry); } if( !empty($stopword_list) ) @@ -77,7 +76,11 @@ function clean_words($entry, &$stopword_list, &$synonym_list) for ($j = 0; $j < count($stopword_list); $j++) { $filter_word = trim(strtolower($stopword_list[$j])); - $entry = preg_replace("/\b" . phpbb_preg_quote($filter_word, "/") . "\b/is", " ", $entry); + + if( ( $filter_word != "and" && $filter_word != "or" && $filter_word != "not" ) || $mode == "post" ) + { + $entry = preg_replace("/\b" . phpbb_preg_quote($filter_word, "/") . "\b/is", " ", $entry); + } } } @@ -86,56 +89,142 @@ function clean_words($entry, &$stopword_list, &$synonym_list) for ($j = 0; $j < count($synonym_list); $j++) { list($replace_synonym, $match_synonym) = split(" ", trim(strtolower($synonym_list[$j]))); - $entry = preg_replace("/\b" . phpbb_preg_quote(trim($match_synonym), "/") . "\b/is", " " . trim($replace_synonym) . " ", $entry); + + if( ( $match_synonym != "and" && $match_synonym != "or" && $match_synonym != "not" && + $replace_synonym != "and" && $replace_synonym != "or" && $replace_synonym != "not" ) || $mode == "post" ) + { + $entry = preg_replace("/\b" . phpbb_preg_quote(trim($match_synonym), "/") . "\b/is", " " . trim($replace_synonym) . " ", $entry); + } } } return $entry; } -function clean_words_search($entry) +function split_words(&$entry) +{ + preg_match_all("/\b(\w[\w']*\w+|\w+?)\b/", $entry, $split_entries); + + return $split_entries[1]; +} + +function add_search_words($post_id, $post_text, $post_title = "") { + global $db, $phpbb_root_path, $board_config, $lang; - $char_match = array("^", "$", "&", "(", ")", "<", ">", "`", "'", "|", ",", "@", "_", "?", "%", "~", ".", "[", "]", "{", "}", ":", "\\", "/", "=", "#", "\"", ";", "!"); - $char_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", "", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " "); + $stopword_array = @file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_stopwords.txt"); + $synonym_array = @file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_synonyms.txt"); - $sgml_match = array(" ", "ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ"); - $sgml_replace = array(" ", "s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y"); + $search_raw_words = array(); + $search_raw_words['text'] = split_words(clean_words("post", $post_text, $stopword_array, $synonym_array)); + $search_raw_words['title'] = split_words(clean_words("post", $post_title, $stopword_array, $synonym_array)); - $accent_match = array("ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ"); - $accent_replace = array("s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y"); + while( list($word_in, $search_matches) = @each($search_raw_words) ) + { + $title_match = ( $word_in == 'title' ) ? 1 : 0; - $entry = " " . strip_tags(strtolower($entry)) . " "; + if( count($search_matches) ) + { + $word = array(); + sort($search_matches); - $entry = str_replace("+", " and ", $entry); - $entry = str_replace("-", " not ", $entry); + $word_text_sql = ""; + for ($i = 0; $i < count($search_matches); $i++) + { + $search_matches[$i] = trim($search_matches[$i]); - $entry = str_replace($sgml_match, $sgml_match, $entry); - $entry = str_replace($accent_match, $accent_replace, $entry); - $entry = str_replace($char_match, $char_replace, $entry); + if( $search_matches[$i] != "" && $search_matches[$i] != $search_matches[$i-1] ) + { + $word[] = $search_matches[$i]; - $entry = preg_replace("/\b[0-9]+\b/", " ", $entry); + if( $word_text_sql != "" ) + { + $word_text_sql .= ", "; + } - return $entry; -} + $word_text_sql .= "'" . $search_matches[$i] . "'"; + } + } -function split_words(&$entry) -{ - preg_match_all("/\b(\w[\w']*\w+|\w+?)\b/", $entry, $split_entries); + $sql = "SELECT word_id, word_text, word_common + FROM " . SEARCH_WORD_TABLE . " + WHERE word_text IN ($word_text_sql)"; + $result = $db->sql_query($sql); + if( !$result ) + { + message_die(GENERAL_ERROR, "Couldn't select words", "", __LINE__, __FILE__, $sql); + } - return $split_entries[1]; + $check_words = array(); + $word_id_list = array(); + if( $word_check_count = $db->sql_numrows($result) ) + { + while( $row = $db->sql_fetchrow($result) ) + { + $check_words[$row['word_text']] = $row['word_common']; + $word_id_list[] = $row['word_id']; + } + } + + $match_word = array(); + for ($i = 0; $i < count($word); $i++) + { + $new_match = true; + $word_common = false; + + if( $word_check_count ) + { + if( isset($check_words[$word[$i]]) ) + { + $new_match = false; + } + } + + if( !$check_words[$word[$i]] ) + { + $match_word[] = "'" . $word[$i] . "'"; + } + + if( $new_match ) + { + $sql = "INSERT INTO " . SEARCH_WORD_TABLE . " (word_text, word_common) + VALUES ('" . $word[$i] . "', 0)"; + $result = $db->sql_query($sql); + if( !$result ) + { + message_die(GENERAL_ERROR, "Couldn't insert new word", "", __LINE__, __FILE__, $sql); + } + } + } + + $word_sql_in = implode(", ", $match_word); + + $sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match) + SELECT $post_id, word_id, $title_match + FROM " . SEARCH_WORD_TABLE . " + WHERE word_text IN ($word_sql_in)"; + $result = $db->sql_query($sql); + if( !$result ) + { + message_die(GENERAL_ERROR, "Couldn't insert new word matches", "", __LINE__, __FILE__, $sql); + } + } + } + + remove_common("single", 40, $word_id_list); + + return; } // // Check if specified words are too common now // -function remove_common($percent, $word_id_list = array()) +function remove_common($mode, $percent, $word_id_list = array()) { global $db; - // 0.01-0.06s - $sql = "SELECT SUM(forum_posts) AS total_posts - FROM " . FORUMS_TABLE ; + $sql = ( $mode == "global" ) ? "SELECT COUNT(DISTINCT post_id) AS total_posts FROM " . SEARCH_MATCH_TABLE : "SELECT SUM(forum_posts) AS total_posts FROM " . FORUMS_TABLE; + $result = $db->sql_query($sql); if( !$result ) { @@ -149,12 +238,11 @@ function remove_common($percent, $word_id_list = array()) $common_threshold = floor($row['total_posts'] * $percent); $word_id_sql = ""; - if( count($word_id_list) ) + if( $mode == "single" && count($word_id_list) ) { $word_id_sql = "WHERE word_id IN (" . implode(", ", $word_id_list) . ") "; } - // 0.020-0.024s $sql = "SELECT word_id FROM " . SEARCH_MATCH_TABLE . " $word_id_sql @@ -166,7 +254,6 @@ function remove_common($percent, $word_id_list = array()) message_die(GENERAL_ERROR, "Couldn't obtain common word list", "", __LINE__, __FILE__, $sql); } - // No matches if( $word_count = $db->sql_numrows($result) ) { $common_word_id_list = array(); @@ -204,79 +291,6 @@ function remove_common($percent, $word_id_list = array()) return $word_count; } -// -// Search complete wordlist for words that are too common -// -function remove_common_global($percent, $delete_common = 0) -{ - global $db; - - $sql = "SELECT COUNT(DISTINCT post_id) as total_posts - FROM " . SEARCH_MATCH_TABLE; - $result = $db->sql_query($sql); - if( !$result ) - { - $error = $db->sql_error(); - die("Couldn't get maximum post ID :: " . $sql . " :: " . $error['message']); - } - $total_posts = $db->sql_fetchrow($result); - $total_posts = $total_posts['total_posts']; - - $common_threshold = floor($total_posts * ( $percent / 100 )); - - $sql = "SELECT word_id - FROM " . SEARCH_MATCH_TABLE . " - GROUP BY word_id - HAVING count(word_id) > $common_threshold"; - $result = $db->sql_query($sql); - if( !$result ) - { - $error = $db->sql_error(); - die("Couldn't obtain common word list :: " . $sql . " :: " . $error['message']); - } - $common_words = $db->sql_numrows($result); - - while($row = $db->sql_fetchrow($result)) - { - $common_word_ids[] = $row['word_id']; - } - $db->sql_freeresult($result); - - if(count($common_word_ids) != 0) - { - $common_word_ids = implode(',',$common_word_ids); - } - else - { - // We didn't remove any common words - return 0; - } - - $sql = "UPDATE ". SEARCH_WORD_TABLE ." - SET word_common = 1 - WHERE word_id IN ($common_word_ids)"; - $result = $db->sql_query($sql); - if( !$result ) - { - $error = $db->sql_error(); - die("Couldn't delete word list entry :: " . $sql . " :: " . $error['message']); - } - - if( $delete_common) - { - $sql = "DELETE FROM ".SEARCH_MATCH_TABLE." - WHERE word_id IN ($common_word_ids)"; - $result = $db->sql_query($sql); - if( !$result ) - { - $error = $db->sql_error(); - die("Couldn't delete word match entry :: " . $sql . " :: " . $error['message']); - } - } - - return $common_words; -} - function remove_unmatched_words() { global $db; @@ -336,7 +350,6 @@ function remove_unmatched_words() case 'mysql': case 'mysql4': - // 0.07s $sql = "SELECT w.word_id FROM " . SEARCH_WORD_TABLE . " w LEFT JOIN " . SEARCH_MATCH_TABLE . " m ON m.word_id = w.word_id @@ -355,7 +368,6 @@ function remove_unmatched_words() if( $word_id_sql ) { - // 0.07s (about 15-20 words) $sql = "DELETE FROM " . SEARCH_WORD_TABLE . " WHERE word_id IN ($word_id_sql)"; $result = $db->sql_query($sql); @@ -381,143 +393,4 @@ function remove_unmatched_words() return $unmatched_count; } -function add_search_words($post_id, $post_text, $post_title = "") -{ - global $db, $phpbb_root_path, $board_config, $lang; - - $stopword_array = @file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_stopwords.txt"); - $synonym_array = @file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_synonyms.txt"); - - $search_text = clean_words($post_text, $stopword_array, $synonym_array); - $search_matches = split_words($search_text); - - if( count($search_matches) ) - { - $word = array(); - sort($search_matches); - - $word_text_sql = ""; - for ($i = 0; $i < count($search_matches); $i++) - { - $search_matches[$i] = trim($search_matches[$i]); - - if( $search_matches[$i] != "" && $search_matches[$i] != $search_matches[$i-1] ) - { - $word[] = $search_matches[$i]; - - if( $word_text_sql != "" ) - { - $word_text_sql .= ", "; - } - - $word_text_sql .= "'" . $search_matches[$i] . "'"; - } - } - - $sql = "SELECT word_id, word_text, word_common - FROM " . SEARCH_WORD_TABLE . " - WHERE word_text IN ($word_text_sql)"; - $result = $db->sql_query($sql); - if( !$result ) - { - message_die(GENERAL_ERROR, "Couldn't select words", "", __LINE__, __FILE__, $sql); - } - - $check_words = array(); - $word_id_list = array(); - if( $word_check_count = $db->sql_numrows($result) ) - { - while( $row = $db->sql_fetchrow($result) ) - { - $check_words[$row['word_text']] = $row['word_common']; - $word_id_list[] = $row['word_id']; - } - } - - $match_word = array(); - for ($i = 0; $i < count($word); $i++) - { - $new_match = true; - $word_common = false; - - if( $word_check_count ) - { - if( isset($check_words[$word[$i]]) ) - { - $new_match = false; - } - } - - if( !$check_words[$word[$i]] ) - { - $match_word[] = "'" . $word[$i] . "'"; - } - - if( $new_match ) - { - $sql = "INSERT INTO " . SEARCH_WORD_TABLE . " (word_text, word_common) - VALUES ('". $word[$i] . "', 0)"; - $result = $db->sql_query($sql); - if( !$result ) - { - message_die(GENERAL_ERROR, "Couldn't insert new word", "", __LINE__, __FILE__, $sql); - } - } - } - - $word_sql_in = implode(", ", $match_word); - - $sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match) - SELECT $post_id, word_id, 0 - FROM " . SEARCH_WORD_TABLE . " - WHERE word_text IN ($word_sql_in)"; - $result = $db->sql_query($sql); - if( !$result ) - { - message_die(GENERAL_ERROR, "Couldn't insert new word matches", "", __LINE__, __FILE__, $sql); - } - } - - remove_common(0.15, $word_id_list); - - return; -} - -function remove_stop_words($entry, &$stopword_list) -{ - - if( !empty($stopword_list) ) - { - for ($j = 0; $j < count($stopword_list); $j++) - { - $filter_word = trim(strtolower($stopword_list[$j])); - if( $filter_word != "and" && $filter_word != "or" && $filter_word != "not" ) - { - $entry = preg_replace("/\b" . phpbb_preg_quote($filter_word, "/") . "\b/is", " ", $entry); - } - } - } - - return $entry; -} - -function replace_synonyms($entry, &$synonym_list) -{ - if( !empty($synonym_list) ) - { - for ($j = 0; $j < count($synonym_list); $j++) - { - list($replace_synonym, $match_synonym) = split(" ", trim(strtolower($synonym_list[$j]))); - - if( $match_synonym != "and" && $match_synonym != "or" && $match_synonym != "not" && - $replace_synonym != "and" && $replace_synonym != "or" && $replace_synonym != "not" ) - { - $entry = preg_replace("/\b" . phpbb_preg_quote(trim($match_synonym), "/") . "\b/is", " " . trim($replace_synonym) . " ", $entry); - } - } - } - - return $entry; -} - -?> +?>
\ No newline at end of file |