-function clean_words($entry, &$stopword_list, &$synonym_list)

-{

- static $init_match = array('^', '$', '&', '(', ')', '<', '>', '`', "'", '|', ',', '@', '_', '?', '%');

- static $init_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", "", " ", " ", " ", " ", " ", " ");

-

- static $later_match = array("-", "~", "+", ".", "[", "]", "{", "}", ":", "\\", "/", "=", "#", "\"", ";", "*", "!");

- static $later_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ");

-

- static $sgml_match = array("&nbsp;", "&szlig;", "&agrave;", "&aacute;", "&acirc;", "&atilde;", "&auml;", "&aring;", "&aelig;", "&ccedil;", "&egrave;", "&eacute;", "&ecirc;", "&euml;", "&igrave;", "&iacute;", "&icirc;", "&iuml;", "&eth;", "&ntilde;", "&ograve;", "&oacute;", "&ocirc;", "&otilde;", "&ouml;", "&oslash;", "&ugrave;", "&uacute;", "&ucirc;", "&uuml;", "&yacute;", "&thorn;", "&yuml;");

- static $sgml_replace = array(" ", "s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y");

-

- static $accent_match = array("ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ");

- static $accent_replace = array("s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y");

-

- $entry = " " . stripslashes(strip_tags(strtolower($entry))) . " ";

-

- $entry = str_replace($sgml_match, $sgml_match, $entry);

- $entry = str_replace($accent_match, $accent_replace, $entry);

-

- $entry = preg_replace("/[\n\r]/is", " ", $entry);

- $entry = preg_replace("/\b[a-z0-9]+:\/\/[a-z0-9\.\-]+(\/[a-z0-9\?\.%_\-\+=&\/]+)?/si", " ", $entry);

-

- $entry = str_replace($init_match, $init_replace, $entry);

-

- $entry = preg_replace("/\[code:[0-9]+:[0-9a-z]{10,}\].*?\[\/code:[0-9]+:[0-9a-z]{10,}\]/is", " ", $entry);

- $entry = preg_replace("/\[img\].*?\[\/img\]/is", " ", $entry);

- $entry = preg_replace("/\[\/?[a-z\*=\+\-]+[0-9a-z]?(\:[a-z0-9]+)?:[a-z0-9]{10,}(\:[a-z0-9]+)?=?.*?\]/si", " ", $entry);

- $entry = preg_replace("/\[\/?[a-z\*]+[=\+\-]?[0-9a-z]+?:[a-z0-9]{10,}[=.*?]?\]/si", " ", $entry);

- $entry = preg_replace("/\[\/?url(=.*?)?\]/si", " ", $entry);

- $entry = preg_replace("/\b[0-9]+\b/si", " ", $entry);

- $entry = preg_replace("/\b&[a-z]+;\b/is", " ", $entry);

- $entry = preg_replace("/\b[a-z0-9]{1,2}?\b/si", " ", $entry);

- $entry = preg_replace("/\b[a-z0-9]{50,}?\b/si", " ", $entry);

-

- $entry = str_replace($later_match, $later_replace, $entry);

-

- if( !empty($stopword_list) )

- {

- for ($j = 0; $j < count($stopword_list); $j++)

- {

- $filter_word = trim(strtolower($stopword_list[$j]));

- $entry = preg_replace("/\b" . phpbb_preg_quote($filter_word, "/") . "\b/is", " ", $entry);

- }

- }

-

- if( !empty($synonym_list) )

- {

- for ($j = 0; $j < count($synonym_list); $j++)

- {

- list($replace_synonym, $match_synonym) = split(" ", trim(strtolower($synonym_list[$j])));

- $entry = preg_replace("/\b" . phpbb_preg_quote(trim($match_synonym), "/") . "\b/is", " " . trim($replace_synonym) . " ", $entry);

- }

- }

-

- return $entry;

-}

-

-function split_words(&$entry)

-{

- $split_entries = array();

- preg_match_all("/\b(\w[\w']*\w+|\w+?)\b/", $entry, $split_entries);

-

- return $split_entries[1];

-}

-

-function remove_common($percent, $word_id_list = array())

-{

- global $db;

-

- // 0.01-0.06s

- $sql = "SELECT SUM(forum_posts) AS total_posts

- FROM " . FORUMS_TABLE ;

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't obtain post count", "", __LINE__, __FILE__, $sql);

- }

-

- $row = $db->sql_fetchrow($result);

-

- if( $row['total_posts'] > 100 )

- {

- $common_threshold = floor($row['total_posts'] * $percent);

-

- $word_id_sql = "";

- if( count($word_id_list) )

- {

- $word_id_sql = "WHERE word_id IN (" . implode(", ", $word_id_list) . ") ";

- }

-

- // 0.020-0.024s

- $sql = "SELECT word_id

- FROM " . SEARCH_MATCH_TABLE . "

- $word_id_sql

- GROUP BY word_id

- HAVING COUNT(word_id) > $common_threshold";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't obtain common word list", "", __LINE__, __FILE__, $sql);

- }

-

- // No matches

- if( $word_count = $db->sql_numrows($result) )

- {

- $common_word_id_list = array();

- while( $row = $db->sql_fetchrow($result) )

- {

- $common_word_id_list[] = $row['word_id'];

- }

-

- $db->sql_freeresult($result);

-

- if( count($common_word_ids) != 0 )

- {

- $common_word_id_list = implode(", ", $common_word_id_list);

-

- $sql = "UPDATE " . SEARCH_WORD_TABLE . "

- SET word_common = " . TRUE . "

- WHERE word_id IN ($common_word_id_list)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't delete word list entry", "", __LINE__, __FILE__, $sql);

- }

-

- $sql = "DELETE FROM " . SEARCH_MATCH_TABLE . "

- WHERE word_id IN ($common_word_id_list)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't delete word match entry", "", __LINE__, __FILE__, $sql);

- }

- }

- }

- }

-

- return $word_count;

-}

-

-function remove_unmatched_words()

-{

- global $db;

-

- switch(SQL_LAYER)

- {

- case 'postgresql':

- $sql = "DELETE FROM " . SEARCH_WORD_TABLE . "

- WHERE word_id NOT IN (

- SELECT word_id

- FROM " . SEARCH_MATCH_TABLE . "

- GROUP BY word_id)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't delete old words from word table", __LINE__, __FILE__, $sql);

- }

-

- $unmatched_count = $db->sql_affectedrows();

-

- break;

-

- case 'oracle':

- $sql = "DELETE FROM " . SEARCH_WORD_TABLE . "

- WHERE word_id IN (

- SELECT w.word_id

- FROM " . SEARCH_WORD_TABLE . " w, " . SEARCH_MATCH_TABLE . " m

- WHERE w.word_id = m.word_id(+)

- AND m.word_id IS NULL)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't delete old words from word table", __LINE__, __FILE__, $sql);

- }

-

- $unmatched_count = $db->sql_affectedrows();

-

- break;

-

- case 'mssql':

- case 'msaccess':

- $sql = "DELETE FROM " . SEARCH_WORD_TABLE . "

- WHERE word_id IN (

- SELECT w.word_id

- FROM " . SEARCH_WORD_TABLE . " w

- LEFT JOIN " . SEARCH_MATCH_TABLE . " m ON m.word_id = w.word_id

- WHERE m.word_id IS NULL)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't delete old words from word table", __LINE__, __FILE__, $sql);

- }

-

- $unmatched_count = $db->sql_affectedrows();

-

- break;

-

- case 'mysql':

- case 'mysql4':

- // 0.07s

- $sql = "SELECT w.word_id

- FROM " . SEARCH_WORD_TABLE . " w

- LEFT JOIN " . SEARCH_MATCH_TABLE . " m ON m.word_id = w.word_id

- WHERE m.word_id IS NULL";

- if( $result = $db->sql_query($sql) )

- {

- if( $unmatched_count = $db->sql_numrows($result) )

- {

- $rowset = array();

- while( $row = $db->sql_fetchrow($result) )

- {

- $rowset[] = $row['word_id'];

- }

-

- $word_id_sql = implode(", ", $rowset);

-

- if( $word_id_sql )

- {

- // 0.07s (about 15-20 words)

- $sql = "DELETE FROM " . SEARCH_WORD_TABLE . "

- WHERE word_id IN ($word_id_sql)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't delete word list entry", "", __LINE__, __FILE__, $sql);

- }

- }

- else

- {

- return 0;

- }

- }

- else

- {

- return 0;

- }

- }

-

- break;

- }

-

- return $unmatched_count;

-}

-

-function add_search_words($post_id, $post_text, $post_title = "")

-{

- global $db, $phpbb_root_path, $board_config, $lang;

-

- $stopword_array = @file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_stopwords.txt");

- $synonym_array = @file($phpbb_root_path . "language/lang_" . $board_config['default_lang'] . "/search_synonyms.txt");

-

- $search_text = clean_words($post_text, $stopword_array, $synonym_array);

- $search_matches = split_words($search_text);

-

- if( count($search_matches) )

- {

- $word = array();

- sort($search_matches);

-

- $word_text_sql = "";

- for ($i = 0; $i < count($search_matches); $i++)

- {

- $search_matches[$i] = trim($search_matches[$i]);

-

- if( $search_matches[$i] != "" && $search_matches[$i] != $search_matches[$i-1] )

- {

- $word[] = $search_matches[$i];

-

- if( $word_text_sql != "" )

- {

- $word_text_sql .= ", ";

- }

-

- $word_text_sql .= "'" . $search_matches[$i] . "'";

- }

- }

-

- $sql = "SELECT word_id, word_text, word_common

- FROM " . SEARCH_WORD_TABLE . "

- WHERE word_text IN ($word_text_sql)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't select words", "", __LINE__, __FILE__, $sql);

- }

-

- $check_words = array();

- $word_id_list = array();

- if( $word_check_count = $db->sql_numrows($result) )

- {

- while( $row = $db->sql_fetchrow($result) )

- {

- $check_words[$row['word_text']] = $row['word_common'];

- $word_id_list[] = $row['word_id'];

- }

- }

-

- $match_word = array();

- for ($i = 0; $i < count($word); $i++)

- {

- $new_match = true;

- $word_common = false;

-

- if( $word_check_count )

- {

- if( isset($check_words[$word[$i]]) )

- {

- $new_match = false;

- }

- }

-

- if( !$check_words[$word[$i]] )

- {

- $match_word[] = "'" . $word[$i] . "'";

- }

-

- if( $new_match )

- {

- $sql = "INSERT INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)

- VALUES ('". $word[$i] . "', 0)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't insert new word", "", __LINE__, __FILE__, $sql);

- }

- }

- }

-

- $word_sql_in = implode(", ", $match_word);

-

- $sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match)

- SELECT $post_id, word_id, 0

- FROM " . SEARCH_WORD_TABLE . "

- WHERE word_text IN ($word_sql_in)";

- $result = $db->sql_query($sql);

- if( !$result )

- {

- message_die(GENERAL_ERROR, "Couldn't insert new word matches", "", __LINE__, __FILE__, $sql);

- }

- }

-

- remove_common(0.15, $word_id_list);

-

- return;

-}