Fix for bug 145588: adds full-text search option for more accurate finding of individual bugs via words that appear in their descriptions/comments/summaries.

r=bbaetz
a=myk

14 files changed, 308 insertions, 31 deletions

diff --git a/Bugzilla/Search.pm b/Bugzilla/Search.pm
index 75cf8bb27..e795f03f3 100644
--- a/Bugzilla/Search.pm
+++ b/Bugzilla/Search.pm
@@ -121,12 +121,23 @@ sub init {
 
     # If the user has selected all of either status or resolution, change to
     # selecting none. This is functionally equivalent, but quite a lot faster.
+    # Also, if the status is __open__ or __closed__, translate those
+    # into their equivalent lists of open and closed statuses.
     if ($params->param('bug_status')) {
         my @bug_statuses = $params->param('bug_status');
-        
-        if (scalar(@bug_statuses) == scalar(@::legal_bug_status)) {
+        if (scalar(@bug_statuses) == scalar(@::legal_bug_status) 
+            || $bug_statuses[0] eq "__all__")
+        {
             $params->delete('bug_status');
         }
+        elsif ($bug_statuses[0] eq '__open__') {
+            $params->param('bug_status', map(&::IsOpenedState($_) ? $_ : undef, 
+                                             @::legal_bug_status));
+        }
+        elsif ($bug_statuses[0] eq "__closed__") {
+            $params->param('bug_status', map(&::IsOpenedState($_) ? undef : $_, 
+                                             @::legal_bug_status));
+        }
     }
     
     if ($params->param('resolution')) {
@@ -288,6 +299,10 @@ sub init {
         }
     }
 
+    if (defined $params->param('content')) {
+        push(@specialchart, ['content', 'matches', $params->param('content')]);
+    }
+
     my $chartid;
     my $sequence = 0;
     # $type_id is used by the code that queries for attachment flags.
@@ -365,6 +380,55 @@ sub init {
              push(@wherepart, "$table.bug_id = bugs.bug_id");
              $term = "$table.bug_when > " . &::SqlQuote(SqlifyDate($v));
          },
+         "^content,matches" => sub {
+             # "content" is an alias for columns containing text for which we
+             # can search a full-text index and retrieve results by relevance, 
+             # currently just bug comments (and summaries to some degree).
+             # There's only one way to search a full-text index
+             # ("MATCH (...) AGAINST (...)"), so we only accept the "matches"
+             # operator, which is specific to full-text index searches.
+
+             # Add the longdescs table to the query so we can search comments.
+             my $table = "longdescs_$chartid";
+             push(@supptables, "INNER JOIN longdescs $table ON bugs.bug_id " . 
+                               "= $table.bug_id");
+             if (Param("insidergroup") 
+                 && !&::UserInGroup(Param("insidergroup")))
+             {
+                 push(@wherepart, "$table.isprivate < 1");
+             }
+             push(@wherepart, "$table.bug_id = bugs.bug_id");
+
+             # Create search terms to add to the SELECT and WHERE clauses.
+             # $term1 searches comments.
+             # $term2 searches summaries, which contributes to the relevance
+             # ranking in SELECT but doesn't limit which bugs get retrieved.
+             my $term1 = "MATCH($table.thetext) AGAINST(".&::SqlQuote($v).")";
+             my $term2 = "MATCH(bugs.short_desc) AGAINST(".&::SqlQuote($v).")";
+
+             # The term to use in the WHERE clause.
+             $term = $term1;
+
+             # In order to sort by relevance, we SELECT the relevance value
+             # and give it an alias so we can add it to the SORT BY clause
+             # when we build that clause in buglist.cgi.  We also flag the
+             # query in Bugzilla with the "sorted_by_relevance" flag
+             # so buglist.cgi knows to sort by relevance instead of anything
+             # else the user selected.
+             #
+             # Note: MySQL calculates relevance for each comment separately,
+             # so we need to do some additional calculations to get an overall
+             # relevance value, which we do by calculating the average (mean)
+             # comment relevance and then adding the summary relevance, if any.
+             # This weights summary relevance heavily, which makes sense
+             # since summaries are short and thus highly significant.
+             #
+             # Note: We should be calculating the average relevance of all
+             # comments for a bug, not just matching comments, but that's hard
+             # (see http://bugzilla.mozilla.org/show_bug.cgi?id=145588#c35).
+             push(@fields, "(SUM($term1)/COUNT($term1) + $term2) AS relevance");
+             $self->{'sorted_by_relevance'} = 1;
+         },
          "^long_?desc," => sub {
              my $table = "longdescs_$chartid";
              push(@supptables, "longdescs $table");
diff --git a/buglist.cgi b/buglist.cgi
index 4beb57a09..bff5e75e0 100755
--- a/buglist.cgi
+++ b/buglist.cgi
@@ -619,6 +619,15 @@ if ($db_order =~ /bugs.target_milestone/) {
     $query =~ s/\sWHERE\s/ LEFT JOIN milestones ms_order ON ms_order.value = bugs.target_milestone AND ms_order.product_id = bugs.product_id WHERE /;
 }
 
+# Even more disgusting hack: if we are doing a full text search,
+# order by relevance instead of anything else, and limit to 200 results.
+if ($search->{'sorted_by_relevance'}) {
+    $db_order = $order = "relevance DESC LIMIT 200";
+    $vars->{'sorted_by_relevance'} = 1;
+}
+
+
+
 $query .= " ORDER BY $db_order " if ($order);
 
 
diff --git a/checksetup.pl b/checksetup.pl
index 8b348211b..b7cffb05d 100755
--- a/checksetup.pl
+++ b/checksetup.pl
@@ -1590,6 +1590,8 @@ $table{bugs} =
     index (target_milestone),
     index (qa_contact),
     index (votes),
+
+    fulltext (short_desc),
     
     unique(alias)';
 
@@ -1618,7 +1620,8 @@ $table{longdescs} =
     isprivate tinyint not null default 0,
     index(bug_id),
     index(who),
-    index(bug_when)';
+    index(bug_when),
+    fulltext (thetext)';
 
 
 $table{components} =
@@ -2043,6 +2046,8 @@ AddFDef("setters.login_name", "Flag Setter", 0);
 AddFDef("work_time", "Hours Worked", 0);
 AddFDef("percentage_complete", "Percentage Complete", 0);
 
+AddFDef("content", "Content", 0);
+
 ###########################################################################
 # Detect changed local settings
 ###########################################################################
@@ -4044,6 +4049,15 @@ if ($sth->rows == 0) {
   print "\n$login is now set up as an administrator account.\n";
 }
 
+# Add fulltext indexes for bug summaries and descriptions/comments.
+if (!defined GetIndexDef('bugs', 'short_desc')) {
+    print "Adding full-text index for short_desc column in bugs table...\n";
+    $dbh->do('ALTER TABLE bugs ADD FULLTEXT (short_desc)');
+}
+if (!defined GetIndexDef('longdescs', 'thetext')) {
+    print "Adding full-text index for thetext column in longdescs table...\n";
+    $dbh->do('ALTER TABLE longdescs ADD FULLTEXT (thetext)');
+}
 
 # 2002 November, myk@mozilla.org, bug 178841:
 #
diff --git a/query.cgi b/query.cgi
index 5e623437c..149d10f76 100755
--- a/query.cgi
+++ b/query.cgi
@@ -130,7 +130,7 @@ sub PrefillForm {
                       "chfieldto", "chfieldvalue", "target_milestone",
                       "email", "emailtype", "emailreporter",
                       "emailassigned_to", "emailcc", "emailqa_contact",
-                      "emaillongdesc",
+                      "emaillongdesc", "content",
                       "changedin", "votes", "short_desc", "short_desc_type",
                       "long_desc", "long_desc_type", "bug_file_loc",
                       "bug_file_loc_type", "status_whiteboard",
@@ -389,6 +389,7 @@ if (($::FORM{'query_format'} || $::FORM{'format'}) eq "create-series") {
 $vars->{'default'} = \%default;
 
 $vars->{'format'} = $::FORM{'format'};
+$vars->{'query_format'} = $::FORM{'query_format'};
 
 # Generate and return the UI (HTML page) from the appropriate template.
 # If we submit back to ourselves (for e.g. boolean charts), we need to
diff --git a/template/en/default/bug/create/create-guided.html.tmpl b/template/en/default/bug/create/create-guided.html.tmpl
index 0d8217ade..fb5828fe8 100644
--- a/template/en/default/bug/create/create-guided.html.tmpl
+++ b/template/en/default/bug/create/create-guided.html.tmpl
@@ -152,26 +152,11 @@ function PutDescription() {
     For example: <tt><b>pop3 mail</b></tt> or <tt><b>copy paste</b></tt>.
   </p>
 
-  <script type="text/javascript" language="JavaScript">
-    [%# Tell QuickSearch to use the custom-supplied load_relative_url()
-        function. This was originally designed for the sidebar, hence the
-        variable name. %]
-    var sidebar = 1;
-
-    function load_relative_url(url) {
-        frames['somebugs'].location.href = url + "&format=simple";
-    }
-  </script>
-
-  <script type="text/javascript" language="JavaScript"
-          src="localconfig.js"></script>
-  <script type="text/javascript" language="JavaScript"
-          src="quicksearch.js"></script>
-
-  <form name="f" action="show_bug.cgi" method="get"
-      onsubmit="QuickSearch(f.id.value); return false;">
-
-    <input type="text" name="id" size="40">
+  <form action="buglist.cgi" method="get" target="somebugs">
+    <input type="hidden" name="format" value="simple">
+    <input type="hidden" name="bug_status" value="__open__">
+    <input type="hidden" name="product" value="[% product FILTER html %]">
+    <input type="text" name="content" size="40">
     <input type="submit" value="Search">
   </form>
 
diff --git a/template/en/default/filterexceptions.pl b/template/en/default/filterexceptions.pl
index 34fc99380..a1f0a89f2 100644
--- a/template/en/default/filterexceptions.pl
+++ b/template/en/default/filterexceptions.pl
@@ -95,6 +95,15 @@
   'button_name', #
 ],
 
+'search/search-specific.html.tmpl' => [
+  's',
+],
+
+'search/tabs.html.tmpl' => [
+  'tab.name',
+  'tab.description',
+],
+
 'request/queue.html.tmpl' => [
   'column_headers.$group_field', 
   'column_headers.$column', 
diff --git a/template/en/default/global/messages.html.tmpl b/template/en/default/global/messages.html.tmpl
index 6cba576c3..e8aa8047f 100644
--- a/template/en/default/global/messages.html.tmpl
+++ b/template/en/default/global/messages.html.tmpl
@@ -59,6 +59,10 @@
     [% link  = "Go back to the query page." %]
     OK, the <b>[% namedcmd FILTER html %]</b> query is gone.
 
+  [% ELSIF message_tag == "buglist_sorted_by_relevance" %]
+    Bugs on this list are sorted by relevance, with the most relevant bugs
+    at the top.  Only the 200 most relevant bugs are shown.
+
   [% ELSIF message_tag == "change_columns" %]
     [% title = "Change columns" %]
     Resubmitting your query with new columns...
diff --git a/template/en/default/list/list.html.tmpl b/template/en/default/list/list.html.tmpl
index 35a80d08a..f02d92904 100644
--- a/template/en/default/list/list.html.tmpl
+++ b/template/en/default/list/list.html.tmpl
@@ -28,6 +28,7 @@
 [% DEFAULT title = "$terms.Bug List" %]
 [% style_urls = [ "css/buglist.css" ] %]
 [% qorder = order FILTER url_quote IF order %]
+[% message = "buglist_sorted_by_relevance" IF sorted_by_relevance %]
 
 
 [%############################################################################%]
diff --git a/template/en/default/list/table.html.tmpl b/template/en/default/list/table.html.tmpl
index 32016390a..99be51257 100644
--- a/template/en/default/list/table.html.tmpl
+++ b/template/en/default/list/table.html.tmpl
@@ -85,8 +85,12 @@
       <th>&nbsp;</th>
       [% END %]
       <th colspan="[% splitheader ? 2 : 1 %]">
-        <a href="buglist.cgi?
-                  [% urlquerypart FILTER html %]&amp;order=bugs.bug_id">ID</a>
+        [% IF sorted_by_relevance %]
+          ID
+        [% ELSE %]
+          <a href="buglist.cgi?
+                    [% urlquerypart FILTER html %]&amp;order=bugs.bug_id">ID</a>
+        [% END %]
       </th>
 
       [% IF splitheader %]
@@ -119,10 +123,14 @@
 
 [% BLOCK columnheader %]
   <th colspan="[% splitheader ? 2 : 1 %]">
-    <a href="buglist.cgi?[% urlquerypart FILTER html %]&amp;order=
-      [% column.name FILTER url_quote FILTER html %]
-      [% ",$qorder" FILTER html IF order %]">
-        [%- abbrev.$id.title || field_descs.$id || column.title -%]</a>
+    [% IF sorted_by_relevance %]
+      [%- abbrev.$id.title || field_descs.$id || column.title -%]
+    [% ELSE %]
+      <a href="buglist.cgi?[% urlquerypart FILTER html %]&amp;order=
+        [% column.name FILTER url_quote FILTER html %]
+        [% ",$qorder" FILTER html IF order %]">
+          [%- abbrev.$id.title || field_descs.$id || column.title -%]</a>
+    [% END %]
   </th>
 [% END %]
 
diff --git a/template/en/default/search/boolean-charts.html.tmpl b/template/en/default/search/boolean-charts.html.tmpl
index 3987352dc..2d73ae4d7 100644
--- a/template/en/default/search/boolean-charts.html.tmpl
+++ b/template/en/default/search/boolean-charts.html.tmpl
@@ -40,7 +40,8 @@
   { name => "changedafter", description => "changed after" },
   { name => "changedfrom", description => "changed from" },
   { name => "changedto", description => "changed to" },
-  { name => "changedby", description => "changed by" } ] %]
+  { name => "changedby", description => "changed by" },
+  { name => "matches", description => "matches" } ] %]
 
   <p>
     <strong>
diff --git a/template/en/default/search/search-advanced.html.tmpl b/template/en/default/search/search-advanced.html.tmpl
index a5fa51d7d..42207a122 100644
--- a/template/en/default/search/search-advanced.html.tmpl
+++ b/template/en/default/search/search-advanced.html.tmpl
@@ -32,9 +32,20 @@
 
 [% PROCESS global/header.html.tmpl
   title = "Search for $terms.bugs"
+  h1 = ""
   onload = "selectProduct(document.forms['queryform']);initHelp();"
+  style = "td.selected_tab {
+             border-width: 2px 2px 0px;
+             border-style: solid; 
+           }
+           td.unselected_tab, td.spacer {
+             border-width: 0px 0px 2px 0px;
+             border-style: solid; 
+           }"
 %]
 
+[% PROCESS search/tabs.html.tmpl %]
+
 [% button_name = "Search" %]
 
 [%# The decent help requires Javascript %]
diff --git a/template/en/default/search/search-specific.html.tmpl b/template/en/default/search/search-specific.html.tmpl
new file mode 100644
index 000000000..72f86fb13
--- /dev/null
+++ b/template/en/default/search/search-specific.html.tmpl
@@ -0,0 +1,101 @@
+<!-- 1.0@bugzilla.org -->
+[%# The contents of this file are subject to the Mozilla Public
+  # License Version 1.1 (the "License"); you may not use this file
+  # except in compliance with the License. You may obtain a copy of
+  # the License at http://www.mozilla.org/MPL/
+  #
+  # Software distributed under the License is distributed on an "AS
+  # IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+  # implied. See the License for the specific language governing
+  # rights and limitations under the License.
+  #
+  # The Original Code is the Bugzilla Bug Tracking System.
+  #
+  # The Initial Developer of the Original Code is Netscape Communications
+  # Corporation. Portions created by Netscape are
+  # Copyright (C) 1998 Netscape Communications Corporation. All
+  # Rights Reserved.
+  #
+  # Contributor(s): Myk Melez <myk@mozilla.org>
+  #%]
+
+[% PROCESS global/header.html.tmpl 
+  title = "Find a Specific Bug"
+  h1 = ""
+  style = "td.selected_tab {
+             border-width: 2px 2px 0px;
+             border-style: solid; 
+           }
+           td.unselected_tab, td.spacer {
+             border-width: 0px 0px 2px 0px;
+             border-style: solid; 
+           }"
+%]
+
+[% PROCESS search/tabs.html.tmpl %]
+
+<p>
+Find a specific bug by entering words that describe it.  Bugzilla will search
+bug summaries, descriptions, and comments for those words and return a list
+of matching bugs sorted by relevance.
+</p>
+
+<p>
+For example, if the bug you are looking for is a browser crash when you go 
+to a secure web site with an embedded Flash animation, you might search for 
+"crash secure SSL flash".
+</p>
+
+<form method="get" action="buglist.cgi">
+<input type="hidden" name="query_format" value="specific">
+
+<table>
+  <tr>
+    <td align="right" valign="baseline">
+      <b><label for="bug_status">Status:</label></b>
+    </td>
+    <td>
+      <select name="bug_status">
+        [% FOREACH s = ['open', 'closed', 'all'] %]
+            <option value="__[% s %]__" 
+                  [% " selected" IF default.bug_status.0 == "__${s}__" %]>
+            [% s %]
+          </option>
+        [% END %]
+      </select>
+    </td>
+  </tr>
+  <tr>
+    <td align="right" valign="baseline">
+      <b><label for="product">Product:</label></b>
+    </td>
+    <td>
+      <select name="product">
+        <option value="">All</option>
+        [% FOREACH p = product %]
+          <option value="[% p.name FILTER html %]"
+            [% " selected" IF lsearch(default.product, p.name) != -1 %]>
+            [% p.name FILTER html %]</option>
+        [% END %]
+      </select>
+    </td>
+  </tr>
+  <tr>
+    <td align="right" valign="baseline">
+      <b><label for="content">Words:</label></b>
+    </td>
+    <td>
+      <input name="content" size="40" 
+             value="[% default.content.0 FILTER html %]">
+    </td>
+  </tr>
+  <tr>
+    <td></td>
+    <td>
+      <input type="submit" value="Search">
+    </td>
+
+</form>
+
+[% PROCESS global/footer.html.tmpl %]
+
diff --git a/template/en/default/search/search.html.tmpl b/template/en/default/search/search.html.tmpl
index a5fa51d7d..42207a122 100644
--- a/template/en/default/search/search.html.tmpl
+++ b/template/en/default/search/search.html.tmpl
@@ -32,9 +32,20 @@
 
 [% PROCESS global/header.html.tmpl
   title = "Search for $terms.bugs"
+  h1 = ""
   onload = "selectProduct(document.forms['queryform']);initHelp();"
+  style = "td.selected_tab {
+             border-width: 2px 2px 0px;
+             border-style: solid; 
+           }
+           td.unselected_tab, td.spacer {
+             border-width: 0px 0px 2px 0px;
+             border-style: solid; 
+           }"
 %]
 
+[% PROCESS search/tabs.html.tmpl %]
+
 [% button_name = "Search" %]
 
 [%# The decent help requires Javascript %]
diff --git a/template/en/default/search/tabs.html.tmpl b/template/en/default/search/tabs.html.tmpl
new file mode 100644
index 000000000..f6c14c0e6
--- /dev/null
+++ b/template/en/default/search/tabs.html.tmpl
@@ -0,0 +1,58 @@
+<!-- 1.0@bugzilla.org -->
+[%# The contents of this file are subject to the Mozilla Public
+  # License Version 1.1 (the "License"); you may not use this file
+  # except in compliance with the License. You may obtain a copy of
+  # the License at http://www.mozilla.org/MPL/
+  #
+  # Software distributed under the License is distributed on an "AS
+  # IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+  # implied. See the License for the specific language governing
+  # rights and limitations under the License.
+  #
+  # The Original Code is the Bugzilla Bug Tracking System.
+  #
+  # The Initial Developer of the Original Code is Netscape Communications
+  # Corporation. Portions created by Netscape are
+  # Copyright (C) 1998 Netscape Communications Corporation. All
+  # Rights Reserved.
+  #
+  # Contributor(s): Gervase Markham <gerv@gerv.net>
+  #                 Myk Melez <myk@mozilla.org>
+  #%]
+
+[%# INTERFACE:
+  # tabs: List of hashes. May not be empty. Each hash has two members:
+  #   name: string. Name of the tab and the format it represents.
+  #   description: string. Description of the tab (used in tab title).
+  #%]
+
+[% tabs = [ { name => '__DEFAULT__', description => "Advanced Search" },
+            { name => 'specific', description => "Find a Specific Bug" } ] %]
+
+[% current_tab = query_format || format || "__DEFAULT__" %]
+
+<center>
+  <table cellspacing="0" cellpadding="10" border="0" width="100%">
+    <tr>
+      <td class="spacer">&nbsp;</td>
+ 
+      [% FOREACH tab = tabs %]
+        [% IF tab.name == current_tab %]
+          <td align="center" bgcolor="lightblue" class="selected_tab">
+            [% tab.description %]
+          </td>
+        [% ELSE %]
+          <td align="center" bgcolor="#BBBBEE" class="unselected_tab">
+            <a href="query.cgi
+              [% IF tab.name != "__DEFAULT__" %]?format=[% tab.name %][% END %]"
+            >
+              [% tab.description %]
+            </a>
+          </td>
+        [% END %]
+       [% END %]
+ 
+       <td class="spacer">&nbsp;</td>
+     </tr>
+   </table>
+</center>