have RDBMS full-text indexer do AND searching [SF#1055435]

Author: Richard Jones

CHANGES.txt

@@ -73,6 +73,7 @@ Fixed:
 - fix indexer searching with no valid words (sf bug 1086787)
 - updated searching / indexing docs
 - fix "(list)" popup when list is one item long (sf bug 1064716)
+- have RDBMS full-text indexer do AND searching (sf bug 1055435)
 
 
 2004-10-26 0.7.9

roundup/backends/back_mysql.py

@@ -105,6 +105,9 @@ def db_exists(config):
 class Database(Database):
     arg = '%s'
 
+    # used by some code to switch styles of query
+    implements_intersect = 0
+
     # Backend for MySQL to use.
     # InnoDB is faster, but if you're running <4.0.16 then you'll need to
     # use BDB to pass all unit tests.

roundup/backends/back_postgresql.py

@@ -81,6 +81,9 @@ def db_exists(config):
 class Database(rdbms_common.Database):
     arg = '%s'
 
+    # used by some code to switch styles of query
+    implements_intersect = 1
+
     def sql_open_connection(self):
         db = rdbms_common.connection_dict(self.config, 'database')
         self.config.logging.getLogger('hyperdb').info('open database %r'%(

roundup/backends/back_sqlite.py

@@ -1,4 +1,4 @@
-# $Id: back_sqlite.py,v 1.36.2.1 2004-12-14 23:23:35 richard Exp $
+# $Id: back_sqlite.py,v 1.36.2.2 2005-01-04 01:33:03 richard Exp $
 '''Implements a backend for SQLite.
 
 See https://pysqlite.sourceforge.net/ for pysqlite info
@@ -25,6 +25,10 @@ def db_nuke(config):
 class Database(rdbms_common.Database):
     # char to use for positional arguments
     arg = '%s'
+
+    # used by some code to switch styles of query
+    implements_intersect = 1
+
     hyperdb_to_sql_datatypes = {
         hyperdb.String : 'VARCHAR(255)',
         hyperdb.Date   : 'VARCHAR(30)',

roundup/backends/indexer_rdbms.py

@@ -79,16 +79,53 @@ def find(self, wordlist):
 
         l = [word.upper() for word in wordlist if 26 > len(word) > 2]
 
-        a = ','.join([self.db.arg] * len(l))
-        sql = 'select distinct(_textid) from __words where _word in (%s)'%a
-        self.db.cursor.execute(sql, tuple(l))
-        r = self.db.cursor.fetchall()
-        if not r:
+        if not l:
             return {}
-        a = ','.join([self.db.arg] * len(r))
-        sql = 'select _class, _itemid, _prop from __textids '\
-            'where _textid in (%s)'%a
-        self.db.cursor.execute(sql, tuple([int(id) for (id,) in r]))
+
+        if self.db.implements_intersect:
+            # simple AND search
+            sql = 'select distinct(_textid) from __words where _word=%s'%self.db.arg
+            sql = '\nINTERSECT\n'.join([sql]*len(l))
+            self.db.cursor.execute(sql, tuple(l))
+            r = self.db.cursor.fetchall()
+            if not r:
+                return {}
+            a = ','.join([self.db.arg] * len(r))
+            sql = 'select _class, _itemid, _prop from __textids '\
+                'where _textid in (%s)'%a
+            self.db.cursor.execute(sql, tuple([int(id) for (id,) in r]))
+
+        else:
+            # A more complex version for MySQL since it doesn't implement INTERSECT
+
+            # Construct SQL statement to join __words table to itself
+            # multiple times.
+            sql = """select distinct(__words1._textid)
+                        from __words as __words1 %s
+                        where __words1._word=%s %s"""
+
+            join_tmpl = ' left join __words as __words%d using (_textid) \n'
+            match_tmpl = ' and __words%d._word=%s \n'
+
+            join_list = []
+            match_list = []
+            for n in xrange(len(l) - 1):
+                join_list.append(join_tmpl % (n + 2))
+                match_list.append(match_tmpl % (n + 2, self.db.arg))
+
+            sql = sql%(' '.join(join_list), self.db.arg, ' '.join(match_list))
+            self.db.cursor.execute(sql, l)
+
+            r = map(lambda x: x[0], self.db.cursor.fetchall())
+            if not r:
+                return {}
+
+            a = ','.join([self.db.arg] * len(r))
+            sql = 'select _class, _itemid, _prop from __textids '\
+                'where _textid in (%s)'%a
+
+            self.db.cursor.execute(sql, tuple(map(int, r)))
+
         # self.search_index has the results as {some id: identifier} ...
         # sigh
         r = {}

test/db_test_base.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
-# $Id: db_test_base.py,v 1.55.2.1 2005-01-03 03:23:38 richard Exp $
+# $Id: db_test_base.py,v 1.55.2.2 2005-01-04 01:33:04 richard Exp $
 
 import unittest, os, shutil, errno, imp, sys, time, pprint
 
@@ -707,6 +707,10 @@ def testIndexerSearching(self):
         self.assertEquals(self.db.indexer.search(['flebble'], self.db.issue),
             {i1: {}, i2: {}})
 
+        # test AND'ing of search terms
+        self.assertEquals(self.db.indexer.search(['frooz', 'flebble'],
+            self.db.issue), {i2: {}})
+
         # unindexed stopword
         self.assertEquals(self.db.indexer.search(['the'], self.db.issue), {})