fix bug introduced in 1.4.5 in RDBMS full-text indexing;

fix tests so the code is actually exercised; fix another bug
discovered in process

Author: Richard Jones

CHANGES.txt

@@ -1,6 +1,11 @@
 This file contains the changes to the Roundup system over time. The entries
 are given with the most recent entry first.
 
+2008-09-01 1.4.6
+Fixed:
+- Bug introduced in 1.4.5 in RDBMS full-text indexing
+
+
 2008-08-19 1.4.5
 Feature:
 - Add use of username/password stored in ~/.netrc in mailgw (sf patch

roundup/backends/indexer_rdbms.py

@@ -1,4 +1,4 @@
-#$Id: indexer_rdbms.py,v 1.17 2008-08-07 22:02:30 richard Exp $
+#$Id: indexer_rdbms.py,v 1.18 2008-09-01 00:43:02 richard Exp $
 ''' This implements the full-text indexer over two RDBMS tables. The first
 is a mapping of words to occurance IDs. The second maps the IDs to (Class,
 propname, itemid) instances.
@@ -41,7 +41,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
         # Ensure all elements of the identifier are strings 'cos the itemid
         # column is varchar even if item ids may be numbers elsewhere in the
         # code. ugh.
-        identifier = map(str, identifier)
+        identifier = tuple(map(str, identifier))
 
         # first, find the id of the (classname, itemid, property)
         a = self.db.arg
@@ -82,12 +82,12 @@ def find(self, wordlist):
         * more rules here
         '''
         if not wordlist:
-            return {}
+            return []
 
         l = [word.upper() for word in wordlist if 26 > len(word) > 2]
 
         if not l:
-            return {}
+            return []
 
         if self.db.implements_intersect:
             # simple AND search
@@ -96,7 +96,7 @@ def find(self, wordlist):
             self.db.cursor.execute(sql, tuple(l))
             r = self.db.cursor.fetchall()
             if not r:
-                return {}
+                return []
             a = ','.join([self.db.arg] * len(r))
             sql = 'select _class, _itemid, _prop from __textids '\
                 'where _textid in (%s)'%a
@@ -125,7 +125,7 @@ def find(self, wordlist):
 
             r = map(lambda x: x[0], self.db.cursor.fetchall())
             if not r:
-                return {}
+                return []
 
             a = ','.join([self.db.arg] * len(r))
             sql = 'select _class, _itemid, _prop from __textids '\

test/test_indexer.py

@@ -18,10 +18,19 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# $Id: test_indexer.py,v 1.10 2006-02-07 04:59:05 richard Exp $
+# $Id: test_indexer.py,v 1.11 2008-09-01 00:43:02 richard Exp $
 
 import os, unittest, shutil
 
+from roundup.backends import get_backend, have_backend
+from roundup.backends.indexer_rdbms import Indexer
+
+# borrow from other tests
+from db_test_base import setupSchema, config
+from test_postgresql import postgresqlOpener
+from test_mysql import mysqlOpener
+from test_sqlite import sqliteOpener
+
 class db:
     class config(dict):
         DATABASE = 'test-index'
@@ -38,29 +47,35 @@ def setUp(self):
         self.dex = Indexer(db)
         self.dex.load_index()
 
+    def assertSeqEqual(self, s1, s2):
+        # first argument is the db result we're testing, second is the desired result
+        # some db results don't have iterable rows, so we have to work around that
+        if [i for x,y in zip(s1, s2) for i,j in enumerate(y) if x[i] != j]:
+            self.fail('contents of %r != %r'%(s1, s2))
+
     def test_basics(self):
         self.dex.add_text(('test', '1', 'foo'), 'a the hello world')
         self.dex.add_text(('test', '2', 'foo'), 'blah blah the world')
-        self.assertEqual(self.dex.find(['world']), [('test', '1', 'foo'),
+        self.assertSeqEqual(self.dex.find(['world']), [('test', '1', 'foo'),
                                                     ('test', '2', 'foo')])
-        self.assertEqual(self.dex.find(['blah']), [('test', '2', 'foo')])
-        self.assertEqual(self.dex.find(['blah', 'hello']), [])
+        self.assertSeqEqual(self.dex.find(['blah']), [('test', '2', 'foo')])
+        self.assertSeqEqual(self.dex.find(['blah', 'hello']), [])
 
     def test_change(self):
         self.dex.add_text(('test', '1', 'foo'), 'a the hello world')
         self.dex.add_text(('test', '2', 'foo'), 'blah blah the world')
-        self.assertEqual(self.dex.find(['world']), [('test', '1', 'foo'),
+        self.assertSeqEqual(self.dex.find(['world']), [('test', '1', 'foo'),
                                                     ('test', '2', 'foo')])
         self.dex.add_text(('test', '1', 'foo'), 'a the hello')
-        self.assertEqual(self.dex.find(['world']), [('test', '2', 'foo')])
+        self.assertSeqEqual(self.dex.find(['world']), [('test', '2', 'foo')])
 
     def test_clear(self):
         self.dex.add_text(('test', '1', 'foo'), 'a the hello world')
         self.dex.add_text(('test', '2', 'foo'), 'blah blah the world')
-        self.assertEqual(self.dex.find(['world']), [('test', '1', 'foo'),
+        self.assertSeqEqual(self.dex.find(['world']), [('test', '1', 'foo'),
                                                     ('test', '2', 'foo')])
         self.dex.add_text(('test', '1', 'foo'), '')
-        self.assertEqual(self.dex.find(['world']), [('test', '2', 'foo')])
+        self.assertSeqEqual(self.dex.find(['world']), [('test', '2', 'foo')])
 
     def tearDown(self):
         shutil.rmtree('test-index')
@@ -75,15 +90,74 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree('test-index')
 
+class RDBMSIndexerTest(IndexerTest):
+    def setUp(self):
+        # remove previous test, ignore errors
+        if os.path.exists(config.DATABASE):
+            shutil.rmtree(config.DATABASE)
+        self.db = self.module.Database(config, 'admin')
+        self.dex = Indexer(self.db)
+    def tearDown(self):
+        if hasattr(self, 'db'):
+            self.db.close()
+        if os.path.exists(config.DATABASE):
+            shutil.rmtree(config.DATABASE)
+
+class postgresqlIndexerTest(postgresqlOpener, RDBMSIndexerTest):
+    def setUp(self):
+        postgresqlOpener.setUp(self)
+        RDBMSIndexerTest.setUp(self)
+    def tearDown(self):
+        RDBMSIndexerTest.tearDown(self)
+        postgresqlOpener.tearDown(self)
+
+class mysqlIndexerTest(mysqlOpener, RDBMSIndexerTest):
+    def setUp(self):
+        mysqlOpener.setUp(self)
+        RDBMSIndexerTest.setUp(self)
+    def tearDown(self):
+        RDBMSIndexerTest.tearDown(self)
+        mysqlOpener.tearDown(self)
+
+class sqliteIndexerTest(sqliteOpener, RDBMSIndexerTest):
+    pass
+
 def test_suite():
     suite = unittest.TestSuite()
+
     suite.addTest(unittest.makeSuite(IndexerTest))
+
     try:
         import xapian
         suite.addTest(unittest.makeSuite(XapianIndexerTest))
     except ImportError:
         print "Skipping Xapian indexer tests"
         pass
+
+    if have_backend('postgresql'):
+        # make sure we start with a clean slate
+        if postgresqlOpener.module.db_exists(config):
+            postgresqlOpener.module.db_nuke(config, 1)
+        suite.addTest(unittest.makeSuite(postgresqlIndexerTest))
+    else:
+        print "Skipping postgresql indexer tests"
+
+    if have_backend('mysql'):
+        # make sure we start with a clean slate
+        if mysqlOpener.module.db_exists(config):
+            mysqlOpener.module.db_nuke(config)
+        suite.addTest(unittest.makeSuite(mysqlIndexerTest))
+    else:
+        print "Skipping mysql indexer tests"
+
+    if have_backend('sqlite'):
+        # make sure we start with a clean slate
+        if sqliteOpener.module.db_exists(config):
+            sqliteOpener.module.db_nuke(config)
+        suite.addTest(unittest.makeSuite(sqliteIndexerTest))
+    else:
+        print "Skipping sqlite indexer tests"
+
     return suite
 
 if __name__ == '__main__':

test/test_sqlite.py

@@ -14,8 +14,8 @@
 # FOR A PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
-# 
-# $Id: test_sqlite.py,v 1.5 2004-11-03 01:34:21 richard Exp $ 
+#
+# $Id: test_sqlite.py,v 1.6 2008-09-01 00:43:02 richard Exp $
 
 import unittest, os, shutil, time
 from roundup.backends import get_backend, have_backend