fixed rdbms searching by ID [SF#666615]

detect corrupted index and raise semi-useful exception [SF#666767]

also some mysql support (in tests)

Author: Richard Jones

CHANGES.txt

@@ -11,6 +11,11 @@ are given with the most recent entry first.
 - applied patch for nicer history display (sf feature 638280)
 
 
+2003-??-?? 0.5.5
+- fixed rdbms searching by ID (sf bug 666615)
+- detect corrupted index and raise semi-useful exception (sf bug 666767)
+
+
 2003-01-10 0.5.4
 - key the templates cache off full path, not filename
 - implemented whole-database locking

TODO.txt

@@ -58,5 +58,7 @@ pending web       multilink item removal action with retirement
 pending web       implement a python dict version of the form values
 pending web       implement the request.url attribute?
 pending web       UNIX init.d script for roundup-server
+
+bug     hyperdb   need unit tests for firing of auditors and reactors
 ======= ========= =============================================================
 

doc/customizing.txt

@@ -2,7 +2,7 @@
 Customising Roundup
 ===================
 
-:Version: $Revision: 1.69 $
+:Version: $Revision: 1.70 $
 
 .. This document borrows from the ZopeBook section on ZPT. The original is at:
    http://www.zope.org/Documentation/Books/ZopeBook/current/ZPT.stx
@@ -999,7 +999,7 @@ forms:
    in place if the "foo" form variable doesn't exist.
 
 **String Expressions** - eg. ``string:hello ${user/name}``
-   These expressions are simple string interpolations (though they can be just
+   These expressions are simple string interpolations - though they can be just
    plain strings with no interpolation if you want. The expression in the
    ``${ ... }`` is just a path expression as above.
 

roundup/backends/__init__.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: __init__.py,v 1.20 2002-10-07 00:52:51 richard Exp $
+# $Id: __init__.py,v 1.21 2003-01-12 23:53:19 richard Exp $
 
 ''' Container for the hyperdb storage backend implementations.
 
@@ -56,6 +56,15 @@
     gadfly = back_gadfly
     __all__.append('gadfly')
 
+try:
+    import MySQLdb
+except ImportError, message:
+    if str(message) != 'No module named MySQLdb': raise
+else:
+    import back_mysql
+    mysql = back_mysql
+    __all__.append('mysql')
+
 try:
     import sqlite
 except ImportError, message:

roundup/backends/back_gadfly.py

@@ -1,4 +1,4 @@
-# $Id: back_gadfly.py,v 1.31 2003-01-08 05:39:40 richard Exp $
+# $Id: back_gadfly.py,v 1.32 2003-01-12 23:53:20 richard Exp $
 ''' Gadlfy relational database hypderb backend.
 
 About Gadfly
@@ -185,6 +185,14 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                 else:
                     where.append('_%s=%s'%(k, a))
                     args.append(date.Interval(v).serialise())
+            elif k == 'id':
+                if isinstance(v, type([])):
+                    s = ','.join([a for x in v])
+                    where.append('%s in (%s)'%(k, s))
+                    args = args + v
+                else:
+                    where.append('%s=%s'%(k, a))
+                    args.append(v)
             else:
                 if isinstance(v, type([])):
                     s = ','.join([a for x in v])

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.27 2003-01-08 05:39:40 richard Exp $
+# $Id: rdbms_common.py,v 1.28 2003-01-12 23:53:20 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -1762,6 +1762,14 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                 else:
                     where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
                     args.append(v)
+            elif k == 'id':
+                if isinstance(v, type([])):
+                    s = ','.join([a for x in v])
+                    where.append('%s in (%s)'%(k, s))
+                    args = args + v
+                else:
+                    where.append('%s=%s'%(k, a))
+                    args.append(v)
             elif isinstance(propclass, String):
                 if not isinstance(v, type([])):
                     v = [v]

roundup/indexer.py

@@ -14,7 +14,7 @@
 #     that promote freedom, but obviously am giving up any rights
 #     to compel such.
 # 
-#$Id: indexer.py,v 1.14 2002-09-25 05:06:14 richard Exp $
+#$Id: indexer.py,v 1.15 2003-01-12 23:53:19 richard Exp $
 '''
 This module provides an indexer class, RoundupIndexer, that stores text
 indices in a roundup instance.  This class makes searching the content of
@@ -214,13 +214,15 @@ def find(self, wordlist):
                 # word outside the bounds of what we index - ignore
                 continue
             word = word.upper()
-            entry = self.words.get(word)    # For each word, get index
-            entries[word] = entry           #   of matching files
-            if not entry:                   # Nothing for this one word (fail)
+            entry = self.words[word]    # For each word, get index
+            entries[word] = entry       #   of matching files
+            if not entry:               # Nothing for this one word (fail)
                 return {}
             if hits is None:
                 hits = {}
                 for k in entry.keys():
+                    if not self.fileids.has_key(k):
+                        raise ValueError, 'Index is corrupted: re-generate it'
                     hits[k] = self.fileids[k]
             else:
                 # Eliminate hits for every non-match

test/test_db.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: test_db.py,v 1.64 2003-01-12 00:43:43 richard Exp $ 
+# $Id: test_db.py,v 1.65 2003-01-12 23:53:20 richard Exp $ 
 
 import unittest, os, shutil, time
 
@@ -570,6 +570,10 @@ def filteringSetup(self):
         self.db.commit()
         return self.assertEqual, self.db.issue.filter
 
+    def testFilteringID(self):
+        ae, filt = self.filteringSetup()
+        ae(filt(None, {'id': '1'}, ('+','id'), (None,None)), ['1'])
+
     def testFilteringString(self):
         ae, filt = self.filteringSetup()
         ae(filt(None, {'title': 'issue one'}, ('+','id'), (None,None)), ['1'])
@@ -698,6 +702,32 @@ def setUp(self):
         setupSchema(self.db, 0, gadfly)
 
 
+class mysqlDBTestCase(anydbmDBTestCase):
+    def setUp(self):
+        from roundup.backends import mysql
+        # remove previous test, ignore errors
+        if os.path.exists(config.DATABASE):
+            shutil.rmtree(config.DATABASE)
+        config.MYSQL_DATABASE='mysql@localhost root rootpasswd'.split()
+        os.makedirs(config.DATABASE + '/files')
+        self.db = mysql.Database(config, 'admin')
+        setupSchema(self.db, 1, mysql)
+
+class mysqlReadOnlyDBTestCase(anydbmReadOnlyDBTestCase):
+    def setUp(self):
+        from roundup.backends import mysql
+        # remove previous test, ignore errors
+        if os.path.exists(config.DATABASE):
+            shutil.rmtree(config.DATABASE)
+        config.MYSQL_DATABASE='mysql@localhost root rootpasswd'.split()
+        os.makedirs(config.DATABASE + '/files')
+        db = mysql.Database(config, 'admin')
+        setupSchema(db, 1, mysql)
+        db.close()
+        self.db = sqlite.Database(config)
+        setupSchema(self.db, 0, mysql)
+
+
 class sqliteDBTestCase(anydbmDBTestCase):
     def setUp(self):
         from roundup.backends import sqlite
@@ -708,9 +738,6 @@ def setUp(self):
         self.db = sqlite.Database(config, 'admin')
         setupSchema(self.db, 1, sqlite)
 
-    def testIDGeneration(self):
-        pass
-
 class sqliteReadOnlyDBTestCase(anydbmReadOnlyDBTestCase):
     def setUp(self):
         from roundup.backends import sqlite
@@ -791,6 +818,11 @@ def suite():
 #    return unittest.TestSuite(l)
 
     from roundup import backends
+#    if hasattr(backends, 'mysql'):
+#        l.append(unittest.makeSuite(mysqlDBTestCase, 'test'))
+#        l.append(unittest.makeSuite(mysqlReadOnlyDBTestCase, 'test'))
+#    return unittest.TestSuite(l)
+
     if hasattr(backends, 'gadfly'):
         l.append(unittest.makeSuite(gadflyDBTestCase, 'test'))
         l.append(unittest.makeSuite(gadflyReadOnlyDBTestCase, 'test'))