merge from HEAD

Author: Richard Jones

CHANGES.txt

@@ -1,7 +1,7 @@
 This file contains the changes to the Roundup system over time. The entries
 are given with the most recent entry first.
 
-2004-??-?? 0.7.6
+2004-07-20 0.7.6
 Fixed:
 - rdbms backend full text search failure after import (sf bug 980314)
 - rdbms backends not filtering correctly on link=None
@@ -16,6 +16,12 @@ Fixed:
 - document the STATIC_FILES config var
 - implement the HTTP HEAD command (sf bug 992544)
 - fix journal export of files to remove content from CSV files
+- API clarification. Previously, the anydbm/bsddb/metakit filter() methods
+  had required exact matches to Multilink argument lists. The RDBMS
+  backends treated Multilink matches like all other data types - matching
+  any of the Multilink argument list is good enough. The latter behaviour
+  is implemented across the board now.
+- fix metakit handling of filter on Link==None
 
 
 2004-06-24 0.7.5

doc/announcement.txt

@@ -16,6 +16,7 @@ from Ka-Ping Yee in the Software Carpentry "Track" design competition.
 - fixed external password source example (sf bug 986601)
 - document the STATIC_FILES config var
 - implement the HTTP HEAD command (sf bug 992544)
+- fix journal export of files to remove content from CSV files
 
 If you're upgrading from an older version of Roundup you *must* follow
 the "Software Upgrade" guidelines given in the maintenance documentation.

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.146.2.14 2004-07-20 05:58:47 richard Exp $
+#$Id: back_anydbm.py,v 1.146.2.15 2004-07-20 23:27:02 richard Exp $
 '''This module defines a backend that saves the hyperdatabase in a
 database chosen by anydbm. It is guaranteed to always be available in python
 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
@@ -943,8 +943,6 @@ def create_inner(self, **propvalues):
                 raise ValueError, 'key property "%s" is required'%key
             if isinstance(prop, Multilink):
                 propvalues[key] = []
-            else:
-                propvalues[key] = None
 
         # done
         self.db.addnode(self.classname, newid, propvalues)
@@ -1588,9 +1586,7 @@ def filter(self, search_matches, filterspec, sort=(None,None),
 
         The filter must match all properties specificed - but if the
         property value to match is a list, any one of the values in the
-        list may match for that property to match. Unless the property
-        is a Multilink, in which case the item's property list must
-        match the filterspec list.
+        list may match for that property to match.
         """
         if __debug__:
             start_t = time.time()
@@ -1718,10 +1714,9 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                             # othewise, make sure this node has each of the
                             # required values
                             for want in v:
-                                if want not in nv:
+                                if want in nv:
+                                    match = True
                                     break
-                            else:
-                                match = True
                     elif t == STRING:
                         if nv is None:
                             nv = ''
@@ -2108,10 +2103,6 @@ def create(self, **propvalues):
         # do the database create
         newid = self.create_inner(**propvalues)
 
-        # and index!
-        self.db.indexer.add_text((self.classname, newid, 'content'), content,
-            mime_type)
-
         # fire reactors
         self.fireReactors('create', newid, None)
 

roundup/backends/back_metakit.py

@@ -1,4 +1,4 @@
-# $Id: back_metakit.py,v 1.70.2.7 2004-07-20 05:58:47 richard Exp $
+# $Id: back_metakit.py,v 1.70.2.8 2004-07-20 23:27:02 richard Exp $
 '''Metakit backend for Roundup, originally by Gordon McMillan.
 
 Known Current Bugs:
@@ -84,6 +84,9 @@ def __init__(self, config, journaltag=None):
         self.indexer = Indexer(self.config.DATABASE, self._db)
         self.security = security.Security(self)
 
+        self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
+            'filtering': 0}
+
         os.umask(0002)
 
     def post_init(self):
@@ -1189,10 +1192,8 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         property value to match is a list, any one of the values in the
         list may match for that property to match.
         '''        
-        # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
-        # filterspec is a dict {propname:value}
-        # sort and group are (dir, prop) where dir is '+', '-' or None
-        #                    and prop is a prop name or None
+        if __debug__:
+            start_t = time.time()
 
         timezone = self.db.getUserTimezone()
 
@@ -1227,10 +1228,14 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                 # transform keys to ids
                 u = []
                 for item in value:
-                    try:
-                        item = int(item)
-                    except (TypeError, ValueError):
-                        item = int(self.db.getclass(prop.classname).lookup(item))
+                    if item is None:
+                        item = -1
+                    else:
+                        try:
+                            item = int(item)
+                        except (TypeError, ValueError):
+                            linkcl = self.db.getclass(prop.classname)
+                            item = int(linkcl.lookup(item))
                     if item == -1:
                         item = 0
                     u.append(item)
@@ -1300,29 +1305,25 @@ def filter(self, search_matches, filterspec, sort=(None,None),
             else:
                 where[propname] = str(value)
         v = self.getview()
-        #print "filter start at  %s" % time.time() 
         if where:
             where_higherbound = where.copy()
             where_higherbound.update(wherehigh)
             v = v.select(where, where_higherbound)
-        #print "filter where at  %s" % time.time() 
 
         if mlcriteria:
             # multilink - if any of the nodeids required by the
             # filterspec aren't in this node's property, then skip it
             def ff(row, ml=mlcriteria):
                 for propname, values in ml.items():
                     sv = getattr(row, propname)
-                    if not values and sv:
-                        return 0
+                    if not values and not sv:
+                        return 1
                     for id in values:
-                        if sv.find(fid=id) == -1:
-                            return 0
-                return 1
+                        if sv.find(fid=id) != -1:
+                            return 1
+                return 0
             iv = v.filter(ff)
             v = v.remapwith(iv)
-
-        #print "filter mlcrit at %s" % time.time() 
 
         if orcriteria:
             def ff(row, crit=orcriteria):
@@ -1335,7 +1336,6 @@ def ff(row, crit=orcriteria):
             iv = v.filter(ff)
             v = v.remapwith(iv)
 
-        #print "filter orcrit at %s" % time.time() 
         if regexes:
             def ff(row, r=regexes):
                 for propname, regex in r.items():
@@ -1346,7 +1346,6 @@ def ff(row, r=regexes):
 
             iv = v.filter(ff)
             v = v.remapwith(iv)
-        #print "filter regexs at %s" % time.time() 
 
         if sort or group:
             sortspec = []
@@ -1381,7 +1380,6 @@ def ff(row, r=regexes):
                     rev.append(prop)
                 sortspec.append(prop)
             v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
-        #print "filter sort   at %s" % time.time() 
 
         rslt = []
         for row in v:
@@ -1391,6 +1389,10 @@ def ff(row, r=regexes):
                     rslt.append(id)
             else:
                 rslt.append(id)
+
+        if __debug__:
+            self.db.stats['filtering'] += (time.time() - start_t)
+
         return rslt
 
     def hasnode(self, nodeid):

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.98.2.18 2004-07-20 05:58:47 richard Exp $
+# $Id: rdbms_common.py,v 1.98.2.19 2004-07-20 23:27:02 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -2268,7 +2268,6 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         cols = ','.join(cols)
         loj = ' '.join(loj)
         sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
-        print sql
         args = tuple(args)
         if __debug__:
             print >>hyperdb.DEBUG, 'filter', (self, sql, args)

setup.py

@@ -16,7 +16,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: setup.py,v 1.64.2.5 2004-06-24 07:59:55 richard Exp $
+# $Id: setup.py,v 1.64.2.6 2004-07-20 23:27:00 richard Exp $
 
 from distutils.core import setup, Extension
 from distutils.util import get_platform
@@ -217,22 +217,21 @@ def main():
 If you're upgrading from an older version of Roundup you *must* follow
 the "Software Upgrade" guidelines given in the maintenance documentation.
 
-0.7.5 is a bug fix release, including:
-
-- force lookup of journal props in anydbm filtering
-- fixed lookup of "missing" Link values for new props in anydbm backend
-- allow list of values for id, Number and Boolean filtering in anydbm
-  backend
-- fixed some more mysql 0.6->0.7 upgrade bugs (sf bug 950410)
-- fixed Boolean values in postgresql (sf bugs 972546 and 972600)
-- fixed -g arg to roundup-server (sf bug 973946)
-- better roundup-server usage string (sf bug 973352)
-- include "context" always, as documented (sf bug 965447)
-- fixed REMOTE_USER (external HTTP Basic auth) (sf bug 977309)
-- fixed roundup-admin "find" to use better value parsing
-- fixed RDBMS Class.find() to handle None value in multiple find
-- export now stores file "content" in separate files in export directory
-
+This is a bug fix release, including:
+
+- rdbms backend full text search failure after import (sf bug 980314)
+- rdbms backends not filtering correctly on link=None
+- fix anydbm journal import (sf bug 983166)
+- handle postgresql bug in SQL generation (sf bug 984591)
+- fix dates-from-Dates (sf bug 984604)
+- fix messageid generated when msgid is None for send_message (sf bug 987933)
+- make user permissions check more sane (fix search page for anonymous)
+- fixed RDBMS filter() for no matches from full-text search (sf bug 990778)
+- fixed DateHTMLProperty for invalid date entry (sf bug 986538)
+- fixed external password source example (sf bug 986601)
+- document the STATIC_FILES config var
+- implement the HTTP HEAD command (sf bug 992544)
+- fix journal export of files to remove content from CSV files
 ''',
         author = "Richard Jones",
         author_email = "[email protected]",

test/db_test_base.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: db_test_base.py,v 1.27.2.9 2004-07-03 23:08:44 richard Exp $ 
+# $Id: db_test_base.py,v 1.27.2.10 2004-07-20 23:27:02 richard Exp $ 
 
 import unittest, os, shutil, errno, imp, sys, time, pprint
 
@@ -691,6 +691,7 @@ def testFileClassReindexing(self):
         i1 = self.db.issue.create(files=[f1, f2])
         self.db.commit()
         d = self.db.indexer.search(['hello'], self.db.issue)
+        self.assert_(d.has_key(i1))
         d[i1]['files'].sort()
         self.assertEquals(d, {i1: {'files': [f1, f2]}})
         self.assertEquals(self.db.indexer.search(['world'], self.db.issue),
@@ -836,11 +837,11 @@ def filteringSetup(self):
         for issue in (
                 {'title': 'issue one', 'status': '2', 'assignedto': '1',
                     'foo': date.Interval('1:10'), 'priority': '3',
-                    'deadline': date.Date('2003-01-01.00:00')},
+                    'deadline': date.Date('2003-02-16.22:50')},
                 {'title': 'issue two', 'status': '1', 'assignedto': '2',
                     'foo': date.Interval('1d'), 'priority': '3',
-                    'deadline': date.Date('2003-02-16.22:50')},
-                    {'title': 'issue three', 'status': '1', 'priority': '2',
+                    'deadline': date.Date('2003-01-01.00:00')},
+                {'title': 'issue three', 'status': '1', 'priority': '2',
                     'nosy': ['1','2'], 'deadline': date.Date('2003-02-18')},
                 {'title': 'non four', 'status': '3',
                     'foo': date.Interval('0:10'), 'priority': '2',
@@ -893,7 +894,7 @@ def testFilteringMultilink(self):
         ae(filt(None, {'nosy': '2'}, ('+','id'), (None,None)), ['3'])
         ae(filt(None, {'nosy': '-1'}, ('+','id'), (None,None)), ['1', '2'])
         ae(filt(None, {'nosy': ['1','2']}, ('+', 'status'),
-            ('-', 'activity')), ['4', '3'])
+            ('-', 'deadline')), ['4', '3'])
 
     def testFilteringMany(self):
         ae, filt = self.filteringSetup()
@@ -903,20 +904,20 @@ def testFilteringMany(self):
     def testFilteringRange(self):
         ae, filt = self.filteringSetup()
         # Date ranges
-        ae(filt(None, {'deadline': 'from 2003-02-10 to 2003-02-23'}), ['2','3'])
-        ae(filt(None, {'deadline': '2003-02-10; 2003-02-23'}), ['2','3'])
-        ae(filt(None, {'deadline': '; 2003-02-16'}), ['1'])
+        ae(filt(None, {'deadline': 'from 2003-02-10 to 2003-02-23'}), ['1','3'])
+        ae(filt(None, {'deadline': '2003-02-10; 2003-02-23'}), ['1','3'])
+        ae(filt(None, {'deadline': '; 2003-02-16'}), ['2'])
         # Lets assume people won't invent a time machine, otherwise this test
         # may fail :)
-        ae(filt(None, {'deadline': 'from 2003-02-16'}), ['2', '3', '4'])
-        ae(filt(None, {'deadline': '2003-02-16;'}), ['2', '3', '4'])
+        ae(filt(None, {'deadline': 'from 2003-02-16'}), ['1', '3', '4'])
+        ae(filt(None, {'deadline': '2003-02-16;'}), ['1', '3', '4'])
         # year and month granularity
         ae(filt(None, {'deadline': '2002'}), [])
         ae(filt(None, {'deadline': '2003'}), ['1', '2', '3'])
         ae(filt(None, {'deadline': '2004'}), ['4'])
-        ae(filt(None, {'deadline': '2003-02'}), ['2', '3'])
+        ae(filt(None, {'deadline': '2003-02'}), ['1', '3'])
         ae(filt(None, {'deadline': '2003-03'}), [])
-        ae(filt(None, {'deadline': '2003-02-16'}), ['2'])
+        ae(filt(None, {'deadline': '2003-02-16'}), ['1'])
         ae(filt(None, {'deadline': '2003-02-17'}), [])
         # Interval ranges
         ae(filt(None, {'foo': 'from 0:50 to 2:00'}), ['1'])
@@ -945,32 +946,33 @@ def testFilteringMultilinkSort(self):
         ae(filt(None, {}, ('-','nosy'), (None,None)), ['3', '4', '1', '2'])
 
     def testFilteringDateSort(self):
-        # '1': '2003-01-01.00:00'
-        # '2': '2003-02-16.22:50'
+        # '1': '2003-02-16.22:50'
+        # '2': '2003-01-01.00:00'
         # '3': '2003-02-18'
         # '4': '2004-03-08'
         ae, filt = self.filteringSetup()
         # ascending
-        ae(filt(None, {}, ('+','deadline'), (None,None)), ['1', '2', '3', '4'])
+        ae(filt(None, {}, ('+','deadline'), (None,None)), ['2', '1', '3', '4'])
         # descending
-        ae(filt(None, {}, ('-','deadline'), (None,None)), ['4', '3', '2', '1'])
+        ae(filt(None, {}, ('-','deadline'), (None,None)), ['4', '3', '1', '2'])
 
     def testFilteringDateSortPriorityGroup(self):
-        # '1': '2003-01-01.00:00'  1 => 2
-        # '2': '2003-02-16.22:50'  3 => 1
+        # '1': '2003-02-16.22:50'  1 => 2
+        # '2': '2003-01-01.00:00'  3 => 1
         # '3': '2003-02-18'        2 => 3
         # '4': '2004-03-08'        1 => 2
         ae, filt = self.filteringSetup()
+
         # ascending
         ae(filt(None, {}, ('+','deadline'), ('+','priority')),
-            ['2', '1', '4', '3'])
+            ['2', '1', '3', '4'])
         ae(filt(None, {}, ('-','deadline'), ('+','priority')),
-            ['2', '4', '1', '3'])
+            ['1', '2', '4', '3'])
         # descending
         ae(filt(None, {}, ('+','deadline'), ('-','priority')),
-            ['3', '1', '4', '2'])
+            ['3', '4', '2', '1'])
         ae(filt(None, {}, ('-','deadline'), ('-','priority')),
-            ['3', '4', '1', '2'])
+            ['4', '3', '1', '2'])
 
 # XXX add sorting tests for other types
 # XXX test auditors and reactors