Verify contents of tracker module when the tracker is opened

Performance improvements in *dbm and sq backends

New benchmark module. To use:

  PYTHONPATH=. python2 test/benchmark.py

(yes, it's a little basic at present ;)

Author: Richard Jones

CHANGES.txt

@@ -17,7 +17,8 @@ are given with the most recent entry first.
 - we now verify instance attributes on instance open and throw a useful error
   if they're not all there
 - sf 611217 ] menu() has problems when labelprop==None
-
+- verify contents of tracker module when the tracker is opened
+- performance improvements in *dbm and sq backends
 
 2002-09-13 0.5.0 beta2
  . all backends now have a .close() method, and it's used everywhere

TODO.txt

@@ -53,5 +53,6 @@ pending web       allow multilink selections to select a "none" element to allow
 bug     mailgw    some f*ked mailers QUOTE their Re; "Re: "[issue1] bla blah""
 bug     docs      need to mention somewhere how sorting works
 bug     web       :multilink isn't working
+bug     docs      mention not putting spaces in tracker URL aliases
 ======= ========= =============================================================
 

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.81 2002-09-19 02:37:41 richard Exp $
+#$Id: back_anydbm.py,v 1.82 2002-09-20 01:20:31 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -236,6 +236,13 @@ def addnode(self, classname, nodeid, node):
         '''
         if __debug__:
             print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
+
+        # add in the "calculated" properties (dupe so we don't affect
+        # calling code's node assumptions)
+        node = node.copy()
+        node['creator'] = self.journaltag
+        node['creation'] = node['activity'] = date.Date()
+
         self.newnodes.setdefault(classname, {})[nodeid] = 1
         self.cache.setdefault(classname, {})[nodeid] = node
         self.savenode(classname, nodeid, node)
@@ -247,6 +254,11 @@ def setnode(self, classname, nodeid, node):
             print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
         self.dirtynodes.setdefault(classname, {})[nodeid] = 1
 
+        # update the activity time (dupe so we don't affect
+        # calling code's node assumptions)
+        node = node.copy()
+        node['activity'] = date.Date()
+
         # can't set without having already loaded the node
         self.cache[classname][nodeid] = node
         self.savenode(classname, nodeid, node)
@@ -975,7 +987,13 @@ def get(self, nodeid, propname, default=_marker, cache=1):
         if propname == 'id':
             return nodeid
 
+        # get the node's dict
+        d = self.db.getnode(self.classname, nodeid, cache=cache)
+
+        # check for one of the special props
         if propname == 'creation':
+            if d.has_key('creation'):
+                return d['creation']
             if not self.do_journal:
                 raise ValueError, 'Journalling is disabled for this class'
             journal = self.db.getjournal(self.classname, nodeid)
@@ -985,6 +1003,8 @@ def get(self, nodeid, propname, default=_marker, cache=1):
                 # on the strange chance that there's no journal
                 return date.Date()
         if propname == 'activity':
+            if d.has_key('activity'):
+                return d['activity']
             if not self.do_journal:
                 raise ValueError, 'Journalling is disabled for this class'
             journal = self.db.getjournal(self.classname, nodeid)
@@ -994,6 +1014,8 @@ def get(self, nodeid, propname, default=_marker, cache=1):
                 # on the strange chance that there's no journal
                 return date.Date()
         if propname == 'creator':
+            if d.has_key('creator'):
+                return d['creator']
             if not self.do_journal:
                 raise ValueError, 'Journalling is disabled for this class'
             journal = self.db.getjournal(self.classname, nodeid)
@@ -1005,9 +1027,6 @@ def get(self, nodeid, propname, default=_marker, cache=1):
         # get the property (raises KeyErorr if invalid)
         prop = self.properties[propname]
 
-        # get the node's dict
-        d = self.db.getnode(self.classname, nodeid, cache=cache)
-
         if not d.has_key(propname):
             if default is _marker:
                 if isinstance(prop, Multilink):
@@ -1103,7 +1122,11 @@ class or a KeyError is raised.
             # this will raise the KeyError if the property isn't valid
             # ... we don't use getprops() here because we only care about
             # the writeable properties.
-            prop = self.properties[propname]
+            try:
+                prop = self.properties[propname]
+            except KeyError:
+                raise KeyError, '"%s" has no property named "%s"'%(
+                    self.classname, propname)
 
             # if the value's the same as the existing value, no sense in
             # doing anything
@@ -1388,7 +1411,8 @@ def lookup(self, keyvalue):
                     return nodeid
         finally:
             cldb.close()
-        raise KeyError, keyvalue
+        raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
+            keyvalue, self.classname)
 
     # change from spec - allows multiple props to match
     def find(self, **propspec):

roundup/backends/back_gadfly.py

@@ -1,4 +1,4 @@
-# $Id: back_gadfly.py,v 1.23 2002-09-19 02:37:41 richard Exp $
+# $Id: back_gadfly.py,v 1.24 2002-09-20 01:20:31 richard Exp $
 __doc__ = '''
 About Gadfly
 ============
@@ -154,7 +154,7 @@ def filter(self, search_matches, filterspec, sort, group):
                 tn = '%s_%s'%(cn, k)
                 frum.append(tn)
                 if isinstance(v, type([])):
-                    s = ','.join([self.arg for x in v])
+                    s = ','.join([a for x in v])
                     where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
                     args = args + v
                 else:
@@ -176,66 +176,50 @@ def filter(self, search_matches, filterspec, sort, group):
             where.append('id in (%s)'%s)
             args = args + v
 
-        # figure the order by clause
+        # "grouping" is just the first-order sorting in the SQL fetch
+        # can modify it...)
         orderby = []
         ordercols = []
+        if group[0] is not None and group[1] is not None:
+            if group[0] != '-':
+                orderby.append('_'+group[1])
+                ordercols.append('_'+group[1])
+            else:
+                orderby.append('_'+group[1]+' desc')
+                ordercols.append('_'+group[1])
+
+        # now add in the sorting
+        group = ''
         if sort[0] is not None and sort[1] is not None:
             direction, colname = sort
             if direction != '-':
-                if colname == 'activity':
-                    orderby.append('activity')
-                    ordercols.append('max(%s__journal.date) as activity'%cn)
-                    frum.append('%s__journal'%cn)
-                    where.append('%s__journal.nodeid = _%s.id'%(cn, cn))
-                elif colname == 'id':
+                if colname == 'id':
                     orderby.append(colname)
-                    ordercols.append(colname)
                 else:
                     orderby.append('_'+colname)
                     ordercols.append('_'+colname)
             else:
-                if colname == 'activity':
-                    orderby.append('activity desc')
-                    ordercols.append('max(%s__journal.date) as activity'%cn)
-                    frum.append('%s__journal'%cn)
-                    where.append('%s__journal.nodeid = _%s.id'%(cn, cn))
-                elif colname == 'id':
+                if colname == 'id':
                     orderby.append(colname+' desc')
                     ordercols.append(colname)
                 else:
                     orderby.append('_'+colname+' desc')
                     ordercols.append('_'+colname)
 
-        # figure the group by clause
-        groupby = []
-        groupcols = []
-        if group[0] is not None and group[1] is not None:
-            if group[0] != '-':
-                groupby.append('_'+group[1])
-                groupcols.append('_'+group[1])
-            else:
-                groupby.append('_'+group[1]+' desc')
-                groupcols.append('_'+group[1])
-
         # construct the SQL
         frum = ','.join(frum)
-        where = ' and '.join(where)
-        cols = []
+        if where:
+            where = ' where ' + (' and '.join(where))
+        else:
+            where = ''
+        cols = ['id']
         if orderby:
             cols = cols + ordercols
             order = ' order by %s'%(','.join(orderby))
         else:
             order = ''
-        if 0: #groupby:
-            cols = cols + groupcols
-            group = ' group by %s'%(','.join(groupby))
-        else:
-            group = ''
-        if 'id' not in cols:
-            cols.append('id')
         cols = ','.join(cols)
-        sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order,
-            group)
+        sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
         args = tuple(args)
         if __debug__:
             print >>hyperdb.DEBUG, 'filter', (self, sql, args)