The whole filter method was replicated in back_mysql.py from rdbms_common.py

I've factored the reason for the difference, mysqls inability to do
subselects. We now have a _subselect method that is overridden in
back_mysql and most of the filter method stays in rdbms_common. This
passed all tests for mysql, postgresql and sqlite with python2.4 and
python2.3.

I've done this as a prerequisite for doing some surgery on filter.

Author: Ralf Schlatterbeck

roundup/backends/back_mysql.py

@@ -1,4 +1,4 @@
-#$Id: back_mysql.py,v 1.68 2006-04-27 03:40:42 richard Exp $
+#$Id: back_mysql.py,v 1.69 2006-07-07 15:04:28 schlatterbeck Exp $
 #
 # Copyright (c) 2003 Martynas Sklyzmantas, Andrey Lebedev <[email protected]>
 #
@@ -550,275 +550,13 @@ def sql_close(self):
                 raise
 
 class MysqlClass:
-    # we're overriding this method for ONE missing bit of functionality.
-    # look for "I can't believe it's not a toy RDBMS" below
-    def filter(self, search_matches, filterspec, sort=(None,None),
-            group=(None,None)):
-        '''Return a list of the ids of the active nodes in this class that
-        match the 'filter' spec, sorted by the group spec and then the
-        sort spec
-
-        "filterspec" is {propname: value(s)}
-
-        "sort" and "group" are (dir, prop) where dir is '+', '-' or None
-        and prop is a prop name or None
-
-        "search_matches" is {nodeid: marker} or None
-
-        The filter must match all properties specificed. If the property
-        value to match is a list:
-
-        1. String properties must match all elements in the list, and
-        2. Other properties must match any of the elements in the list.
+    def _subselect (self, cn, tn) :
+        ''' "I can't believe it's not a toy RDBMS"
+           see, even toy RDBMSes like gadfly and sqlite can do sub-selects...
         '''
-        # we can't match anything if search_matches is empty
-        if search_matches == {}:
-            return []
-
-        if __debug__:
-            start_t = time.time()
-
-        cn = self.classname
-
-        # vars to hold the components of the SQL statement
-        frum = ['_'+cn] # FROM clauses
-        loj = []        # LEFT OUTER JOIN clauses
-        where = []      # WHERE clauses
-        args = []       # *any* positional arguments
-        a = self.db.arg
-
-        # figure the WHERE clause from the filterspec
-        props = self.getprops()
-        mlfilt = 0      # are we joining with Multilink tables?
-        for k, v in filterspec.items():
-            propclass = props[k]
-            # now do other where clause stuff
-            if isinstance(propclass, Multilink):
-                mlfilt = 1
-                tn = '%s_%s'%(cn, k)
-                if v in ('-1', ['-1']):
-                    # only match rows that have count(linkid)=0 in the
-                    # corresponding multilink table)
-
-                    # "I can't believe it's not a toy RDBMS"
-                    # see, even toy RDBMSes like gadfly and sqlite can do
-                    # sub-selects...
-                    self.db.sql('select nodeid from %s'%tn)
-                    s = ','.join([x[0] for x in self.db.sql_fetchall()])
-
-                    where.append('_%s.id not in (%s)'%(cn, s))
-                elif isinstance(v, type([])):
-                    frum.append(tn)
-                    s = ','.join([a for x in v])
-                    where.append('_%s.id=%s.nodeid and %s.linkid in (%s)'%(cn,
-                        tn, tn, s))
-                    args = args + v
-                else:
-                    frum.append(tn)
-                    where.append('_%s.id=%s.nodeid and %s.linkid=%s'%(cn, tn,
-                        tn, a))
-                    args.append(v)
-            elif k == 'id':
-                if isinstance(v, type([])):
-                    s = ','.join([a for x in v])
-                    where.append('_%s.%s in (%s)'%(cn, k, s))
-                    args = args + v
-                else:
-                    where.append('_%s.%s=%s'%(cn, k, a))
-                    args.append(v)
-            elif isinstance(propclass, String):
-                if not isinstance(v, type([])):
-                    v = [v]
-
-                # Quote the bits in the string that need it and then embed
-                # in a "substring" search. Note - need to quote the '%' so
-                # they make it through the python layer happily
-                v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
-
-                # now add to the where clause
-                where.append('('
-                    +' and '.join(["_%s._%s LIKE '%s'"%(cn, k, s) for s in v])
-                    +')')
-                # note: args are embedded in the query string now
-            elif isinstance(propclass, Link):
-                if isinstance(v, type([])):
-                    d = {}
-                    for entry in v:
-                        if entry == '-1':
-                            entry = None
-                        d[entry] = entry
-                    l = []
-                    if d.has_key(None) or not d:
-                        del d[None]
-                        l.append('_%s._%s is NULL'%(cn, k))
-                    if d:
-                        v = d.keys()
-                        s = ','.join([a for x in v])
-                        l.append('(_%s._%s in (%s))'%(cn, k, s))
-                        args = args + v
-                    if l:
-                        where.append('(' + ' or '.join(l) +')')
-                else:
-                    if v in ('-1', None):
-                        v = None
-                        where.append('_%s._%s is NULL'%(cn, k))
-                    else:
-                        where.append('_%s._%s=%s'%(cn, k, a))
-                        args.append(v)
-            elif isinstance(propclass, Date):
-                dc = self.db.hyperdb_to_sql_value[hyperdb.Date]
-                if isinstance(v, type([])):
-                    s = ','.join([a for x in v])
-                    where.append('_%s._%s in (%s)'%(cn, k, s))
-                    args = args + [dc(date.Date(x)) for x in v]
-                else:
-                    try:
-                        # Try to filter on range of dates
-                        date_rng = propclass.range_from_raw (v, self.db)
-                        if date_rng.from_value:
-                            where.append('_%s._%s >= %s'%(cn, k, a))
-                            args.append(dc(date_rng.from_value))
-                        if date_rng.to_value:
-                            where.append('_%s._%s <= %s'%(cn, k, a))
-                            args.append(dc(date_rng.to_value))
-                    except ValueError:
-                        # If range creation fails - ignore that search parameter
-                        pass
-            elif isinstance(propclass, Interval):
-                # filter using the __<prop>_int__ column
-                if isinstance(v, type([])):
-                    s = ','.join([a for x in v])
-                    where.append('_%s.__%s_int__ in (%s)'%(cn, k, s))
-                    args = args + [date.Interval(x).as_seconds() for x in v]
-                else:
-                    try:
-                        # Try to filter on range of intervals
-                        date_rng = Range(v, date.Interval)
-                        if date_rng.from_value:
-                            where.append('_%s.__%s_int__ >= %s'%(cn, k, a))
-                            args.append(date_rng.from_value.as_seconds())
-                        if date_rng.to_value:
-                            where.append('_%s.__%s_int__ <= %s'%(cn, k, a))
-                            args.append(date_rng.to_value.as_seconds())
-                    except ValueError:
-                        # If range creation fails - ignore that search parameter
-                        pass
-            else:
-                if isinstance(v, type([])):
-                    s = ','.join([a for x in v])
-                    where.append('_%s._%s in (%s)'%(cn, k, s))
-                    args = args + v
-                else:
-                    where.append('_%s._%s=%s'%(cn, k, a))
-                    args.append(v)
-
-        # don't match retired nodes
-        where.append('_%s.__retired__ <> 1'%cn)
-
-        # add results of full text search
-        if search_matches is not None:
-            v = search_matches.keys()
-            s = ','.join([a for x in v])
-            where.append('_%s.id in (%s)'%(cn, s))
-            args = args + v
-
-        # sanity check: sorting *and* grouping on the same property?
-        if group[1] == sort[1]:
-            sort = (None, None)
-
-        # "grouping" is just the first-order sorting in the SQL fetch
-        orderby = []
-        ordercols = []
-        mlsort = []
-        rhsnum = 0
-        for sortby in group, sort:
-            sdir, prop = sortby
-            if sdir and prop:
-                if isinstance(props[prop], Multilink):
-                    mlsort.append(sortby)
-                    continue
-                elif isinstance(props[prop], Interval):
-                    # use the int column for sorting
-                    o = '__'+prop+'_int__'
-                    ordercols.append(o)
-                elif isinstance(props[prop], Link):
-                    # determine whether the linked Class has an order property
-                    lcn = props[prop].classname
-                    link = self.db.classes[lcn]
-                    o = '_%s._%s'%(cn, prop)
-                    op = link.orderprop ()
-                    if op != 'id':
-                        tn = '_' + lcn
-                        rhs = 'rhs%s_'%rhsnum
-                        rhsnum += 1
-                        loj.append('LEFT OUTER JOIN %s as %s on %s=%s.id'%(
-                            tn, rhs, o, rhs))
-                        o = '%s._%s'%(rhs, op)
-                    ordercols.append(o)
-                elif prop == 'id':
-                    o = '_%s.id'%cn
-                else:
-                    o = '_%s._%s'%(cn, prop)
-                    ordercols.append(o)
-                if sdir == '-':
-                    o += ' desc'
-                orderby.append(o)
-
-        # construct the SQL
-        frum = ','.join(frum)
-        if where:
-            where = ' where ' + (' and '.join(where))
-        else:
-            where = ''
-        if mlfilt:
-            # we're joining tables on the id, so we will get dupes if we
-            # don't distinct()
-            cols = ['distinct(_%s.id)'%cn]
-        else:
-            cols = ['_%s.id'%cn]
-        if orderby:
-            cols = cols + ordercols
-            order = ' order by %s'%(','.join(orderby))
-        else:
-            order = ''
-        cols = ','.join(cols)
-        loj = ' '.join(loj)
-        sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
-        args = tuple(args)
-        self.db.sql(sql, args)
-        l = self.db.cursor.fetchall()
-
-        # return the IDs (the first column)
-        # XXX numeric ids
-        l = [str(row[0]) for row in l]
-
-        if not mlsort:
-            if __debug__:
-                self.db.stats['filtering'] += (time.time() - start_t)
-            return l
-
-        # ergh. someone wants to sort by a multilink.
-        r = []
-        for id in l:
-            m = []
-            for ml in mlsort:
-                m.append(self.get(id, ml[1]))
-            r.append((id, m))
-        i = 0
-        for sortby in mlsort:
-            def sortfun(a, b, dir=sortby[i], i=i):
-                if dir == '-':
-                    return cmp(b[1][i], a[1][i])
-                else:
-                    return cmp(a[1][i], b[1][i])
-            r.sort(sortfun)
-            i += 1
-        r = [i[0] for i in r]
-
-        if __debug__:
-            self.db.stats['filtering'] += (time.time() - start_t)
-
-        return r
+        self.db.sql('select nodeid from %s'%tn)
+        s = ','.join([x[0] for x in self.db.sql_fetchall()])
+        return '_%s.id not in (%s)'%(cn, s)
 
 class Class(MysqlClass, rdbms_common.Class):
     pass

roundup/backends/rdbms_common.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
-#$Id: rdbms_common.py,v 1.170 2006-04-27 05:15:17 richard Exp $
+#$Id: rdbms_common.py,v 1.171 2006-07-07 15:04:28 schlatterbeck Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -2014,6 +2014,14 @@ def getnodeids(self, retired=None):
         ids = [str(x[0]) for x in self.db.cursor.fetchall()]
         return ids
 
+    def _subselect (self, cn, tn) :
+        '''Create a subselect. This is factored out because some
+           databases (hmm only one, so far) doesn't support subselects
+           look for "I can't believe it's not a toy RDBMS" in the mysql
+           backend.
+        '''
+        return '_%s.id not in (select nodeid from %s)'%(cn, tn)
+
     def filter(self, search_matches, filterspec, sort=(None,None),
             group=(None,None)):
         '''Return a list of the ids of the active nodes in this class that
@@ -2061,8 +2069,7 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                 if v in ('-1', ['-1']):
                     # only match rows that have count(linkid)=0 in the
                     # corresponding multilink table)
-                    where.append('_%s.id not in (select nodeid from %s)'%(cn,
-                        tn))
+                    where.append(self._subselect(cn, tn))
                 elif isinstance(v, type([])):
                     frum.append(tn)
                     s = ','.join([a for x in v])
@@ -2126,7 +2133,7 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                 if isinstance(v, type([])):
                     s = ','.join([a for x in v])
                     where.append('_%s._%s in (%s)'%(cn, k, s))
-                    args = args + [dc(date.Date(v)) for x in v]
+                    args = args + [dc(date.Date(x)) for x in v]
                 else:
                     try:
                         # Try to filter on range of dates