Add a 'retired' parameter to Class.filter

.. to allow searching for retired, non-retired or all (retired and
non-retired) items similar to the argument of the same name to
Class.getnodeids. This is 'False' by default (finding only non-retired
items for backwards compatibility) and can be set to None (for finding
retired and non-retired items) or True (for finding only retired items).

Author: schlatterbeck

CHANGES.txt

@@ -222,6 +222,12 @@ Features:
   bounced back to the sender with an error. Enable by configuring the
   convert_htmltotext option in your upgraded config.ini. (Initial
   patch by Igor Ippolitov merged with changes by John Rouillard.)
+- Add a 'retired' parameter to Class.filter to allow searching for
+  retired, non-retired or all (retired and non-retired) items similar
+  to the argument of the same name to Class.getnodeids. This is 'False'
+  by default (finding only non-retired items for backwards
+  compatibility) and can be set to None (for finding retired and
+  non-retired items) or True (for finding only retired items).
 
 Fixed:
 

roundup/backends/back_anydbm.py

@@ -1683,8 +1683,8 @@ def getnodeids(self, db=None, retired=None):
         return res
 
     def _filter(self, search_matches, filterspec, proptree,
-            num_re = re.compile('^\d+$')):
-        """Return a list of the ids of the active nodes in this class that
+            num_re = re.compile('^\d+$'), retired=False):
+        """Return a list of the ids of the nodes in this class that
         match the 'filter' spec, sorted by the group spec and then the
         sort spec.
 
@@ -1695,6 +1695,9 @@ def _filter(self, search_matches, filterspec, proptree,
 
         "search_matches" is a sequence type or None
 
+        "retired" specifies if we should find only non-retired nodes
+        (default) or only retired node (value True) or all nodes.
+
         The filter must match all properties specificed. If the property
         value to match is a list:
 
@@ -1795,16 +1798,14 @@ def _filter(self, search_matches, filterspec, proptree,
 
         filterspec = l
 
-        # now, find all the nodes that are active and pass filtering
+        # now, find all the nodes that pass filtering
         matches = []
         cldb = self.db.getclassdb(cn)
         t = 0
         try:
             # TODO: only full-scan once (use items())
-            for nodeid in self.getnodeids(cldb):
+            for nodeid in self.getnodeids(cldb, retired=retired):
                 node = self.db.getnode(cn, nodeid, cldb)
-                if self.db.RETIRED_FLAG in node:
-                    continue
                 # apply filter
                 for t, k, v in filterspec:
                     # handle the id prop

roundup/backends/back_sqlite.py

@@ -439,12 +439,12 @@ def load_journal(self, classname, cols, nodeid):
 
 class sqliteClass:
     def filter(self, search_matches, filterspec, sort=(None,None),
-            group=(None,None)):
+            group=(None,None), retired=False):
         """ If there's NO matches to a fetch, sqlite returns NULL
             instead of nothing
         """
         return [f for f in rdbms_common.Class.filter(self, search_matches,
-            filterspec, sort=sort, group=group) if f]
+            filterspec, sort=sort, group=group, retired=retired) if f]
 
 class Class(sqliteClass, rdbms_common.Class):
     pass

roundup/backends/rdbms_common.py

@@ -2381,7 +2381,8 @@ def collect_values(n): values.append(n.x)
                 multilink_table, ','.join([self.db.arg] * len(v)))
             return where, v, True # True to indicate original
 
-    def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
+    def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0,
+                     retired=False):
         """ Compute the proptree and the SQL/ARGS for a filter.
         For argument description see filter below.
         We return a 3-tuple, the proptree, the sql and the sql-args
@@ -2650,7 +2651,11 @@ def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
         props = self.getprops()
 
         # don't match retired nodes
-        where.append('_%s.__retired__=0'%icn)
+        if retired is not None:
+            op = '='
+            if retired:
+                op = '!='
+            where.append('_%s.__retired__%s0'%(icn, op))
 
         # add results of full text search
         if search_matches is not None:
@@ -2688,7 +2693,8 @@ def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
         __traceback_info__ = (sql, args)
         return proptree, sql, args
 
-    def filter(self, search_matches, filterspec, sort=[], group=[]):
+    def filter(self, search_matches, filterspec, sort=[], group=[],
+               retired=False):
         """Return a list of the ids of the active nodes in this class that
         match the 'filter' spec, sorted by the group spec and then the
         sort spec
@@ -2711,7 +2717,8 @@ def filter(self, search_matches, filterspec, sort=[], group=[]):
         if __debug__:
             start_t = time.time()
 
-        sq = self._filter_sql (search_matches, filterspec, sort, group)
+        sq = self._filter_sql (search_matches, filterspec, sort, group,
+                               retired=retired)
         # nothing to match?
         if sq is None:
             return []
@@ -2734,7 +2741,8 @@ def filter(self, search_matches, filterspec, sort=[], group=[]):
             self.db.stats['filtering'] += (time.time() - start_t)
         return l
 
-    def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
+    def filter_iter(self, search_matches, filterspec, sort=[], group=[],
+                    retired=False):
         """Iterator similar to filter above with same args.
         Limitation: We don't sort on multilinks.
         This uses an optimisation: We put all nodes that are in the
@@ -2743,7 +2751,8 @@ def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
         a join) from the database because the nodes are already in the
         cache. We're using our own temporary cursor.
         """
-        sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
+        sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1,
+                              retired=retired)
         # nothing to match?
         if sq is None:
             return

roundup/hyperdb.py

@@ -452,7 +452,7 @@ def ancestors(self):
             yield p
             p = p.parent
 
-    def search(self, search_matches=None, sort=True):
+    def search(self, search_matches=None, sort=True, retired=False):
         """ Recursively search for the given properties in a proptree.
         Once all properties are non-transitive, the search generates a
         simple _filter call which does the real work
@@ -463,7 +463,8 @@ def search(self, search_matches=None, sort=True):
                 if p.children:
                     p.search(sort = False)
                 filterspec[p.name] = p.val
-        self.val = self.cls._filter(search_matches, filterspec, sort and self)
+        self.val = self.cls._filter(search_matches, filterspec, sort and self,
+                                    retired=retired)
         return self.val
 
     def sort (self, ids=None):
@@ -1255,7 +1256,7 @@ def find(self, **propspec):
         raise NotImplementedError
 
     def _filter(self, search_matches, filterspec, sort=(None,None),
-            group=(None,None)):
+            group=(None,None), retired=False):
         """For some backends this implements the non-transitive
         search, for more information see the filter method.
         """
@@ -1346,7 +1347,8 @@ def _sortattr(self, sort=[], group=[]):
             sortattr.append(('+', 'id'))
         return sortattr
 
-    def filter(self, search_matches, filterspec, sort=[], group=[]):
+    def filter(self, search_matches, filterspec, sort=[], group=[],
+               retired=False):
         """Return a list of the ids of the active nodes in this class that
         match the 'filter' spec, sorted by the group spec and then the
         sort spec.
@@ -1382,7 +1384,7 @@ def filter(self, search_matches, filterspec, sort=[], group=[]):
         """
         sortattr = self._sortattr(sort = sort, group = group)
         proptree = self._proptree(filterspec, sortattr)
-        proptree.search(search_matches)
+        proptree.search(search_matches, retired=retired)
         return proptree.sort()
 
     # non-optimized filter_iter, a backend may chose to implement a

test/db_test_base.py

@@ -2166,6 +2166,27 @@ def testFilteringSortId(self):
             ae(filt(None, {}, ('+','id')),
                 ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'])
 
+    def testFilteringRetiredString(self):
+        ae, filter, filter_iter = self.filteringSetup()
+        self.db.issue.retire('1')
+        self.db.commit()
+        r = { None: (['1'], ['1'], ['1'], ['1', '2', '3'], [])
+            , True: (['1'], ['1'], ['1'], ['1'], [])
+            , False: ([], [], [], ['2', '3'], [])
+            }
+        for filt in filter, filter_iter:
+            for retire in True, False, None:
+                ae(filt(None, {'title': ['one']}, ('+','id'),
+                   retired=retire), r[retire][0])
+                ae(filt(None, {'title': ['issue one']}, ('+','id'),
+                   retired=retire), r[retire][1])
+                ae(filt(None, {'title': ['issue', 'one']}, ('+','id'),
+                   retired=retire), r[retire][2])
+                ae(filt(None, {'title': ['issue']}, ('+','id'),
+                   retired=retire), r[retire][3])
+                ae(filt(None, {'title': ['one', 'two']}, ('+','id'),
+                   retired=retire), r[retire][4])
+
 # XXX add sorting tests for other types
 
     # nuke and re-create db for restore