Make rev multilink expressions work for anydbm

Author: schlatterbeck

roundup/backends/back_anydbm.py

@@ -28,6 +28,7 @@
 from roundup.anypy.strings import b2s, bs2b, repr_export, eval_import, is_us
 
 from roundup import hyperdb, date, password, roundupdb, security, support
+from roundup.mlink_expr import Expression
 from roundup.backends import locking
 from roundup.i18n import _
 
@@ -48,101 +49,6 @@ def db_exists(config):
 def db_nuke(config):
     shutil.rmtree(config.DATABASE)
 
-class Binary:
-
-    def __init__(self, x, y):
-        self.x = x
-        self.y = y
-
-    def visit(self, visitor):
-        self.x.visit(visitor)
-        self.y.visit(visitor)
-
-class Unary:
-
-    def __init__(self, x):
-        self.x = x
-
-    def generate(self, atom):
-        return atom(self)
-
-    def visit(self, visitor):
-        self.x.visit(visitor)
-
-class Equals(Unary):
-
-    def evaluate(self, v):
-        return self.x in v
-
-    def visit(self, visitor):
-        visitor(self)
-
-class Empty(Unary):
-
-    def evaluate(self, v):
-        return not v
-
-    def visit(self, visitor):
-        visitor(self)
-
-class Not(Unary):
-
-    def evaluate(self, v):
-        return not self.x.evaluate(v)
-
-    def generate(self, atom):
-        return "NOT(%s)" % self.x.generate(atom)
-
-class Or(Binary):
-
-    def evaluate(self, v):
-        return self.x.evaluate(v) or self.y.evaluate(v)
-
-    def generate(self, atom):
-        return "(%s)OR(%s)" % (
-            self.x.generate(atom),
-            self.y.generate(atom))
-
-class And(Binary):
-
-    def evaluate(self, v):
-        return self.x.evaluate(v) and self.y.evaluate(v)
-
-    def generate(self, atom):
-        return "(%s)AND(%s)" % (
-            self.x.generate(atom),
-            self.y.generate(atom))
-
-def compile_expression(opcodes):
-
-    stack = []
-    push, pop = stack.append, stack.pop
-    for opcode in opcodes:
-        if   opcode == -1: push(Empty(opcode))
-        elif opcode == -2: push(Not(pop()))
-        elif opcode == -3: push(And(pop(), pop()))
-        elif opcode == -4: push(Or(pop(), pop()))
-        else:              push(Equals(opcode))
-
-    return pop()
-
-class Expression:
-
-    def __init__(self, v):
-        try:
-            opcodes = [int(x) for x in v]
-            if min(opcodes) >= -1:
-                raise ValueError()
-
-            compiled = compile_expression(opcodes)
-            self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
-        except:
-            if '-1' in v:
-                v = [x for x in v if int(x) > 0]
-                self.evaluate = lambda x: bool(set(x) & set(v)) or not x
-            else:
-                self.evaluate = lambda x: bool(set(x) & set(v))
-
 #
 # Now the database
 #

roundup/backends/rdbms_common.py

@@ -66,7 +66,7 @@
 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
 from roundup.date import Range
 
-from roundup.backends.back_anydbm import compile_expression
+from roundup.mlink_expr import compile_expression
 from roundup.anypy.strings import b2s, bs2b, us2s, repr_export, eval_import
 
 from hashlib import md5

roundup/hyperdb.py

@@ -28,6 +28,7 @@
 # roundup modules
 from . import date, password
 from .support import ensureParentsExist, PrioList
+from roundup.mlink_expr import Expression
 from roundup.i18n import _
 from roundup.cgi.exceptions import DetectorError
 from roundup.anypy.cmp_ import NoneAndDictComparable
@@ -590,24 +591,50 @@ def search(self, search_matches=None, sort=True, retired=False):
                     cl = p.propclass.rev_property.cls
                     if not isinstance(p.val, type([])):
                         p.val = [p.val]
-                    if p.val == ['-1'] :
-                        s1 = set(self.cls.getnodeids(retired=False))
-                        s2 = set()
+                    nval = [int(i) for i in p.val]
+                    pval = [str(i) for i in nval if i >= 0]
+                    items = set()
+                    if not nval or min(nval) >= -1:
+                        if -1 in nval:
+                            s1 = set(self.cls.getnodeids(retired=False))
+                            s2 = set()
+                            for id in cl.getnodeids(retired=False):
+                                node = cl.getnode(id)
+                                if node[pn]:
+                                    if isinstance(node[pn], type([])):
+                                        s2.update(node[pn])
+                                    else:
+                                        s2.add(node[pn])
+                            items |= s1.difference(s2)
+                        if isinstance(p.propclass.rev_property, Link):
+                            items |= set(cl.get(x, pn) for x in pval
+                                if not cl.is_retired(x))
+                        else:
+                            items |= set().union(*(cl.get(x, pn) for x in pval
+                                if not cl.is_retired(x)))
+                    else:
+                        # Expression: materialize rev multilinks and run
+                        # expression on them
+                        expr = Expression(nval)
+                        by_id = {}
                         for id in cl.getnodeids(retired=False):
+                            by_id[id] = set()
+                        items = set()
+                        for id in self.cls.getnodeids(retired=False):
                             node = cl.getnode(id)
                             if node[pn]:
-                                if isinstance(node [pn], type([])):
-                                    s2.update(node [pn])
-                                else:
-                                    s2.add(node [pn])
-                        items = s1.difference(s2)
-                    elif isinstance(p.propclass.rev_property, Link):
-                        items = set(cl.get(x, pn) for x in p.val
-                            if not cl.is_retired(x))
-                    else:
-                        items = set().union(*(cl.get(x, pn) for x in p.val
-                            if not cl.is_retired(x)))
-                    filterspec[p.name] = list(sorted(items))
+                                v = node[pn]
+                                if not isinstance(v, type([])):
+                                    v = [v]
+                                for x in v:
+                                    if x not in by_id:
+                                        by_id[x] = set()
+                                    by_id[x].add(id)
+                        for k in by_id:
+                            if expr.evaluate(by_id[k]):
+                                items.add(k)
+
+                    filterspec[p.name] = list(sorted(items, key=int))
                 elif isinstance(p.val, type([])):
                     exact = []
                     subst = []

roundup/mlink_expr.py

@@ -0,0 +1,103 @@
+#
+# Copyright: 2010 Intevation GmbH.
+#            2021 Ralf Schlatterbeck, [email protected].
+# 
+
+# This module is Free Software under the Roundup licensing,
+# see the COPYING.txt file coming with Roundup.
+#
+
+class Binary:
+
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+    def visit(self, visitor):
+        self.x.visit(visitor)
+        self.y.visit(visitor)
+
+class Unary:
+
+    def __init__(self, x):
+        self.x = x
+
+    def generate(self, atom):
+        return atom(self)
+
+    def visit(self, visitor):
+        self.x.visit(visitor)
+
+class Equals(Unary):
+
+    def evaluate(self, v):
+        return self.x in v
+
+    def visit(self, visitor):
+        visitor(self)
+
+class Empty(Unary):
+
+    def evaluate(self, v):
+        return not v
+
+    def visit(self, visitor):
+        visitor(self)
+
+class Not(Unary):
+
+    def evaluate(self, v):
+        return not self.x.evaluate(v)
+
+    def generate(self, atom):
+        return "NOT(%s)" % self.x.generate(atom)
+
+class Or(Binary):
+
+    def evaluate(self, v):
+        return self.x.evaluate(v) or self.y.evaluate(v)
+
+    def generate(self, atom):
+        return "(%s)OR(%s)" % (
+            self.x.generate(atom),
+            self.y.generate(atom))
+
+class And(Binary):
+
+    def evaluate(self, v):
+        return self.x.evaluate(v) and self.y.evaluate(v)
+
+    def generate(self, atom):
+        return "(%s)AND(%s)" % (
+            self.x.generate(atom),
+            self.y.generate(atom))
+
+def compile_expression(opcodes):
+
+    stack = []
+    push, pop = stack.append, stack.pop
+    for opcode in opcodes:
+        if   opcode == -1: push(Empty(opcode))
+        elif opcode == -2: push(Not(pop()))
+        elif opcode == -3: push(And(pop(), pop()))
+        elif opcode == -4: push(Or(pop(), pop()))
+        else:              push(Equals(opcode))
+
+    return pop()
+
+class Expression:
+
+    def __init__(self, v):
+        try:
+            opcodes = [int(x) for x in v]
+            if min(opcodes) >= -1:
+                raise ValueError()
+
+            compiled = compile_expression(opcodes)
+            self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
+        except:
+            if '-1' in v:
+                v = [x for x in v if int(x) > 0]
+                self.evaluate = lambda x: bool(set(x) & set(v)) or not x
+            else:
+                self.evaluate = lambda x: bool(set(x) & set(v))

test/db_test_base.py

@@ -2038,8 +2038,6 @@ def ls(x):
         self.assertEqual(ls(self.db.user.get('4', ni)), ['1'])
         self.assertEqual(ls(self.db.user.get('5', ni)), ['7'])
 
-    # Currently only works for sql databases
-    @pytest.mark.xfail
     def testFilteringRevMultilinkExpression(self):
         ae, iiter = self.filteringSetupTransitiveSearch('user')
         ni = 'nosy_issues'