Allow to define reverse Multilinks

Now it's possible to specify a rev_multilink parameter when creating
Link or Multilink properties. The parameter takes a property name to be
inserted into the linked-to class. It allows to navigate from the other
side of the link as if it where a forward Multilink using the existing
data structures.

Author: schlatterbeck

CHANGES.txt

@@ -74,6 +74,13 @@ Features:
   with new RoundupException (inheriting from Exception) for most
   roundup exceptions. (John Rouillard and Ralf Schlatterbeck on
   request from Robert Klonner.)
+- When defining Link or Multilink properties in the schema, it's now
+  possible to add a parameter rev_multilink that accepts a property name
+  to be inserted into the linked-to class. So this creates a reverse
+  Multilink property in the linked-to class. This Multilink is read-only
+  (cannot be updated) but can be used in filter -- and thus in normal
+  index templates as well as in the REST and XMLRPC APIs. (Ralf
+  Schlatterbeck)
 
 Fixed:
 

doc/customizing.txt

@@ -716,6 +716,14 @@ behaviour:
     (like, e.g., product codes). For these roundup needs to match the
     numeric name and should never match an ID. In this case you can set
     ``try_id_parsing='no'``.
+  - The ``rev_multilink`` option takes a property name to be inserted
+    into the linked-to class. This property is a Multilink property that
+    links back to the current class. The new Multilink is read-only (it
+    is automagically modified if the Link or Multilink property defining
+    it is modified). The new property can be used in normal searches
+    using the "filter" method of the Class. This means it can be used
+    like other Multilink properties when searching (in an index
+    template) or via the REST- and XMLRPC-APIs.
   - The ``msg_header_property`` is used by the mail gateway when sending
     out messages. When a link or multilink property of an issue changes,
     roundup creates email headers of the form::

roundup/backends/back_anydbm.py

@@ -200,6 +200,7 @@ def __init__(self, config, journaltag=None):
     def post_init(self):
         """Called once the schema initialisation has finished.
         """
+        super(Database, self).post_init()
         # reindex the db if necessary
         if self.indexer.should_reindex():
             self.reindex()
@@ -915,6 +916,12 @@ def create_inner(self, **propvalues):
              'creation' in propvalues or 'activity' in propvalues):
             raise KeyError('"creator", "actor", "creation" and '
                 '"activity" are reserved')
+
+        for p in propvalues:
+            prop = self.properties[p]
+            if prop.computed:
+                raise KeyError('"%s" is a computed property'%p)
+
         # new node's id
         newid = self.db.newid(self.classname)
 
@@ -1197,6 +1204,11 @@ def set_inner(self, nodeid, **propvalues):
         if 'id' in propvalues:
             raise KeyError('"id" is reserved')
 
+        for p in propvalues:
+            prop = self.properties[p]
+            if prop.computed:
+                raise KeyError('"%s" is a computed property'%p)
+
         if self.db.journaltag is None:
             raise hyperdb.DatabaseError(_('Database open read-only'))
 
@@ -1556,6 +1568,8 @@ def find(self, **propspec):
 
             db.issue.find(messages='1')
             db.issue.find(messages={'1':1,'3':1}, files={'7':1})
+            db.issue.find(messages=('1','3'), files=('7',))
+            db.issue.find(messages=['1','3'], files=['7'])
         """
         for propname, itemids in propspec.items():
             # check the prop is OK
@@ -1574,7 +1588,10 @@ def find(self, **propspec):
                     continue
                 for propname, itemids in propspec.items():
                     if type(itemids) is not type({}):
-                        itemids = {itemids:1}
+                        if itemids is None or isinstance(itemids, type("")):
+                            itemids = {itemids:1}
+                        else:
+                            itemids = dict.fromkeys(itemids)
 
                     # special case if the item doesn't have this property
                     if propname not in item:
@@ -1744,12 +1761,17 @@ def _filter(self, search_matches, filterspec, proptree,
                         u.append(entry)
                     l.append((LINK, k, u))
                 elif isinstance(propclass, hyperdb.Multilink):
-                    # the value -1 is a special "not set" sentinel
-                    if v in ('-1', ['-1']):
-                        v = []
-                    elif type(v) is not type([]):
-                        v = [v]
-                    l.append((MULTILINK, k, v))
+                    # If it's a reverse multilink, we've already
+                    # computed the ids of our own class.
+                    if propclass.rev_property:
+                        l.append((OTHER, 'id', v))
+                    else:
+                        # the value -1 is a special "not set" sentinel
+                        if v in ('-1', ['-1']):
+                            v = []
+                        elif type(v) is not type([]):
+                            v = [v]
+                        l.append((MULTILINK, k, v))
                 elif isinstance(propclass, hyperdb.String) and k != 'id':
                     if type(v) is not type([]):
                         v = [v]

roundup/backends/back_mysql.py

@@ -605,12 +605,12 @@ def supports_subselects(self):
         # TODO: AFAIK its version dependent for MySQL
         return False
 
-    def _subselect(self, classname, multilink_table):
+    def _subselect(self, classname, multilink_table, nodeid_name):
         ''' "I can't believe it's not a toy RDBMS"
            see, even toy RDBMSes like gadfly and sqlite can do sub-selects...
         '''
-        self.db.sql('select nodeid from %s'%multilink_table)
-        s = ','.join([x[0] for x in self.db.sql_fetchall()])
+        self.db.sql('select %s from %s'%(nodeid_name, multilink_table))
+        s = ','.join([str(x[0]) for x in self.db.sql_fetchall()])
         return '_%s.id not in (%s)'%(classname, s)
 
     def create_inner(self, **propvalues):

roundup/backends/blobfiles.py

@@ -34,7 +34,7 @@ def files_in_dir(dir):
             num_files = num_files + files_in_dir(full_filename)
     return num_files
 
-class FileStorage:
+class FileStorage(object):
     """Store files in some directory structure
 
     Some databases do not permit the storage of arbitrary data (i.e.,

roundup/backends/rdbms_common.py

@@ -293,6 +293,7 @@ def post_init(self):
             We should now confirm that the schema defined by our "classes"
             attribute actually matches the schema in the database.
         """
+        super(Database, self).post_init()
 
         # upgrade the database for column type changes, new internal
         # tables, etc.
@@ -512,6 +513,10 @@ def determine_columns(self, properties):
         mls = []
         # add the multilinks separately
         for col, prop in properties:
+            # Computed props are not in the db
+            if prop.computed:
+                continue
+
             if isinstance(prop, Multilink):
                 mls.append(col)
                 continue
@@ -1128,8 +1133,11 @@ def _materialize_multilink(self, classname, nodeid, node, propname):
         """ evaluation of single Multilink (lazy eval may have skipped this)
         """
         if propname not in node:
-            sql = 'select linkid from %s_%s where nodeid=%s' % (
-                classname, propname, self.arg)
+            prop = self.getclass(classname).properties[propname]
+            tn  = prop.table_name
+            lid = prop.linkid_name
+            nid = prop.nodeid_name
+            sql = 'select %s from %s where %s=%s' % (lid, tn, nid, self.arg)
             self.sql(sql, (nodeid,))
             # extract the first column from the result
             # XXX numeric ids
@@ -1536,7 +1544,8 @@ def schema(self):
         """ A dumpable version of the schema that we can store in the
             database
         """
-        return (self.key, [(x, repr(y)) for x, y in self.properties.items()])
+        return (self.key,
+          [(x, repr(y)) for x, y in self.properties.items() if not y.computed])
 
     def enableJournalling(self):
         """Turn journalling on for this class
@@ -1585,6 +1594,11 @@ def create_inner(self, **propvalues):
             raise KeyError('"creator", "actor", "creation" and '
                            '"activity" are reserved')
 
+        for p in propvalues:
+            prop = self.properties[p]
+            if prop.computed:
+                raise KeyError('"%s" is a computed property'%p)
+
         # new node's id
         newid = self.db.newid(self.classname)
 
@@ -1814,6 +1828,11 @@ def set_inner(self, nodeid, **propvalues):
         if 'id' in propvalues:
             raise KeyError('"id" is reserved')
 
+        for p in propvalues:
+            prop = self.properties[p]
+            if prop.computed:
+                raise KeyError('"%s" is a computed property'%p)
+
         if self.db.journaltag is None:
             raise DatabaseError(_('Database open read-only'))
 
@@ -2312,13 +2331,13 @@ def getnodeids(self, retired=None):
         ids = [str(x[0]) for x in self.db.cursor.fetchall()]
         return ids
 
-    def _subselect(self, classname, multilink_table):
+    def _subselect(self, classname, multilink_table, nodeid_name):
         """Create a subselect. This is factored out because some
            databases (hmm only one, so far) doesn't support subselects
            look for "I can't believe it's not a toy RDBMS" in the mysql
            backend.
         """
-        return '_%s.id not in (select nodeid from %s)'%(classname,
+        return '_%s.id not in (select %s from %s)'%(classname, nodeid_name,
             multilink_table)
 
     # Some DBs order NULL values last. Set this variable in the backend
@@ -2379,7 +2398,8 @@ def _filter_multilink_expression_fallback(
         # we have ids of the classname table
         return ids.where("_%s.id" % classname, self.db.arg)
 
-    def _filter_multilink_expression(self, classname, multilink_table, v):
+    def _filter_multilink_expression(self, classname, multilink_table,
+        linkid_name, nodeid_name, v):
         """ Filters out elements of the classname table that do not
             match the given expression.
             Returns tuple of 'WHERE' introns for the overall filter.
@@ -2397,9 +2417,8 @@ def _filter_multilink_expression(self, classname, multilink_table, v):
                     classname, multilink_table, expr)
 
             atom = \
-                "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
-                self.db.arg,
-                multilink_table)
+                "%s IN(SELECT %s FROM %s WHERE %s=a.id)" % (
+                self.db.arg, linkid_name, multilink_table, nodeid_name)
 
             intron = \
                 "_%(classname)s.id in (SELECT id " \
@@ -2414,8 +2433,8 @@ def collect_values(n): values.append(n.x)
             return intron, values
         except:
             # original behavior
-            where = "%s.linkid in (%s)" % (
-                multilink_table, ','.join([self.db.arg] * len(v)))
+            where = "%s.%s in (%s)" % (
+                multilink_table, linkid_name, ','.join([self.db.arg] * len(v)))
             return where, v, True # True to indicate original
 
     def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0,
@@ -2479,34 +2498,36 @@ def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0,
             if isinstance(propclass, Multilink):
                 if 'search' in p.need_for:
                     mlfilt = 1
-                    tn = '%s_%s'%(pcn, k)
+                    tn = propclass.table_name
+                    nid = propclass.nodeid_name
+                    lid = propclass.linkid_name
                     if v in ('-1', ['-1'], []):
                         # only match rows that have count(linkid)=0 in the
                         # corresponding multilink table)
-                        where.append(self._subselect(pcn, tn))
+                        where.append(self._subselect(pcn, tn, nid))
                     else:
                         frum.append(tn)
                         gen_join = True
 
                         if p.has_values and isinstance(v, type([])):
                             result = self._filter_multilink_expression(pln,
-                                tn, v)
+                                tn, lid, nid, v)
                             # XXX: We dont need an id join if we used the filter
                             gen_join = len(result) == 3
 
                         if gen_join:
-                            where.append('_%s.id=%s.nodeid'%(pln,tn))
+                            where.append('_%s.id=%s.%s'%(pln, tn, nid))
 
                         if p.children:
                             frum.append('_%s as _%s' % (cn, ln))
-                            where.append('%s.linkid=_%s.id'%(tn, ln))
+                            where.append('%s.%s=_%s.id'%(tn, lid, ln))
 
                         if p.has_values:
                             if isinstance(v, type([])):
                                 where.append(result[0])
                                 args += result[1]
                             else:
-                                where.append('%s.linkid=%s'%(tn, a))
+                                where.append('%s.%s=%s'%(tn, lid, a))
                                 args.append(v)
                 if 'sort' in p.need_for:
                     assert not p.attr_sort_done and not p.sort_ids_needed