Implement exact string search

.. in the 'filter' method of hyperdb.Class (and the corresponding
backend implementations).

Author: schlatterbeck

CHANGES.txt

@@ -85,6 +85,7 @@ Features:
 - issue2551043: Add X-Roundup-issue-id email header. Add a new header
   to make it easier to filter notification emails without having to
   parse the subject line. (John Rouillard)
+- The database filter function now can also do an exact string search.
 
 Fixed:
 

doc/design.txt

@@ -515,26 +515,45 @@ Here is the interface provided by the hyperdatabase::
                 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
             """
 
-        def filter(self, search_matches, filterspec, sort, group):
+        def filter(self, search_matches, filterspec, sort, group,
+                   retired, exact_match_spec):
             """Return a list of the ids of the active nodes in this class that
             match the 'filter' spec, sorted by the group spec and then the
-            sort spec.
+            sort spec. The arguments sort, group, retired, and
+            exact_match_spec are optional.
 
-            "search_matches" is a container type
+            "search_matches" is a container type which by default is
+            None and optionally contains IDs of items to match. If
+            non-empty only IDs of the initial set are returned.
 
             "filterspec" is {propname: value(s)}
+            "exact_match_spec" is the same format as "filterspec" but
+            specifies exact match for the given propnames. This only
+            makes a difference for String properties, these specify case
+            insensitive substring search when in "filterspec" and exact
+            match when in exact_match_spec.
 
             "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
             or None and prop is a prop name or None. Note that for
             backward-compatibility reasons a single (dir, prop) tuple is
             also allowed.
 
+            The parameter retired when set to False, returns only live
+            (un-retired) results. When setting it to True, only retired
+            items are returned. If None, both retired and unretired
+            items are returned. The default is False, i.e. only live
+            items are returned by default.
+
             The filter must match all properties specificed. If the property
             value to match is a list:
 
             1. String properties must match all elements in the list, and
             2. Other properties must match any of the elements in the list.
 
+            This also means that for strings in exact_match_spec it
+            doesn't make sense to specify multiple values because those
+            cannot all be matched.
+
             The propname in filterspec and prop in a sort/group spec may be
             transitive, i.e., it may contain properties of the form
             link.link.link.name, e.g. you can search for all issues where

roundup/backends/back_anydbm.py

@@ -25,7 +25,7 @@
 import os, marshal, re, weakref, string, copy, time, shutil, logging
 
 from roundup.anypy.dbm_ import anydbm, whichdb
-from roundup.anypy.strings import b2s, bs2b, repr_export, eval_import
+from roundup.anypy.strings import b2s, bs2b, repr_export, eval_import, is_us
 
 from roundup import hyperdb, date, password, roundupdb, security, support
 from roundup.backends import locking
@@ -1690,12 +1690,16 @@ def getnodeids(self, db=None, retired=None):
         return res
 
     def _filter(self, search_matches, filterspec, proptree,
-            num_re = re.compile(r'^\d+$'), retired=False):
+            num_re = re.compile(r'^\d+$'), retired=False,
+            exact_match_spec={}):
         """Return a list of the ids of the nodes in this class that
         match the 'filter' spec, sorted by the group spec and then the
         sort spec.
 
         "filterspec" is {propname: value(s)}
+        same for "exact_match_spec". The latter specifies exact matching
+        for String type while String types in "filterspec" are searched
+        for as case insensitive substring match.
 
         "sort" and "group" are (dir, prop) where dir is '+', '-' or None
         and prop is a prop name or None
@@ -1726,82 +1730,86 @@ def _filter(self, search_matches, filterspec, proptree,
         INTERVAL = 'spec:interval'
         OTHER = 'spec:other'
 
-        for k, v in filterspec.items():
-            propclass = props[k]
-            if isinstance(propclass, hyperdb.Link):
-                if type(v) is not type([]):
-                    v = [v]
-                u = []
-                for entry in v:
+        for exact, filtertype in enumerate((filterspec, exact_match_spec)):
+            for k, v in filtertype.items():
+                propclass = props[k]
+                if isinstance(propclass, hyperdb.Link):
+                    if type(v) is not type([]):
+                        v = [v]
+                    u = []
+                    for entry in v:
+                        # the value -1 is a special "not set" sentinel
+                        if entry == '-1':
+                            entry = None
+                        u.append(entry)
+                    l.append((LINK, k, u))
+                elif isinstance(propclass, hyperdb.Multilink):
                     # the value -1 is a special "not set" sentinel
-                    if entry == '-1':
-                        entry = None
-                    u.append(entry)
-                l.append((LINK, k, u))
-            elif isinstance(propclass, hyperdb.Multilink):
-                # the value -1 is a special "not set" sentinel
-                if v in ('-1', ['-1']):
-                    v = []
-                elif type(v) is not type([]):
-                    v = [v]
-                l.append((MULTILINK, k, v))
-            elif isinstance(propclass, hyperdb.String) and k != 'id':
-                if type(v) is not type([]):
-                    v = [v]
-                for v in v:
-                    # simple glob searching
-                    v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
-                    v = v.replace('?', '.')
-                    v = v.replace('*', '.*?')
-                    l.append((STRING, k, re.compile(v, re.I)))
-            elif isinstance(propclass, hyperdb.Date):
-                try:
-                    date_rng = propclass.range_from_raw(v, self.db)
-                    l.append((DATE, k, date_rng))
-                except ValueError:
-                    # If range creation fails - ignore that search parameter
-                    pass
-            elif isinstance(propclass, hyperdb.Interval):
-                try:
-                    intv_rng = date.Range(v, date.Interval)
-                    l.append((INTERVAL, k, intv_rng))
-                except ValueError:
-                    # If range creation fails - ignore that search parameter
-                    pass
-
-            elif isinstance(propclass, hyperdb.Boolean):
-                if type(v) == type(""):
-                    v = v.split(',')
-                if type(v) != type([]):
-                    v = [v]
-                bv = []
-                for val in v:
-                    if type(val) is type(''):
-                        bv.append(propclass.from_raw (val))
-                    else:
-                        bv.append(val)
-                l.append((OTHER, k, bv))
-
-            elif k == 'id':
-                if type(v) != type([]):
-                    v = v.split(',')
-                l.append((OTHER, k, [str(int(val)) for val in v]))
-
-            elif isinstance(propclass, hyperdb.Number):
-                if type(v) != type([]):
-                    try :
+                    if v in ('-1', ['-1']):
+                        v = []
+                    elif type(v) is not type([]):
+                        v = [v]
+                    l.append((MULTILINK, k, v))
+                elif isinstance(propclass, hyperdb.String) and k != 'id':
+                    if type(v) is not type([]):
+                        v = [v]
+                    for x in v:
+                        if exact:
+                            l.append((STRING, k, x))
+                        else:
+                            # simple glob searching
+                            x = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', x)
+                            x = x.replace('?', '.')
+                            x = x.replace('*', '.*?')
+                            l.append((STRING, k, re.compile(x, re.I)))
+                elif isinstance(propclass, hyperdb.Date):
+                    try:
+                        date_rng = propclass.range_from_raw(v, self.db)
+                        l.append((DATE, k, date_rng))
+                    except ValueError:
+                        # If range creation fails - ignore that search parameter
+                        pass
+                elif isinstance(propclass, hyperdb.Interval):
+                    try:
+                        intv_rng = date.Range(v, date.Interval)
+                        l.append((INTERVAL, k, intv_rng))
+                    except ValueError:
+                        # If range creation fails - ignore that search parameter
+                        pass
+
+                elif isinstance(propclass, hyperdb.Boolean):
+                    if type(v) == type(""):
                         v = v.split(',')
-                    except AttributeError :
+                    if type(v) != type([]):
                         v = [v]
-                l.append((OTHER, k, [float(val) for val in v]))
+                    bv = []
+                    for val in v:
+                        if type(val) is type(''):
+                            bv.append(propclass.from_raw (val))
+                        else:
+                            bv.append(val)
+                    l.append((OTHER, k, bv))
 
-            elif isinstance(propclass, hyperdb.Integer):
-                if type(v) != type([]):
-                    try :
+                elif k == 'id':
+                    if type(v) != type([]):
                         v = v.split(',')
-                    except AttributeError :
-                        v = [v]
-                l.append((OTHER, k, [int(val) for val in v]))
+                    l.append((OTHER, k, [str(int(val)) for val in v]))
+
+                elif isinstance(propclass, hyperdb.Number):
+                    if type(v) != type([]):
+                        try :
+                            v = v.split(',')
+                        except AttributeError :
+                            v = [v]
+                    l.append((OTHER, k, [float(val) for val in v]))
+
+                elif isinstance(propclass, hyperdb.Integer):
+                    if type(v) != type([]):
+                        try :
+                            v = v.split(',')
+                        except AttributeError :
+                            v = [v]
+                    l.append((OTHER, k, [int(val) for val in v]))
 
         filterspec = l
 
@@ -1848,8 +1856,12 @@ def _filter(self, search_matches, filterspec, proptree,
                     elif t == STRING:
                         if nv is None:
                             nv = ''
-                        # RE search
-                        match = v.search(nv)
+                        if is_us(v):
+                            # Exact match
+                            match = (nv == v)
+                        else:
+                            # RE search
+                            match = v.search(nv)
                     elif t == DATE or t == INTERVAL:
                         if nv is None:
                             match = v is None

roundup/backends/back_mysql.py

@@ -592,6 +592,7 @@ def sql_close(self):
                 raise
 
 class MysqlClass:
+    case_sensitive_equal = 'COLLATE utf8_bin ='
 
     def supports_subselects(self):
         # TODO: AFAIK its version dependent for MySQL

roundup/backends/back_sqlite.py

@@ -455,12 +455,13 @@ def load_journal(self, classname, cols, nodeid):
 
 class sqliteClass:
     def filter(self, search_matches, filterspec, sort=(None,None),
-            group=(None,None), retired=False):
+            group=(None,None), retired=False, exact_match_spec={}):
         """ If there's NO matches to a fetch, sqlite returns NULL
             instead of nothing
         """
         return [f for f in rdbms_common.Class.filter(self, search_matches,
-            filterspec, sort=sort, group=group, retired=retired) if f]
+            filterspec, sort=sort, group=group, retired=retired,
+            exact_match_spec=exact_match_spec) if f]
 
 class Class(sqliteClass, rdbms_common.Class):
     pass