Fix SQL wildcards in string search

Fix String search with special SQL wildcard characters in LIKE/ILIKE
clause and add testcase.

Author: schlatterbeck

CHANGES.txt

@@ -41,6 +41,8 @@ Fixed:
   (Thomas Arendsen Hein)
 - issue2550822: Fix showing more than one additional property in class menu
   (James Mack)
+- Fix String search with special SQL wildcard characters in LIKE/ILIKE
+  clause and add testcase (Ralf Schlatterbeck)
 
 
 2013-07-06: 1.5.0

roundup/backends/rdbms_common.py

@@ -240,10 +240,16 @@ def sql_fetchiter(self):
             if not row: break
             yield row
 
-    def sql_stringquote(self, value):
-        """ Quote the string so it's safe to put in the 'sql quotes'
+    def search_stringquote(self, value):
+        """ Quote a search string to escape magic search characters
+            '%' and '_', also need to quote '\' (first)
+            Then put '%' around resulting string for LIKE (or ILIKE) operator
         """
-        return re.sub("'", "''", str(value))
+        v = value.replace('\\', '\\\\')
+        v = v.replace('%', '\\%')
+        v = v.replace('_', '\\_')
+        return '%' + v + '%'
+
 
     def init_dbschema(self):
         self.database_schema = {
@@ -1463,6 +1469,10 @@ class Class(hyperdb.Class):
         All methods except __repr__ and getnode must be implemented by a
         concrete backend Class.
     """
+    # For many databases the LIKE operator ignores case.
+    # Postgres and Oracle have an ILIKE operator to support this.
+    # We define the default here, can be changed in derivative class
+    case_insensitive_like = 'LIKE'
 
     def schema(self):
         """ A dumpable version of the schema that we can store in the
@@ -2433,23 +2443,24 @@ def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
                     if not isinstance(v, type([])):
                         v = [v]
 
-                    # Quote the bits in the string that need it and then embed
-                    # in a "substring" search. Note - need to quote the '%' so
-                    # they make it through the python layer happily
-                    v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
+                    # Quote special search characters '%' and '_' for
+                    # correct matching with LIKE/ILIKE
+                    # Note that we now pass the elements of v as query
+                    # arguments and don't interpolate the quoted string
+                    # into the sql statement. Should be safer.
+                    v = [self.db.search_stringquote(s) for s in v]
 
                     # now add to the where clause
                     where.append('('
-                        +' and '.join(["_%s._%s %s '%s'"%(
+                        +' and '.join(["_%s._%s %s %s ESCAPE %s"%(
                                     pln,
                                     k,
-                                    # For many databases the LIKE operator
-                                    # ignores case.  Postgres and Oracle have
-                                    # an ILIKE operator to support this.
-                                    getattr(self,'case_insensitive_like','LIKE'),
-                                    s) for s in v])
+                                    self.case_insensitive_like,
+                                    a,
+                                    a) for s in v])
                         +')')
-                    # note: args are embedded in the query string now
+                    for vv in v:
+                        args.extend((vv, '\\'))
                 if 'sort' in p.need_for:
                     oc = ac = 'lower(_%s._%s)'%(pln, k)
             elif isinstance(propclass, Link):

test/db_test_base.py

@@ -1252,6 +1252,22 @@ def testFilteringStringCase(self):
             ae(filt(None, {'title': ['One', 'Two']}, ('+','id'), (None,None)),
                 [])
 
+    def testFilteringSpecialChars(self):
+        """ Special characters in SQL search are '%' and '_', some used
+            to lead to a traceback.
+        """
+        ae, filter, filter_iter = self.filteringSetup()
+        self.db.issue.set('1', title="With % symbol")
+        self.db.issue.set('2', title="With _ symbol")
+        self.db.issue.set('3', title="With \\ symbol")
+        self.db.issue.set('4', title="With ' symbol")
+        d = dict (status = '1')
+        for filt in filter, filter_iter:
+            ae(filt(None, dict(title='%'), ('+','id'), (None,None)), ['1'])
+            ae(filt(None, dict(title='_'), ('+','id'), (None,None)), ['2'])
+            ae(filt(None, dict(title='\\'), ('+','id'), (None,None)), ['3'])
+            ae(filt(None, dict(title="'"), ('+','id'), (None,None)), ['4'])
+
     def testFilteringLink(self):
         ae, filter, filter_iter = self.filteringSetup()
         a = 'assignedto'