store Intervals as two columns (and other fixes

Author: Richard Jones

CHANGES.txt

@@ -31,6 +31,7 @@ Fixed:
 - sqlite backend had stopped using the global lock
 - better check for anonymous viewing of user items (sf bug 933510)
 - stop Interval from displaying an empty string (sf bug 934022)
+- fixed storage of some datatypes in some RDBMS backends
 
 
 2004-03-27 0.7.0b2

TODO.txt

@@ -1,8 +1,5 @@
 This file contains items that need doing before the next release:
 
-. make Intervals store timestamps, not strings
-
-
 Optionally:
 - have rdbms backends look up the journal for actor if it's not set
 - migrate to numeric ID values (fixes bug 817217)

roundup/backends/back_mysql.py

@@ -126,7 +126,7 @@ class Database(Database):
         # no fractional seconds for MySQL
         hyperdb.Date   : lambda x: x.formal(sep=' '),
         hyperdb.Link   : int,
-        hyperdb.Interval  : lambda x: x.serialise(),
+        hyperdb.Interval  : str,
         hyperdb.Password  : str,
         hyperdb.Boolean   : int,
         hyperdb.Number    : lambda x: x,
@@ -190,7 +190,7 @@ def create_version_2_tables(self):
         sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
         self.cursor.execute(sql, ('__textids', 1))
 
-    def add_actor_column(self):
+    def add_new_columns_v2(self):
         '''While we're adding the actor column, we need to update the
         tables to have the correct datatypes.'''
         for klass in self.classes.values():
@@ -274,8 +274,15 @@ def add_actor_column(self):
                     # convert to new MySQL data type
                     prop = properties[name]
                     if v is not None:
-                        v = self.hyperdb_to_sql_value[prop.__class__](v)
-                    l.append(v)
+                        e = self.hyperdb_to_sql_value[prop.__class__](v)
+                    l.append(e)
+
+                    # Intervals store the seconds value too
+                    if isinstance(prop, Interval):
+                        if v is not None:
+                            l.append(v.as_seconds())
+                        else:
+                            l.append(e)
 
             self.drop_class_table_indexes(cn, old_spec[0])
 
@@ -595,25 +602,24 @@ def filter(self, search_matches, filterspec, sort=(None,None),
                         # If range creation fails - ignore that search parameter
                         pass                        
             elif isinstance(propclass, Interval):
+                # filter using the __<prop>_int__ column
                 if isinstance(v, type([])):
                     s = ','.join([a for x in v])
-                    where.append('_%s in (%s)'%(k, s))
-                    args = args + [date.Interval(x).serialise() for x in v]
+                    where.append('__%s_int__ in (%s)'%(k, s))
+                    args = args + [date.Interval(x).as_seconds() for x in v]
                 else:
                     try:
                         # Try to filter on range of intervals
                         date_rng = Range(v, date.Interval)
-                        if (date_rng.from_value):
-                            where.append('_%s >= %s'%(k, a))
-                            args.append(date_rng.from_value.serialise())
-                        if (date_rng.to_value):
-                            where.append('_%s <= %s'%(k, a))
-                            args.append(date_rng.to_value.serialise())
+                        if date_rng.from_value:
+                            where.append('__%s_int__ >= %s'%(k, a))
+                            args.append(date_rng.from_value.as_seconds())
+                        if date_rng.to_value:
+                            where.append('__%s_int__ <= %s'%(k, a))
+                            args.append(date_rng.to_value.as_seconds())
                     except ValueError:
                         # If range creation fails - ignore that search parameter
                         pass                        
-                    #where.append('_%s=%s'%(k, a))
-                    #args.append(date.Interval(v).serialise())
             else:
                 if isinstance(v, type([])):
                     s = ','.join([a for x in v])
@@ -634,49 +640,42 @@ def filter(self, search_matches, filterspec, sort=(None,None),
             args = args + v
 
         # "grouping" is just the first-order sorting in the SQL fetch
-        # can modify it...)
         orderby = []
         ordercols = []
-        if group[0] is not None and group[1] is not None:
-            if group[0] != '-':
-                orderby.append('_'+group[1])
-                ordercols.append('_'+group[1])
-            else:
-                orderby.append('_'+group[1]+' desc')
-                ordercols.append('_'+group[1])
-
-        # now add in the sorting
-        group = ''
-        if sort[0] is not None and sort[1] is not None:
-            direction, colname = sort
-            if direction != '-':
-                if colname == 'id':
-                    orderby.append(colname)
-                else:
-                    orderby.append('_'+colname)
-                    ordercols.append('_'+colname)
-            else:
-                if colname == 'id':
-                    orderby.append(colname+' desc')
-                    ordercols.append(colname)
+        mlsort = []
+        for sortby in group, sort:
+            sdir, prop = sortby
+            if sdir and prop:
+                if isinstance(props[prop], Multilink):
+                    mlsort.append(sortby)
+                    continue
+                elif isinstance(props[prop], Interval):
+                    # use the int column for sorting
+                    o = '__'+prop+'_int__'
+                    ordercols.append(o)
+                elif prop == 'id':
+                    o = 'id'
                 else:
-                    orderby.append('_'+colname+' desc')
-                    ordercols.append('_'+colname)
+                    o = '_'+prop
+                    ordercols.append(o)
+                if sdir == '-':
+                    o += ' desc'
+                orderby.append(o)
 
         # construct the SQL
         frum = ','.join(frum)
         if where:
             where = ' where ' + (' and '.join(where))
         else:
             where = ''
-        cols = ['id']
+        cols = ['distinct(id)']
         if orderby:
             cols = cols + ordercols
             order = ' order by %s'%(','.join(orderby))
         else:
             order = ''
         cols = ','.join(cols)
-        sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
+        sql = 'select %s from %s %s%s'%(cols, frum, where, order)
         args = tuple(args)
         if __debug__:
             print >>hyperdb.DEBUG, 'filter', (self, sql, args)
@@ -685,7 +684,28 @@ def filter(self, search_matches, filterspec, sort=(None,None),
 
         # return the IDs (the first column)
         # XXX numeric ids
-        return [str(row[0]) for row in l]
+        l = [str(row[0]) for row in l]
+
+        if not mlsort:
+            return l
+
+        # ergh. someone wants to sort by a multilink.
+        r = []
+        for id in l:
+            m = []
+            for ml in mlsort:
+                m.append(self.get(id, ml[1]))
+            r.append((id, m))
+        i = 0
+        for sortby in mlsort:
+            def sortfun(a, b, dir=sortby[i]):
+                if dir == '-':
+                    return cmp(b[1][i], a[1][i])
+                else:
+                    return cmp(a[1][i], b[1][i])
+            r.sort(sortfun)
+            i += 1
+        return [i[0] for i in r]
 
 class Class(MysqlClass, rdbms_common.Class):
     pass

roundup/backends/back_sqlite.py

@@ -1,4 +1,4 @@
-# $Id: back_sqlite.py,v 1.24 2004-04-07 01:12:26 richard Exp $
+# $Id: back_sqlite.py,v 1.25 2004-04-18 05:31:02 richard Exp $
 '''Implements a backend for SQLite.
 
 See https://pysqlite.sourceforge.net/ for pysqlite info
@@ -32,16 +32,15 @@ class Database(rdbms_common.Database):
         hyperdb.String : str,
         hyperdb.Date   : lambda x: x.serialise(),
         hyperdb.Link   : int,
-        hyperdb.Interval  : lambda x: x.serialise(),
+        hyperdb.Interval  : str,
         hyperdb.Password  : str,
         hyperdb.Boolean   : int,
         hyperdb.Number    : lambda x: x,
     }
     sql_to_hyperdb_value = {
         hyperdb.String : str,
         hyperdb.Date   : lambda x: date.Date(str(x)),
-#        hyperdb.Link   : int,      # XXX numeric ids
-        hyperdb.Link   : str,
+        hyperdb.Link   : str, # XXX numeric ids
         hyperdb.Interval  : date.Interval,
         hyperdb.Password  : lambda x: password.Password(encrypted=x),
         hyperdb.Boolean   : int,
@@ -95,17 +94,17 @@ def create_version_2_tables(self):
         sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
         self.cursor.execute(sql, ('__textids', 1))
 
-    def add_actor_column(self):
+    def add_new_columns_v2(self):
         # update existing tables to have the new actor column
         tables = self.database_schema['tables']
         for classname, spec in self.classes.items():
             if tables.has_key(classname):
                 dbspec = tables[classname]
-                self.update_class(spec, dbspec, force=1, adding_actor=1)
+                self.update_class(spec, dbspec, force=1, adding_v2=1)
                 # we've updated - don't try again
                 tables[classname] = spec.schema()
 
-    def update_class(self, spec, old_spec, force=0, adding_actor=0):
+    def update_class(self, spec, old_spec, force=0, adding_v2=0):
         ''' Determine the differences between the current spec and the
             database version of the spec, and update where necessary.
 
@@ -141,7 +140,7 @@ def update_class(self, spec, old_spec, force=0, adding_actor=0):
         old_has = old_has.has_key
 
         # now figure how we populate the new table
-        if adding_actor:
+        if adding_v2:
             fetch = ['_activity', '_creation', '_creator']
         else:
             fetch = ['_actor', '_activity', '_creation', '_creator']
@@ -201,14 +200,41 @@ def update_class(self, spec, old_spec, force=0, adding_actor=0):
         self.create_class_table(spec)
 
         if olddata:
+            inscols = []
+            for propname,x in new_spec[1]:
+                prop = properties[propname]
+                if isinstance(prop, hyperdb.Multilink):
+                    continue
+                elif isinstance(prop, hyperdb.Interval):
+                    inscols.append('_'+propname)
+                    inscols.append('__'+propname+'_int__')
+                elif old_has(propname):
+                    # we copy this col over from the old table
+                    inscols.append('_'+propname)
+
             # do the insert of the old data - the new columns will have
             # NULL values
-            args = ','.join([self.arg for x in fetch])
-            sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
+            args = ','.join([self.arg for x in inscols])
+            cols = ','.join(inscols)
+            sql = 'insert into _%s (%s) values (%s)'%(cn, cols, args)
             if __debug__:
                 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
             for entry in olddata:
-                self.cursor.execute(sql, tuple(entry))
+                d = []
+                for name in inscols:
+                    # generate the new value for the Interval int column
+                    if name.endswith('_int__'):
+                        name = name[2:-6]
+                        if entry.has_key(name):
+                            v = hyperdb.Interval(entry[name]).as_seconds()
+                        else:
+                            v = None
+                    elif entry.has_key(name):
+                        v = entry[name]
+                    else:
+                        v = None
+                    d.append(v)
+                self.cursor.execute(sql, tuple(d))
 
         return 1