Fixed serialisation problem...

Richard Jones · Richard Jones · commit 439eb02549fb · 2002-04-03T05:54:31.000Z
...by moving the serialisation step out of the hyperdb.Class (get,
set) into the hyperdb.Database.

Also fixed htmltemplate after the showid changes I made yesterday.

Unit tests for all of the above written.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,7 +1,7 @@
 This file contains the changes to the Roundup system over time. The entries
 are given with the most recent entry first.
 
-0.4.2
+2002-04-?? 0.4.2
 Feature:
  . link() htmltemplate function now has a "showid" option for links and
    multilinks. When true, it only displays the linked node id as the anchor
@@ -18,6 +18,7 @@ Feature:
 
 Fixed:
  . stop sending blank (whitespace-only) notes
+ . cleanup of serialisation for database storage
 
 
 2002-03-25 - 0.4.1
diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.30 2002-02-27 03:40:59 richard Exp $
+#$Id: back_anydbm.py,v 1.31 2002-04-03 05:54:31 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -177,6 +177,7 @@ def setnode(self, classname, nodeid, node):
         if hyperdb.DEBUG:
             print 'setnode', (self, classname, nodeid, node)
         self.dirtynodes.setdefault(classname, {})[nodeid] = 1
+
         # can't set without having already loaded the node
         self.cache[classname][nodeid] = node
         self.savenode(classname, nodeid, node)
@@ -204,9 +205,17 @@ def getnode(self, classname, nodeid, db=None, cache=1):
             db = self.getclassdb(classname)
         if not db.has_key(nodeid):
             raise IndexError, "no such %s %s"%(classname, nodeid)
+
+        # decode
         res = marshal.loads(db[nodeid])
+
+        # reverse the serialisation
+        res = self.unserialise(classname, res)
+
+        # store off in the cache
         if cache:
             cache[nodeid] = res
+
         return res
 
     def hasnode(self, classname, nodeid, db=None):
@@ -380,11 +389,17 @@ def _doSaveNode(self, classname, nodeid, node):
             db = self.databases[db_name] = self.getclassdb(classname, 'c')
 
         # now save the marshalled data
-        db[nodeid] = marshal.dumps(node)
+        db[nodeid] = marshal.dumps(self.serialise(classname, node))
 
     def _doSaveJournal(self, classname, nodeid, action, params):
+        # serialise first
+        if action in ('set', 'create'):
+            params = self.serialise(classname, params)
+
+        # create the journal entry
         entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
             params)
+
         if hyperdb.DEBUG:
             print '_doSaveJournal', entry
 
@@ -397,11 +412,13 @@ def _doSaveJournal(self, classname, nodeid, action, params):
 
         # now insert the journal entry
         if db.has_key(nodeid):
+            # append to existing
             s = db[nodeid]
             l = marshal.loads(s)
             l.append(entry)
         else:
             l = [entry]
+
         db[nodeid] = marshal.dumps(l)
 
     def _doStoreFile(self, name, **databases):
@@ -425,6 +442,9 @@ def rollback(self):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.30  2002/02/27 03:40:59  richard
+#Ran it through pychecker, made fixes
+#
 #Revision 1.29  2002/02/25 14:34:31  grubert
 # . use blobfiles in back_anydbm which is used in back_bsddb.
 #   change test_db as dirlist does not work for subdirectories.
diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_bsddb.py,v 1.16 2002-02-27 03:40:59 richard Exp $
+#$Id: back_bsddb.py,v 1.17 2002-04-03 05:54:31 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in BSDDB.
 '''
@@ -95,20 +95,33 @@ def getjournal(self, classname, nodeid):
         return res
 
     def _doSaveJournal(self, classname, nodeid, action, params):
+        # serialise first
+        if action in ('set', 'create'):
+            params = self.serialise(classname, params)
+
         entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
             params)
+
+        if hyperdb.DEBUG:
+            print '_doSaveJournal', entry
+
         db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c')
+
         if db.has_key(nodeid):
             s = db[nodeid]
             l = marshal.loads(s)
             l.append(entry)
         else:
             l = [entry]
+
         db[nodeid] = marshal.dumps(l)
         db.close()
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.16  2002/02/27 03:40:59  richard
+#Ran it through pychecker, made fixes
+#
 #Revision 1.15  2002/02/16 09:15:33  richard
 #forgot to patch bsddb backend too
 #
diff --git a/roundup/htmltemplate.py b/roundup/htmltemplate.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: htmltemplate.py,v 1.85 2002-04-02 01:40:58 richard Exp $
+# $Id: htmltemplate.py,v 1.86 2002-04-03 05:54:31 richard Exp $
 
 __doc__ = """
 Template engine.
@@ -357,10 +357,11 @@ def do_link(self, property=None, is_download=0, showid=0):
             linkvalue = cgi.escape(linkcl.get(value, k))
             if showid:
                 label = value
-		title = ' title="%s"'%linkvalue
-		# note ... this should be urllib.quote(linkcl.get(value, k))
+                title = ' title="%s"'%linkvalue
+                # note ... this should be urllib.quote(linkcl.get(value, k))
             else:
                 label = linkvalue
+                title = ''
             if is_download:
                 return '<a href="%s%s/%s"%s>%s</a>'%(linkname, value,
                     linkvalue, title, label)
@@ -376,9 +377,10 @@ def do_link(self, property=None, is_download=0, showid=0):
                 if showid:
                     label = value
                     title = ' title="%s"'%linkvalue
-		    # note ... this should be urllib.quote(linkcl.get(value, k))
+                    # note ... this should be urllib.quote(linkcl.get(value, k))
                 else:
                     label = linkvalue
+                    title = ''
                 if is_download:
                     l.append('<a href="%s%s/%s"%s>%s</a>'%(linkname, value,
                         linkvalue, title, label))
@@ -1126,6 +1128,12 @@ def render(self, form):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.85  2002/04/02 01:40:58  richard
+#  . link() htmltemplate function now has a "showid" option for links and
+#    multilinks. When true, it only displays the linked node id as the anchor
+#    text. The link value is displayed as a tooltip using the title anchor
+#    attribute.
+#
 # Revision 1.84  2002/03/29 19:41:48  rochecompaan
 #  . Fixed display of mutlilink properties when using the template
 #    functions, menu and plain.
diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: hyperdb.py,v 1.59 2002-03-12 22:52:26 richard Exp $
+# $Id: hyperdb.py,v 1.60 2002-04-03 05:54:31 richard Exp $
 
 __doc__ = """
 Hyperdatabase implementation, especially field types.
@@ -171,11 +171,51 @@ def addnode(self, classname, nodeid, node):
         '''
         raise NotImplementedError
 
+    def serialise(self, classname, node):
+        '''Copy the node contents, converting non-marshallable data into
+           marshallable data.
+        '''
+        if DEBUG: print 'serialise', classname, node
+        properties = self.getclass(classname).getprops()
+        d = {}
+        for k, v in node.items():
+            prop = properties[k]
+
+            if isinstance(prop, Password):
+                d[k] = str(v)
+            elif isinstance(prop, Date) and v is not None:
+                d[k] = v.get_tuple()
+            elif isinstance(prop, Interval) and v is not None:
+                d[k] = v.get_tuple()
+            else:
+                d[k] = v
+        return d
+
     def setnode(self, classname, nodeid, node):
         '''Change the specified node.
         '''
         raise NotImplementedError
 
+    def unserialise(self, classname, node):
+        '''Decode the marshalled node data
+        '''
+        if DEBUG: print 'unserialise', classname, node
+        properties = self.getclass(classname).getprops()
+        d = {}
+        for k, v in node.items():
+            prop = properties[k]
+            if isinstance(prop, Date) and v is not None:
+                d[k] = date.Date(v)
+            elif isinstance(prop, Interval) and v is not None:
+                d[k] = date.Interval(v)
+            elif isinstance(prop, Password):
+                p = password.Password()
+                p.unpack(v)
+                d[k] = p
+            else:
+                d[k] = v
+        return d
+
     def getnode(self, classname, nodeid, db=None, cache=1):
         '''Get a node from the database.
         '''
@@ -396,17 +436,6 @@ def create(self, **propvalues):
                 # TODO: None isn't right here, I think...
                 propvalues[key] = None
 
-        # convert all data to strings
-        for key, prop in self.properties.items():
-            if isinstance(prop, Date):
-                if propvalues[key] is not None:
-                    propvalues[key] = propvalues[key].get_tuple()
-            elif isinstance(prop, Interval):
-                if propvalues[key] is not None:
-                    propvalues[key] = propvalues[key].get_tuple()
-            elif isinstance(prop, Password):
-                propvalues[key] = str(propvalues[key])
-
         # done
         self.db.addnode(self.classname, newid, propvalues)
         self.db.addjournal(self.classname, newid, 'create', propvalues)
@@ -443,20 +472,6 @@ def get(self, nodeid, propname, default=_marker, cache=1):
             else:
                 return default
 
-        # possibly convert the marshalled data to instances
-        if isinstance(prop, Date):
-            if d[propname] is None:
-                return None
-            return date.Date(d[propname])
-        elif isinstance(prop, Interval):
-            if d[propname] is None:
-                return None
-            return date.Interval(d[propname])
-        elif isinstance(prop, Password):
-            p = password.Password()
-            p.unpack(d[propname])
-            return p
-
         return d[propname]
 
     # XXX not in spec
@@ -605,17 +620,17 @@ class or a KeyError is raised.
             elif isinstance(prop, Password):
                 if not isinstance(value, password.Password):
                     raise TypeError, 'new property "%s" not a Password'% key
-                propvalues[key] = value = str(value)
+                propvalues[key] = value
 
             elif value is not None and isinstance(prop, Date):
                 if not isinstance(value, date.Date):
                     raise TypeError, 'new property "%s" not a Date'% key
-                propvalues[key] = value = value.get_tuple()
+                propvalues[key] = value
 
             elif value is not None and isinstance(prop, Interval):
                 if not isinstance(value, date.Interval):
                     raise TypeError, 'new property "%s" not an Interval'% key
-                propvalues[key] = value = value.get_tuple()
+                propvalues[key] = value
 
             node[key] = value
 
@@ -1097,6 +1112,9 @@ def Choice(name, db, *options):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.59  2002/03/12 22:52:26  richard
+# more pychecker warnings removed
+#
 # Revision 1.58  2002/02/27 03:23:16  richard
 # Ran it through pychecker, made fixes
 #
diff --git a/test/test_db.py b/test/test_db.py
@@ -15,14 +15,14 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: test_db.py,v 1.19 2002-02-25 14:34:31 grubert Exp $ 
+# $Id: test_db.py,v 1.20 2002-04-03 05:54:31 richard Exp $ 
 
 import unittest, os, shutil
 
 from roundup.hyperdb import String, Password, Link, Multilink, Date, \
     Interval, Class, DatabaseError
 from roundup.roundupdb import FileClass
-from roundup import date
+from roundup import date, password
 
 def setupSchema(db, create):
     status = Class(db, "status", name=String())
@@ -84,7 +84,11 @@ def testChanges(self):
 
         a = self.db.issue.get('5', "deadline")
         self.db.issue.set('5', deadline=date.Date())
-        self.assertNotEqual(a, self.db.issue.get('5', "deadline"))
+        b = self.db.issue.get('5', "deadline")
+        self.db.commit()
+        self.assertNotEqual(a, b)
+        self.assertNotEqual(b, date.Date('1970-1-1 00:00:00'))
+        self.db.issue.set('5', deadline=date.Date())
 
         a = self.db.issue.get('5', "foo")
         self.db.issue.set('5', foo=date.Interval('-1d'))
@@ -98,6 +102,17 @@ def testChanges(self):
         self.db.status.history('1')
         self.db.status.history('2')
 
+    def testSerialisation(self):
+        self.db.issue.create(title="spam", status='1',
+            deadline=date.Date(), foo=date.Interval('-1d'))
+        self.db.commit()
+        assert isinstance(self.db.issue.get('1', 'deadline'), date.Date)
+        assert isinstance(self.db.issue.get('1', 'foo'), date.Interval)
+        self.db.user.create(username="fozzy",
+            password=password.Password('t. bear'))
+        self.db.commit()
+        assert isinstance(self.db.user.get('1', 'password'), password.Password)
+
     def testTransactions(self):
         # remember the number of items we started
         num_issues = len(self.db.issue.list())
@@ -324,7 +339,8 @@ def setUp(self):
 
 
 def suite():
-    l = [unittest.makeSuite(anydbmDBTestCase, 'test'),
+    l = [
+         unittest.makeSuite(anydbmDBTestCase, 'test'),
          unittest.makeSuite(anydbmReadOnlyDBTestCase, 'test')
     ]
 
@@ -346,6 +362,11 @@ def suite():
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.19  2002/02/25 14:34:31  grubert
+#  . use blobfiles in back_anydbm which is used in back_bsddb.
+#    change test_db as dirlist does not work for subdirectories.
+#    ATTENTION: blobfiles now creates subdirectories for files.
+#
 # Revision 1.18  2002/01/22 07:21:13  richard
 # . fixed back_bsddb so it passed the journal tests
 #
diff --git a/test/test_htmltemplate.py b/test/test_htmltemplate.py
