Moved over to using marshal in the bsddb and anydbm backends.

roundup-admin now has a "freshen" command that'll load/save all nodes (not
 retired - mod hyperdb.Class.list() so it lists retired nodes)

Author: Richard Jones

roundup-admin

@@ -1,6 +1,6 @@
 #! /usr/bin/python
 
-# $Id: roundup-admin,v 1.1 2001-07-23 03:46:48 richard Exp $
+# $Id: roundup-admin,v 1.2 2001-07-23 08:20:44 richard Exp $
 
 import sys
 if int(sys.version[0]) < 2:
@@ -254,6 +254,16 @@ def main():
             classname, nodeid = roundupdb.splitDesignator(designator)
             db.getclass(classname).retire(nodeid)
 
+    elif command == 'freshen':
+        n, db = determineLogin(instance, argv, n)
+        for classname, cl in db.classes.items():
+            properties = cl.properties.keys()
+            for nodeid in cl.list():
+                node = {}
+                for name in properties:
+                    node[name] = cl.get(nodeid, name)
+                db.setnode(classname, nodeid, node)
+
     else:
         print "Unknown command '%s'"%command
         usage()
@@ -267,6 +277,9 @@ if __name__ == '__main__':
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.1  2001/07/23 03:46:48  richard
+# moving the bin files to facilitate out-of-the-boxness
+#
 # Revision 1.1  2001/07/22 11:15:45  richard
 # More Grande Splite stuff
 #

roundup/backends/back_anydbm.py

@@ -1,6 +1,6 @@
-#$Id: back_anydbm.py,v 1.1 2001-07-23 07:22:13 richard Exp $
+#$Id: back_anydbm.py,v 1.2 2001-07-23 08:20:44 richard Exp $
 
-import anydbm, os, cPickle
+import anydbm, os, marshal
 from roundup import hyperdb, date
 
 #
@@ -68,11 +68,24 @@ def getclassdb(self, classname, mode='r'):
         path = os.path.join(os.getcwd(), self.dir, 'nodes.%s'%classname)
         return anydbm.open(path, mode)
 
+    #
+    # Nodes
+    #
     def addnode(self, classname, nodeid, node):
         ''' add the specified node to its class's db
         '''
         db = self.getclassdb(classname, 'c')
-        db[nodeid] = cPickle.dumps(node, 1)
+
+        # convert the instance data to builtin types
+        properties = self.classes[classname].properties
+        for key in properties.keys():
+            if properties[key].isDateType:
+                node[key] = node[key].get_tuple()
+            elif properties[key].isIntervalType:
+                node[key] = node[key].get_tuple()
+
+        # now save the marshalled data
+        db[nodeid] = marshal.dumps(node)
         db.close()
     setnode = addnode
 
@@ -82,7 +95,16 @@ def getnode(self, classname, nodeid, cldb=None):
         db = cldb or self.getclassdb(classname)
         if not db.has_key(nodeid):
             raise IndexError, nodeid
-        res = cPickle.loads(db[nodeid])
+        res = marshal.loads(db[nodeid])
+
+        # convert the marshalled data to instances
+        properties = self.classes[classname].properties
+        for key in res.keys():
+            if properties[key].isDateType:
+                res[key] = date.Date(res[key])
+            elif properties[key].isIntervalType:
+                res[key] = date.Interval(res[key])
+
         if not cldb: db.close()
         return res
 
@@ -117,22 +139,35 @@ def addjournal(self, classname, nodeid, action, params):
             'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
             'retire' -- 'params' is None
         '''
-        entry = (nodeid, date.Date(), self.journaltag, action, params)
+        entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
+            params)
         db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'c')
         if db.has_key(nodeid):
             s = db[nodeid]
-            l = cPickle.loads(db[nodeid])
+            l = marshal.loads(db[nodeid])
             l.append(entry)
         else:
             l = [entry]
-        db[nodeid] = cPickle.dumps(l)
+        db[nodeid] = marshal.dumps(l)
         db.close()
 
     def getjournal(self, classname, nodeid):
         ''' get the journal for id
         '''
-        db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'r')
-        res = cPickle.loads(db[nodeid])
+        # attempt to open the journal - in some rare cases, the journal may
+        # not exist
+        try:
+            db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname),
+                'r')
+        except anydbm.open, error:
+            if error.args[0] != 2: raise
+            return []
+        journal = marshal.loads(db[nodeid])
+        res = []
+        for entry in journal:
+            (nodeid, date_stamp, self.journaltag, action, params) = entry
+            date_obj = date.Date(date_stamp)
+            res.append((nodeid, date_obj, self.journaltag, action, params))
         db.close()
         return res
 
@@ -162,7 +197,4 @@ def rollback(self):
 
 #
 #$Log: not supported by cvs2svn $
-#Revision 1.1  2001/07/23 07:15:57  richard
-#Moved the backends into the backends package. Anydbm hasn't been tested at all.
-#
 #

roundup/backends/back_bsddb.py

@@ -1,8 +1,6 @@
-#$Id: back_bsddb.py,v 1.2 2001-07-23 07:56:05 richard Exp $
+#$Id: back_bsddb.py,v 1.3 2001-07-23 08:20:44 richard Exp $
 
 import bsddb, os, marshal
-# handle the older cPickle'd data
-import cPickle
 from roundup import hyperdb, date
 
 #
@@ -77,14 +75,17 @@ def addnode(self, classname, nodeid, node):
         ''' add the specified node to its class's db
         '''
         db = self.getclassdb(classname, 'c')
+
         # convert the instance data to builtin types
         properties = self.classes[classname].properties
-        for key in res.keys():
+        for key in properties.keys():
             if properties[key].isDateType:
-                res[key] = res[key].get_tuple()
+                node[key] = node[key].get_tuple()
             elif properties[key].isIntervalType:
-                res[key] = res[key].get_tuple()
-        db[nodeid] = marshal.dumps(node, 1)
+                node[key] = node[key].get_tuple()
+
+        # now save the marshalled data
+        db[nodeid] = marshal.dumps(node)
         db.close()
     setnode = addnode
 
@@ -94,20 +95,15 @@ def getnode(self, classname, nodeid, cldb=None):
         db = cldb or self.getclassdb(classname)
         if not db.has_key(nodeid):
             raise IndexError, nodeid
-        try:
-            res = marshal.loads(db[nodeid])
-            # convert the marshalled data to instances
-            properties = self.classes[classname].properties
-            for key in res.keys():
-                if properties[key].isDateType:
-                    res[key] = date.Date(res[key])
-                elif properties[key].isIntervalType:
-                    res[key] = date.Interval(res[key])
-        except ValueError, message:
-            if str(message) != 'bad marshal data':
-                raise
-            # handle the older cPickle'd data
-            res = cPickle.loads(db[nodeid])
+        res = marshal.loads(db[nodeid])
+
+        # convert the marshalled data to instances
+        properties = self.classes[classname].properties
+        for key in res.keys():
+            if properties[key].isDateType:
+                res[key] = date.Date(res[key])
+            elif properties[key].isIntervalType:
+                res[key] = date.Interval(res[key])
 
         if not cldb: db.close()
         return res
@@ -143,7 +139,7 @@ def addjournal(self, classname, nodeid, action, params):
             'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
             'retire' -- 'params' is None
         '''
-        entry = (nodeid, date.Date().journal_tuple(), self.journaltag, action,
+        entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
             params)
         db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c')
         if db.has_key(nodeid):
@@ -170,7 +166,7 @@ def getjournal(self, classname, nodeid):
         res = []
         for entry in journal:
             (nodeid, date_stamp, self.journaltag, action, params) = entry
-            date_obj = date.Date(set=date_stamp)
+            date_obj = date.Date(date_stamp)
             res.append((nodeid, date_obj, self.journaltag, action, params))
         db.close()
         return res
@@ -201,6 +197,9 @@ def rollback(self):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.2  2001/07/23 07:56:05  richard
+#Storing only marshallable data in the db - no nasty pickled class references.
+#
 #Revision 1.1  2001/07/23 07:22:13  richard
 #*sigh* some databases have _foo.so as their underlying implementation.
 #This time for sure, Rocky.