merge from HEAD

Author: Richard Jones

CHANGES.txt

@@ -15,6 +15,7 @@ Fixed:
 - fixed REMOTE_USER (external HTTP Basic auth) (sf bug 977309)
 - fixed roundup-admin "find" to use better value parsing
 - fixed RDBMS Class.find() to handle None value in multiple find
+- export now stores file "content" in separate files in export directory
 
 
 2004-06-10 0.7.4

roundup/admin.py

@@ -16,7 +16,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: admin.py,v 1.68.2.2 2004-06-23 22:59:17 richard Exp $
+# $Id: admin.py,v 1.68.2.3 2004-06-24 07:14:22 richard Exp $
 
 '''Administration commands for maintaining Roundup trackers.
 '''
@@ -1031,15 +1031,16 @@ def do_export(self, args):
             writer = rcsv.writer(f, rcsv.colon_separated)
 
             properties = cl.getprops()
-            propnames = properties.keys()
-            propnames.sort()
+            propnames = cl.export_propnames()
             fields = propnames[:]
             fields.append('is retired')
             writer.writerow(fields)
 
             # all nodes for this class
             for nodeid in cl.getnodeids():
                 writer.writerow(cl.export_list(propnames, nodeid))
+                if hasattr(cl, 'export_files'):
+                    cl.export_files(dir, nodeid)
 
             # close this file
             f.close()
@@ -1077,7 +1078,11 @@ class to import.
             raise UsageError, _(rcsv.error)
         from roundup import hyperdb
 
-        for file in os.listdir(args[0]):
+        # directory to import from
+        dir = args[0]
+
+        # import all the files
+        for file in os.listdir(dir):
             classname, ext = os.path.splitext(file)
             # we only care about CSV files
             if ext != '.csv' or classname.endswith('-journals'):
@@ -1086,7 +1091,7 @@ class to import.
             cl = self.get_class(classname)
 
             # ensure that the properties and the CSV file headings match
-            f = open(os.path.join(args[0], file))
+            f = open(os.path.join(dir, file))
             reader = rcsv.reader(f, rcsv.colon_separated)
             file_props = None
             maxid = 1
@@ -1096,7 +1101,11 @@ class to import.
                     file_props = r
                     continue
                 # do the import and figure the current highest nodeid
-                maxid = max(maxid, int(cl.import_list(file_props, r)))
+                nodeid = int(cl.import_list(file_props, r))
+                if hasattr(cl, 'import_files'):
+                    cl.import_files(dir, nodeid)
+                maxid = max(maxid, nodeid)
+
             f.close()
 
             # import the journals

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.146.2.10 2004-06-21 05:42:45 richard Exp $
+#$Id: back_anydbm.py,v 1.146.2.11 2004-06-24 07:14:26 richard Exp $
 '''This module defines a backend that saves the hyperdatabase in a
 database chosen by anydbm. It is guaranteed to always be available in python
 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
@@ -2118,25 +2118,6 @@ def create(self, **propvalues):
         self.db.storefile(self.classname, newid, None, content)
         return newid
 
-    def import_list(self, propnames, proplist):
-        ''' Trap the "content" property...
-        '''
-        # dupe this list so we don't affect others
-        propnames = propnames[:]
-
-        # extract the "content" property from the proplist
-        i = propnames.index('content')
-        content = eval(proplist[i])
-        del propnames[i]
-        del proplist[i]
-
-        # do the normal import
-        newid = Class.import_list(self, propnames, proplist)
-
-        # save off the "content" file
-        self.db.storefile(self.classname, newid, None, content)
-        return newid
-
     def get(self, nodeid, propname, default=_marker, cache=1):
         ''' Trap the content propname and get it from the file
 
@@ -2194,7 +2175,6 @@ def getprops(self, protected=1):
         d['content'] = hyperdb.String()
         return d
 
-
 # deviation from spec - was called ItemClass
 class IssueClass(Class, roundupdb.IssueClass):
     # Overridden methods:

roundup/backends/back_metakit.py

@@ -1,4 +1,4 @@
-# $Id: back_metakit.py,v 1.70.2.3 2004-06-08 05:34:21 richard Exp $
+# $Id: back_metakit.py,v 1.70.2.4 2004-06-24 07:14:47 richard Exp $
 '''Metakit backend for Roundup, originally by Gordon McMillan.
 
 Known Current Bugs:
@@ -44,7 +44,7 @@
 from roundup import hyperdb, date, password, roundupdb, security
 import metakit
 from sessions_dbm import Sessions, OneTimeKeys
-import re, marshal, os, sys, time, calendar
+import re, marshal, os, sys, time, calendar, shutil
 from indexer_dbm import Indexer
 import locking
 from roundup.date import Range
@@ -496,7 +496,7 @@ def create_inner(self, **propvalues):
        ndx = self.getview(READWRITE).append(rowdict)
        propvalues['#ISNEW'] = 1
        try:
-           self.set(str(newid), **propvalues)
+           self.set_inner(str(newid), **propvalues)
        except Exception:
            self.maxid -= 1
            raise
@@ -801,7 +801,7 @@ def set_inner(self, nodeid, **propvalues):
             oldnode[key] = oldvalue
 
         # nothing to do?
-        if not propvalues:
+        if not isnew and not propvalues:
             return propvalues, oldnode
         if not propvalues.has_key('activity'):
             row.activity = int(time.time())
@@ -1832,6 +1832,29 @@ def gen_filename(self, nodeid):
             os.makedirs(d)
         return os.path.join(d, nm)
 
+    def export_files(self, dirname, nodeid):
+        ''' Export the "content" property as a file, not csv column
+        '''
+        source = self.gen_filename(nodeid)
+        x, filename = os.path.split(source)
+        x, subdir = os.path.split(x)
+        dest = os.path.join(dirname, self.classname+'-files', subdir, filename)
+        if not os.path.exists(os.path.dirname(dest)):
+            os.makedirs(os.path.dirname(dest))
+        shutil.copyfile(source, dest)
+
+    def import_files(self, dirname, nodeid):
+        ''' Import the "content" property as a file
+        '''
+        dest = self.gen_filename(nodeid)
+        x, filename = os.path.split(dest)
+        x, subdir = os.path.split(x)
+        source = os.path.join(dirname, self.classname+'-files', subdir,
+            filename)
+        if not os.path.exists(os.path.dirname(dest)):
+            os.makedirs(os.path.dirname(dest))
+        shutil.copyfile(source, dest)
+
     def get(self, nodeid, propname, default=_marker, cache=1):
         if propname == 'content':
             poss_msg = 'Possibly an access right configuration problem.'

roundup/backends/blobfiles.py

@@ -15,15 +15,15 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: blobfiles.py,v 1.12 2004-03-19 04:47:59 richard Exp $
+#$Id: blobfiles.py,v 1.12.2.1 2004-06-24 07:14:48 richard Exp $
 '''This module exports file storage for roundup backends.
 Files are stored into a directory hierarchy.
 '''
 __docformat__ = 'restructuredtext'
 
 import os
 
-def files_in_dir(dir):       
+def files_in_dir(dir):
     if not os.path.exists(dir):
         return 0
     num_files = 0

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.98.2.10 2004-06-23 23:21:32 richard Exp $
+# $Id: rdbms_common.py,v 1.98.2.11 2004-06-24 07:14:48 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -2575,25 +2575,6 @@ def create(self, **propvalues):
         self.db.storefile(self.classname, newid, None, content)
         return newid
 
-    def import_list(self, propnames, proplist):
-        ''' Trap the "content" property...
-        '''
-        # dupe this list so we don't affect others
-        propnames = propnames[:]
-
-        # extract the "content" property from the proplist
-        i = propnames.index('content')
-        content = eval(proplist[i])
-        del propnames[i]
-        del proplist[i]
-
-        # do the normal import
-        newid = Class.import_list(self, propnames, proplist)
-
-        # save off the "content" file
-        self.db.storefile(self.classname, newid, None, content)
-        return newid
-
     _marker = []
     def get(self, nodeid, propname, default=_marker, cache=1):
         ''' Trap the content propname and get it from the file

roundup/hyperdb.py

@@ -15,14 +15,14 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: hyperdb.py,v 1.97.2.1 2004-05-18 21:50:30 richard Exp $
+# $Id: hyperdb.py,v 1.97.2.2 2004-06-24 07:14:23 richard Exp $
 
 """Hyperdatabase implementation, especially field types.
 """
 __docformat__ = 'restructuredtext'
 
 # standard python modules
-import sys, os, time, re
+import sys, os, time, re, shutil
 
 # roundup modules
 import date, password
@@ -588,6 +588,13 @@ def safeget(self, nodeid, propname, default=None):
         except IndexError:
             return default            
 
+    def export_propnames(self):
+        '''List the property names for export from this Class.'''
+        propnames = self.getprops().keys()
+        propnames.sort()
+        return propnames
+
+
 class HyperdbValueError(ValueError):
     ''' Error converting a raw value into a Hyperdb value '''
     pass
@@ -758,7 +765,36 @@ class FileClass:
     ''' A class that requires the "content" property and stores it on
         disk.
     '''
-    pass
+    def export_propnames(self):
+        ''' Don't export the "content" property
+        '''
+        propnames = self.getprops().keys()
+        propnames.remove('content')
+        propnames.sort()
+        return propnames
+
+    def export_files(self, dirname, nodeid):
+        ''' Export the "content" property as a file, not csv column
+        '''
+        source = self.db.filename(self.classname, nodeid)
+        x, filename = os.path.split(source)
+        x, subdir = os.path.split(x)
+        dest = os.path.join(dirname, self.classname+'-files', subdir, filename)
+        if not os.path.exists(os.path.dirname(dest)):
+            os.makedirs(os.path.dirname(dest))
+        shutil.copyfile(source, dest)
+
+    def import_files(self, dirname, nodeid):
+        ''' Import the "content" property as a file
+        '''
+        dest = self.db.filename(self.classname, nodeid)
+        x, filename = os.path.split(dest)
+        x, subdir = os.path.split(x)
+        source = os.path.join(dirname, self.classname+'-files', subdir,
+            filename)
+        if not os.path.exists(os.path.dirname(dest)):
+            os.makedirs(os.path.dirname(dest))
+        shutil.copyfile(source, dest)
 
 class Node:
     ''' A convenience wrapper for the given node

test/db_test_base.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: db_test_base.py,v 1.27.2.6 2004-06-23 23:21:32 richard Exp $ 
+# $Id: db_test_base.py,v 1.27.2.7 2004-06-24 07:14:49 richard Exp $ 
 
 import unittest, os, shutil, errno, imp, sys, time, pprint
 
@@ -846,6 +846,8 @@ def filteringSetup(self):
                     'foo': date.Interval('0:10'), 'priority': '1',
                     'nosy': ['1'], 'deadline': date.Date('2004-03-08')}):
             self.db.issue.create(**issue)
+        file_content = ''.join([chr(i) for i in range(255)])
+        self.db.file.create(content=file_content)
         self.db.commit()
         return self.assertEqual, self.db.issue.filter
 
@@ -981,34 +983,43 @@ def testImportExport(self):
                 for name in klass.getprops().keys():
                     it[name] = klass.get(id, name)
 
-        # grab the export
-        export = {}
-        journals = {}
-        for cn,klass in self.db.classes.items():
-            names = klass.getprops().keys()
-            cl = export[cn] = [names+['is retired']]
-            for id in klass.getnodeids():
-                cl.append(klass.export_list(names, id))
-            journals[cn] = klass.export_journals()
-
-        # shut down this db and nuke it
-        self.db.close()
-        self.nuke_database()
-
-        # open a new, empty database
-        os.makedirs(config.DATABASE + '/files')
-        self.db = self.module.Database(config, 'admin')
-        setupSchema(self.db, 0, self.module)
-
-        # import
-        for cn, items in export.items():
-            klass = self.db.classes[cn]
-            names = items[0]
-            maxid = 1
-            for itemprops in items[1:]:
-                maxid = max(maxid, int(klass.import_list(names, itemprops)))
-            self.db.setid(cn, str(maxid+1))
-            klass.import_journals(journals[cn])
+        os.mkdir('_test_export')
+        try:
+            # grab the export
+            export = {}
+            journals = {}
+            for cn,klass in self.db.classes.items():
+                names = klass.export_propnames()
+                cl = export[cn] = [names+['is retired']]
+                for id in klass.getnodeids():
+                    cl.append(klass.export_list(names, id))
+                    if hasattr(klass, 'export_files'):
+                        klass.export_files('_test_export', id)
+                journals[cn] = klass.export_journals()
+
+            # shut down this db and nuke it
+            self.db.close()
+            self.nuke_database()
+
+            # open a new, empty database
+            os.makedirs(config.DATABASE + '/files')
+            self.db = self.module.Database(config, 'admin')
+            setupSchema(self.db, 0, self.module)
+
+            # import
+            for cn, items in export.items():
+                klass = self.db.classes[cn]
+                names = items[0]
+                maxid = 1
+                for itemprops in items[1:]:
+                    id = int(klass.import_list(names, itemprops))
+                    if hasattr(klass, 'import_files'):
+                        klass.import_files('_test_export', id)
+                    maxid = max(maxid, id)
+                self.db.setid(cn, str(maxid+1))
+                klass.import_journals(journals[cn])
+        finally:
+            shutil.rmtree('_test_export')
 
         # compare with snapshot of the database
         for cn, items in orig.items():