First cut at exporting/importing file content from and to the hyperdb.

Unfortunately, I currently set the content using "Class.set", which also
updates the "activity". So testImportExport is broken for tsearch2 right now.

Also did bits of refactoring in hyperdb.FileClass, a tiny step towards making
in-database files backend & indexer-independent.

Author: Johannes Gijsbers

roundup/backends/back_tsearch2.py

@@ -5,6 +5,7 @@
 import psycopg
 
 from roundup import hyperdb
+from roundup.support import ensureParentsExist
 from roundup.backends import back_postgresql, tsearch2_setup, indexer_rdbms
 from roundup.backends.back_postgresql import db_create, db_nuke, db_command
 from roundup.backends.back_postgresql import pg_command, db_exists, Class, IssueClass, FileClass
@@ -198,11 +199,13 @@ def tsearchQuery(self, classname, search_terms):
 
         return [(classname, nodeid) for nodeid in nodeids]
 
-# XXX: we can't reuse hyperdb.FileClass for importing/exporting, so file
-# contents will end up in CSV exports for now. Not sure whether this is
-# truly a problem. If it is, we should write an importer/exporter that
-# converts from the database to the filesystem and vice versa
-class FileClass(Class):
+class FileClass(hyperdb.FileClass, Class):
+    '''This class defines a large chunk of data. To support this, it has a
+       mandatory String property "content" which is typically saved off
+       externally to the hyperdb.
+
+       However, this implementation just stores it in the hyperdb.
+    '''
     def __init__(self, db, classname, **properties):
         '''The newly-created class automatically includes the "content" property.,
         '''
@@ -215,3 +218,22 @@ def create(self, **propvalues):
         if 'type' in self.getprops() and not propvalues.get('type'):
             propvalues['type'] = self.default_mime_type
         return Class.create(self, **propvalues)
+
+    def export_files(self, dirname, nodeid):
+        dest = self.exportFilename(dirname, nodeid)
+        ensureParentsExist(dest)
+        fp = open(dest, "w")
+        fp.write(self.get(nodeid, "content", default=''))
+        fp.close()
+
+    # XXX: this unfortunately sets the activity on the node to "now", causing
+    # testImportExport to fail. We either need to create a way to disable
+    # setting activity/actor or we need to override import_list so it already
+    # includes the "content" property. However, that would mean changing its
+    # call signature, as we need to know `dirname`.
+    def import_files(self, dirname, nodeid):
+        source = self.exportFilename(dirname, nodeid)
+
+        fp = open(source, "r")
+        self.set(nodeid, content=fp.read())
+        fp.close()

roundup/backends/blobfiles.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: blobfiles.py,v 1.17 2004-11-25 23:53:31 richard Exp $
+#$Id: blobfiles.py,v 1.18 2004-12-18 11:12:04 jlgijsbers Exp $
 '''This module exports file storage for roundup backends.
 Files are stored into a directory hierarchy.
 '''
@@ -36,25 +36,30 @@ def files_in_dir(dir):
     return num_files
 
 class FileStorage:
-    """Store files in some directory structure"""
-    def filename(self, classname, nodeid, property=None, create=0):
-        '''Determine what the filename for the given node and optionally 
-        property is.
-
-        Try a variety of different filenames - the file could be in the
-        usual place, or it could be in a temp file pre-commit *or* it
-        could be in an old-style, backwards-compatible flat directory.
-        '''
+    """Store files in some directory structure"""    
+    def subdirFilename(self, classname, nodeid, property=None):
+        """Determine what the filename and subdir for nodeid + classname is."""
         if property:
             name = '%s%s.%s'%(classname, nodeid, property)
         else:
             # roundupdb.FileClass never specified the property name, so don't 
             # include it
             name = '%s%s'%(classname, nodeid)
-
+        
         # have a separate subdir for every thousand messages
         subdir = str(int(nodeid) / 1000)
-        filename  = os.path.join(self.dir, 'files', classname, subdir, name)
+        return os.path.join(subdir, name)
+    
+    def filename(self, classname, nodeid, property=None, create=0):
+        '''Determine what the filename for the given node and optionally 
+        property is.
+
+        Try a variety of different filenames - the file could be in the
+        usual place, or it could be in a temp file pre-commit *or* it
+        could be in an old-style, backwards-compatible flat directory.
+        '''
+        filename  = os.path.join(self.dir, 'files', classname,
+                                 self.subdirFilename(classname, nodeid, property))
         if create or os.path.exists(filename):
             return filename
 

roundup/hyperdb.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
-# $Id: hyperdb.py,v 1.107 2004-11-25 22:51:06 richard Exp $
+# $Id: hyperdb.py,v 1.108 2004-12-18 11:12:03 jlgijsbers Exp $
 
 """Hyperdatabase implementation, especially field types.
 """
@@ -26,6 +26,7 @@
 
 # roundup modules
 import date, password
+from support import ensureParentsExist
 
 #
 # Types
@@ -746,27 +747,26 @@ def export_propnames(self):
         propnames.sort()
         return propnames
 
+    def exportFilename(self, dirname, nodeid):
+        subdir_filename = self.db.subdirFilename(self.classname, nodeid)
+        return os.path.join(dirname, self.classname+'-files', subdir_filename)
+
     def export_files(self, dirname, nodeid):
         ''' Export the "content" property as a file, not csv column
         '''
         source = self.db.filename(self.classname, nodeid)
-        x, filename = os.path.split(source)
-        x, subdir = os.path.split(x)
-        dest = os.path.join(dirname, self.classname+'-files', subdir, filename)
-        if not os.path.exists(os.path.dirname(dest)):
-            os.makedirs(os.path.dirname(dest))
-        shutil.copyfile(source, dest)
+
+        dest = self.exportFilename(dirname, nodeid)
+        ensureParentsExist(dest)
+        shutil.copyfile(source, dest)    
 
     def import_files(self, dirname, nodeid):
         ''' Import the "content" property as a file
         '''
+        source = self.exportFilename(dirname, nodeid)
+
         dest = self.db.filename(self.classname, nodeid, create=1)
-        x, filename = os.path.split(dest)
-        x, subdir = os.path.split(x)
-        source = os.path.join(dirname, self.classname+'-files', subdir,
-            filename)
-        if not os.path.exists(os.path.dirname(dest)):
-            os.makedirs(os.path.dirname(dest))
+        ensureParentsExist(dest)
         shutil.copyfile(source, dest)
 
 class Node:

roundup/support.py

@@ -4,6 +4,8 @@
 
 __docformat__ = 'restructuredtext'
 
+import os
+
 class TruthDict:
     '''Returns True for valid keys, False for others.
     '''
@@ -18,4 +20,8 @@ def __init__(self, keys):
     def __getitem__(self, name):
         return self.keys.has_key(name)
 
+def ensureParentsExist(dest):
+    if not os.path.exists(os.path.dirname(dest)):
+        os.makedirs(os.path.dirname(dest))
+
 # vim: set et sts=4 sw=4 :