use blobfiles in back_anydbm which is used in back_bsddb.

   change test_db as dirlist does not work for subdirectories.
   ATTENTION: blobfiles now creates subdirectories for files.

Author: Engelbert Gruber

CHANGES.txt

@@ -3,6 +3,9 @@ are given with the most recent entry first.
 
 2002-02-?? - 0.4.1
 Feature:
+ . use blobfiles in back_anydbm which is used in back_bsddb.
+   change test_db as dirlist does not work for subdirectories.
+   ATTENTION: blobfiles now creates subdirectories for files.
  . add module blobfiles in backends with file access functions.
  . roundup db catch only IOError in getfile.
  . roundup db catches retrieving not existing files.

MIGRATION.txt

@@ -9,6 +9,16 @@ This file contains information for users upgrading from:
   0.3.x -> 0.4.x
   0.2.x -> 0.3.x
 
+From CVS
+========
+
+Files storage
+-------------
+
+Messages and files from newly created issues will be put into subdierectories
+in thousands e.g. msg123 will be put into files/msg/0/msg123, file2003
+will go into files/file/2/file2003. Previous messages are still found, but
+could be put into this structure.
 
 Migrating from 0.4.0 to 0.4.1
 =============================

README.txt

@@ -51,8 +51,6 @@ in general:
   . more back-ends
 hyperdb:
   . more efficient reverse lookups
-roundupdb:
-  . split the file storage into multiple dirs?
 roundup-server:
   . check the source file timestamps before reloading
 cgi_client

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.28 2002-02-16 09:14:17 richard Exp $
+#$Id: back_anydbm.py,v 1.29 2002-02-25 14:34:31 grubert Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -25,11 +25,12 @@
 
 import whichdb, anydbm, os, marshal
 from roundup import hyperdb, date, password
+from blobfiles import FileStorage
 
 #
 # Now the database
 #
-class Database(hyperdb.Database):
+class Database(FileStorage, hyperdb.Database):
     """A database for storing records containing flexible data types.
 
     Transaction stuff TODO:
@@ -250,36 +251,7 @@ def getnodeids(self, classname, db=None):
 
     #
     # Files - special node properties
-    #
-    def filename(self, classname, nodeid, property=None):
-        '''Determine what the filename for the given node and optionally property is.
-        '''
-        # TODO: split into multiple files directories
-        if property:
-            return os.path.join(self.dir, 'files', '%s%s.%s'%(classname,
-                nodeid, property))
-        else:
-            # roundupdb.FileClass never specified the property name, so don't include it
-            return os.path.join(self.dir, 'files', '%s%s'%(classname,
-                nodeid))
-
-    def storefile(self, classname, nodeid, property, content):
-        '''Store the content of the file in the database. The property may be None, in
-           which case the filename does not indicate which property is being saved.
-        '''
-        name = self.filename(classname, nodeid, property)
-        open(name + '.tmp', 'wb').write(content)
-        self.transactions.append((self._doStoreFile, (name, )))
-
-    def getfile(self, classname, nodeid, property):
-        '''Store the content of the file in the database.
-        '''
-        filename = self.filename(classname, nodeid, property)
-        try:
-            return open(filename, 'rb').read()
-        except:
-            return open(filename+'.tmp', 'rb').read()
-
+    # inherited from FileStorage
 
     #
     # Journal
@@ -453,6 +425,9 @@ def rollback(self):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.28  2002/02/16 09:14:17  richard
+# . #514854 ] History: "User" is always ticket creator
+#
 #Revision 1.27  2002/01/22 07:21:13  richard
 #. fixed back_bsddb so it passed the journal tests
 #

test/test_db.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: test_db.py,v 1.18 2002-01-22 07:21:13 richard Exp $ 
+# $Id: test_db.py,v 1.19 2002-02-25 14:34:31 grubert Exp $ 
 
 import unittest, os, shutil
 
@@ -99,12 +99,9 @@ def testChanges(self):
         self.db.status.history('2')
 
     def testTransactions(self):
+        # remember the number of items we started
         num_issues = len(self.db.issue.list())
-        files_dir = os.path.join('_test_dir', 'files')
-        if os.path.exists(files_dir):
-            num_files = len(os.listdir(files_dir))
-        else:
-            num_files = 0
+        num_files = self.db.numfiles()
         self.db.issue.create(title="don't commit me!", status='1')
         self.assertNotEqual(num_issues, len(self.db.issue.list()))
         self.db.rollback()
@@ -117,15 +114,18 @@ def testTransactions(self):
         self.assertNotEqual(num_issues, len(self.db.issue.list()))
         self.db.file.create(name="test", type="text/plain", content="hi")
         self.db.rollback()
-        self.assertEqual(num_files, len(os.listdir(files_dir)))
-        self.db.file.create(name="test", type="text/plain", content="hi")
-        self.db.commit()
-        self.assertNotEqual(num_files, len(os.listdir(files_dir)))
-        num_files2 = len(os.listdir(files_dir))
+        self.assertEqual(num_files, self.db.numfiles())
+        for i in range(10):
+            self.db.file.create(name="test", type="text/plain", 
+                    content="hi %d"%(i))
+            self.db.commit()
+        num_files2 = self.db.numfiles()
+        self.assertNotEqual(num_files, num_files2)
         self.db.file.create(name="test", type="text/plain", content="hi")
         self.db.rollback()
-        self.assertNotEqual(num_files, len(os.listdir(files_dir)))
-        self.assertEqual(num_files2, len(os.listdir(files_dir)))
+        self.assertNotEqual(num_files, self.db.numfiles())
+        self.assertEqual(num_files2, self.db.numfiles())
+            
 
 
     def testExceptions(self):
@@ -346,6 +346,14 @@ def suite():
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.18  2002/01/22 07:21:13  richard
+# . fixed back_bsddb so it passed the journal tests
+#
+# ... it didn't seem happy using the back_anydbm _open method, which is odd.
+# Yet another occurrance of whichdb not being able to recognise older bsddb
+# databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
+# process.
+#
 # Revision 1.17  2002/01/22 05:06:09  rochecompaan
 # We need to keep the last 'set' entry in the journal to preserve
 # information on 'activity' for nodes.