New config option csv_field_size:

Pythons csv module (which is used for export/import) has a new field
size limit starting with python2.5.  We now issue a warning during
export if the limit is too small and use the csv_field_size
configuration during import to set the limit for the csv module.

Author: Ralf Schlatterbeck

CHANGES.txt

@@ -12,6 +12,11 @@ Fixes:
   and stopwords (thanks Thomas Arendsen Hein, Bernhard Reiter)(issue 2550584)
 - fixed typos in the installation instructions (thanks Thomas Arendsen Hein)
   (issue 2550573) 
+- New config option csv_field_size: Pythons csv module (which is used
+  for export/import) has a new field size limit starting with python2.5.
+  We now issue a warning during export if the limit is too small and use
+  the csv_field_size configuration during import to set the limit for
+  the csv module.
 
 2009-08-10 1.4.9 (r4346)
 

roundup/admin.py

@@ -1099,6 +1099,9 @@ class colon_separated(csv.excel):
         if not os.path.exists(dir):
             os.makedirs(dir)
 
+        # maximum csv field length exceeding configured size?
+        max_len = self.db.config.CSV_FIELD_SIZE
+
         # do all the classes specified
         for classname in classes:
             cl = self.get_class(classname)
@@ -1121,7 +1124,18 @@ class colon_separated(csv.excel):
                 if self.verbose:
                     sys.stdout.write('\rExporting %s - %s'%(classname, nodeid))
                     sys.stdout.flush()
-                writer.writerow(cl.export_list(propnames, nodeid))
+                node = cl.getnode(nodeid)
+                exp = cl.export_list(propnames, nodeid)
+                lensum = sum (len (repr(node[p])) for p in propnames)
+                # for a safe upper bound of field length we add
+                # difference between CSV len and sum of all field lengths
+                d = sum (len(x) for x in exp) - lensum
+                assert (d > 0)
+                for p in propnames:
+                    ll = len(repr(node[p])) + d
+                    if ll > max_len:
+                        max_len = ll
+                writer.writerow(exp)
                 if export_files and hasattr(cl, 'export_files'):
                     cl.export_files(dir, nodeid)
 
@@ -1136,6 +1150,9 @@ class colon_separated(csv.excel):
             journals = csv.writer(jf, colon_separated)
             map(journals.writerow, cl.export_journals())
             jf.close()
+        if max_len > self.db.config.CSV_FIELD_SIZE:
+            print >> sys.stderr, \
+                "Warning: config csv_field_size should be at least %s"%max_len
         return 0
 
     def do_exporttables(self, args):
@@ -1177,6 +1194,9 @@ def do_import(self, args):
             raise UsageError, _('Not enough arguments supplied')
         from roundup import hyperdb
 
+        if hasattr (csv, 'field_size_limit'):
+            csv.field_size_limit(self.db.config.CSV_FIELD_SIZE)
+
         # directory to import from
         dir = args[0]
 
@@ -1212,7 +1232,7 @@ class colon_separated(csv.excel):
                 if hasattr(cl, 'import_files'):
                     cl.import_files(dir, nodeid)
                 maxid = max(maxid, int(nodeid))
-            print
+            print >> sys.stdout
             f.close()
 
             # import the journals
@@ -1222,7 +1242,7 @@ class colon_separated(csv.excel):
             f.close()
 
             # set the id counter
-            print 'setting', classname, maxid+1
+            print >> sys.stdout, 'setting', classname, maxid+1
             self.db.setid(classname, str(maxid+1))
 
         self.db_uncommitted = True

roundup/configuration.py

@@ -530,6 +530,13 @@ def str2value(self, value):
             "stop-words (eg. A,AND,ARE,AS,AT,BE,BUT,BY, ...)"),
         (OctalNumberOption, "umask", "02",
             "Defines the file creation mode mask."),
+        (IntegerNumberOption, 'csv_field_size', '131072',
+            "Maximum size of a csv-field during import. Roundups export\n"
+            "format is a csv (comma separated values) variant. The csv\n"
+            "reader has a limit on the size of individual fields\n"
+            "starting with python 2.5. Set this to a higher value if you\n"
+            "get the error 'Error: field larger than field limit' during\n"
+            "import."),
     )),
     ("tracker", (
         (Option, "name", "Roundup issue tracker",

test/db_test_base.py

@@ -1613,6 +1613,18 @@ def testFilteringSortId(self):
 
 # XXX add sorting tests for other types
 
+    # nuke and re-create db for restore
+    def nukeAndCreate(self):
+        # shut down this db and nuke it
+        self.db.close()
+        self.nuke_database()
+
+        # open a new, empty database
+        os.makedirs(config.DATABASE + '/files')
+        self.db = self.module.Database(config, 'admin')
+        setupSchema(self.db, 0, self.module)
+
+
     def testImportExport(self):
         # use the filtering setup to create a bunch of items
         ae, filt = self.filteringSetup()
@@ -1660,14 +1672,7 @@ def testImportExport(self):
                         klass.export_files('_test_export', id)
                 journals[cn] = klass.export_journals()
 
-            # shut down this db and nuke it
-            self.db.close()
-            self.nuke_database()
-
-            # open a new, empty database
-            os.makedirs(config.DATABASE + '/files')
-            self.db = self.module.Database(config, 'admin')
-            setupSchema(self.db, 0, self.module)
+            self.nukeAndCreate()
 
             # import
             for cn, items in export.items():
@@ -1730,6 +1735,58 @@ def testImportExport(self):
         newid = self.db.user.create(username='testing')
         assert newid > maxid
 
+    # test import/export via admin interface
+    def testAdminImportExport(self):
+        import roundup.admin
+        import csv
+        # use the filtering setup to create a bunch of items
+        ae, filt = self.filteringSetup()
+        # create large field
+        self.db.priority.create(name = 'X' * 500)
+        self.db.config.CSV_FIELD_SIZE = 400
+        self.db.commit()
+        output = []
+        # ugly hack to get stderr output and disable stdout output
+        # during regression test. Depends on roundup.admin not using
+        # anything but stdout/stderr from sys (which is currently the
+        # case)
+        def stderrwrite(s):
+            output.append(s)
+        roundup.admin.sys = MockNull ()
+        try:
+            roundup.admin.sys.stderr.write = stderrwrite
+            tool = roundup.admin.AdminTool()
+            home = '.'
+            tool.tracker_home = home
+            tool.db = self.db
+            tool.verbose = False
+            tool.do_export (['_test_export'])
+            self.assertEqual(len(output), 2)
+            self.assertEqual(output [1], '\n')
+            self.failUnless(output [0].startswith
+                ('Warning: config csv_field_size should be at least'))
+            self.failUnless(int(output[0].split()[-1]) > 500)
+
+            if hasattr(roundup.admin.csv, 'field_size_limit'):
+                self.nukeAndCreate()
+                self.db.config.CSV_FIELD_SIZE = 400
+                tool = roundup.admin.AdminTool()
+                tool.tracker_home = home
+                tool.db = self.db
+                tool.verbose = False
+                self.assertRaises(csv.Error, tool.do_import, ['_test_export'])
+
+            self.nukeAndCreate()
+            self.db.config.CSV_FIELD_SIZE = 3200
+            tool = roundup.admin.AdminTool()
+            tool.tracker_home = home
+            tool.db = self.db
+            tool.verbose = False
+            tool.do_import(['_test_export'])
+        finally:
+            roundup.admin.sys = sys
+            shutil.rmtree('_test_export')
+
     def testAddProperty(self):
         self.db.issue.create(title="spam", status='1')
         self.db.commit()