relaxed CVS importing (feature [SF#693277])

Author: Richard Jones

CHANGES.txt

@@ -40,6 +40,7 @@ Feature:
   file serving
 - added Node.get() method
 - nicer page titles (sf feature 65197)
+- relaxed CVS importing (sf feature 693277)
 
 
 Fixed:

roundup/admin.py

@@ -16,7 +16,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: admin.py,v 1.41 2003-03-06 06:06:49 richard Exp $
+# $Id: admin.py,v 1.42 2003-03-06 07:33:29 richard Exp $
 
 '''Administration commands for maintaining Roundup trackers.
 '''
@@ -921,7 +921,9 @@ def do_export(self, args):
             properties = cl.getprops()
             propnames = properties.keys()
             propnames.sort()
-            print >> f, p.join(propnames)
+            l = propnames[:]
+            l.append('is retired')
+            print >> f, p.join(l)
 
             # all nodes for this class (not using list() 'cos it doesn't
             # include retired nodes)
@@ -965,14 +967,16 @@ class to import.
             cl = self.get_class(classname)
             p = csv.parser(field_sep=':')
             file_props = p.parse(f.readline())
-            properties = cl.getprops()
-            propnames = properties.keys()
-            propnames.sort()
-            m = file_props[:]
-            m.sort()
-            if m != propnames:
-                raise UsageError, _('Import file doesn\'t define the same '
-                    'properties as "%(arg0)s".')%{'arg0': args[0]}
+
+# XXX we don't _really_ need to do this...
+#            properties = cl.getprops()
+#            propnames = properties.keys()
+#            propnames.sort()
+#            m = file_props[:]
+#            m.sort()
+#            if m != propnames:
+#                raise UsageError, _('Import file doesn\'t define the same '
+#                    'properties as "%(arg0)s".')%{'arg0': args[0]}
 
             # loop through the file and create a node for each entry
             maxid = 1
@@ -989,7 +993,7 @@ class to import.
                         raise ValueError, "Unexpected EOF during CSV parse"
 
                 # do the import and figure the current highest nodeid
-                maxid = max(maxid, int(cl.import_list(propnames, l)))
+                maxid = max(maxid, int(cl.import_list(file_props, l)))
 
             print 'setting', classname, maxid+1
             self.db.setid(classname, str(maxid+1))

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.108 2003-03-03 21:05:17 richard Exp $
+#$Id: back_anydbm.py,v 1.109 2003-03-06 07:33:29 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -920,22 +920,29 @@ def import_list(self, propnames, proplist):
 
         # make the new node's property map
         d = {}
+        newid = None
         for i in range(len(propnames)):
-            # Use eval to reverse the repr() used to output the CSV
-            value = eval(proplist[i])
-
             # Figure the property for this column
             propname = propnames[i]
-            prop = properties[propname]
+
+            # Use eval to reverse the repr() used to output the CSV
+            value = eval(proplist[i])
 
             # "unmarshal" where necessary
             if propname == 'id':
                 newid = value
                 continue
+            elif propname == 'is retired':
+                # is the item retired?
+                if int(value):
+                    d[self.db.RETIRED_FLAG] = 1
+                continue
             elif value is None:
                 # don't set Nones
                 continue
-            elif isinstance(prop, hyperdb.Date):
+
+            prop = properties[propname]
+            if isinstance(prop, hyperdb.Date):
                 value = date.Date(value)
             elif isinstance(prop, hyperdb.Interval):
                 value = date.Interval(value)
@@ -945,9 +952,9 @@ def import_list(self, propnames, proplist):
                 value = pwd
             d[propname] = value
 
-        # check retired flag
-        if int(proplist[-1]):
-            d[self.db.RETIRED_FLAG] = 1
+        # get a new id if necessary
+        if newid is None:
+            newid = self.db.newid(self.classname)
 
         # add the node and journal
         self.db.addnode(self.classname, newid, d)

roundup/backends/back_metakit.py

@@ -1106,11 +1106,18 @@ def import_list(self, propnames, proplist):
             value = eval(proplist[i])
             if not value:
                 continue
+
             propname = propnames[i]
-            prop = properties[propname]
             if propname == 'id':
                 newid = value = int(value)
-            elif isinstance(prop, hyperdb.Date):
+            elif propname == 'is retired':
+                # is the item retired?
+                if int(value):
+                    d['_isdel'] = 1
+                continue
+
+            prop = properties[propname]
+            if isinstance(prop, hyperdb.Date):
                 value = int(calendar.timegm(value))
             elif isinstance(prop, hyperdb.Interval):
                 value = str(date.Interval(value))
@@ -1124,16 +1131,23 @@ def import_list(self, propnames, proplist):
                 # we handle multilinks separately
                 continue
             d[propname] = value
-        # is the item retired?
-        if int(proplist[-1]):
-            d['_isdel'] = 1
+
+        # possibly make a new node
+        if not d.has_key('id'):
+            d['id'] = newid = self.maxid
+            self.maxid += 1
+
+        # save off the node
         view.append(d)
 
+        # fix up multilinks
         ndx = view.find(id=newid)
         row = view[ndx]
         for i in range(len(propnames)):
             value = eval(proplist[i])
             propname = propnames[i]
+            if propname == 'is retired':
+                continue
             prop = properties[propname]
             if not isinstance(prop, hyperdb.Multilink):
                 continue

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.39 2003-03-06 06:03:51 richard Exp $
+# $Id: rdbms_common.py,v 1.40 2003-03-06 07:33:29 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -450,8 +450,10 @@ def addnode(self, classname, nodeid, node):
 
         # default the non-multilink columns
         for col, prop in cl.properties.items():
-            if not isinstance(col, Multilink):
-                if not node.has_key(col):
+            if not node.has_key(col):
+                if isinstance(prop, Multilink):
+                    node[col] = []
+                else:
                     node[col] = None
 
         # clear this node out of the cache if it's in there
@@ -1100,19 +1102,27 @@ def import_list(self, propnames, proplist):
 
         # make the new node's property map
         d = {}
+        retire = 0
+        newid = None
         for i in range(len(propnames)):
             # Use eval to reverse the repr() used to output the CSV
             value = eval(proplist[i])
 
             # Figure the property for this column
             propname = propnames[i]
-            prop = properties[propname]
 
             # "unmarshal" where necessary
             if propname == 'id':
                 newid = value
                 continue
-            elif value is None:
+            elif propname == 'is retired':
+                # is the item retired?
+                if int(value):
+                    retire = 1
+                continue
+
+            prop = properties[propname]
+            if value is None:
                 # don't set Nones
                 continue
             elif isinstance(prop, hyperdb.Date):
@@ -1125,8 +1135,12 @@ def import_list(self, propnames, proplist):
                 value = pwd
             d[propname] = value
 
+        # get a new id if necessary
+        if newid is None:
+            newid = self.db.newid(self.classname)
+
         # retire?
-        if int(proplist[-1]):
+        if retire:
             # use the arg for __retired__ to cope with any odd database type
             # conversion (hello, sqlite)
             sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,