full database export and import is done

Author: Richard Jones

roundup/admin.py

@@ -16,7 +16,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: admin.py,v 1.22 2002-08-16 04:26:42 richard Exp $
+# $Id: admin.py,v 1.23 2002-08-19 02:53:27 richard Exp $
 
 import sys, os, getpass, getopt, re, UserDict, shlex, shutil
 try:
@@ -808,13 +808,19 @@ def do_retire(self, args):
         return 0
 
     def do_export(self, args):
-        '''Usage: export [class[,class]] destination_dir
+        '''Usage: export [class[,class]] export_dir
         Export the database to tab-separated-value files.
 
         This action exports the current data from the database into
         tab-separated-value files that are placed in the nominated destination
         directory. The journals are not exported.
         '''
+        # we need the CSV module
+        if csv is None:
+            raise UsageError, \
+                _('Sorry, you need the csv module to use this function.\n'
+                'Get it from: http://www.object-craft.com.au/projects/csv/')
+
         # grab the directory to export to
         if len(args) < 1:
             raise UsageError, _('Not enough arguments supplied')
@@ -827,56 +833,38 @@ def do_export(self, args):
             classes = self.db.classes.keys()
 
         # use the csv parser if we can - it's faster
-        if csv is not None:
-            p = csv.parser(field_sep=':')
+        p = csv.parser(field_sep=':')
 
         # do all the classes specified
         for classname in classes:
             cl = self.get_class(classname)
             f = open(os.path.join(dir, classname+'.csv'), 'w')
-            f.write(':'.join(cl.properties.keys()) + '\n')
+            properties = cl.getprops()
+            propnames = properties.keys()
+            propnames.sort()
+            print >> f, p.join(propnames)
 
             # all nodes for this class
-            properties = cl.getprops()
             for nodeid in cl.list():
-                l = []
-                for prop, proptype in properties:
-                    value = cl.get(nodeid, prop)
-                    # convert data where needed
-                    if isinstance(proptype, hyperdb.Date):
-                        value = value.get_tuple()
-                    elif isinstance(proptype, hyperdb.Interval):
-                        value = value.get_tuple()
-                    elif isinstance(proptype, hyperdb.Password):
-                        value = str(value)
-                    l.append(repr(value))
-
-                # now write
-                if csv is not None:
-                   f.write(p.join(l) + '\n')
-                else:
-                   # escape the individual entries to they're valid CSV
-                   m = []
-                   for entry in l:
-                      if '"' in entry:
-                          entry = '""'.join(entry.split('"'))
-                      if ':' in entry:
-                          entry = '"%s"'%entry
-                      m.append(entry)
-                   f.write(':'.join(m) + '\n')
+                print >>f, p.join(cl.export_list(propnames, nodeid))
         return 0
 
     def do_import(self, args):
-        '''Usage: import class file
-        Import the contents of the tab-separated-value file.
-
-        The file must define the same properties as the class (including having
-        a "header" line with those property names.) The new nodes are added to
-        the existing database - if you want to create a new database using the
-        imported data, then create a new database (or, tediously, retire all
-        the old data.)
+        '''Usage: import import_dir
+        Import a database from the directory containing CSV files, one per
+        class to import.
+
+        The files must define the same properties as the class (including having
+        a "header" line with those property names.)
+
+        The imported nodes will have the same nodeid as defined in the
+        import file, thus replacing any existing content.
+
+        XXX The new nodes are added to the existing database - if you want to
+        XXX create a new database using the imported data, then create a new
+        XXX database (or, tediously, retire all the old data.)
         '''
-        if len(args) < 2:
+        if len(args) < 1:
             raise UsageError, _('Not enough arguments supplied')
         if csv is None:
             raise UsageError, \
@@ -885,56 +873,44 @@ def do_import(self, args):
 
         from roundup import hyperdb
 
-        # ensure that the properties and the CSV file headings match
-        classname = args[0]
-        cl = self.get_class(classname)
-        f = open(args[1])
-        p = csv.parser(field_sep=':')
-        file_props = p.parse(f.readline())
-        props = cl.properties.keys()
-        m = file_props[:]
-        m.sort()
-        props.sort()
-        if m != props:
-            raise UsageError, _('Import file doesn\'t define the same '
-                'properties as "%(arg0)s".')%{'arg0': args[0]}
-
-        # loop through the file and create a node for each entry
-        n = range(len(props))
-        while 1:
-            line = f.readline()
-            if not line: break
+        for file in os.listdir(args[0]):
+            f = open(os.path.join(args[0], file))
 
-            # parse lines until we get a complete entry
+            # get the classname
+            classname = os.path.splitext(file)[0]
+
+            # ensure that the properties and the CSV file headings match
+            cl = self.get_class(classname)
+            p = csv.parser(field_sep=':')
+            file_props = p.parse(f.readline())
+            properties = cl.getprops()
+            propnames = properties.keys()
+            propnames.sort()
+            m = file_props[:]
+            m.sort()
+            if m != propnames:
+                raise UsageError, _('Import file doesn\'t define the same '
+                    'properties as "%(arg0)s".')%{'arg0': args[0]}
+
+            # loop through the file and create a node for each entry
+            maxid = 1
             while 1:
-                l = p.parse(line)
-                if l: break
                 line = f.readline()
-                if not line:
-                    raise ValueError, "Unexpected EOF during CSV parse"
-
-            # make the new node's property map
-            d = {}
-            for i in n:
-                # Use eval to reverse the repr() used to output the CSV
-                value = eval(l[i])
-                # Figure the property for this column
-                key = file_props[i]
-                proptype = cl.properties[key]
-                # Convert for property type
-                if isinstance(proptype, hyperdb.Date):
-                    value = date.Date(value)
-                elif isinstance(proptype, hyperdb.Interval):
-                    value = date.Interval(value)
-                elif isinstance(proptype, hyperdb.Password):
-                    pwd = password.Password()
-                    pwd.unpack(value)
-                    value = pwd
-                if value is not None:
-                    d[key] = value
-
-            # and create the new node
-            apply(cl.create, (), d)
+                if not line: break
+
+                # parse lines until we get a complete entry
+                while 1:
+                    l = p.parse(line)
+                    if l: break
+                    line = f.readline()
+                    if not line:
+                        raise ValueError, "Unexpected EOF during CSV parse"
+
+                # do the import and figure the current highest nodeid
+                maxid = max(maxid, int(cl.import_list(propnames, l)))
+
+            print 'setting', classname, maxid
+            self.db.setid(classname, str(maxid))
         return 0
 
     def do_pack(self, args):
@@ -1170,6 +1146,9 @@ def main(self):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.22  2002/08/16 04:26:42  richard
+# moving towards full database export
+#
 # Revision 1.21  2002/08/01 01:07:37  richard
 # include info about new user roles
 #

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.60 2002-08-19 00:23:19 richard Exp $
+#$Id: back_anydbm.py,v 1.61 2002-08-19 02:53:27 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -215,6 +215,16 @@ def newid(self, classname):
         release_lock(lock)
         return newid
 
+    def setid(self, classname, setid):
+        ''' Set the id counter: used during import of database
+        '''
+        # open the ids DB - create if if doesn't exist
+        lock = self.lockdb('_ids')
+        db = self.opendb('_ids', 'c')
+        db[classname] = str(setid)
+        db.close()
+        release_lock(lock)
+
     #
     # Nodes
     #
@@ -593,13 +603,27 @@ def getCachedJournalDB(self, classname):
         return self.databases[db_name]
 
     def doSaveJournal(self, classname, nodeid, action, params):
-        # serialise first
+        # handle supply of the special journalling parameters (usually
+        # supplied on importing an existing database)
+        if params.has_key('creator'):
+            journaltag = self.user.get(params['creator'], 'username')
+            del params['creator']
+        else:
+            journaltag = self.journaltag
+        if params.has_key('created'):
+            journaldate = params['created'].get_tuple()
+            del params['created']
+        else:
+            journaldate = date.Date().get_tuple()
+        if params.has_key('activity'):
+            del params['activity']
+
+        # serialise the parameters now
         if action in ('set', 'create'):
             params = self.serialise(classname, params)
 
         # create the journal entry
-        entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
-            params)
+        entry = (nodeid, journaldate, journaltag, action, params)
 
         if __debug__:
             print >>hyperdb.DEBUG, 'doSaveJournal', entry
@@ -845,6 +869,67 @@ def create(self, **propvalues):
 
         return newid
 
+    def export_list(self, propnames, nodeid):
+        ''' Export a node - generate a list of CSV-able data in the order
+            specified by propnames for the given node.
+        '''
+        properties = self.getprops()
+        l = []
+        for prop in propnames:
+            proptype = properties[prop]
+            value = self.get(nodeid, prop)
+            # "marshal" data where needed
+            if isinstance(proptype, hyperdb.Date):
+                value = value.get_tuple()
+            elif isinstance(proptype, hyperdb.Interval):
+                value = value.get_tuple()
+            elif isinstance(proptype, hyperdb.Password):
+                value = str(value)
+            l.append(repr(value))
+        return l
+
+    def import_list(self, propnames, proplist):
+        ''' Import a node - all information including "id" is present and
+            should not be sanity checked. Triggers are not triggered. The
+            journal should be initialised using the "creator" and "created"
+            information.
+
+            Return the nodeid of the node imported.
+        '''
+        if self.db.journaltag is None:
+            raise DatabaseError, 'Database open read-only'
+        properties = self.getprops()
+
+        # make the new node's property map
+        d = {}
+        for i in range(len(propnames)):
+            # Use eval to reverse the repr() used to output the CSV
+            value = eval(proplist[i])
+
+            # Figure the property for this column
+            propname = propnames[i]
+            prop = properties[propname]
+
+            # "unmarshal" where necessary
+            if propname == 'id':
+                newid = value
+                continue
+            elif isinstance(prop, hyperdb.Date):
+                value = date.Date(value)
+            elif isinstance(prop, hyperdb.Interval):
+                value = date.Interval(value)
+            elif isinstance(prop, hyperdb.Password):
+                pwd = password.Password()
+                pwd.unpack(value)
+                value = pwd
+            if value is not None:
+                d[propname] = value
+
+        # add
+        self.db.addnode(self.classname, newid, d)
+        self.db.addjournal(self.classname, newid, 'create', d)
+        return newid
+
     def get(self, nodeid, propname, default=_marker, cache=1):
         """Get the value of a property on an existing node of this class.
 
@@ -1730,6 +1815,25 @@ def create(self, **propvalues):
         self.db.storefile(self.classname, newid, None, content)
         return newid
 
+    def import_list(self, propnames, proplist):
+        ''' Trap the "content" property...
+        '''
+        # dupe this list so we don't affect others
+        propnames = propnames[:]
+
+        # extract the "content" property from the proplist
+        i = propnames.index('content')
+        content = proplist[i]
+        del propnames[i]
+        del proplist[i]
+
+        # do the normal import
+        newid = Class.import_list(self, propnames, proplist)
+
+        # save off the "content" file
+        self.db.storefile(self.classname, newid, None, content)
+        return newid
+
     def get(self, nodeid, propname, default=_marker, cache=1):
         ''' trap the content propname and get it from the file
         '''
@@ -1803,6 +1907,9 @@ def __init__(self, db, classname, **properties):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.60  2002/08/19 00:23:19  richard
+#handle "unset" initial Link values (!)
+#
 #Revision 1.59  2002/08/16 04:28:13  richard
 #added is_retired query to Class
 #