fix some column datatypes in postgresql and mysql

(bugs [SF#962611], [SF#959177] and [SF#964231])

Author: Richard Jones

CHANGES.txt

@@ -18,6 +18,8 @@ Fixed:
 - fixed grouping by a NULL Link value
 - fixed anydbm import/export (sf bugs 965216, 964457, 964450)
 - fix python 2.3.3 strftime deprecation warning (sf patch 968398)
+- fix some column datatypes in postgresql and mysql (sf bugs 962611,
+  959177 and 964231)
 
 
 2004-05-28 0.7.3

roundup/backends/back_mysql.py

@@ -337,6 +337,13 @@ def add_new_columns_v2(self):
             # change things 
             self.database_schema['tables'][cn] = klass.schema()
 
+    def fix_version_2_tables(self):
+        # Convert journal date column to TIMESTAMP, params column to TEXT
+        self._convert_journal_tables()
+
+        # Convert all String properties to TEXT
+        self._convert_string_properties()
+
     def __repr__(self):
         return '<myroundsql 0x%x>'%id(self)
 
@@ -353,11 +360,7 @@ def sql_index_exists(self, table_name, index_name):
                 return 1
         return 0
 
-    def save_dbschema(self, schema):
-        s = repr(self.database_schema)
-        self.sql('INSERT INTO schema VALUES (%s)', (s,))
-    
-    def create_class_table(self, spec):
+    def create_class_table(self, spec, create_sequence=True):
         cols, mls = self.determine_columns(spec.properties.items())
 
         # add on our special columns
@@ -391,11 +394,14 @@ def drop_class_table_indexes(self, cn, key):
             self.cursor.execute(index_sql)
 
     def create_journal_table(self, spec):
+        ''' create the journal table for a class given the spec and 
+            already-determined cols
+        '''
         # journal table
         cols = ','.join(['%s varchar'%x
             for x in 'nodeid date tag action params'.split()])
         sql = '''create table %s__journal (
-            nodeid integer, date timestamp, tag varchar(255),
+            nodeid integer, date datetime, tag varchar(255),
             action varchar(255), params text) type=%s'''%(
             spec.classname, self.mysql_backend)
         if __debug__:

roundup/backends/back_postgresql.py

@@ -122,12 +122,12 @@ def open_connection(self):
     def create_version_2_tables(self):
         # OTK store
         self.cursor.execute('''CREATE TABLE otks (otk_key VARCHAR(255),
-            otk_value VARCHAR(255), otk_time FLOAT(20))''')
+            otk_value VARCHAR(255), otk_time REAL)''')
         self.cursor.execute('CREATE INDEX otks_key_idx ON otks(otk_key)')
 
         # Sessions store
         self.cursor.execute('''CREATE TABLE sessions (
-            session_key VARCHAR(255), session_time FLOAT(20),
+            session_key VARCHAR(255), session_time REAL,
             session_value VARCHAR(255))''')
         self.cursor.execute('''CREATE INDEX sessions_key_idx ON
             sessions(session_key)''')
@@ -141,6 +141,24 @@ def create_version_2_tables(self):
             _textid integer)''')
         self.cursor.execute('CREATE INDEX words_word_idx ON __words(_word)')
 
+    def fix_version_2_tables(self):
+        # Convert journal date column to TIMESTAMP, params column to TEXT
+        self._convert_journal_tables()
+
+        # Convert all String properties to TEXT
+        self._convert_string_properties()
+
+        # convert session / OTK *_time columns to REAL
+        c = self.cursor
+        for name in ('otk', 'session'):
+            c.execute('drop index %ss_key_idx'%name)
+            c.execute('drop table %ss'%name)
+            c.execute('''CREATE TABLE %ss (%s_key VARCHAR(255),
+                %s_value VARCHAR(255), %s_time REAL)'''%(name, name, name,
+                name))
+            c.execute('CREATE INDEX %ss_key_idx ON %ss(%s_key)'%(name, name,
+                name))
+
     def add_actor_column(self):
         # update existing tables to have the new actor column
         tables = self.database_schema['tables']
@@ -162,11 +180,12 @@ def sql_index_exists(self, table_name, index_name):
         self.cursor.execute(sql, (table_name, index_name))
         return self.cursor.fetchone()[0]
 
-    def create_class_table(self, spec):
-        sql = 'CREATE SEQUENCE _%s_ids'%spec.classname
-        if __debug__:
-            print >>hyperdb.DEBUG, 'create_class_table', (self, sql)
-        self.cursor.execute(sql)
+    def create_class_table(self, spec, create_sequence=True):
+        if create_sequence:
+            sql = 'CREATE SEQUENCE _%s_ids'%spec.classname
+            if __debug__:
+                print >>hyperdb.DEBUG, 'create_class_table', (self, sql)
+            self.cursor.execute(sql)
 
         return rdbms_common.Database.create_class_table(self, spec)
 
@@ -181,25 +200,6 @@ def drop_class_table(self, cn):
             print >>hyperdb.DEBUG, 'drop_class', (self, sql)
         self.cursor.execute(sql)
 
-    def create_journal_table(self, spec):
-        cols = ',' . join(['"%s" VARCHAR(255)'%x
-          for x in 'nodeid date tag action params' . split()])
-        sql  = 'CREATE TABLE "%s__journal" (%s)'%(spec.classname, cols)
-        if __debug__:
-            print >>hyperdb.DEBUG, 'create_journal_table', (self, sql)
-        self.cursor.execute(sql)
-        self.create_journal_table_indexes(spec)
-
-    def create_multilink_table(self, spec, ml):
-        sql = '''CREATE TABLE "%s_%s" (linkid VARCHAR(255),
-            nodeid VARCHAR(255))'''%(spec.classname, ml)
-
-        if __debug__:
-            print >>hyperdb.DEBUG, 'create_class', (self, sql)
-
-        self.cursor.execute(sql)
-        self.create_multilink_table_indexes(spec, ml)
-
     def newid(self, classname):
         sql = "select nextval('_%s_ids') from dual"%classname
         if __debug__:

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.104 2004-05-29 01:35:27 richard Exp $
+# $Id: rdbms_common.py,v 1.105 2004-06-09 05:13:14 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -143,10 +143,11 @@ def load_dbschema(self):
         else:
             self.database_schema = {}
 
-    def save_dbschema(self, schema):
+    def save_dbschema(self):
         ''' Save the schema definition that the database currently implements
         '''
         s = repr(self.database_schema)
+        self.sql('delete from schema')
         self.sql('insert into schema values (%s)', (s,))
 
     def post_init(self):
@@ -178,8 +179,7 @@ def post_init(self):
 
         # update the database version of the schema
         if save:
-            self.sql('delete from schema')
-            self.save_dbschema(self.database_schema)
+            self.save_dbschema()
 
         # reindex the db if necessary
         if self.indexer.should_reindex():
@@ -190,7 +190,7 @@ def post_init(self):
 
     # update this number when we need to make changes to the SQL structure
     # of the backen database
-    current_db_version = 2
+    current_db_version = 3
     def upgrade_db(self):
         ''' Update the SQL database to reflect changes in the backend code.
 
@@ -213,9 +213,65 @@ def upgrade_db(self):
             # database
             self.create_version_2_tables()
 
+        if version == 2:
+            self.fix_version_2_tables()
+
         self.database_schema['version'] = self.current_db_version
         return 1
 
+    def fix_version_2_tables(self):
+        '''Default (used by sqlite): NOOP'''
+        pass
+
+    def _convert_journal_tables(self):
+        '''Get current journal table contents, drop the table and re-create'''
+        c = self.cursor
+        cols = ','.join('nodeid date tag action params'.split())
+        for klass in self.classes.values():
+            # slurp and drop
+            sql = 'select %s from %s__journal order by date'%(cols,
+                klass.classname)
+            c.execute(sql)
+            contents = c.fetchall()
+            self.drop_journal_table_indexes(klass.classname)
+            c.execute('drop table %s__journal'%klass.classname)
+
+            # re-create and re-populate
+            self.create_journal_table(klass)
+            a = self.arg
+            sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
+                klass.classname, cols, a, a, a, a, a)
+            for row in contents:
+                # no data conversion needed
+                self.cursor.execute(sql, row)
+
+    def _convert_string_properties(self):
+        '''Get current Class tables that contain String properties, and
+        convert the VARCHAR columns to TEXT'''
+        c = self.cursor
+        for klass in self.classes.values():
+            # slurp and drop
+            cols, mls = self.determine_columns(klass.properties.items())
+            scols = ','.join([i[0] for i in cols])
+            sql = 'select id,%s from _%s'%(scols, klass.classname)
+            c.execute(sql)
+            contents = c.fetchall()
+            self.drop_class_table_indexes(klass.classname, klass.getkey())
+            c.execute('drop table _%s'%klass.classname)
+
+            # re-create and re-populate
+            self.create_class_table(klass, create_sequence=False)
+            a = ','.join([self.arg for i in range(len(cols)+1)])
+            sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
+                scols, a)
+            for row in contents:
+                l = []
+                for entry in row:
+                    # mysql will already be a string - psql needs "help"
+                    if entry is not None and not isinstance(entry, type('')):
+                        entry = str(entry)
+                    l.append(entry)
+                self.cursor.execute(sql, l)
 
     def refresh_database(self):
         self.post_init()
@@ -227,7 +283,7 @@ def reindex(self):
         self.indexer.save_index()
 
     hyperdb_to_sql_datatypes = {
-        hyperdb.String : 'VARCHAR(255)',
+        hyperdb.String : 'TEXT',
         hyperdb.Date   : 'TIMESTAMP',
         hyperdb.Link   : 'INTEGER',
         hyperdb.Interval  : 'VARCHAR(255)',
@@ -466,7 +522,7 @@ def create_multilink_table(self, spec, ml):
             given by the spec
         '''
         # create the table
-        sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
+        sql = 'create table %s_%s (linkid varchar(255), nodeid varchar(255))'%(
             spec.classname, ml)
         if __debug__:
             print >>hyperdb.DEBUG, 'create_class', (self, sql)