issue2550853 - better error handling and cleanup on some postgres

  tests by Stuart McGraw.

My postgres tests passed both before and after I applied his patch,
but the code change seems rational.

issue2086536 - back_postgresql: fixing pg_command and prefering
  psycopg2. Patch done by Philipp Gortan (mephinet). His patch
  also improves handling of retryable errors. Applied and
  edited by John Rouillard. Edits included removing support for
  psycopg1. See:
    https://sourceforge.net/p/roundup/mailman/message/32855027/
  for rational for dropping it

Again all 137 postgres tests pass.

Author: rouilj

CHANGES.txt

@@ -221,6 +221,15 @@ Fixed:
   String fileds is 'no' not 'yes'. So to get a new string field into
   the full text/all text index you need to use String(indexme='yes').
   Reported by Michael Belleville. (John Rouillard)
+- issue2550853 - better error handling and cleanup on some postgres
+  tests by Stuart McGraw.
+- issue2086536 - back_postgresql: fixing pg_command and prefering
+  psycopg2. Patch done by Philipp Gortan (mephinet). His patch
+  also improves handling of retryable errors. Applied and
+  edited by John Rouillard. Edits included removing support for
+  psycopg1. See:
+    https://sourceforge.net/p/roundup/mailman/message/32855027/
+  for rational for dropping it.
 
 2016-01-11: 1.5.1
 

roundup/backends/back_postgresql.py

@@ -12,27 +12,16 @@
 ISOLATION_LEVEL_READ_COMMITTED = None
 ISOLATION_LEVEL_REPEATABLE_READ = None
 ISOLATION_LEVEL_SERIALIZABLE = None
-try:
-    import psycopg
-    from psycopg import QuotedString
-    from psycopg import ProgrammingError
-    TransactionRollbackError = ProgrammingError
-    try:
-        from psycopg.extensions import ISOLATION_LEVEL_READ_UNCOMMITTED
-        from psycopg.extensions import ISOLATION_LEVEL_READ_COMMITTED
-        from psycopg.extensions import ISOLATION_LEVEL_REPEATABLE_READ
-        from psycopg.extensions import ISOLATION_LEVEL_SERIALIZABLE
-    except ImportError:
-        pass
-except:
-    from psycopg2 import psycopg1 as psycopg
-    from psycopg2.extensions import QuotedString
-    from psycopg2.extensions import ISOLATION_LEVEL_READ_UNCOMMITTED
-    from psycopg2.extensions import ISOLATION_LEVEL_READ_COMMITTED
-    from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
-    from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
-    from psycopg2.psycopg1 import ProgrammingError
-    from psycopg2.extensions import TransactionRollbackError
+
+from psycopg2 import psycopg1 as psycopg
+from psycopg2.extensions import QuotedString
+from psycopg2.extensions import ISOLATION_LEVEL_READ_UNCOMMITTED
+from psycopg2.extensions import ISOLATION_LEVEL_READ_COMMITTED
+from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
+from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
+from psycopg2.psycopg1 import ProgrammingError
+from psycopg2.extensions import TransactionRollbackError
+
 import logging
 
 from roundup import hyperdb, date
@@ -108,23 +97,18 @@ def pg_command(cursor, command):
     '''
     try:
         cursor.execute(command)
-    except psycopg.ProgrammingError, err:
+    except psycopg.DatabaseError, err:
         response = str(err).split('\n')[0]
-        if response.find('FATAL') != -1:
-            raise RuntimeError(response)
-        else:
-            msgs = [
+        if "FATAL" not in response :
+            msgs = (
                 'is being accessed by other users',
                 'could not serialize access due to concurrent update',
-            ]
-            can_retry = 0
-            for msg in msgs:
-                if response.find(msg) == -1:
-                    can_retry = 1
-            if can_retry:
-                time.sleep(1)
-                return 0
-            raise RuntimeError(response)
+            )
+            for msg in msgs :
+                if msg in response :
+                    time.sleep(0.1)
+                    return 0
+        raise RuntimeError (response)
     return 1
 
 def db_exists(config):
@@ -194,7 +178,7 @@ def open_connection(self):
 
         try:
             self.load_dbschema()
-        except psycopg.ProgrammingError, message:
+        except ProgrammingError, message:
             if str(message).find('schema') == -1:
                 raise
             self.rollback()
@@ -267,7 +251,7 @@ def sql_commit(self, fail_ok=False):
 
         try:
             self.conn.commit()
-        except psycopg.ProgrammingError, message:
+        except ProgrammingError, message:
             # we've been instructed that this commit is allowed to fail
             if fail_ok and str(message).endswith('could not serialize '
                     'access due to concurrent update'):

test/test_postgresql.py

@@ -25,6 +25,7 @@
 from db_test_base import ClassicInitBase, setupTracker
 
 from roundup.backends import get_backend, have_backend
+from roundup.backends.back_postgresql import psycopg
 
 if not have_backend('postgresql'):
     # FIX: workaround for a bug in pytest.mark.skip():
@@ -100,8 +101,12 @@ def setUp(self):
         db.close()
 
     def tearDown(self):
-        self.db1.close()
-        self.db2.close()
+        try:
+            self.db1.close()
+            self.db2.close()
+        except psycopg.InterfaceError, exc:
+            if 'connection already closed' in str(exc): pass
+            else: raise
         ClassicInitBase.tearDown(self)
         postgresqlOpener.tearDown(self)
 
@@ -117,9 +122,15 @@ def _test_journal(self, expected_journal):
         db1.commit()
         db1.close()
 
-        db2.issue.set (id, title='t2')
-        db2.commit()
-        db2.close()
+        # Test testConcurrentRepeatableRead is expected to raise
+        # an error when the db2.issue.set() call is executed. 
+        try:
+            db2.issue.set (id, title='t2')
+            db2.commit()    
+        finally:
+            # Make sure that the db2 connection is closed, even when
+            # an error is raised.
+            db2.close()
         self.db = self.tracker.open('admin')
         journal = self.db.getjournal('issue', id)
         for n, line in enumerate(journal):