Use server-side cursor for postgres in some cases

In filter, filter_iter, and _materialize_multilinks, use named cursor
with postgresql. This turns of client-side cursor handling and avoids
*large* roundup process (or wsgi process) in case of large results.
Fixes issue2551114.

Author: schlatterbeck

CHANGES.txt

@@ -66,6 +66,10 @@ Fixed:
 - in rest interface, fix uncaught exceptions when parsing invalid
   Content-Type and Accept headers. Document response formats more
   fully in doc/rest.txt.
+- in filter, filter_iter and _materialize_multilinks, use named cursor
+  with postgresql. This turns of client-side cursor handling and avoids
+  *large* roundup process (or wsgi process) in case of large results.
+  Fixes issue2551114.
 
 Features:
 - issue2550522 - Add 'filter' command to command-line

roundup/backends/back_postgresql.py

@@ -47,7 +47,7 @@ def connection_dict(config, dbnamestr=None):
 def db_create(config):
     """Clear all database contents and drop database itself"""
     command = "CREATE DATABASE \"%s\" WITH ENCODING='UNICODE'"%config.RDBMS_NAME
-    if config.RDBMS_TEMPLATE :
+    if config.RDBMS_TEMPLATE:
         command = command + " TEMPLATE=%s" % config.RDBMS_TEMPLATE
     logging.getLogger('roundup.hyperdb').info(command)
     db_command(config, command)
@@ -99,13 +99,13 @@ def pg_command(cursor, command):
         cursor.execute(command)
     except psycopg2.DatabaseError as err:
         response = str(err).split('\n')[0]
-        if "FATAL" not in response :
+        if "FATAL" not in response:
             msgs = (
                 'is being accessed by other users',
                 'could not serialize access due to concurrent update',
             )
-            for msg in msgs :
-                if msg in response :
+            for msg in msgs:
+                if msg in response:
                     time.sleep(0.1)
                     return 0
         raise RuntimeError (response)
@@ -167,6 +167,18 @@ def sql_open_connection(self):
 
         return (conn, cursor)
 
+    def sql_new_cursor(self, name='default', conn=None, *args, **kw):
+        """ Create new cursor, this may need additional parameters for
+            performance optimization for different backends.
+        """
+        use_name = self.config.RDBMS_SERVERSIDE_CURSOR
+        kw = {}
+        if use_name:
+            kw['name'] = name
+        if conn is None:
+            conn = self.conn
+        return conn.cursor(*args, **kw)
+
     def open_connection(self):
         if not db_exists(self.config):
             db_create(self.config)

roundup/backends/rdbms_common.py

@@ -1146,14 +1146,14 @@ def _materialize_multilink(self, classname, nodeid, node, propname):
                     joi = ', %s' % tn2
                     w = ' and %s.%s=%s.id and %s.__retired__=0'%(tn, lid,
                         tn2, tn2)
+            cursor = self.sql_new_cursor(name='_materialize_multilink')
             sql = 'select %s from %s%s where %s=%s%s' %(lid, tn, joi, nid,
                 self.arg, w)
-            self.sql(sql, (nodeid,))
-            # extract the first column from the result
-            # XXX numeric ids
-            items = [int(x[0]) for x in self.cursor.fetchall()]
-            items.sort()
-            node[propname] = [str(x) for x in items]
+            self.sql(sql, (nodeid,), cursor)
+            # Reduce this to only the first row (the ID), this can save a
+            # lot of space for large query results (not using fetchall)
+            node[propname] = [str(x) for x in sorted(int(r[0]) for r in cursor)]
+            cursor.close()
 
     def _materialize_multilinks(self, classname, nodeid, node, props=None):
         """ get all Multilinks of a node (lazy eval may have skipped this)
@@ -1461,6 +1461,14 @@ def sql_commit(self):
         # open a new cursor for subsequent work
         self.cursor = self.conn.cursor()
 
+    def sql_new_cursor(self, conn=None, *args, **kw):
+        """ Create new cursor, this may need additional parameters for
+            performance optimization for different backends.
+        """
+        if conn is None:
+            conn = self.conn
+        return conn.cursor()
+
     def commit(self):
         """ Commit the current transactions.
 
@@ -2854,18 +2862,26 @@ def filter(self, search_matches, filterspec, sort=[], group=[],
             return []
         proptree, sql, args = sq
 
-        self.db.sql(sql, args)
-        l = self.db.sql_fetchall()
+        cursor = self.db.sql_new_cursor(name='filter')
+        self.db.sql(sql, args, cursor)
+        # Reduce this to only the first row (the ID), this can save a
+        # lot of space for large query results (not using fetchall)
+        # We cannot do this if sorting by multilink
+        if proptree.tree_sort_done:
+            l = [str(row[0]) for row in cursor]
+        else:
+            l = cursor.fetchall()
+        cursor.close()
 
+        # Multilink sorting
         # Compute values needed for sorting in proptree.sort
-        for p in proptree:
-            if hasattr(p, 'auxcol'):
-                p.sort_ids = [row[p.auxcol] for row in l]
-                p.sort_result = p._sort_repr (p.propclass.sort_repr, p.sort_ids)
-        # return the IDs (the first column)
-        # XXX numeric ids
-        l = [str(row[0]) for row in l]
-        l = proptree.sort (l)
+        if not proptree.tree_sort_done:
+            for p in proptree:
+                if hasattr(p, 'auxcol'):
+                    p.sort_ids = [row[p.auxcol] for row in l]
+                    p.sort_result = p._sort_repr \
+                        (p.propclass.sort_repr, p.sort_ids)
+            l = proptree.sort ([str(row[0]) for row in l])
 
         if __debug__:
             self.db.stats['filtering'] += (time.time() - start_t)
@@ -2890,7 +2906,7 @@ def filter_iter(self, search_matches, filterspec, sort=[], group=[],
         if sq is None:
             return
         proptree, sql, args = sq
-        cursor = self.db.conn.cursor()
+        cursor = self.db.sql_new_cursor(name='filter_iter')
         self.db.sql(sql, args, cursor)
         classes = {}
         for p in proptree:
@@ -2922,6 +2938,7 @@ def filter_iter(self, search_matches, filterspec, sort=[], group=[],
                     node[propname] = value
                 self.db._cache_save(key, node)
             yield str(row[0])
+        cursor.close()
 
     def filter_sql(self, sql):
         """Return a list of the ids of the items in this class that match

roundup/configuration.py

@@ -1082,6 +1082,10 @@ def str2value(self, value):
             "Database isolation level, currently supported for\n"
             "PostgreSQL and mysql. See, e.g.,\n"
             "http://www.postgresql.org/docs/9.1/static/transaction-iso.html"),
+        (BooleanOption, "serverside_cursor", "yes",
+            "Set the database cursor for filter queries to serverside\n"
+            "cursor, this avoids caching large amounts of data in the\n"
+            "client. This option only applies for the postgresql backend."),
     ), "Settings in this section (except for backend) are used"
         " by RDBMS backends only."
     ),