merge from trunk. travisci xenil plus more

Author: rouilj

.travis.yml

@@ -1,18 +1,12 @@
+# check syntax using:
+# https://config.travis-ci.com/explore
+
 os: linux
 
 language: python
 
 cache: pip
 
-python:
-  - 2.7
-  - 3.4
-  - 3.6
-  - 3.7
-  - 3.8
-  - 3.9-dev
-  - nightly
-
 #I would like to build and test the maint-1.6 and trunk/default
 #but we need different environments for these:
 #  maint-1.6 only python 2, install only psycopg2 version with support for
@@ -23,29 +17,18 @@ branches:
 #    - maint-1.6
 
 dist:
- - xenial
-
-# Commented out stanza for bionic 18.04. Currently testing on
-# xenial 16.04.
-# Consider move to this after 2.1.0 release. Python 3.4
-# is not supported on bionic and 3.4 is obsolete. Was retained
-# because 3.4 was EPEL version for centos 7. With centos demise,
-# remove it from 'python:' settings and test earliest still supported
-# release, last two production releases and nightly to cut down on cost
-# of testing.
-# dist:
-#   - bionic
-#
-# python:
-#   - 2.7
-#   - 3.6
-#   - 3.8
-#   - 3.9-dev
-#   - nightly
-#
-# services:
-#   - mysql
-#   - postgresql
+  - bionic
+
+python:
+  - 2.7
+  - 3.9-dev
+  - 3.8
+  - 3.6
+  - nightly
+
+services:
+  - mysql
+  - postgresql
 
 jobs:
     allow_failures:  # nightly not ready for prime time yet.
@@ -102,7 +85,7 @@ before_install:
   - GPGME_VERSION=1.11.1
   - cd /tmp
   - curl -s -O https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-$GPGME_VERSION.tar.bz2
-  - tar -jxvf gpgme-$GPGME_VERSION.tar.bz2
+  - tar -jxf gpgme-$GPGME_VERSION.tar.bz2
   - cd gpgme-$GPGME_VERSION
   - ./configure --prefix=$VIRTUAL_ENV
   - make && make install
@@ -137,11 +120,27 @@ before_script:
   #   needed for test_mysql.mysqlDBTest.testFilteringSpecialChars
   - sed -i 's/CREATE DATABASE \%s/CREATE DATABASE \%s COLLATE utf8_general_ci/' roundup/backends/back_mysql.py
 
+  # build the .mo translation files and install them into a tree
+  # (locale/locale under roundup directory root) 
+  # suitable for use by gettext.
+  - (cd locale; make local_install; ls -lR locale/de/LC_MESSAGES)
+
 script:
   - PATH=$VIRTUAL_ENV/bin:$PATH
   - export LD_LIBRARY_PATH=$VIRTUAL_ENV/lib:$LD_LIBRARY_PATH
-  - py.test -v --maxfail=20 test/ --cov=roundup
-  
+  - if [[ "$TRAVIS_PYTHON_VERSION" != "2."* ]]; then
+    py.test 
+      -W default
+      -W "ignore:SelectableGroups:DeprecationWarning" 
+      -W "ignore:the imp module:DeprecationWarning:gpg.gpgme:15" 
+      -W "ignore:'U' mode::docutils.io" 
+      -W "ignore:unclosed:ResourceWarning:roundup.roundup.demo" 
+      -W "ignore:unclosed file:ResourceWarning:enum" 
+      -v --maxfail=20 test/ --cov=roundup;
+    fi
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "2."* ]]; then
+      py.test -v --maxfail=20 test/ --cov=roundup;
+    fi
 
 after_success:
   - codecov

CHANGES.txt

@@ -15,6 +15,17 @@ onwards Python 3.4 and later are also supported.
 
 Fixed:
 
+- issue2551161 - Fix ResourceWarnings when running with -W default.
+  Cleaned up leaking file descriptors from zopetal pre-compile, python
+  module compile and loading localization file. (John Rouillard) 
+- When using roundup-server with native SSL, only accept TLS v1.2.
+  Previously it used to accept only TLS v1.1. 1.1 is deprecated by
+  chrome. I don't expect this to be a major problem since a front
+  end server (apache, Nginx...) is usually customer facing and
+  terminates SSL.
+- Fix hang when valid user without authorization for REST tries to use
+  the rest interface.
+
 Features:
 
 - issue2551147 - Enable compression of http responses in roundup.

issues/extensions/templating.py

@@ -0,0 +1,104 @@
+import logging
+logger = logging.getLogger('extension')
+
+import sys
+from roundup import __version__ as roundup_version
+def AboutPage(db):
+    "report useful info about this tracker"
+
+    def is_module_loaded(module):
+        modules = sys.modules.keys()
+        return module in modules
+
+    def get_status_of_module(module, prefix=None, version=True):
+        modules = sys.modules.keys()
+        is_enabled = module in modules
+        if is_enabled:
+            if module == 'pyme':
+                from pyme import version
+                version="version %s"%version.versionstr
+            elif module == 'pychart':
+                from pychart import version
+                version="version %s"%version.version
+            elif module == 'sqlite3':
+                from sqlite3 import version
+                version="version %s"%version
+            else:
+                if version:
+                    m = __import__(module)
+                    try:
+                        version="version %s"%m.__version__
+                    except AttributeError:
+                        version="version unavailable - exception thrown"
+                else:
+                    version="version unavailable"
+
+            if prefix:
+                return "%s %s %s enabled: %s"%(prefix, module, version, is_enabled)
+            else:
+                return "Module: %s %s enabled: %s"%(module, version, is_enabled)
+        else:
+            if prefix:
+                return "%s %s enabled: %s"%(prefix, module, is_enabled)
+            else:
+                return "Module: %s enabled: %s"%(module, is_enabled)
+
+    info = []
+
+    info.append("Tracker name: %s<br>"%db.config['TRACKER_NAME'])
+
+    info.append("<h2>Operating environment</h2>")
+    info.append('<a href="http://roundup.sourceforge.net/">Roundup</a> version: %s<br>'%roundup_version)
+    info.append("Python Version: %s<br>"%sys.version)
+
+    info.append("<h2>Configuration</h2>")
+
+    backend = db.config['RDBMS_BACKEND']
+    info.append("Roundup backend: %s<br>"%backend)
+    if backend != 'anydbm':
+        info.append("Roundup db cache: %s<br>"%db.config['RDBMS_CACHE_SIZE'])
+        info.append("Roundup isolation_level: %s<br>"%db.config['RDBMS_ISOLATION_LEVEL'])
+
+    info.append("Roundup template: %s<br>"%db.config['TEMPLATE_ENGINE'])
+
+    info.append("<h2>Database modules</h2>")
+    info.append(get_status_of_module('anydbm', version=False) + "<br>")
+    info.append(get_status_of_module('sqlite3') + "<br>")
+    info.append(get_status_of_module('MySQLdb') + "<br>")
+    info.append(get_status_of_module('psycopg2') + "<br>")
+
+    info.append("<h2>Other modules</h2>")
+
+    info.append(get_status_of_module('pytz') + "<br>")
+    if is_module_loaded('xapian'):
+        info.append(get_status_of_module('xapian', prefix="Test indexer:") +
+                    "<br>")
+    elif is_module_loaded('whoosh'):
+        info.append(get_status_of_module('whoosh', prefix="Test indexer:") +
+                    "<br>")
+    else:
+        info.append("Text indexer: Native enabled: True<br>")
+
+    info.append(get_status_of_module('pyme') + "<br>")
+    info.append(get_status_of_module('OpenSSL') + "<br>")
+    info.append(get_status_of_module('pychart') + "<br>")
+
+    info.append(get_status_of_module('jinja2') + "<br>")
+
+    if db._db.getuid() == "1":
+        #may leak sensitive info about system, directory paths etc.
+        #and keys so require admin user access. Consider expanding
+        #to Admin rights for tracker.
+        info.append("")
+        info.append("Module Path: %r"%sys.path)
+
+        info.append("<h2>Environment Variables</h2>")
+        info.append("<pre>") # include pre to prevent wrapping of values
+        for key in db._client.env.keys():
+            info.append("%s=%s"%(key,db._client.env[key]) + "<br>")
+        info.append("</pre>")
+    return "\n".join(info)
+
+def init(instance):
+    instance.registerUtil('AboutPage', AboutPage)
+ 

locale/GNUmakefile

@@ -50,6 +50,13 @@ template:
 	  --copyright-holder="See Roundup README.txt" \
 	  -o $(TEMPLATE) $(SOURCES)
 
+local_install: dist
+	for file in $(MO_FILES); do \
+	   lang=`basename $$file .mo`; \
+	   mkdir -p locale/$$lang/LC_MESSAGES; \
+	   cp $$file locale/$$lang/LC_MESSAGES/roundup.mo; \
+	done
+
 # helps to check template file before check in
 diff:
 	svn diff roundup.pot|grep -v '^[-+]#'|vim -Rv -

roundup/admin.py

@@ -1310,64 +1310,60 @@ class colon_separated(csv.excel):
                 sys.stdout.write('Exporting %s WITHOUT the files\r\n' %
                     classname)
 
-            f = open(os.path.join(dir, classname+'.csv'), 'w')
-            writer = csv.writer(f, colon_separated)
-
-            properties = cl.getprops()
-            propnames = cl.export_propnames()
-            fields = propnames[:]
-            fields.append('is retired')
-            writer.writerow(fields)
-
-            # If a node has a key, sort all nodes by key
-            # with retired nodes first. Retired nodes
-            # must occur before a non-retired node with
-            # the same key. Otherwise you get an
-            # IntegrityError: UNIQUE constraint failed:
-            #     _class.__retired__, _<class>._<keyname>
-            # on imports to rdbms.
-            all_nodes = cl.getnodeids()
-
-            classkey = cl.getkey()
-            if classkey: # False sorts before True, so negate is_retired
-                keysort = lambda i: (cl.get(i, classkey),
-                                     not cl.is_retired(i))
-                all_nodes.sort(key=keysort)
-            # if there is no classkey no need to sort
-
-            for nodeid in all_nodes:
-                if self.verbose:
-                    sys.stdout.write('\rExporting %s - %s' % 
-                                     (classname, nodeid))
-                    sys.stdout.flush()
-                node = cl.getnode(nodeid)
-                exp = cl.export_list(propnames, nodeid)
-                lensum = sum([len(repr_export(node[p])) for p in propnames])
-                # for a safe upper bound of field length we add
-                # difference between CSV len and sum of all field lengths
-                d = sum([len(x) for x in exp]) - lensum
-                if not d > 0:
-                    raise AssertionError("Bad assertion d > 0")
-                for p in propnames:
-                    ll = len(repr_export(node[p])) + d
-                    if ll > max_len:
-                        max_len = ll
-                writer.writerow(exp)
-                if export_files and hasattr(cl, 'export_files'):
-                    cl.export_files(dir, nodeid)
-
-            # close this file
-            f.close()
+            with open(os.path.join(dir, classname+'.csv'), 'w') as f:
+                writer = csv.writer(f, colon_separated)
+
+                properties = cl.getprops()
+                propnames = cl.export_propnames()
+                fields = propnames[:]
+                fields.append('is retired')
+                writer.writerow(fields)
+
+                # If a node has a key, sort all nodes by key
+                # with retired nodes first. Retired nodes
+                # must occur before a non-retired node with
+                # the same key. Otherwise you get an
+                # IntegrityError: UNIQUE constraint failed:
+                #     _class.__retired__, _<class>._<keyname>
+                # on imports to rdbms.
+                all_nodes = cl.getnodeids()
+
+                classkey = cl.getkey()
+                if classkey: # False sorts before True, so negate is_retired
+                    keysort = lambda i: (cl.get(i, classkey),
+                                         not cl.is_retired(i))
+                    all_nodes.sort(key=keysort)
+                # if there is no classkey no need to sort
+
+                for nodeid in all_nodes:
+                    if self.verbose:
+                        sys.stdout.write('\rExporting %s - %s' % 
+                                         (classname, nodeid))
+                        sys.stdout.flush()
+                    node = cl.getnode(nodeid)
+                    exp = cl.export_list(propnames, nodeid)
+                    lensum = sum([len(repr_export(node[p])) for p in propnames])
+                    # for a safe upper bound of field length we add
+                    # difference between CSV len and sum of all field lengths
+                    d = sum([len(x) for x in exp]) - lensum
+                    if not d > 0:
+                        raise AssertionError("Bad assertion d > 0")
+                    for p in propnames:
+                        ll = len(repr_export(node[p])) + d
+                        if ll > max_len:
+                            max_len = ll
+                    writer.writerow(exp)
+                    if export_files and hasattr(cl, 'export_files'):
+                        cl.export_files(dir, nodeid)
 
             # export the journals
-            jf = open(os.path.join(dir, classname+'-journals.csv'), 'w')
-            if self.verbose:
-                sys.stdout.write("\nExporting Journal for %s\n" % classname)
-                sys.stdout.flush()
-            journals = csv.writer(jf, colon_separated)
-            for row in cl.export_journals():
-                journals.writerow(row)
-            jf.close()
+            with open(os.path.join(dir, classname+'-journals.csv'), 'w') as jf:
+                if self.verbose:
+                    sys.stdout.write("\nExporting Journal for %s\n" % classname)
+                    sys.stdout.flush()
+                journals = csv.writer(jf, colon_separated)
+                for row in cl.export_journals():
+                    journals.writerow(row)
         if max_len > self.db.config.CSV_FIELD_SIZE:
             print("Warning: config csv_field_size should be at least %s" %
                   max_len, file=sys.stderr)

roundup/backends/blobfiles.py

@@ -332,7 +332,9 @@ def storefile(self, classname, nodeid, property, content):
         # in multi-tracker (i.e. multi-umask) or modpython scenarios
         # the umask may have changed since last we set it.
         os.umask(self.umask)
-        open(name, 'wb').write(content)
+        fd = open(name, 'wb')
+        fd.write(content)
+        fd.close()
 
     def getfile(self, classname, nodeid, property):
         """Get the content of the file in the database.

roundup/backends/indexer_dbm.py

@@ -51,7 +51,9 @@ def __init__(self, db):
             # for now the file itself is a flag
             self.force_reindex()
         elif os.path.exists(version):
-            version = open(version).read()
+            fd = open(version)
+            version = fd.read()
+            fd.close()
             # check the value and reindex if it's not the latest
             if version.strip() != '1':
                 self.force_reindex()
@@ -63,7 +65,9 @@ def force_reindex(self):
             shutil.rmtree(self.indexdb_path)
         os.makedirs(self.indexdb_path)
         os.chmod(self.indexdb_path, 0o775)  # nosec - allow group write
-        open(os.path.join(self.indexdb_path, 'version'), 'w').write('1\n')
+        fd = open(os.path.join(self.indexdb_path, 'version'), 'w')
+        fd.write('1\n')
+        fd.close()
         self.reindex = 1
         self.changed = 1
 
@@ -260,6 +264,7 @@ def save_index(self):
             filename = self.indexdb + initchar
             pickle_fh = open(filename, 'wb')
             pickle_fh.write(zlib.compress(pickle_str))
+            pickle_fh.close()
             os.chmod(filename, 0o664)
 
         # save done