merge from default branch. Fix travis.ci so CI builds don't error out

Author: rouilj

.travis.yml

@@ -16,12 +16,13 @@ branches:
 #    - default
 #    - maint-1.6
 
-dist:
-  - bionic
+dist: focal
 
+#  - pypy3
 python:
   - 2.7
-  - 3.9-dev
+  - 3.10.0
+  - 3.9
   - 3.8
   - 3.6
   - nightly
@@ -33,11 +34,12 @@ services:
 jobs:
     allow_failures:  # nightly not ready for prime time yet.
       - python: nightly
+      - python: pypy3
 
 addons:
   apt:
-    sources:
-      - sourceline: ppa:xapian-backports/ppa
+    #sources:
+    #  - sourceline: ppa:xapian-backports/ppa
 
     packages:
       # Required to build/install the xapian-binding
@@ -48,8 +50,12 @@ addons:
       - gpgsm
 
 before_install:
-  # Sphinx required to build the xapian python bindings
-  - pip install sphinx==1.8.5
+  - echo "$TRAVIS_PYTHON_VERSION"
+  # Sphinx required to build the xapian python bindings. Use 1.8.5 on
+  # older python and newest on newer.
+  - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then pip install sphinx; fi
   - XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1)
   - cd /tmp
   - curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz
@@ -59,12 +65,13 @@ before_install:
   - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python; fi
   - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python3; fi
   - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python3; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python3; fi
   - make && make install
 
   - PATH=$VIRTUAL_ENV/bin:$PATH
 
   # libgpg-error
-  - LIBGPG_ERROR_VERSION=1.32
+  - LIBGPG_ERROR_VERSION=1.43
   - cd /tmp
   - curl -s -O https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-$LIBGPG_ERROR_VERSION.tar.bz2
   - tar -jxvf libgpg-error-$LIBGPG_ERROR_VERSION.tar.bz2
@@ -73,7 +80,7 @@ before_install:
   - make && make install
 
   # libassuan
-  - LIBASSUAN_VERSION=2.5.1
+  - LIBASSUAN_VERSION=2.5.5
   - cd /tmp
   - curl -s -O https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-$LIBASSUAN_VERSION.tar.bz2
   - tar -jxvf libassuan-$LIBASSUAN_VERSION.tar.bz2
@@ -82,7 +89,7 @@ before_install:
   - make && make install
 
   # gpgme
-  - GPGME_VERSION=1.11.1
+  - GPGME_VERSION=1.16.0
   - cd /tmp
   - curl -s -O https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-$GPGME_VERSION.tar.bz2
   - tar -jxf gpgme-$GPGME_VERSION.tar.bz2
@@ -100,24 +107,31 @@ install:
   - pip install gpg pytz whoosh pyjwt requests
   - pip install pytest-cov codecov
   - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install docutils; fi
-  - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install mistune; fi
+  - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install mistune==0.8.4; fi
   - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* && $TRAVIS_PYTHON_VERSION != "2."* ]]; then pip install Markdown; fi
   - pip install markdown2
-  - pip install brotli zstd
+  - pip install brotli
+  # zstd fails to build under python nightly aborting test.
+  # allow testing to still happen if the optional package doesn't install.
+  - pip install zstd || true
 
 before_script:
   # set up mysql database
   - sudo sed -i -e '/^\[mysqld\]/,/^\[mysql/s/^max_allowed_packet.*/max_allowed_packet = 500M/' /etc/mysql/my.cnf
   - cat /etc/mysql/my.cnf
   - sudo service mysql restart
-  - mysql -u root -e 'GRANT ALL ON rounduptest.* TO rounduptest@localhost IDENTIFIED BY "rounduptest";'
+  - mysql -u root -e 'CREATE USER "rounduptest"@"localhost" IDENTIFIED WITH mysql_native_password BY "rounduptest"; GRANT ALL on rounduptest.* TO "rounduptest"@"localhost";'
 
+  # Disable fsync for speed, don't care about data durability when testing
+  - sudo sed -i -e '$a\fsync = off' /etc/postgresql/*/*/postgresql.conf
   - sudo service postgresql restart; sleep 30
   # set up postgresql database
   - psql -c "CREATE ROLE rounduptest WITH CREATEDB LOGIN PASSWORD 'rounduptest';" -U postgres
 
   # HACK: workaround mysql bug: http://bugs.mysql.com/bug.php?id=74901
   #   needed for test_mysql.mysqlDBTest.testFilteringSpecialChars
+  # plus others. Otherwise we get:
+  # COLLATION 'utf8_bin' is not valid for CHARACTER SET 'utf8mb4'
   - sed -i 's/CREATE DATABASE \%s/CREATE DATABASE \%s COLLATE utf8_general_ci/' roundup/backends/back_mysql.py
 
   # build the .mo translation files and install them into a tree
@@ -141,6 +155,7 @@ script:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2."* ]]; then
       py.test -v --maxfail=20 test/ --cov=roundup;
     fi
+  - ./setup.py build_doc
 
 after_success:
   - codecov

CHANGES.txt

@@ -22,9 +22,62 @@ Fixed:
   Previously it used to accept only TLS v1.1. 1.1 is deprecated by
   chrome. I don't expect this to be a major problem since a front
   end server (apache, Nginx...) is usually customer facing and
-  terminates SSL.
+  terminates SSL.  (John Rouillard)
 - Fix hang when valid user without authorization for REST tries to use
-  the rest interface.
+  the rest interface.  (John Rouillard)
+- Remove Content-Type and make sure no content is returned by OPTIONS
+  request in REST interface. (John Rouillard)
+- In write_html set the Content-Length when response is not
+  encoded/compressed. (John Rouillard)
+- In REST interface do not raise UsageError for invalid api version.
+  Return json error with proper message. Fixes crash. (John Rouillard)
+- In REST interface, allow extensions on URI less than 6 characters in
+  length. All other paths with a . in then will be passed through
+  without change. This allows items like a JWT to be passed as a path
+  element. (John Rouillard)
+- issue2551167 - pip install in containerized environments puts
+  template and locale files under site-packages where roundup can't find
+  them. Change code to find them under site-packages.
+- REST replace hard coded list of child endpoints for /rest/ with list
+  pulled from registered endpoints. So newly added endpoints are
+  shown. (John Rouillard)
+- issue2551107 - Handle representation of long int in history params
+  for python3. Causes SyntaxError crash when showing history due to
+  long int e.g. 2345L. This is not a problem for roundup trackers
+  created using 1.2.0 or newer. The fix may have predated the 1.2.0
+  release but where the fix actually landed (representing id as a
+  string and not as an int) is unknown.
+- issue2551175 - Make ETag content-encoding aware. HTTP ETag headers
+  now include a suffix indicating the content-encoding used to send
+  the data per rfc7232. Properly validate any form of ETag suffixed or
+  non-suffixed for If-Match.
+- issue2551178 - fix Traceback in Apache WSGI - during file upload
+- issue2551179 - make roundup-demo initialize templates using
+  config_ini.ini overrides. Needed for jinja to set template lang etc.
+  Recognize minimal template when presented with a full
+  path. (John Kristensen (jerrykan) and John Rouillard)
+- handle configparser.InterpolationSyntaxError raised if value
+  has a single %. Seems to afect python 3 only. Reported by
+  nomicon on IRC. (John Rouillard)
+- add random delay to session database retry code between 0 and .125
+  seconds. This seems to help reduce stalled connections when a
+  number of connections are made at the same time. Log remaining
+  retries once 5 of them have been used. (John Rouillard)
+- issue2551169 - setup.py enters endless loop on gentoo linux python2
+  installation. Fixed.
+- issue2551185 - must set PYTHONPATH=... python2 setup.py install
+  --prefix=/tmp/r2. Force insert --old-and-unmangable to get it
+  to use a classic installer and not an easy install. This only
+  affects python2.
+- issue2551186 - Python versions >= 3.3 no longer use socket.sslerror.
+  Andrew (kragacles) patched uses of socket.sslerror in mailgy.py.
+  Patch adapted to allow trapping sslerror under both python2 and 3.
+  (John Rouillard)
+- issue2551142 - postgresql reworked to use savepoint/"rollback to"
+  rather than commit()/rollback(). Using savepoint should be faster.
+- issue2551196 - Unset labelprop of a Multilink can lead to Python
+  error when using context/history. (reported and initial patch: Nagy
+  Gabor, John Rouillard)
 
 Features:
 
@@ -42,6 +95,50 @@ Features:
   responsive templates already have this feature.
 - issue2550917 - Add a: "Welcome user, you have logged in" ok_message
   on login. (Ashley Burke)
+- enable HTTP/1.1 for roundup-server. This enables keep-alive for
+  faster response/loading. Also eliminates stalls when the front end web
+  server uses http 1.1 but the roundup-server uses 1.0. New option
+  "-V HTTP/1.0" can turn it off. (John Rouillard)
+- issue2551163 - add scripts/Docker/Dockerfile to provide basic support for
+  containerization. See installation.txt for details. (John Rouillard)
+- issue2551163 - add scripts/Docker/docker-compose.yml to get a
+  mysql/roundup deployment. (Norbert Schlemmer, modified by John
+  Rouilard)
+- REST add openapi_doc decorator to add openapi_doc to
+  endpoints. Decorate a couple of examples. (John Rouillard)
+- REST when incorrect method is used, report allowed methods in error
+  message as well as in an Allow header. (John Rouillard)
+- REST change response to invalid attribute specified in path.  Return
+  400 code not 405 code for this case and improve error. (John
+  Rouillard)
+- REST correct values for some Access-Control-Allow-Methods and
+  Access-Control-Allow-Headers headers. (John Rouillard)
+- issue2550991 - define default cache control settings for javascript
+  and css assets. (John Rouillard)
+- issue2551181 - fragments can be appended to designators. So
+  issue23#msg24 could jump to the element with id msg24 in issue 23.
+  Before this patch you would have two links issue23 and msg24
+  separated by # (John Rouillard).
+- added small utility script to dump dbm based tracker databases
+  (e.g. db/sessions). (John Rouillard)
+- issue2551182 - Enhance configuration module to allow loading values
+  from an external file. Secrets (passwords, secrets) can specify
+  file using file:// or file:///. The first line of the file is used
+  as the secret. This allows committing config.ini to a VCS. (John
+  Rouillard)
+- Added xapian indexer to Docker container. (John Rouillard)
+- Add support for indexer type native-fts to use FTS5 for sqlite
+  databases. (John Rouillard)
+- Add support for indexer type native-fts to use PostreSQL's full text
+  search. (John Rouillard)
+- Add better error display to the user. Needed to expose errors in fts5
+  search syntax to the user while also displaying the template page
+  structure. (John Rouillard)
+- issue2551189 - increase size of words in full text index.
+  Many terms (like exception names or symbolic constants) are larger
+  than 25. Also German words are long. Since there is little chance of
+  fixing German to shorten their words, change indexer maxlength to 50.
+  (Thomas Arendsen Hein provided patch; patch reworked John Rouillard)
 
 2021-07-13 2.1.0