feat: Celery support and asynchronous draft submission API

.github/workflows/build-celery-worker.yml

@@ -0,0 +1,47 @@
+name: Build Celery Worker Docker Image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'jennifer/submit-async'
+    paths:
+      - 'requirements.txt'
+      - 'dev/celery/**'
+      - '.github/workflows/build-celery-worker.yml'
+
+  workflow_dispatch: 
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Docker Build & Push
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: dev/celery/Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+#        tags: ghcr.io/ietf-tools/datatracker-celery:latest
+        tags: ghcr.io/painless-security/datatracker-celery:latest
+

.github/workflows/build-mq-broker.yml

@@ -0,0 +1,46 @@
+name: Build MQ Broker Docker Image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'jennifer/submit-async'
+    paths:
+      - 'dev/mq/**'
+      - '.github/workflows/build-mq-worker.yml'
+
+  workflow_dispatch: 
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Docker Build & Push
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: dev/mq/Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+#        tags: ghcr.io/ietf-tools/datatracker-mq:latest
+        tags: ghcr.io/painless-security/datatracker-mq:latest
+

dev/INSTALL

@@ -29,38 +29,78 @@ General Instructions for Deployment of a New Release
       python3.9 -mvenv env
       source env/bin/activate
       pip install -r requirements.txt
+      pip freeze > frozen-requirements.txt
+
+    (The pip freeze command records the exact versions of the Python libraries that pip installed.
+     This is used by the celery docker container to ensure it uses the same library versions as
+     the datatracker service.)
 
  5. Move static files into place for CDN (/a/www/www6s/lib/dt):
 
       ietf/manage.py collectstatic
 
  6. Run system checks (which patches the just installed modules)::
 
-     ietf/manage.py check
+      ietf/manage.py check
+
+ 7. Switch to the docker directory and update images:
+
+      cd /a/docker/datatracker-cel
+      docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback
+      docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback
+      docker-compose pull
+
+ 8. Stop and remove the async task containers:
+    Wait for this to finish cleanly. It may take up to about 10 minutes for the 'stop' command to
+    complete if a long-running task is in progress.
+
+      docker-compose down
+
+ 9. Stop the datatracker 
+    (consider doing this with a second shell at ietfa to avoid the exit and shift back to wwwrun)
+
+      exit
+      sudo systemctl stop datatracker.socket datatracker.service
+      sudo su - -s /bin/bash wwwrun
 
- 7. Run migrations:  
+ 10. Return to the release directory and run migrations:
 
+      cd /a/www/ietf-datatracker/${releasenumber}
       ietf/manage.py migrate
 
       Take note if any migrations were executed.
 
- 8. Back out one directory level, then re-point the 'web' symlink::
+ 11. Back out one directory level, then re-point the 'web' symlink::
 
       cd ..
       rm ./web; ln -s ${releasenumber} web
 
- 9. Reload the datatracker service (it is no longer necessary to restart apache) ::
+ 12. Start the datatracker service (it is no longer necessary to restart apache) ::
 
       exit # or CTRL-D, back to root level shell
-      systemctl restart datatracker
+      sudo systemctl start datatracker.service datatracker.socket
+
+ 13. Start async task worker and message broker:
 
- 10. Verify operation: 
+      cd /a/docker/datatracker-cel
+      bash startcommand
+
+ 14. Verify operation:
 
       http://datatracker.ietf.org/
 
- 11. If install failed and there were no migrations at step 7, revert web symlink and repeat the restart in step 9.
-     If there were migrations at step 7, they will need to be reversed before the restart at step 9. If it's not obvious
-     what to do to reverse the migrations, contact the dev team.
+ 15. If install failed and there were no migrations at step 9, revert web symlink and docker update and repeat the
+     restart in steps 11 and 12. To revert the docker update:
+
+          cd /a/docker/datatracker-cel
+          docker-compose down
+          docker image rm ghcr.io/ietf-tools/datatracker-celery:latest ghcr.io/ietf-tools/datatracker-mq:latest
+          docker image tag datatracker-celery-fallback ghcr.io/ietf-tools/datatracker-celery:latest
+          docker image tag datatracker-mq-fallback ghcr.io/ietf-tools/datatracker-mq:latest
+          cd -
+
+     If there were migrations at step 10, they will need to be reversed before the restart at step 12.
+     If it's not obvious what to do to reverse the migrations, contact the dev team.
 
 
 Patching a Production Release
@@ -95,8 +135,17 @@ The following process should be used:
  6. Edit ``.../ietf/__init__.py`` in the new patched release to indicate the patch
     version in the ``__patch__`` string.
 
- 7. Change the 'web' symlink, reload etc. as described in
+ 7. Stop the async task container (this may take a few minutes if tasks are in progress):
+
+      cd /a/docker/datatracker-cel
+      docker-compose stop celery
+
+ 8. Change the 'web' symlink, reload etc. as described in
     `General Instructions for Deployment of a New Release`_.
 
+ 9. Start async task worker:
+
+      cd /a/docker/datatracker-cel
+      bash startcommand
 
 

dev/celery/Dockerfile

@@ -0,0 +1,20 @@
+# Dockerfile for celery worker
+#
+FROM ghcr.io/ietf-tools/datatracker-app-base:latest
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get purge -y imagemagick imagemagick-6-common
+
+# Copy the startup file
+COPY dev/celery/docker-init.sh /docker-init.sh
+RUN sed -i 's/\r$//' /docker-init.sh && \
+    chmod +x /docker-init.sh
+
+# Install current datatracker python dependencies
+COPY requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
+RUN rm -rf /tmp/pip-tmp
+
+ENTRYPOINT [ "/docker-init.sh" ]

dev/celery/docker-init.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Environment parameters:
+#
+#   CELERY_APP - name of application to pass to celery (defaults to ietf)
+#
+#   CELERY_ROLE - 'worker' or 'beat' (defaults to 'worker')
+#
+#   CELERY_UID - numeric uid for the celery worker process
+#
+#   CELERY_GID - numeric gid for the celery worker process
+#
+#   UPDATES_REQUIREMENTS_FROM - path, relative to /workspace mount, to a pip requirements
+#       file that should be installed at container startup. Default is no package install/update.
+#
+#   DEBUG_TERM_TIMING - if non-empty, writes debug messages during shutdown after a TERM signal
+#
+WORKSPACEDIR="/workspace"
+CELERY_ROLE="${CELERY_ROLE:-worker}"
+
+cd "$WORKSPACEDIR" || exit 255
+
+if [[ -n "${UPDATE_REQUIREMENTS_FROM}" ]]; then
+  reqs_file="${WORKSPACEDIR}/${UPDATE_REQUIREMENTS_FROM}"
+  echo "Updating requirements from ${reqs_file}..."
+  pip install --upgrade -r "${reqs_file}"
+fi
+
+if [[ "${CELERY_ROLE}" == "worker" ]]; then
+    echo "Running initial checks..."
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check
+fi
+
+CELERY_OPTS=( "${CELERY_ROLE}" )
+if [[ -n "${CELERY_UID}" ]]; then
+  # ensure that some group with the necessary GID exists in container
+  if ! id "${CELERY_UID}" ; then
+    adduser --system --uid "${CELERY_UID}" --no-create-home --disabled-login "celery-user-${CELERY_UID}"
+  fi
+  CELERY_OPTS+=("--uid=${CELERY_UID}")
+fi
+
+if [[ -n "${CELERY_GID}" ]]; then
+  # ensure that some group with the necessary GID exists in container
+  if ! getent group "${CELERY_GID}" ; then
+    addgroup --gid "${CELERY_GID}" "celery-group-${CELERY_GID}"
+  fi
+  CELERY_OPTS+=("--gid=${CELERY_GID}")
+fi
+
+log_term_timing_msgs () {
+  # output periodic debug message
+  while true; do
+    echo "Waiting for celery worker shutdown ($(date --utc --iso-8601=ns))"
+    sleep 0.5s
+  done
+}
+
+cleanup () {
+  # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
+  if [[ -n "${celery_pid}" ]]; then
+    echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..."
+    kill -TERM "${celery_pid}"
+    if [[ -n "${DEBUG_TERM_TIMING}" ]]; then
+      log_term_timing_msgs &
+    fi
+    wait "${celery_pid}"
+  fi
+}
+
+trap 'trap "" TERM; cleanup' TERM
+# start celery in the background so we can trap the TERM signal
+celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &
+celery_pid=$!
+wait "${celery_pid}"

dev/mq/Dockerfile

@@ -0,0 +1,17 @@
+# Dockerfile for RabbitMQ worker
+#
+FROM rabbitmq:3-alpine
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
+
+# Copy the startup file
+COPY dev/mq/ietf-rabbitmq-server.bash /ietf-rabbitmq-server.bash
+RUN sed -i 's/\r$//' /ietf-rabbitmq-server.bash && \
+    chmod +x /ietf-rabbitmq-server.bash
+
+# Put the rabbitmq.conf in the conf.d so it runs after 10-defaults.conf.
+# Can override this for an individual container by mounting additional
+# config files in /etc/rabbitmq/conf.d.
+COPY dev/mq/rabbitmq.conf /etc/rabbitmq/conf.d/20-ietf-config.conf
+COPY dev/mq/definitions.json /definitions.json
+
+CMD ["/ietf-rabbitmq-server.bash"]

dev/mq/definitions.json

@@ -0,0 +1,30 @@
+{
+  "permissions": [
+    {
+      "configure": ".*",
+      "read": ".*",
+      "user": "datatracker",
+      "vhost": "dt",
+      "write": ".*"
+    }
+  ],
+  "users": [
+    {
+      "hashing_algorithm": "rabbit_password_hashing_sha256",
+      "limits": {},
+      "name": "datatracker",
+      "password_hash": "",
+      "tags": []
+    }
+  ],
+  "vhosts": [
+    {
+      "limits": [],
+      "metadata": {
+        "description": "",
+        "tags": []
+      },
+      "name": "dt"
+    }
+  ]
+}

dev/mq/ietf-rabbitmq-server.bash

@@ -0,0 +1,22 @@
+#!/bin/bash -x
+#
+# Environment parameters:
+#
+#   CELERY_PASSWORD - password for the datatracker celery user
+#
+export RABBITMQ_PID_FILE=/tmp/rabbitmq.pid
+
+update_celery_password () {
+  rabbitmqctl wait "${RABBITMQ_PID_FILE}" --timeout 300
+  rabbitmqctl await_startup --timeout 300
+  if [[ -n "${CELERY_PASSWORD}" ]]; then
+    rabbitmqctl change_password datatracker <<END
+${CELERY_PASSWORD}
+END
+  else
+    rabbitmqctl clear_password datatracker
+  fi
+}
+
+update_celery_password &
+exec rabbitmq-server "$@"

dev/mq/rabbitmq.conf

@@ -0,0 +1,18 @@
+# prevent guest from logging in over tcp
+loopback_users.guest = true
+
+# load saved definitions
+load_definitions = /definitions.json
+
+# Ensure that enough disk is available to flush to disk. To do this, need to limit the
+# memory available to the container to something reasonable. See
+# https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage
+# for recommendations.
+
+# 1-1.5 times the memory available to the container is adequate for disk limit
+disk_free_limit.absolute = 6000MB
+
+# This should be ~40% of the memory available to the container. Use an
+# absolute number because relative will be proprtional to the full machine
+# memory.
+vm_memory_high_watermark.absolute = 1600MB