ci: merge main to release

.editorconfig

@@ -50,3 +50,9 @@ indent_size = 2
 
 [ietf/**.html]
 insert_final_newline = false
+
+# Settings for Kubernetes yaml
+# ---------------------------------------------------------
+# Use 2-space indents
+[k8s/**.yaml]
+indent_size = 2

.github/workflows/build.yml

@@ -17,15 +17,15 @@ on:
         default: true
         required: true
         type: boolean
-      sandboxNoDbRefresh:
-        description: 'Sandbox Disable Daily DB Refresh'
+      deploy:
+        description: 'Deploy to Staging / Prod'
         default: false
         required: true
         type: boolean
-      legacySandbox:
-        description: 'Deploy to Legacy Sandbox'
+      sandboxNoDbRefresh:
+        description: 'Sandbox Disable Daily DB Refresh'
         default: false
-        required: false
+        required: true
         type: boolean
       skiptests:
         description: 'Skip Tests'
@@ -161,7 +161,7 @@ jobs:
 
     - name: Download a Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.4
+      uses: actions/download-artifact@v4.1.7
       with:
         name: coverage
 
@@ -220,7 +220,7 @@ jobs:
         .devcontainer
         .github
         .vscode
-        helm
+        k8s
         playwright
         svn-history
         docker-compose.yml
@@ -323,7 +323,7 @@ jobs:
     steps:      
     - name: Notify on Slack (Success)
       if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.25.0
+      uses: slackapi/slack-github-action@v1.26.0
       with:
         channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
         payload: |
@@ -346,7 +346,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
     - name: Notify on Slack (Failure)
       if: ${{ contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.25.0
+      uses: slackapi/slack-github-action@v1.26.0
       with:
         channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
         payload: |
@@ -385,7 +385,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Download a Release Artifact
-      uses: actions/download-artifact@v4.1.4
+      uses: actions/download-artifact@v4.1.7
       with:
         name: release-${{ env.PKG_VERSION }}
 
@@ -407,62 +407,39 @@ jobs:
         DEBIAN_FRONTEND: noninteractive
       run: |
         docker image prune -a -f
-
-  legacySandbox:
-    name: Deploy to Legacy Sandbox
-    if: ${{ !failure() && !cancelled() && github.event.inputs.legacySandbox == 'true' }}
+
+  # -----------------------------------------------------------------
+  # STAGING
+  # -----------------------------------------------------------------
+  staging:
+    name: Deploy to Staging
+    if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }}
     needs: [prepare, release]
-    runs-on: [self-hosted, legacy-sandbox-server]
+    runs-on: ubuntu-latest
     environment:
-      name: legacy-sandbox
-      url: "https://sandbox.ietf.org"
+      name: staging
     env:
       PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
 
     steps:
-    - name: Download a Release Artifact
-      uses: actions/download-artifact@v4.1.4
-      with:
-        name: release-${{ env.PKG_VERSION }}
-        path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
+      - name: Deploy to staging
+        run: |
+          curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}'
 
-    - name: Extract Release
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
-      run: |
-        echo "Extracting release tarball..."
-        tar xzf release.tar.gz
-        echo "Deleting release tarball..."
-        rm -rf release.tar.gz
-
-    - name: Setup Environment
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
-      run: |
-        echo "Copying settings from previous deploy..."
-        cp ../web/ietf/settings_local.py ietf/
-        rsync -a ../web/test/ test/
-        echo "Installing Python dependencies..."
-        python3.9 -mvenv env
-        source env/bin/activate
-        pip install -r requirements.txt
-        pip freeze > frozen-requirements.txt
-        echo "Collecting static..."
-        ietf/manage.py collectstatic
-        echo "Running checks..."
-        ietf/manage.py check
-
-    - name: Update Docker Containers
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      working-directory: /a/docker/datatracker
-      run: |
-        echo "Pulling latest docker images..."
-        docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback
-        docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback
-        docker-compose pull
-        # echo "Shutting down containers..."
-        # docker-compose down -t 300
-
+  # -----------------------------------------------------------------
+  # PROD
+  # -----------------------------------------------------------------
+  prod:
+    name: Deploy to Production
+    if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }}
+    needs: [staging]
+    runs-on: ubuntu-latest
+    environment:
+      name: production
+    env:
+      PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
+
+    steps:
+      - name: Deploy to production
+        run: |
+          curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}'

.github/workflows/tests.yml

@@ -59,7 +59,7 @@ jobs:
         path: geckodriver.log
 
     - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v4.2.0
+      uses: codecov/codecov-action@v4.3.1
       with:
         files: coverage.xml
 

bin/daily

@@ -17,16 +17,10 @@ cd $DTDIR/
 
 logger -p user.info -t cron "Running $DTDIR/bin/daily"
 
-# Run the hourly jobs first
-$DTDIR/bin/hourly
-
 # Set up the virtual environment
 source $DTDIR/env/bin/activate
 
 
-# Update our information about the current version of some commands we use
-$DTDIR/ietf/manage.py update_external_command_info
-
 # Get IANA-registered yang models
 #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
 # Hardcode the rsync target to avoid any unwanted deletes:
@@ -43,9 +37,5 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0
 # Re-run yang checks on active documents
 $DTDIR/ietf/manage.py run_yang_model_checks -v0
 
-# Expire last calls
-# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
-$DTDIR/ietf/bin/expire-last-calls
-
 # Purge older PersonApiKeyEvents
 $DTDIR/ietf/manage.py purge_old_personal_api_key_events 14

bin/dump-to-names-json

@@ -10,7 +10,7 @@
 
 set -x
 ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType       \
-        mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo    \
+        mailtrigger.MailTrigger mailtrigger.Recipient name                      \
         group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate            \
         | jq --sort-keys "sort_by(.model, .pk)"                                 \
         | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json

dev/build/Dockerfile

@@ -1,17 +1,39 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:latest
-LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-COPY . .
-COPY ./dev/build/start.sh ./start.sh
-RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt
-RUN chmod +x start.sh && \
-    chmod +x docker/scripts/app-create-dirs.sh && \
-    sh ./docker/scripts/app-create-dirs.sh
-
-VOLUME [ "/assets" ]
-
-EXPOSE 8000
-
-CMD ["./start.sh"]
+FROM ghcr.io/ietf-tools/datatracker-app-base:latest
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# uid 498 = wwwrun and gid 496 = www on ietfa
+RUN groupadd -g 1000 datatracker && \
+    useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker
+
+RUN apt-get purge -y imagemagick imagemagick-6-common
+
+# Install libreoffice (needed via PPT2PDF_COMMAND)
+RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \
+    apt-get update && \
+    apt-get -qyt bullseye-backports install libreoffice-nogui
+
+COPY . .
+COPY ./dev/build/start.sh ./start.sh
+COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh
+COPY ./dev/build/celery-start.sh ./celery-start.sh
+
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \
+    echo '# empty' > ietf/settings_local.py && \
+    ietf/manage.py patch_libraries && \
+    rm -f ietf/settings_local.py
+
+RUN chmod +x start.sh && \
+    chmod +x datatracker-start.sh && \
+    chmod +x celery-start.sh && \
+    chmod +x docker/scripts/app-create-dirs.sh && \
+    sh ./docker/scripts/app-create-dirs.sh
+
+RUN mkdir -p /a
+
+VOLUME [ "/a" ]
+
+EXPOSE 8000
+
+CMD ["./start.sh"]

dev/build/celery-start.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Run a celery worker
+#
+echo "Running Datatracker checks..."
+./ietf/manage.py check
+
+cleanup () {
+  # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
+  if [[ -n "${celery_pid}" ]]; then
+    echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..."
+    kill -TERM "${celery_pid}"
+    wait "${celery_pid}"
+  fi
+}
+
+trap 'trap "" TERM; cleanup' TERM
+
+# start celery in the background so we can trap the TERM signal
+celery "$@" &
+celery_pid=$!
+wait "${celery_pid}"

dev/build/datatracker-start.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+echo "Running Datatracker checks..."
+./ietf/manage.py check
+
+echo "Running Datatracker migrations..."
+./ietf/manage.py migrate --settings=settings_local
+
+echo "Starting Datatracker..."
+
+gunicorn \
+          --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \
+          --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \
+          --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \
+          --bind :8000 \
+          --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \
+          ietf.wsgi:application

dev/build/start.sh

@@ -1,10 +1,20 @@
 #!/bin/bash
-
-echo "Running Datatracker checks..."
-./ietf/manage.py check
-
-echo "Running Datatracker migrations..."
-./ietf/manage.py migrate --settings=settings_local
-
-echo "Starting Datatracker..."
-./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local
+#
+# Environment config:
+#
+#  CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker)
+#
+case "${CONTAINER_ROLE:-datatracker}" in
+    datatracker)
+        exec ./datatracker-start.sh
+        ;;
+    celery)
+        exec ./celery-start.sh --app=ietf worker
+        ;;
+    beat)
+        exec ./celery-start.sh --app=ietf beat
+        ;;
+    *)
+        echo "Unknown role '${CONTAINER_ROLE}'"
+        exit 255       
+esac

dev/deploy-to-container/cli.js

@@ -3,7 +3,7 @@
 import Docker from 'dockerode'
 import path from 'path'
 import fs from 'fs-extra'
-import tar from 'tar'
+import * as tar from 'tar'
 import yargs from 'yargs/yargs'
 import { hideBin } from 'yargs/helpers'
 import slugify from 'slugify'