diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 4a4394fca0c..1b0855cc470 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: token: ${{ secrets.GH_COMMON_TOKEN }} @@ -31,17 +31,17 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index f49a11af19c..14c4b1a135a 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,20 +20,20 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 4de861dbcd3..ef7ed2f65c0 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,23 +24,32 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set rabbitmq version + id: rabbitmq-version + run: | + if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then + echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT + else + echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT + fi + - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -48,7 +57,7 @@ jobs: file: dev/mq/Dockerfile platforms: linux/amd64,linux/arm64 push: true - build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }} + build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} tags: | - ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }} + ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} ghcr.io/ietf-tools/datatracker-mq:latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d97889fbb8e..8ec806b2291 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -98,7 +98,7 @@ jobs: echo "IS_RELEASE=true" >> $GITHUB_ENV - name: Create Draft Release - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.ref_name == 'release' }} with: prerelease: true @@ -164,7 +164,7 @@ jobs: TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -175,7 +175,7 @@ jobs: node-version: 18.x - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" @@ -186,7 +186,7 @@ jobs: - name: Download a Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@v6.0.0 with: name: coverage @@ -253,10 +253,10 @@ jobs: EOL - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -267,7 +267,7 @@ jobs: run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV - name: Build Images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -291,7 +291,7 @@ jobs: - name: Download Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@v6.0.0 with: name: coverage @@ -315,7 +315,7 @@ jobs: histCoveragePath: historical-coverage.json - name: Create Release - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ env.SHOULD_DEPLOY == 'true' }} with: allowUpdates: true @@ -328,7 +328,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} with: allowUpdates: true @@ -341,7 +341,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: release-${{ env.PKG_VERSION }} path: /home/runner/work/release/release.tar.gz @@ -360,7 +360,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage @@ -375,7 +375,7 @@ jobs: value: "Completed" - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage @@ -403,7 +403,7 @@ jobs: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: main diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 278bd8af2f3..5349f1ac7a6 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3444c03b5e0..bc20779ae6c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,12 +26,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 6d0683c471c..e255b270ff9 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 4cfbf6365b4..cd986f06f33 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,17 +29,17 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 836314bac09..be7b834b7af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -68,7 +68,7 @@ jobs: coverage xml - name: Upload geckodriver.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ failure() }} with: name: geckodriverlog @@ -87,7 +87,7 @@ jobs: mv latest-coverage.json coverage.json - name: Upload Coverage Results as Build Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} with: name: coverage @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: @@ -121,7 +121,7 @@ jobs: npx playwright test --project=${{ matrix.project }} - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: @@ -143,7 +143,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -180,7 +180,7 @@ jobs: npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index b62109f0e21..a68f170c4b5 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.9", + "dockerode": "^4.0.10", "fs-extra": "^11.3.4", "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", "slugify": "1.6.8", - "tar": "^7.5.11", + "tar": "^7.5.12", "yargs": "^17.7.2" }, "engines": { @@ -160,7 +160,6 @@ "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", "dependencies": { "safer-buffer": "~2.1.0" } @@ -188,7 +187,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", "dependencies": { "tweetnacl": "^0.14.3" } @@ -227,9 +225,9 @@ } }, "node_modules/buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true, "engines": { "node": ">=10.0.0" @@ -284,10 +282,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dependencies": { "ms": "^2.1.3" }, @@ -301,10 +298,9 @@ } }, "node_modules/docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", - "license": "Apache-2.0", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "dependencies": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -316,14 +312,14 @@ } }, "node_modules/dockerode": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", - "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" @@ -464,14 +460,12 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", - "license": "MIT", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "node_modules/nanoid": { @@ -580,8 +574,7 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/slugify": { "version": "1.6.8", @@ -594,13 +587,12 @@ "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", - "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", - "license": "ISC" + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "node_modules/ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "hasInstallScript": true, "dependencies": { "asn1": "^0.2.6", @@ -611,7 +603,7 @@ }, "optionalDependencies": { "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "node_modules/string_decoder": { @@ -647,9 +639,9 @@ } }, "node_modules/tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.12", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz", + "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -698,8 +690,7 @@ "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "node_modules/undici-types": { "version": "6.20.0", @@ -949,9 +940,9 @@ } }, "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true }, "chownr": { @@ -993,17 +984,17 @@ } }, "debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { "ms": "^2.1.3" } }, "docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "requires": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -1012,14 +1003,14 @@ } }, "dockerode": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", - "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" @@ -1126,9 +1117,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "nanoid": { @@ -1213,14 +1204,14 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "requires": { "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2", "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "string_decoder": { @@ -1250,9 +1241,9 @@ } }, "tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.12", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz", + "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 1c95a4540ca..aa9e82dbdf8 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.9", + "dockerode": "^4.0.10", "fs-extra": "^11.3.4", "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", "slugify": "1.6.8", - "tar": "^7.5.11", + "tar": "^7.5.12", "yargs": "^17.7.2" }, "engines": { diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 701f05eece8..397ca05d9b0 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -20,7 +20,7 @@ RfcAuthor, ) from ietf.doc.serializers import RfcAuthorSerializer -from ietf.doc.tasks import update_rfc_searchindex_task +from ietf.doc.tasks import trigger_red_precomputer_task, update_rfc_searchindex_task from ietf.doc.utils import ( default_consensus, prettify_std_name, @@ -683,7 +683,18 @@ def update(self, instance, validated_data): stale_subseries_relations.delete() if len(rfc_events) > 0: rfc.save_with_history(rfc_events) - + # Gather obs and updates in both directions as a title/author change to + # this doc affects the info rendering of all of the other RFCs + needs_updating = sorted( + [ + d.rfc_number + for d in [rfc] + + rfc.related_that_doc(("obs", "updates")) + + rfc.related_that(("obs", "updates")) + ] + ) + trigger_red_precomputer_task.delay(rfc_number_list=needs_updating) + # Update the search index also update_rfc_searchindex_task.delay(rfc.rfc_number) return rfc diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py index ed326be451d..167ffcd3eec 100644 --- a/ietf/api/tests_serializers_rpc.py +++ b/ietf/api/tests_serializers_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2026, All Rights Reserved + from unittest import mock from django.utils import timezone @@ -35,8 +36,21 @@ def test_create(self): serializer.save() @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") - def test_update(self, mock_update_searchindex_task): + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_update(self, mock_trigger_red_task, mock_update_searchindex_task): + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) rfc = WgRfcFactory(pages=10) + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) serializer = EditableRfcSerializer( instance=rfc, data={ @@ -59,11 +73,6 @@ def test_update(self, mock_update_searchindex_task): ) self.assertTrue(serializer.is_valid()) result = serializer.save() - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) result.refresh_from_db() self.assertEqual(result.title, "Yadda yadda yadda") self.assertEqual( @@ -91,12 +100,42 @@ def test_update(self, mock_update_searchindex_task): result.part_of(), [Document.objects.get(name="fyi999")], ) + # Confirm that red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was triggered correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") - def test_partial_update(self, mock_update_searchindex_task): + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_partial_update(self, mock_trigger_red_task, mock_update_searchindex_task): # We could test other permutations of fields, but authors is a partial update # we know we are going to use, so verifying that one in particular. + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) serializer = EditableRfcSerializer( partial=True, instance=rfc, @@ -113,11 +152,6 @@ def test_partial_update(self, mock_update_searchindex_task): ) self.assertTrue(serializer.is_valid()) result = serializer.save() - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) result.refresh_from_db() self.assertEqual(rfc.title, "padawan") self.assertEqual( @@ -140,8 +174,27 @@ def test_partial_update(self, mock_update_searchindex_task): self.assertEqual(result.pages, 10) self.assertEqual(result.std_level_id, "ps") self.assertEqual(result.part_of(), []) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) # Test only a field on the Document itself to be sure that it works + mock_trigger_red_task.delay.reset_mock() + mock_update_searchindex_task.delay.reset_mock() serializer = EditableRfcSerializer( partial=True, instance=rfc, @@ -151,3 +204,14 @@ def test_partial_update(self, mock_update_searchindex_task): result = serializer.save() result.refresh_from_db() self.assertEqual(rfc.title, "jedi master") + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index a679e747891..0db67e126f7 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -196,8 +196,15 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") - def test_upload_rfc_files(self, mock_update_searchindex_task): + @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") + def test_upload_rfc_files( + self, + mock_trigger_red_task, + mock_update_searchindex_task, + mock_rebuild_relations, + ): def _valid_post_data(): """Generate a valid post data dict @@ -218,7 +225,14 @@ def _valid_post_data(): } url = urlreverse("ietf.api.purple_api.upload_rfc_files") + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + rfc = WgRfcFactory() + for r in obsoletes: + rfc.relateddocument_set.create(relationship_id="obs", target=r) + for r in updates: + rfc.relateddocument_set.create(relationship_id="updates", target=r) assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" with TemporaryDirectory() as rfc_dir: settings.RFC_PATH = rfc_dir # affects overridden settings @@ -303,6 +317,7 @@ def _valid_post_data(): blob_in_the_way.delete() # valid post + mock_trigger_red_task.delay.reset_mock() r = self.client.post( url, _valid_post_data(), @@ -310,7 +325,6 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) - self.assertTrue(mock_update_searchindex_task.delay.called) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), @@ -350,6 +364,23 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend( + [d.rfc_number for d in updates + obsoletes] + ) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index cb6a59a1674..1e96118e581 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,7 +38,12 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage -from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task +from ietf.doc.tasks import ( + signal_update_rfc_metadata_task, + rebuild_reference_relations_task, + trigger_red_precomputer_task, + update_rfc_searchindex_task, +) from ietf.person.models import Email, Person from ietf.sync.tasks import create_rfc_index_task @@ -205,19 +210,19 @@ def submitted_to_rpc(self, request): Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. """ ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) - irtf_iab_ise_docs = Q( + irtf_iab_ise_editorial_docs = Q( states__type_id__in=[ "draft-stream-iab", "draft-stream-irtf", "draft-stream-ise", + "draft-stream-editorial", ], states__slug__in=["rfc-edit"], ) - # TODO: Need a way to talk about editorial stream docs docs = ( self.get_queryset() .filter(type_id="draft") - .filter(ietf_docs | irtf_iab_ise_docs) + .filter(ietf_docs | irtf_iab_ise_editorial_docs) ) serializer = self.get_serializer(docs, many=True) return Response(serializer.data) @@ -516,7 +521,16 @@ def post(self, request): destination.parent.mkdir() shutil.move(ftm, destination) + # Trigger red precomputer + needs_updating = [rfc.rfc_number] + for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]): + needs_updating.append(rel.target.rfc_number) + trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) + # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) + # Trigger reference relation srebuild + rebuild_reference_relations_task.delay(doc_names=[rfc.name]) + return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a38cd5eb5cf..19edb390149 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -7,11 +7,14 @@ import debug # pyflakes:ignore from celery import shared_task +from celery.exceptions import MaxRetriesExceededError from pathlib import Path from django.conf import settings from django.utils import timezone +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today @@ -169,6 +172,20 @@ def signal_update_rfc_metadata_task(rfc_number_list=()): signal_update_rfc_metadata(rfc_number_list) +@shared_task(bind=True) +def trigger_red_precomputer_task(self, rfc_number_list=()): + if not rfcs_are_in_r2(rfc_number_list): + log.log(f"Objects are not yet in R2 for RFCs {rfc_number_list}") + try: + countdown = getattr(settings, "RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", 10) + max_retries = getattr(settings, "RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", 12) + self.retry(countdown=countdown, max_retries=max_retries) + except MaxRetriesExceededError: + log.log(f"Gave up waiting for objects in R2 for RFCs {rfc_number_list}") + else: + trigger_red_precomputer(rfc_number_list) + + @shared_task(bind=True) def update_rfc_searchindex_task(self, rfc_number: int): """Update the search index for one RFC""" diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index a2784bc85e0..ba672cd8479 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -1,15 +1,23 @@ # Copyright The IETF Trust 2020, All Rights Reserved import datetime +from io import BytesIO + +import mock import debug # pyflakes:ignore +import requests from pathlib import Path from unittest.mock import call, patch from django.conf import settings +from django.core.files.storage import storages from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone + +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory @@ -17,11 +25,12 @@ from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory -from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor +from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor, StoredObject from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, last_ballot_doc_revision) +from ietf.doc.storage_utils import store_str from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -559,3 +568,132 @@ def test_last_ballot_doc_revision(self): nobody = PersonFactory() self.assertIsNone(last_ballot_doc_revision(doc, nobody)) self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) + + +class UtilsRedTests(TestCase): + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer_not_configured(self, mock_post, mock_log): + with override_settings(): + try: + del settings.CUSTOM_SETTING_NAME + except AttributeError: + pass + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + mock_log.reset_mock() + with override_settings(TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL=None): + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertFalse(mock_post.called) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=requests.Timeout()) + def test_trigger_red_precomputer_swallows_timeout_exception( + self, mock_post, mock_log + ): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertFalse(exception_raised) + self.assertEqual(mock_log.call_count, 2) + # only checking the last log call + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + self.assertIn("POST request timed out", mock_args[0]) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=Exception()) + def test_trigger_red_precomputer_does_not_swallow_too_much(self, mock_post): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertTrue(exception_raised) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + DEFAULT_REQUESTS_TIMEOUT=314159265, + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer(self, mock_post, mock_log): + mock_post.return_value = mock.Mock(status_code=200) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertTrue(mock_post.called) + _, mock_kwargs = mock_post.call_args + self.assertIn("url", mock_kwargs) + self.assertEqual(mock_kwargs["url"], "urlbits") + self.assertIn("json", mock_kwargs) + self.assertEqual(mock_kwargs["json"], {"rfcs": "1,2,3"}) + self.assertIn("timeout", mock_kwargs) + self.assertEqual(mock_kwargs["timeout"], 314159265) + self.assertEqual(mock_log.call_count, 1) # Not testing the first info log value + mock_log.reset_mock() + mock_post.reset_mock() + mock_post.return_value = mock.Mock( + status_code=500, + ) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 2) + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + expected = f"POST request failed for {settings.TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL} : status_code=500" + self.assertEqual(mock_args[0], expected) + + +class UtilsR2TestCase(TestCase): + def test_rfcs_are_in_r2(self): + rfcs = WgRfcFactory.create_batch(2) + rfc_name_list = [rfc.name for rfc in rfcs] + rfc_number_list = [rfc.rfc_number for rfc in rfcs] + r2_rfc_bucket = storages["r2-rfc"] + # Right now the various doc Factories do not populate any content + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 0, + ) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + for rfc in rfcs: + store_str( + kind="rfc", + name=f"testartifact/{rfc.name}.testartifact", + content="", + doc_name=rfc.name, + doc_rev=None, + ) + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 2, + ) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[0].name}.testartifact", BytesIO(b"")) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[1].name}.testartifact", BytesIO(b"")) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + + + diff --git a/ietf/doc/utils_r2.py b/ietf/doc/utils_r2.py new file mode 100644 index 00000000000..53fb978303d --- /dev/null +++ b/ietf/doc/utils_r2.py @@ -0,0 +1,17 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.core.files.storage import storages + +from ietf.doc.models import StoredObject + + +def rfcs_are_in_r2(rfc_number_list=()): + r2_rfc_bucket = storages["r2-rfc"] + for rfc_number in rfc_number_list: + stored_objects = StoredObject.objects.filter( + store="rfc", doc_name=f"rfc{rfc_number}" + ) + for stored_object in stored_objects: + if not r2_rfc_bucket.exists(stored_object.name): + return False + return True diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py new file mode 100644 index 00000000000..bcda893dca9 --- /dev/null +++ b/ietf/doc/utils_red.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def trigger_red_precomputer(rfc_number_list=()): + url = getattr(settings, "TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) + if url is not None: + payload = { + "rfcs": ",".join([str(n) for n in rfc_number_list]), + } + try: + log(f"Triggering red precompute multiple for RFCs {rfc_number_list}") + response = requests.post( + url=url, + json=payload, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log(f"POST request timed out for {url} : {e}") + return + if response.status_code != 200: + log( + f"POST request failed for {url} : status_code={response.status_code}" + ) + else: + log("No URL configured for triggering red precompute multiple, skipping") diff --git a/ietf/group/views.py b/ietf/group/views.py index efe3eca15d5..8561a5059fc 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -245,10 +245,19 @@ def active_review_dirs(request): return render(request, 'group/active_review_dirs.html', {'dirs' : dirs }) def active_teams(request): - teams = Group.objects.filter(type="team", state="active").order_by("name") + parent_type_order = {"area": 1, "adm": 3, None: 4} + + def team_sort_key(group): + type_id = group.parent.type_id if group.parent else None + return (parent_type_order.get(type_id, 2), group.parent.name if group.parent else "", group.name) + + teams = sorted( + Group.objects.filter(type="team", state="active").select_related("parent"), + key=team_sort_key, + ) for group in teams: group.chairs = sorted(roles(group, "chair"), key=extract_last_name) - return render(request, 'group/active_teams.html', {'teams' : teams }) + return render(request, 'group/active_teams.html', {'teams': teams}) def active_iab(request): iabgroups = Group.objects.filter(type__in=("program","iabasg","iabworkshop"), state="active").order_by("-type_id","name") diff --git a/ietf/settings.py b/ietf/settings.py index e0b4f201182..40a4cb5c56e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2025, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -13,6 +13,7 @@ import warnings from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +from django.http import UnreadablePostError # DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) @@ -236,153 +237,112 @@ FILE_UPLOAD_PERMISSIONS = 0o644 -# ------------------------------------------------------------------------ -# Django/Python Logging Framework Modifications -# Filter out "Invalid HTTP_HOST" emails -# Based on http://www.tiwoc.de/blog/2013/03/django-prevent-email-notification-on-suspiciousoperation/ -from django.core.exceptions import SuspiciousOperation -def skip_suspicious_operations(record): - if record.exc_info: - exc_value = record.exc_info[1] - if isinstance(exc_value, SuspiciousOperation): - return False - return True +# +# Logging config +# -# Filter out UreadablePostError: -from django.http import UnreadablePostError +# Callback to filter out UnreadablePostError: def skip_unreadable_post(record): if record.exc_info: - exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable + exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable if isinstance(exc_value, UnreadablePostError): return False return True -# Copied from DEFAULT_LOGGING as of Django 1.10.5 on 22 Feb 2017, and modified -# to incorporate html logging, invalid http_host filtering, and more. -# Changes from the default has comments. - -# The Python logging flow is as follows: -# (see https://docs.python.org/2.7/howto/logging.html#logging-flow) -# -# Init: get a Logger: logger = logging.getLogger(name) -# -# Logging call, e.g. logger.error(level, msg, *args, exc_info=(...), extra={...}) -# --> Logger (discard if level too low for this logger) -# (create log record from level, msg, args, exc_info, extra) -# --> Filters (discard if any filter attach to logger rejects record) -# --> Handlers (discard if level too low for handler) -# --> Filters (discard if any filter attached to handler rejects record) -# --> Formatter (format log record and emit) -# - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - # - 'loggers': { - 'django': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'ERROR', + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "celery": { + "handlers": ["console"], + "level": "INFO", }, - 'django.server': { - 'handlers': ['django.server'], - 'level': 'INFO', + "datatracker": { + "handlers": ["console"], + "level": "INFO", }, - 'django.security': { - 'handlers': ['console', ], - 'level': 'INFO', + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", }, - 'oidc_provider': { - 'handlers': ['console', ], - 'level': 'DEBUG', + "django.request": {"level": "ERROR"}, # only log 5xx, ignore 4xx + "django.security": { + # SuspiciousOperation errors - log to console only + "handlers": ["console"], + "propagate": False, # no further handling please }, - 'datatracker': { - 'handlers': ['console'], - 'level': 'INFO', + "django.server": { + # Only used by Django's runserver development server + "handlers": ["django.server"], + "level": "INFO", }, - 'celery': { - 'handlers': ['console'], - 'level': 'INFO', + "oidc_provider": { + "handlers": ["console"], + "level": "DEBUG", }, }, - # - # No logger filters - # - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'debug_console': { - # Active only when DEBUG=True - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "debug_console": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'django.server': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'django.server', + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", }, - 'mail_admins': { - 'level': 'ERROR', - 'filters': [ - 'require_debug_false', - 'skip_suspicious_operations', # custom - 'skip_unreadable_posts', # custom + "mail_admins": { + "level": "ERROR", + "filters": [ + "require_debug_false", + "skip_unreadable_posts", ], - 'class': 'django.utils.log.AdminEmailHandler', - 'include_html': True, # non-default - } + "class": "django.utils.log.AdminEmailHandler", + "include_html": True, + }, }, - # # All these are used by handlers - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', - }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", }, - # custom filter, function defined above: - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", }, # custom filter, function defined above: - 'skip_unreadable_posts': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_unreadable_post, + "skip_unreadable_posts": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_unreadable_post, }, }, - # And finally the formatters - 'formatters': { - 'django.server': { - '()': 'django.utils.log.ServerFormatter', - 'format': '[%(server_time)s] %(message)s', + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[%(server_time)s] %(message)s", }, - 'plain': { - 'style': '{', - 'format': '{levelname}: {name}:{lineno}: {message}', + "plain": { + "style": "{", + "format": "{levelname}: {name}:{lineno}: {message}", }, - 'json' : { + "json": { "class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter", "style": "{", - "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}", - } + "format": ( + "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}" + "{process}{status_code}" + ), + }, }, } -# End logging -# ------------------------------------------------------------------------ - X_FRAME_OPTIONS = 'SAMEORIGIN' CSRF_TRUSTED_ORIGINS = [ diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 1f5a7e8ddca..e7ebc13eb28 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -115,8 +115,12 @@ def tempdir_with_cleanup(**kwargs): except NameError: pass -# Use InMemoryStorage for red bucket storage +# Use InMemoryStorage for red bucket and r2-rfc storages STORAGES["red_bucket"] = { "BACKEND": "django.core.files.storage.InMemoryStorage", "OPTIONS": {"location": "red_bucket"}, } +STORAGES["r2-rfc"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "r2-rfc"}, +} diff --git a/ietf/static/js/document_html.js b/ietf/static/js/document_html.js index 6e8861739a6..3e609f3965d 100644 --- a/ietf/static/js/document_html.js +++ b/ietf/static/js/document_html.js @@ -117,4 +117,83 @@ document.addEventListener("DOMContentLoaded", function (event) { } }); } + + // Rewrite these CSS properties so that the values are available for restyling. + document.querySelectorAll("svg [style]").forEach(el => { + // Push these CSS properties into their own attributes + const SVG_PRESENTATION_ATTRS = new Set([ + 'alignment-baseline', 'baseline-shift', 'clip', 'clip-path', 'clip-rule', + 'color', 'color-interpolation', 'color-interpolation-filters', + 'color-rendering', 'cursor', 'direction', 'display', 'dominant-baseline', + 'fill', 'fill-opacity', 'fill-rule', 'filter', 'flood-color', + 'flood-opacity', 'font-family', 'font-size', 'font-size-adjust', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', + 'image-rendering', 'letter-spacing', 'lighting-color', 'marker-end', + 'marker-mid', 'marker-start', 'mask', 'opacity', 'overflow', 'paint-order', + 'pointer-events', 'shape-rendering', 'stop-color', 'stop-opacity', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', + 'text-anchor', 'text-decoration', 'text-rendering', 'unicode-bidi', + 'vector-effect', 'visibility', 'word-spacing', 'writing-mode', + ]); + + // Simple CSS splitter: respects quoted strings and parens so semicolons + // inside url(...) or "..." don't get treated as declaration boundaries. + function parseDeclarations(styleText) { + const decls = []; + let buf = ''; + let inStr = false; + let strChar = ''; + let escaped = false; + let depth = 0; + + for (const ch of styleText) { + if (inStr) { + if (escaped) { + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else if (ch === strChar) { + inStr = false; + } + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } else if (ch === ';' && depth === 0) { + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + buf = ''; + continue; + } + buf += ch; + } + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + return decls; + } + + const remainder = []; + for (const decl of parseDeclarations(el.getAttribute('style'))) { + const [prop, val] = decl.split(":", 2).map(v => v.trim()); + if (val && !/!important$/.test(val) && SVG_PRESENTATION_ATTRS.has(prop)) { + el.setAttribute(prop, val); + } else { + remainder.push(decl); + } + } + + if (remainder.length > 0) { + el.setAttribute('style', remainder.join('; ')); + } else { + el.removeAttribute('style'); + } + }); }); diff --git a/ietf/templates/group/active_teams.html b/ietf/templates/group/active_teams.html index 502d971a206..771dfda290a 100644 --- a/ietf/templates/group/active_teams.html +++ b/ietf/templates/group/active_teams.html @@ -16,21 +16,29 @@

Active teams

Chairs - - {% for group in teams %} - - - {{ group.acronym }} - - {{ group.name }} - - {% for chair in group.chairs %} - {% person_link chair.person %}{% if not forloop.last %},{% endif %} - {% endfor %} - - - {% endfor %} - + {% regroup teams by parent as grouped_teams %} + {% for group_entry in grouped_teams %} + + + {% if group_entry.grouper %}{{ group_entry.grouper.name }}{% else %}Other{% endif %} + + + + {% for group in group_entry.list %} + + + {{ group.acronym }} + + {{ group.name }} + + {% for chair in group.chairs %} + {% person_link chair.person %}{% if not forloop.last %},{% endif %} + {% endfor %} + + + {% endfor %} + + {% endfor %} {% endblock %} {% block js %} diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 8c0c66cdf26..323b7fd45a8 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -80,6 +80,16 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None) +if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None) +if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: + RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) +if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL + # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true"