diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 35172aa299..a2abe089ce 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: token: ${{ secrets.GH_COMMON_TOKEN }} diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 14c4b1a135..429aa08b00 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index b297e34b47..7832e65a3a 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,7 +24,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU uses: docker/setup-qemu-action@v4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a89bac46e7..5088d763e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,7 @@ -name: Build and Release -run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} +name: Build Dev and Deploy +run-name: Dev Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} on: - push: - branches: [release] - workflow_dispatch: inputs: deploy: @@ -14,24 +11,28 @@ on: type: choice options: - Skip - - Staging Only - - Staging + Prod - dev: - description: 'Deploy to Dev' - default: true + - Dev + - Staging + deployStrategy: + description: 'Deploy Strategy (Staging Only)' + default: 'Recreate' required: true - type: boolean + type: choice + options: + - Recreate + - RollingUpdate + - Current devNoDbRefresh: - description: 'Dev Disable Daily DB Refresh' + description: 'Disable Daily DB Refresh (Dev Only)' default: false required: true type: boolean - skiptests: + skipTests: description: 'Skip Tests' default: false required: true type: boolean - skiparm: + skipArmBuild: description: 'Skip ARM64 Build' default: false required: true @@ -56,34 +57,21 @@ jobs: # PREPARE # ----------------------------------------------------------------- prepare: - name: Prepare Release + name: Prepare runs-on: ubuntu-latest outputs: - should_deploy: ${{ steps.buildvars.outputs.should_deploy }} - pkg_version: ${{ steps.buildvars.outputs.pkg_version }} - from_tag: ${{ steps.semver.outputs.nextStrict }} - to_tag: ${{ steps.semver.outputs.current }} - base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false - - name: Get Next Version (Prod) - if: ${{ github.ref_name == 'release' }} - id: semver - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - patchList: fix, bugfix, perf, refactor, test, tests, chore - - name: Get Dev Version - if: ${{ github.ref_name != 'release' }} - id: semverdev + id: semver uses: ietf-tools/semver-action@v1 with: token: ${{ github.token }} @@ -91,260 +79,54 @@ jobs: skipInvalidTags: true noVersionBumpBehavior: 'current' noNewCommitBehavior: 'current' - - - name: Set Release Flag - if: ${{ github.ref_name == 'release' }} - run: | - echo "IS_RELEASE=true" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.ref_name == 'release' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ steps.semver.outputs.nextStrict }} - name: ${{ steps.semver.outputs.nextStrict }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} - name: Set Build Variables id: buildvars run: | - if [[ $IS_RELEASE ]]; then - echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}" - echo "should_deploy=true" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT - echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" - else - echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" - echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" - fi + SHORT_SHA="${GITHUB_SHA:0:7}" + echo "Will set build version to: ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" + echo "buildVersion=${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" >> $GITHUB_OUTPUT + echo "::notice::Non-production build ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA created using branch $GITHUB_REF_NAME" - name: Get Base Image Target Version id: baseimgversion run: | - echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- - tests: name: Run Tests - uses: ./.github/workflows/tests.yml - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: ./.github/workflows/reusable-tests.yml + if: ${{ inputs.skipTests == 'false' }} needs: [prepare] secrets: inherit with: - ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + ignoreLowerCoverage: ${{ inputs.ignoreLowerCoverage == 'true' }} skipSelenium: true - targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} # ----------------------------------------------------------------- - # RELEASE + # BUILD IMAGE # ----------------------------------------------------------------- - release: - name: Make Release + build: + name: Build Image + uses: ./.github/workflows/reusable-build.yml if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: - group: hperf-8c32r permissions: contents: write packages: write - env: - SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - FROM_TAG: ${{needs.prepare.outputs.from_tag}} - TO_TAG: ${{needs.prepare.outputs.to_tag}} - TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Setup Node.js environment - uses: actions/setup-node@v6 - with: - node-version: 18.x - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - name: Setup AWS CLI - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2.22.35 - - - name: Download a Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - run: | - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - - name: Set Production Flags - if: ${{ env.SHOULD_DEPLOY == 'true' }} - run: | - echo "Setting production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - - - name: Make Release Tarball - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . - - - name: Collect + Push Statics - env: - DEBIAN_FRONTEND: noninteractive - AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - run: | - echo "Collecting statics..." - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - - - name: Augment dockerignore for docker image build - env: - DEBIAN_FRONTEND: noninteractive - run: | - cat >> .dockerignore <> $GITHUB_ENV - - - name: Build Images - uses: docker/build-push-action@v7 - env: - DOCKER_BUILD_SUMMARY: false - with: - context: . - file: dev/build/Dockerfile - platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} - push: true - tags: | - ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - - - name: Update CHANGELOG - id: changelog - uses: Requarks/changelog-action@v1 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - token: ${{ github.token }} - fromTag: ${{ env.FROM_TAG }} - toTag: ${{ env.TO_TAG }} - writeToFile: false - - - name: Download Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Prepare Coverage Action - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - working-directory: ./dev/coverage-action - run: npm install - - - name: Process Coverage Stats + Chart - id: covprocess - uses: ./dev/coverage-action/ - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - with: - token: ${{ github.token }} - tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} - repoCommon: common - version: ${{needs.prepare.outputs.pkg_version}} - changelog: ${{ steps.changelog.outputs.changes }} - summary: '' - coverageResultsPath: coverage.json - histCoveragePath: historical-coverage.json - - - name: Create Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - allowUpdates: true - makeLatest: true - draft: false - tag: ${{ env.PKG_VERSION }} - name: ${{ env.PKG_VERSION }} - body: ${{ steps.covprocess.outputs.changelog }} - artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} - with: - allowUpdates: true - tag: baseline - omitBodyDuringUpdate: true - omitNameDuringUpdate: true - omitPrereleaseDuringUpdate: true - replacesArtifacts: true - artifacts: "coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v7 - with: - name: release-${{ env.PKG_VERSION }} - path: /home/runner/work/release/release.tar.gz + secrets: inherit + with: + buildVersion: ${{ needs.prepare.outputs.buildVersion }} + isReleaseBuild: false + previousVersion: ${{ needs.prepare.outputs.previousVersion }} + handleCoverageResults: ${{ inputs.skipTests == 'false' }} + updateCoverageBaseline: ${{ inputs.updateCoverage }} + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + skipArmBuild: ${{ inputs.skipArmBuild }} # ----------------------------------------------------------------- # NOTIFY @@ -352,10 +134,10 @@ jobs: notify: name: Notify if: ${{ always() }} - needs: [prepare, tests, release] + needs: [prepare, tests, build] runs-on: ubuntu-latest env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Notify on Slack (Success) @@ -366,7 +148,7 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "28a745" fields: @@ -381,29 +163,29 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "a82929" fields: - title: "Status" short: true - value: "Failed" + value: "Failed 😱" # ----------------------------------------------------------------- # DEV # ----------------------------------------------------------------- dev: name: Deploy to Dev - if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Dev' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: dev env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: main @@ -424,7 +206,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' waitForCompletionTimeout: 60m # ----------------------------------------------------------------- @@ -432,13 +214,13 @@ jobs: # ----------------------------------------------------------------- staging: name: Deploy to Staging - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Staging' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: staging env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Refresh Staging DB @@ -458,29 +240,5 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - waitForCompletionTimeout: 30m - - # ----------------------------------------------------------------- - # PROD - # ----------------------------------------------------------------- - prod: - name: Deploy to Production - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, staging] - runs-on: ubuntu-latest - environment: - name: production - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to production - uses: ietf-tools/workflow-dispatch-action@v1 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 5349f1ac7a..50ec4c1943 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false @@ -38,7 +38,7 @@ jobs: # ----------------------------------------------------------------- tests: name: Run Tests - uses: ./.github/workflows/tests.yml + uses: ./.github/workflows/reusable-tests.yml needs: [prepare] with: ignoreLowerCoverage: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bc20779ae6..15b2231ce3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index e255b270ff..7393e83b82 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index cd986f06f3..83121354e6 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,7 +29,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Login to GitHub Container Registry uses: docker/login-action@v4 diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index 22652dab88..b90300c09b 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -16,9 +16,10 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: ietf-tools/lock-threads@v3.1.1 + - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} + process-only: 'issues, prs' issue-inactive-days: 7 pr-inactive-days: 3 log-output: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..2de02eb285 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,209 @@ +name: Make Release and Deploy +run-name: RELEASE ${{ github.run_number }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} + + steps: + - name: Ensure release branch + if: github.ref != 'refs/heads/release' + run: | + echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" + exit 1 + + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1 + with: + token: ${{ github.token }} + branch: release + skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore + + - name: Create Draft Release + uses: ncipollo/release-action@v1.21.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Build Variables + id: buildvars + run: | + echo "Will set build version to: ${{ steps.semver.outputs.nextStrict }}" + echo "buildVersion=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT + echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD RELEASE + # ----------------------------------------------------------------- + build: + name: Build Release Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{needs.prepare.outputs.buildVersion}} + isReleaseBuild: true + previousVersion: ${{needs.prepare.outputs.previousVersion}} + handleCoverageResults: true + updateCoverageBaseline: true + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() }} + needs: [prepare, staging] + runs-on: ubuntu-latest + environment: + name: production + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Deploy to production + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 0000000000..4d9a3b4bf1 --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,226 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + buildVersion: + required: true + type: string + isReleaseBuild: + default: false + required: false + type: boolean + previousVersion: + required: false + type: string + handleCoverageResults: + default: true + required: false + type: boolean + updateCoverageBaseline: + default: false + required: false + type: boolean + baseImageVersion: + default: latest + required: false + type: string + skipArmBuild: + default: false + required: false + type: boolean + +jobs: + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + runs-on: + group: hperf-8c32r + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 18.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Setup AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2.22.35 + + - name: Download a Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Make Build + env: + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "BUILD_VERSION: ${{ inputs.buildVersion }}" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/${{ inputs.buildVersion }}/ + yarn legacy:build + echo "Setting version ${{ inputs.buildVersion }}..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '${{ inputs.buildVersion }}'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ inputs.isReleaseBuild }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/${{ inputs.buildVersion }} --only-show-errors + + - name: Augment dockerignore for docker image build + env: + DEBIAN_FRONTEND: noninteractive + run: | + cat >> .dockerignore <> $GITHUB_ENV + + - name: Build Images + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/build/Dockerfile + platforms: ${{ inputs.skipArmBuild && 'linux/amd64' || 'linux/amd64,linux/arm64' }} + push: true + tags: | + ghcr.io/ietf-tools/datatracker:${{ inputs.buildVersion }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} + + - name: Update CHANGELOG + id: changelog + uses: Requarks/changelog-action@v1 + if: ${{ inputs.isReleaseBuild }} + with: + token: ${{ github.token }} + fromTag: ${{ inputs.buildVersion }} + toTag: ${{ inputs.previousVersion }} + writeToFile: false + + - name: Download Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Prepare Coverage Action + if: ${{ inputs.handleCoverageResults }} + working-directory: ./dev/coverage-action + run: npm install + + - name: Process Coverage Stats + Chart + id: covprocess + uses: ./dev/coverage-action/ + if: ${{ inputs.handleCoverageResults }} + with: + token: ${{ github.token }} + tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} + repoCommon: common + version: ${{ inputs.buildVersion }} + changelog: ${{ steps.changelog.outputs.changes }} + summary: '' + coverageResultsPath: coverage.json + histCoveragePath: historical-coverage.json + + - name: Create Release + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.isReleaseBuild }} + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ inputs.buildVersion }} + name: ${{ inputs.buildVersion }} + body: ${{ steps.covprocess.outputs.changelog }} + artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Baseline Coverage + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.updateCoverageBaseline }} + with: + allowUpdates: true + tag: baseline + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + replacesArtifacts: true + artifacts: "coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ inputs.buildVersion }} + path: /home/runner/work/release/release.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/reusable-tests.yml similarity index 98% rename from .github/workflows/tests.yml rename to .github/workflows/reusable-tests.yml index ad2e35408d..10d04c8265 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-node@v6 with: @@ -144,7 +144,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml deleted file mode 100644 index 833ca89bef..0000000000 --- a/.github/workflows/tests-az.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Tests (Azure Test) - -on: - workflow_dispatch: - -jobs: - main: - name: Run Tests on Azure temp VM - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - name: Launch VM on Azure - id: azlaunch - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_TESTS_APP_ID }} -p ${{ secrets.AZ_TESTS_PWD }} --tenant ${{ secrets.AZ_TESTS_TENANT_ID }} - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatrackerTests \ - --name tmpGhaVM2 \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D4as_v5 \ - --max-price -1 \ - --os-disk-size-gb 30 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - - - name: Remote SSH into VM - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - host: ${{ steps.azlaunch.outputs.ipaddr }} - port: 22 - username: azureuser - command_timeout: 60m - key_path: ${{ github.workspace }}/prvkey.key - envs: GITHUB_TOKEN - script_stop: true - script: | - export DEBIAN_FRONTEND=noninteractive - lsb_release -a - sudo apt-get update - sudo apt-get upgrade -y - - echo "Installing Docker..." - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh - - echo "Starting Containers..." - sudo docker network create dtnet - sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & - wait - - echo "Cloning datatracker repo..." - sudo docker exec app git clone --depth=1 https://github.com/ietf-tools/datatracker.git . - echo "Prepare tests..." - sudo docker exec app chmod +x ./dev/tests/prepare.sh - sudo docker exec app sh ./dev/tests/prepare.sh - echo "Running checks..." - sudo docker exec app ietf/manage.py check - sudo docker exec app ietf/manage.py migrate --fake-initial - echo "Running tests..." - sudo docker exec app ietf/manage.py test -v2 --validate-html-harder --settings=settings_test - - - name: Destroy VM + resources - if: always() - shell: pwsh - run: | - echo "Destroying VM..." - az vm delete -g ghaDatatrackerTests -n tmpGhaVM2 --yes --force-deletion true - - $resourceOrderRemovalOrder = [ordered]@{ - "Microsoft.Compute/virtualMachines" = 0 - "Microsoft.Compute/disks" = 1 - "Microsoft.Network/networkInterfaces" = 2 - "Microsoft.Network/publicIpAddresses" = 3 - "Microsoft.Network/networkSecurityGroups" = 4 - "Microsoft.Network/virtualNetworks" = 5 - } - echo "Fetching remaining resources..." - $resources = az resource list --resource-group ghaDatatrackerTests | ConvertFrom-Json - - $orderedResources = $resources - | Sort-Object @{ - Expression = {$resourceOrderRemovalOrder[$_.type]} - Descending = $False - } - - echo "Deleting remaining resources..." - $orderedResources | ForEach-Object { - az resource delete --resource-group ghaDatatrackerTests --ids $_.id --verbose - } - - echo "Logout from Azure..." - az logout diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index a02ab2ff06..0a9dde04eb 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -1,6 +1,7 @@ upstream datatracker_backend { server 127.0.0.1:8001; - keepalive 0; # default = 32 since nginx 1.29.7 +# Uncomment when changing to nginx 1.29.7 or later. +# keepalive 0; # default = 32 since nginx 1.29.7 } server { diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index d4562f97dd..00733717c4 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -177,8 +177,15 @@ def dehydrate(self, bundle, for_list=True): return dehydrated + +# XML 1.0 forbids all control characters except tab (#x9), LF (#xA), and CR (#xD). +# Replace each with its Unicode control picture (U+2400 + codepoint) so the +# substitution is lossless and the result is valid XML. +_XML_INVALID_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") + + class Serializer(tastypie.serializers.Serializer): - OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + OPTION_ESCAPE_XML_INVALID = "datatracker-escape-xml-invalid" def format_datetime(self, data): return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" @@ -186,18 +193,17 @@ def format_datetime(self, data): def to_simple(self, data, options): options = options or {} simple_data = super().to_simple(data, options) - if ( - options.get(self.OPTION_ESCAPE_NULLS, False) - and isinstance(simple_data, str) - ): - # replace nulls with unicode "symbol for null character", \u2400 - simple_data = simple_data.replace("\x00", "\u2400") + if options.get(self.OPTION_ESCAPE_XML_INVALID, False) and isinstance(simple_data, str): + # Replace control chars invalid in XML 1.0 with their Unicode + # control pictures (U+2400-U+241F) so lxml won't reject the string. + simple_data = _XML_INVALID_CTRL_RE.sub( + lambda m: chr(ord(m.group()) + 0x2400), simple_data + ) return simple_data def to_etree(self, data, options=None, name=None, depth=0): - # lxml does not escape nulls on its own, so ask to_simple() to do it. - # This is mostly (only?) an issue when generating errors responses for - # fuzzers. + # lxml rejects control characters that are invalid in XML 1.0. + # Ask to_simple() to escape them before they reach lxml. options = options or {} - options[self.OPTION_ESCAPE_NULLS] = True + options[self.OPTION_ESCAPE_XML_INVALID] = True return super().to_etree(data, options, name, depth) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 2a44791a5c..887969cec1 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1542,20 +1542,27 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) - def test_serializer_to_etree_handles_nulls(self): - """Serializer to_etree() should handle a null character""" + def test_serializer_to_etree_handles_xml_invalid_control_chars(self): + """Serializer to_etree() must not raise ValueError for any XML-invalid control character.""" serializer = Serializer() + # Ordinary strings and strings with valid whitespace must pass through unchanged. try: - serializer.to_etree("string with no nulls in it") + serializer.to_etree("string with no special chars") + serializer.to_etree("tab\there lf\nhere cr\rhere") except ValueError: self.fail("serializer.to_etree raised ValueError on an ordinary string") - try: - serializer.to_etree("string with a \x00 in it") - except ValueError: - self.fail( - "serializer.to_etree raised ValueError on a string " - "containing a null character" - ) + # Every control character that XML 1.0 forbids must be escaped rather than + # causing a ValueError. This is the class of characters that triggered the + # production exception (lxml.etree._utf8 rejects them all). + invalid_chars = [chr(c) for c in list(range(0x00, 0x09)) + [0x0b, 0x0c] + list(range(0x0e, 0x20))] + for ch in invalid_chars: + try: + serializer.to_etree(f"string with {ch!r} in it") + except ValueError: + self.fail( + f"serializer.to_etree raised ValueError on a string " + f"containing control character U+{ord(ch):04X}" + ) class RfcdiffSupportTests(TestCase): diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 156bac4e77..3685ab6551 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -163,7 +163,7 @@ class DocumentInfo(models.Model): @property def doi(self) -> str | None: if self.type_id == "rfc" and self.rfc_number is not None: - return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number}" return None def file_extension(self): diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py index b94fc2cbea..ec3cd85893 100644 --- a/ietf/doc/tests_utils_rfc_json.py +++ b/ietf/doc/tests_utils_rfc_json.py @@ -13,6 +13,8 @@ PublishedRfcDocEventFactory, RfcAuthorFactory, RfcFactory, + RgRfcFactory, + WgDraftFactory, WgRfcFactory, ) from ietf.doc.models import RelatedDocument @@ -198,6 +200,19 @@ def test_errata_url_set_when_errata_exist(self): f"https://www.rfc-editor.org/errata/rfc{rfc.rfc_number}", ) + def test_draft_field_includes_revision(self): + """draft field is '-' when the RFC originated from a draft.""" + draft = WgDraftFactory(rev="07") + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["draft"], f"{draft.name}-07") + def test_errata_url_none_when_no_errata(self): """errata_url is None when errata.json has no entries for the RFC.""" rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc @@ -245,6 +260,19 @@ def test_april_first_date_format(self): self.assertEqual(data["pub_date"], "1 April 2020") + def test_empty_keywords_filtered(self): + """Empty-string keywords are stripped from the keywords list.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(keywords=["foo", "", "bar"]), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["keywords"], ["foo", "bar"]) + def test_non_april_first_april_date(self): """An April publication that is NOT in the April Fools list gets 'April YYYY'.""" rfc = PublishedRfcDocEventFactory( @@ -260,7 +288,7 @@ def test_non_april_first_april_date(self): self.assertEqual(data["pub_date"], "April 2020") def test_source_ietf_wg(self): - """IETF-stream WG RFC: source is 'acronym (area)'.""" + """IETF-stream WG RFC: source is the group's full name.""" area = GroupFactory(type_id="area") wg = GroupFactory(type_id="wg", parent=area) rfc = PublishedRfcDocEventFactory( @@ -272,7 +300,7 @@ def test_source_ietf_wg(self): generate_rfc_json(rfc.rfc_number) data = _read_json(rfc.rfc_number) - self.assertEqual(data["source"], f"{wg.acronym} ({area.acronym})") + self.assertEqual(data["source"], wg.name) def test_source_ietf_no_wg(self): """IETF-stream individual RFC (group acronym 'none'): source is 'IETF - NON WORKING GROUP'.""" @@ -293,6 +321,22 @@ def test_source_ietf_no_wg(self): self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + def test_source_ietf_area(self): + """IETF-stream RFC with area-type group: source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + area.parent = GroupFactory() + area.save() + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(group=area, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + def test_source_iab(self): """IAB-stream RFC: source is 'IAB'.""" rfc = PublishedRfcDocEventFactory( @@ -319,6 +363,30 @@ def test_source_ise(self): self.assertEqual(data["source"], "INDEPENDENT") + def test_source_irtf_rg(self): + """IRTF-stream RG RFC: source is the group's full name.""" + rfc = PublishedRfcDocEventFactory(doc=RgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], rfc.group.name) + + def test_source_irtf_no_rg(self): + """IRTF-stream RFC with no specific RG (group acronym 'none'): source is 'IRTF'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="irtf", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IRTF") + def test_pub_levels_passed_in(self): """When pub_levels is passed in, get_publication_std_levels() is not called.""" rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py index 6030a7064c..1f13455686 100644 --- a/ietf/doc/utils_rfc_json.py +++ b/ietf/doc/utils_rfc_json.py @@ -55,7 +55,7 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: # draft name draft_doc = rfc.came_from_draft() - draft = draft_doc.name if draft_doc else None + draft = f"{draft_doc.name}-{draft_doc.rev}" if draft_doc else None # authors: ordered list of display strings authors = [] @@ -105,7 +105,6 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: stream_slug = rfc.stream.slug group_acronym = rfc.group.acronym - area_acronym = None if stream_slug == "ietf": if rfc.group.parent is None: assertion("rfc.group.parent is not None") @@ -113,23 +112,22 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" ) return - else: - area_acronym = rfc.group.parent.acronym if stream_slug == "ise": source = "INDEPENDENT" elif stream_slug == "iab": source = "IAB" elif stream_slug == "ietf" and ( - group_acronym in ("none", "gen") or not area_acronym + group_acronym == "none" or rfc.group.type_id == "area" ): source = "IETF - NON WORKING GROUP" - elif group_acronym not in ("none", ""): - source = group_acronym - if stream_slug == "ietf" and area_acronym: - source += f" ({area_acronym})" - elif stream_slug: - source += f" ({stream_slug})" + elif stream_slug == "irtf": + if group_acronym == "none": + source = "IRTF" + else: + source = rfc.group.name + elif group_acronym not in ("none", "") and stream_slug in ["ietf", "editorial"]: + source = rfc.group.name elif stream_slug: source = "Legacy" if stream_slug == "legacy" else stream_slug.upper() else: @@ -207,7 +205,7 @@ def _rfc_list(qs, attr): "source": source, "abstract": rfc.abstract, "pub_date": pub_date, - "keywords": rfc.keywords, + "keywords": [kw for kw in rfc.keywords if kw], "obsoletes": obsoletes, "obsoleted_by": obsoleted_by, "updates": updates, diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 3f24e2e3d6..97ec7ebdb1 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, WgRfcFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -396,6 +396,7 @@ def test_group_documents(self): draft7 = WgDraftFactory(group=group) draft7.set_state(State.objects.get(type='draft', slug='expired')) draft7.set_state(State.objects.get(type='draft-stream-%s' % draft7.stream_id, slug='dead')) # Expired WG draft, marked as dead + rfc = WgRfcFactory(group=group) clist = CommunityList.objects.get(group=group) related_docs_rule = clist.searchrule_set.get(rule_type='name_contains') @@ -426,6 +427,12 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # RFC rows must use the RFC number as the sort key so that numeric sort + # is not disrupted by the page-count text that precedes the name in the cell. + # Draft rows must use the document name. + self.assertTrue(q(f'td.doc[data-sort-number="{rfc.rfc_number}"]')) + self.assertTrue(q(f'td.doc[data-sort-number="{draft.name}"]')) + # Let's also check the IRTF stream rg = GroupFactory(type_id='rg') setup_default_community_list_for_group(rg) diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 0df667fbd2..30d51cddd0 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs): | Q(name="atlarge", group__acronym="irsg") ), "RSAB Member": Q(name="member", group__acronym="rsab"), + "LLC Staff": Q(name="member", group__acronym="llc-staff"), "Robot": Q(name="robot", group__acronym="secretariat"), } diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 210788ce07..28152ef79b 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1640,6 +1640,20 @@ def test_feedback_topic_badges(self): q = PyQuery(response.content) self.assertEqual( len(q('.text-bg-success')), 0 ) + def test_feedback_index_sort_keys(self): + url = reverse('ietf.nomcom.views.view_feedback', kwargs={'year': self.nc.year()}) + login_testing_unauthorized(self, self.member.user.username, url) + provide_private_key_to_test_client(self) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + # Feedback count cells must carry a numeric data-sort-number so that + # a "New" badge appearing before the count doesn't corrupt the sort key. + sort_cells = q('td[data-sort-number]') + self.assertTrue(len(sort_cells) > 0) + for cell in sort_cells.items(): + self.assertRegex(cell.attr('data-sort-number'), r'^\d+$') + class NewActiveNomComTests(TestCase): def setUp(self): diff --git a/ietf/settings.py b/ietf/settings.py index 95f2ffefd7..d509a877c6 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -329,7 +329,8 @@ def skip_unreadable_post(record): "formatters": { "django.server": { "()": "django.utils.log.ServerFormatter", - "format": "[%(server_time)s] %(message)s", + "format": "[{server_time}] {message}", + "style": "{", }, "plain": { "style": "{", @@ -976,6 +977,10 @@ def skip_unreadable_post(record): ) RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES +# Paths in the red bucket +RFCINDEX_INPUT_PATH = "other/" +RFCINDEX_OUTPUT_PATH = "other/" + IDSUBMIT_MAX_DRAFT_SIZE = { 'txt': 2*1024*1024, # Max size of txt draft file in bytes 'xml': 3*1024*1024, # Max size of xml draft file in bytes diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index dc5b5d6ae8..d8e8741702 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,8 +1,10 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved import calendar -import json +import csv import datetime +import io +import json from django.http import Http404 from pyquery import PyQuery @@ -18,10 +20,12 @@ import ietf.stats.views -from ietf.group.factories import RoleFactory -from ietf.person.factories import PersonFactory +from ietf.doc.factories import NewRevisionDocEventFactory +from ietf.group.factories import GroupFactory, RoleFactory +from ietf.person.factories import EmailFactory, PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory +from ietf.submit.factories import SubmissionFactory from ietf.utils.timezone import date_today @@ -183,3 +187,139 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + +class AnnualReportInputsTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + self.non_member = PersonFactory() + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + + def test_access_unauthenticated(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + def test_access_non_member_forbidden(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + self.client.login( + username=self.non_member.user.username, + password=f"{self.non_member.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + + def test_access_member_allowed(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + + def test_default_year(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], datetime.date.today().year - 1) + + def test_year_param_redirects_to_year_url(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url, {"year": "2022"}) + self.assertRedirects( + r, + urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2022"}), + ) + + def test_summary_counts(self): + self._member_login() + year = 2021 + # author1 has a matching Person record; author2 does not + EmailFactory(address="author1@example.com") + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 6, 1), + submitter_email="submitter@example.com", + ) + sub.authors = [ + {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []}, + {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + # Same draft, second revision — should count once + extra = NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 15, tzinfo=datetime.timezone.utc), + doc=extra.doc, + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], year) + self.assertEqual(r.context["author_count"], 2) + self.assertEqual(r.context["author_person_count"], 1) + self.assertEqual(r.context["author_noperson_count"], 1) + self.assertEqual(r.context["submitter_count"], 1) + self.assertEqual(r.context["submitter_person_count"], 0) + self.assertEqual(r.context["submitter_noperson_count"], 1) + self.assertEqual(r.context["draft_count"], 3) + + def test_download_authors_csv(self): + self._member_login() + year = 2020 + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + ) + sub.authors = [ + {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "authors"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"authors-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvauthor@example.com", rows[0]) + + def test_download_submitters_csv(self): + self._member_login() + year = 2020 + SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + submitter_email="csvsubmitter@example.com", + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "submitters"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"submitters-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvsubmitter@example.com", rows[0]) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 01b8758c84..3bb107813a 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,4 +15,5 @@ url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), + url(r"^annual_report_inputs/(?:(?P\d{4})/)?$", views.annual_report_inputs), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index d61c9cab64..fe2fa82f55 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,8 +1,9 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- import calendar +import csv import datetime import itertools import json @@ -12,7 +13,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.cache import cache -from django.http import HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render, get_object_or_404 from django.urls import reverse as urlreverse from django.db.models import Count @@ -28,7 +29,7 @@ from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName from ietf.meeting.models import Registration, Meeting -from ietf.ietfauth.utils import has_role +from ietf.ietfauth.utils import has_role, role_required from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO from ietf.meeting.helpers import get_current_ietf_meeting_num @@ -957,3 +958,46 @@ def time_key_fn(t): "possible_states": possible_states, "selected_state": selected_state, }) + + +@role_required("LLC Staff") +def annual_report_inputs(request, year=None): + if year is None and "year" in request.GET: + return HttpResponseRedirect( + urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}) + ) + year = int(year) if year else datetime.date.today().year - 1 + + from ietf.doc.models import NewRevisionDocEvent + from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people + + download = request.GET.get("download") + if download in ("authors", "submitters"): + addresses = authors_by_year(year) if download == "authors" else submitters_by_year(year) + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="{download}-{year}.csv"' + writer = csv.writer(response) + writer.writerow(sorted(addresses)) + return response + + authors = authors_by_year(year) + submitters = submitters_by_year(year) + author_persons, author_nopersons = unique_people(authors) + submitter_persons, submitter_nopersons = unique_people(submitters) + + draft_count = len(set( + NewRevisionDocEvent.objects.filter( + doc__type_id="draft", time__year=year + ).values_list("doc__name", flat=True) + )) + + return render(request, "stats/annual_report_inputs.html", { + "year": year, + "author_count": len(authors), + "submitter_count": len(submitters), + "author_person_count": author_persons.count(), + "author_noperson_count": len(author_nopersons), + "submitter_person_count": submitter_persons.count(), + "submitter_noperson_count": len(submitter_nopersons), + "draft_count": draft_count, + }) diff --git a/ietf/submit/factories.py b/ietf/submit/factories.py index 1076434b82..69888a055d 100644 --- a/ietf/submit/factories.py +++ b/ietf/submit/factories.py @@ -21,6 +21,9 @@ class Meta: class SubmissionFactory(factory.django.DjangoModelFactory): state_id = 'uploaded' + submitter_name = factory.Faker("name") + submitter_email = factory.Faker("email") + submitter = factory.LazyAttribute(lambda o: f"{o.submitter_name} <{o.submitter_email}>") @factory.lazy_attribute_sequence def name(self, n): @@ -32,3 +35,4 @@ def auth_key(self): class Meta: model = Submission + exclude = ("submitter_name", "submitter_email") diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index be55a6866e..f47974f900 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -48,9 +48,17 @@ def errata_url(rfc: Document): return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") +def red_bucket_input_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_INPUT_PATH) / filename) + + +def red_bucket_output_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_OUTPUT_PATH) / filename) + + def save_to_red_bucket(filename: str, content: str | bytes): red_bucket = storages["red_bucket"] - bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + bucket_path = red_bucket_output_path(filename) if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): # Django 4.2's FileSystemStorage does not support allow_overwrite. red_bucket.delete(bucket_path) @@ -87,8 +95,7 @@ class UnusableRfcNumber: def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: - FILENAME = "unusable-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("unusable-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -115,8 +122,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: def get_april1_rfc_numbers() -> Container[int]: - FILENAME = "april-first-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("april-first-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -139,8 +145,7 @@ def get_april1_rfc_numbers() -> Container[int]: def get_publication_std_levels() -> dict[int, StdLevelName]: - FILENAME = "publication-std-levels.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("publication-std-levels.json") values: dict[int, StdLevelName] = {} try: with storages["red_bucket"].open(bucket_path) as urn_file: diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 2b70924db3..226b41af94 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -28,9 +28,11 @@ get_april1_rfc_numbers, get_publication_std_levels, get_unusable_rfc_numbers, + red_bucket_input_path, + red_bucket_output_path, + save_to_filesystem, save_to_red_bucket, subseries_text_line, - save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -139,7 +141,10 @@ def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", stripped_contents, ) - self.assertIn("1 April 2020", contents) # from the April 1 RFC + # "1 April 2020" may be split across a line wrap (e.g. "1 April\n 2020") + # when the randomly-generated title is long enough to push the date off the line. + # assertRegex handles both wrapped and non-wrapped cases explicitly. + self.assertRegex(contents, r"1\s+April\s+2020") # from the April 1 RFC self.assertIn( f"{self.rfc.rfc_number} {self.rfc.title}", stripped_contents, @@ -398,25 +403,47 @@ def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): ) +@override_settings(RFCINDEX_INPUT_PATH="input/", RFCINDEX_OUTPUT_PATH="output/") class HelperTests(TestCase): + INPUT_PATH = "input" + OUTPUT_PATH = "output" + def test_format_rfc_number(self): self.assertEqual(format_rfc_number(10), "10") with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): self.assertEqual(format_rfc_number(10), "0010") + def test_red_bucket_input_path(self): + with override_settings(RFCINDEX_INPUT_PATH="bar"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + with override_settings(RFCINDEX_INPUT_PATH="bar/"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + + def test_red_bucket_output_path(self): + self.assertEqual(red_bucket_input_path("foo"), f"{self.INPUT_PATH}/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar/"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + def test_save_to_red_bucket(self): red_bucket = storages["red_bucket"] with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): save_to_red_bucket("test", "contents \U0001f600") # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") - red_bucket.delete("test") # clean up like a good child + red_bucket.delete(f"{self.OUTPUT_PATH}/test") # clean up like a good child + # check that we can override the path + with override_settings(RFCINDEX_OUTPUT_PATH="fruit"): + save_to_red_bucket("test", "content") + self.assertTrue(red_bucket.exists("fruit/test")) + red_bucket.delete("fruit/test") # clean up like a good child def test_save_to_filesystem(self): rfc_path = Path(settings.RFC_PATH) @@ -442,30 +469,36 @@ def test_get_unusable_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_unusable_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/unusable-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_unusable_rfc_numbers() - red_bucket.delete("unusable-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/unusable-rfc-numbers.json") def test_get_april1_rfc_numbers_raises(self): """get_april1_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError): get_april1_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/april-first-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_april1_rfc_numbers() - red_bucket.delete("april-first-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/april-first-rfc-numbers.json") def test_get_publication_std_levels_raises(self): """get_publication_std_levels should bail on errors""" with self.assertRaises(FileNotFoundError): get_publication_std_levels() red_bucket = storages["red_bucket"] - red_bucket.save("publication-std-levels.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/publication-std-levels.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() - red_bucket.delete("publication-std-levels.json") + red_bucket.delete(f"{self.INPUT_PATH}/publication-std-levels.json") def test_subseries_text_line(self): text = "foobar" diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 43ca025e28..b6b744ec21 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} {% origin %} {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %} @@ -453,6 +453,14 @@ {% endif %} + {% if user|has_role:"LLC Staff" %} +
  • + + Annual report inputs + +
    • + {% endif %}
  • diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html index 476c81f598..cb50a1b214 100644 --- a/ietf/templates/doc/search/search_result_row.html +++ b/ietf/templates/doc/search/search_result_row.html @@ -45,7 +45,7 @@ {% endfor %} - + {% if doc.pages %}{{ doc.pages }} page{{ doc.pages|pluralize }}{% endif %}
    diff --git a/ietf/templates/nomcom/view_feedback.html b/ietf/templates/nomcom/view_feedback.html index 93d61b7f42..0758329d60 100644 --- a/ietf/templates/nomcom/view_feedback.html +++ b/ietf/templates/nomcom/view_feedback.html @@ -43,7 +43,7 @@

    Declined each nominated position

     {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} @@ -82,7 +82,7 @@

    Feedback related to topics

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 0000000000..15add7ece3 --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    +
    +

    Summary

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 0000000000..9a969a5217 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved + +from typing import List, Set, Tuple +from django.db.models import QuerySet + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" + addresses = set() + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): + addresses.update([a["email"] for a in submission.authors]) + return addresses + + +def submitters_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year, state="posted" + ).values_list("submitter", flat=True) + ] + ) + + +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lengths of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (persons, set(addresses) - set(known_email)) + + diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index 6a8f4529a8..4e1ee27895 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -171,6 +171,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "acronym": rfc.group.acronym, "name": rfc.group.name, "full": f"{rfc.group.acronym} - {rfc.group.name}", + "type": rfc.group.type.slug, } if ( rfc.group.parent is not None diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 0000000000..83daa15cc1 --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,189 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + authors2020, + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + authors2021, + set( + [ + "author0@example.net", + "author0@example.com", + "author2@example.net", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + sub2020, + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 6e63001e02..ef8c259933 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: auth - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index af2bb6295c..5183893bc8 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: datatracker - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 20c5252ff0..5dc31bac0e 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -472,8 +472,10 @@ def _multiline_to_list(s): ), } RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default -RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") -RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") +if "DATATRACKER_RFCINDEX_OUTPUT_PATH" in os.environ: + RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH") +if "DATATRACKER_RFCINDEX_INPUT_PATH" in os.environ: + RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH") # Configure the blobdb app for artifact storage _blobdb_replication_enabled = (