diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 35172aa2996..a2abe089ce6 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: token: ${{ secrets.GH_COMMON_TOKEN }} diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 14c4b1a135a..429aa08b006 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index b297e34b476..7832e65a3a3 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,7 +24,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU uses: docker/setup-qemu-action@v4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a89bac46e74..5088d763e92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,7 @@ -name: Build and Release -run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} +name: Build Dev and Deploy +run-name: Dev Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} on: - push: - branches: [release] - workflow_dispatch: inputs: deploy: @@ -14,24 +11,28 @@ on: type: choice options: - Skip - - Staging Only - - Staging + Prod - dev: - description: 'Deploy to Dev' - default: true + - Dev + - Staging + deployStrategy: + description: 'Deploy Strategy (Staging Only)' + default: 'Recreate' required: true - type: boolean + type: choice + options: + - Recreate + - RollingUpdate + - Current devNoDbRefresh: - description: 'Dev Disable Daily DB Refresh' + description: 'Disable Daily DB Refresh (Dev Only)' default: false required: true type: boolean - skiptests: + skipTests: description: 'Skip Tests' default: false required: true type: boolean - skiparm: + skipArmBuild: description: 'Skip ARM64 Build' default: false required: true @@ -56,34 +57,21 @@ jobs: # PREPARE # ----------------------------------------------------------------- prepare: - name: Prepare Release + name: Prepare runs-on: ubuntu-latest outputs: - should_deploy: ${{ steps.buildvars.outputs.should_deploy }} - pkg_version: ${{ steps.buildvars.outputs.pkg_version }} - from_tag: ${{ steps.semver.outputs.nextStrict }} - to_tag: ${{ steps.semver.outputs.current }} - base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false - - name: Get Next Version (Prod) - if: ${{ github.ref_name == 'release' }} - id: semver - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - patchList: fix, bugfix, perf, refactor, test, tests, chore - - name: Get Dev Version - if: ${{ github.ref_name != 'release' }} - id: semverdev + id: semver uses: ietf-tools/semver-action@v1 with: token: ${{ github.token }} @@ -91,260 +79,54 @@ jobs: skipInvalidTags: true noVersionBumpBehavior: 'current' noNewCommitBehavior: 'current' - - - name: Set Release Flag - if: ${{ github.ref_name == 'release' }} - run: | - echo "IS_RELEASE=true" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.ref_name == 'release' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ steps.semver.outputs.nextStrict }} - name: ${{ steps.semver.outputs.nextStrict }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} - name: Set Build Variables id: buildvars run: | - if [[ $IS_RELEASE ]]; then - echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}" - echo "should_deploy=true" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT - echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" - else - echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" - echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" - fi + SHORT_SHA="${GITHUB_SHA:0:7}" + echo "Will set build version to: ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" + echo "buildVersion=${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" >> $GITHUB_OUTPUT + echo "::notice::Non-production build ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA created using branch $GITHUB_REF_NAME" - name: Get Base Image Target Version id: baseimgversion run: | - echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- - tests: name: Run Tests - uses: ./.github/workflows/tests.yml - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: ./.github/workflows/reusable-tests.yml + if: ${{ inputs.skipTests == 'false' }} needs: [prepare] secrets: inherit with: - ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + ignoreLowerCoverage: ${{ inputs.ignoreLowerCoverage == 'true' }} skipSelenium: true - targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} # ----------------------------------------------------------------- - # RELEASE + # BUILD IMAGE # ----------------------------------------------------------------- - release: - name: Make Release + build: + name: Build Image + uses: ./.github/workflows/reusable-build.yml if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: - group: hperf-8c32r permissions: contents: write packages: write - env: - SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - FROM_TAG: ${{needs.prepare.outputs.from_tag}} - TO_TAG: ${{needs.prepare.outputs.to_tag}} - TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Setup Node.js environment - uses: actions/setup-node@v6 - with: - node-version: 18.x - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - name: Setup AWS CLI - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2.22.35 - - - name: Download a Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - run: | - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - - name: Set Production Flags - if: ${{ env.SHOULD_DEPLOY == 'true' }} - run: | - echo "Setting production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - - - name: Make Release Tarball - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . - - - name: Collect + Push Statics - env: - DEBIAN_FRONTEND: noninteractive - AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - run: | - echo "Collecting statics..." - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - - - name: Augment dockerignore for docker image build - env: - DEBIAN_FRONTEND: noninteractive - run: | - cat >> .dockerignore <> $GITHUB_ENV - - - name: Build Images - uses: docker/build-push-action@v7 - env: - DOCKER_BUILD_SUMMARY: false - with: - context: . - file: dev/build/Dockerfile - platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} - push: true - tags: | - ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - - - name: Update CHANGELOG - id: changelog - uses: Requarks/changelog-action@v1 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - token: ${{ github.token }} - fromTag: ${{ env.FROM_TAG }} - toTag: ${{ env.TO_TAG }} - writeToFile: false - - - name: Download Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Prepare Coverage Action - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - working-directory: ./dev/coverage-action - run: npm install - - - name: Process Coverage Stats + Chart - id: covprocess - uses: ./dev/coverage-action/ - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - with: - token: ${{ github.token }} - tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} - repoCommon: common - version: ${{needs.prepare.outputs.pkg_version}} - changelog: ${{ steps.changelog.outputs.changes }} - summary: '' - coverageResultsPath: coverage.json - histCoveragePath: historical-coverage.json - - - name: Create Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - allowUpdates: true - makeLatest: true - draft: false - tag: ${{ env.PKG_VERSION }} - name: ${{ env.PKG_VERSION }} - body: ${{ steps.covprocess.outputs.changelog }} - artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} - with: - allowUpdates: true - tag: baseline - omitBodyDuringUpdate: true - omitNameDuringUpdate: true - omitPrereleaseDuringUpdate: true - replacesArtifacts: true - artifacts: "coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v7 - with: - name: release-${{ env.PKG_VERSION }} - path: /home/runner/work/release/release.tar.gz + secrets: inherit + with: + buildVersion: ${{ needs.prepare.outputs.buildVersion }} + isReleaseBuild: false + previousVersion: ${{ needs.prepare.outputs.previousVersion }} + handleCoverageResults: ${{ inputs.skipTests == 'false' }} + updateCoverageBaseline: ${{ inputs.updateCoverage }} + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + skipArmBuild: ${{ inputs.skipArmBuild }} # ----------------------------------------------------------------- # NOTIFY @@ -352,10 +134,10 @@ jobs: notify: name: Notify if: ${{ always() }} - needs: [prepare, tests, release] + needs: [prepare, tests, build] runs-on: ubuntu-latest env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Notify on Slack (Success) @@ -366,7 +148,7 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "28a745" fields: @@ -381,29 +163,29 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "a82929" fields: - title: "Status" short: true - value: "Failed" + value: "Failed 😱" # ----------------------------------------------------------------- # DEV # ----------------------------------------------------------------- dev: name: Deploy to Dev - if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Dev' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: dev env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: main @@ -424,7 +206,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' waitForCompletionTimeout: 60m # ----------------------------------------------------------------- @@ -432,13 +214,13 @@ jobs: # ----------------------------------------------------------------- staging: name: Deploy to Staging - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Staging' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: staging env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Refresh Staging DB @@ -458,29 +240,5 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - waitForCompletionTimeout: 30m - - # ----------------------------------------------------------------- - # PROD - # ----------------------------------------------------------------- - prod: - name: Deploy to Production - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, staging] - runs-on: ubuntu-latest - environment: - name: production - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to production - uses: ietf-tools/workflow-dispatch-action@v1 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 5349f1ac7a6..50ec4c1943b 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false @@ -38,7 +38,7 @@ jobs: # ----------------------------------------------------------------- tests: name: Run Tests - uses: ./.github/workflows/tests.yml + uses: ./.github/workflows/reusable-tests.yml needs: [prepare] with: ignoreLowerCoverage: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bc20779ae6c..15b2231ce39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index e255b270ff9..7393e83b829 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index cd986f06f33..83121354e64 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,7 +29,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Login to GitHub Container Registry uses: docker/login-action@v4 diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index 22652dab889..b90300c09b2 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -16,9 +16,10 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: ietf-tools/lock-threads@v3.1.1 + - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} + process-only: 'issues, prs' issue-inactive-days: 7 pr-inactive-days: 3 log-output: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..2de02eb2852 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,209 @@ +name: Make Release and Deploy +run-name: RELEASE ${{ github.run_number }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} + + steps: + - name: Ensure release branch + if: github.ref != 'refs/heads/release' + run: | + echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" + exit 1 + + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1 + with: + token: ${{ github.token }} + branch: release + skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore + + - name: Create Draft Release + uses: ncipollo/release-action@v1.21.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Build Variables + id: buildvars + run: | + echo "Will set build version to: ${{ steps.semver.outputs.nextStrict }}" + echo "buildVersion=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT + echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD RELEASE + # ----------------------------------------------------------------- + build: + name: Build Release Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{needs.prepare.outputs.buildVersion}} + isReleaseBuild: true + previousVersion: ${{needs.prepare.outputs.previousVersion}} + handleCoverageResults: true + updateCoverageBaseline: true + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() }} + needs: [prepare, staging] + runs-on: ubuntu-latest + environment: + name: production + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Deploy to production + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 00000000000..4d9a3b4bf11 --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,226 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + buildVersion: + required: true + type: string + isReleaseBuild: + default: false + required: false + type: boolean + previousVersion: + required: false + type: string + handleCoverageResults: + default: true + required: false + type: boolean + updateCoverageBaseline: + default: false + required: false + type: boolean + baseImageVersion: + default: latest + required: false + type: string + skipArmBuild: + default: false + required: false + type: boolean + +jobs: + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + runs-on: + group: hperf-8c32r + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 18.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Setup AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2.22.35 + + - name: Download a Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Make Build + env: + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "BUILD_VERSION: ${{ inputs.buildVersion }}" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/${{ inputs.buildVersion }}/ + yarn legacy:build + echo "Setting version ${{ inputs.buildVersion }}..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '${{ inputs.buildVersion }}'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ inputs.isReleaseBuild }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/${{ inputs.buildVersion }} --only-show-errors + + - name: Augment dockerignore for docker image build + env: + DEBIAN_FRONTEND: noninteractive + run: | + cat >> .dockerignore <> $GITHUB_ENV + + - name: Build Images + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/build/Dockerfile + platforms: ${{ inputs.skipArmBuild && 'linux/amd64' || 'linux/amd64,linux/arm64' }} + push: true + tags: | + ghcr.io/ietf-tools/datatracker:${{ inputs.buildVersion }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} + + - name: Update CHANGELOG + id: changelog + uses: Requarks/changelog-action@v1 + if: ${{ inputs.isReleaseBuild }} + with: + token: ${{ github.token }} + fromTag: ${{ inputs.buildVersion }} + toTag: ${{ inputs.previousVersion }} + writeToFile: false + + - name: Download Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Prepare Coverage Action + if: ${{ inputs.handleCoverageResults }} + working-directory: ./dev/coverage-action + run: npm install + + - name: Process Coverage Stats + Chart + id: covprocess + uses: ./dev/coverage-action/ + if: ${{ inputs.handleCoverageResults }} + with: + token: ${{ github.token }} + tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} + repoCommon: common + version: ${{ inputs.buildVersion }} + changelog: ${{ steps.changelog.outputs.changes }} + summary: '' + coverageResultsPath: coverage.json + histCoveragePath: historical-coverage.json + + - name: Create Release + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.isReleaseBuild }} + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ inputs.buildVersion }} + name: ${{ inputs.buildVersion }} + body: ${{ steps.covprocess.outputs.changelog }} + artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Baseline Coverage + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.updateCoverageBaseline }} + with: + allowUpdates: true + tag: baseline + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + replacesArtifacts: true + artifacts: "coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ inputs.buildVersion }} + path: /home/runner/work/release/release.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/reusable-tests.yml similarity index 98% rename from .github/workflows/tests.yml rename to .github/workflows/reusable-tests.yml index ad2e35408d4..10d04c8265e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-node@v6 with: @@ -144,7 +144,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml deleted file mode 100644 index 833ca89befd..00000000000 --- a/.github/workflows/tests-az.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Tests (Azure Test) - -on: - workflow_dispatch: - -jobs: - main: - name: Run Tests on Azure temp VM - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - name: Launch VM on Azure - id: azlaunch - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_TESTS_APP_ID }} -p ${{ secrets.AZ_TESTS_PWD }} --tenant ${{ secrets.AZ_TESTS_TENANT_ID }} - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatrackerTests \ - --name tmpGhaVM2 \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D4as_v5 \ - --max-price -1 \ - --os-disk-size-gb 30 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - - - name: Remote SSH into VM - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - host: ${{ steps.azlaunch.outputs.ipaddr }} - port: 22 - username: azureuser - command_timeout: 60m - key_path: ${{ github.workspace }}/prvkey.key - envs: GITHUB_TOKEN - script_stop: true - script: | - export DEBIAN_FRONTEND=noninteractive - lsb_release -a - sudo apt-get update - sudo apt-get upgrade -y - - echo "Installing Docker..." - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh - - echo "Starting Containers..." - sudo docker network create dtnet - sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & - wait - - echo "Cloning datatracker repo..." - sudo docker exec app git clone --depth=1 https://github.com/ietf-tools/datatracker.git . - echo "Prepare tests..." - sudo docker exec app chmod +x ./dev/tests/prepare.sh - sudo docker exec app sh ./dev/tests/prepare.sh - echo "Running checks..." - sudo docker exec app ietf/manage.py check - sudo docker exec app ietf/manage.py migrate --fake-initial - echo "Running tests..." - sudo docker exec app ietf/manage.py test -v2 --validate-html-harder --settings=settings_test - - - name: Destroy VM + resources - if: always() - shell: pwsh - run: | - echo "Destroying VM..." - az vm delete -g ghaDatatrackerTests -n tmpGhaVM2 --yes --force-deletion true - - $resourceOrderRemovalOrder = [ordered]@{ - "Microsoft.Compute/virtualMachines" = 0 - "Microsoft.Compute/disks" = 1 - "Microsoft.Network/networkInterfaces" = 2 - "Microsoft.Network/publicIpAddresses" = 3 - "Microsoft.Network/networkSecurityGroups" = 4 - "Microsoft.Network/virtualNetworks" = 5 - } - echo "Fetching remaining resources..." - $resources = az resource list --resource-group ghaDatatrackerTests | ConvertFrom-Json - - $orderedResources = $resources - | Sort-Object @{ - Expression = {$resourceOrderRemovalOrder[$_.type]} - Descending = $False - } - - echo "Deleting remaining resources..." - $orderedResources | ForEach-Object { - az resource delete --resource-group ghaDatatrackerTests --ids $_.id --verbose - } - - echo "Logout from Azure..." - az logout diff --git a/.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip b/.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip new file mode 100644 index 00000000000..65b1d3941c4 Binary files /dev/null and b/.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip differ diff --git a/.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip b/.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip new file mode 100644 index 00000000000..e4d1e3fd1df Binary files /dev/null and b/.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip differ diff --git a/.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip b/.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip new file mode 100644 index 00000000000..1babeca0a97 Binary files /dev/null and b/.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip differ diff --git a/.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip b/.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip new file mode 100644 index 00000000000..2dba0d6d9f9 Binary files /dev/null and b/.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip differ diff --git a/.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip b/.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip new file mode 100644 index 00000000000..16cdf883994 Binary files /dev/null and b/.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip differ diff --git a/.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip b/.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip new file mode 100644 index 00000000000..edf50232bc1 Binary files /dev/null and b/.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip differ diff --git a/.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip b/.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip new file mode 100644 index 00000000000..36d13b6aaf3 Binary files /dev/null and b/.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip differ diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 1e44f477612..72bdb7a7ae1 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260527T1529 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 02841510191..7ca47fd1979 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260527T1529 +20260605T2314 diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index e1f739970bd..0a9dde04eb4 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -1,5 +1,7 @@ upstream datatracker_backend { server 127.0.0.1:8001; +# Uncomment when changing to nginx 1.29.7 or later. +# keepalive 0; # default = 32 since nginx 1.29.7 } server { diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index d4562f97ddf..f4bfe8330ee 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -10,6 +10,7 @@ from django.apps import apps as django_apps from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponseNotAllowed from django.utils.module_loading import autodiscover_modules @@ -50,6 +51,9 @@ def autodiscover(): class ModelResource(tastypie.resources.ModelResource): + def post_detail(self, request, **kwargs): + return HttpResponseNotAllowed(["GET"]) + def generate_cache_key(self, *args, **kwargs): """ Creates a unique-enough cache key. @@ -177,8 +181,15 @@ def dehydrate(self, bundle, for_list=True): return dehydrated + +# XML 1.0 forbids all control characters except tab (#x9), LF (#xA), and CR (#xD). +# Replace each with its Unicode control picture (U+2400 + codepoint) so the +# substitution is lossless and the result is valid XML. +_XML_INVALID_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") + + class Serializer(tastypie.serializers.Serializer): - OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + OPTION_ESCAPE_XML_INVALID = "datatracker-escape-xml-invalid" def format_datetime(self, data): return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" @@ -186,18 +197,17 @@ def format_datetime(self, data): def to_simple(self, data, options): options = options or {} simple_data = super().to_simple(data, options) - if ( - options.get(self.OPTION_ESCAPE_NULLS, False) - and isinstance(simple_data, str) - ): - # replace nulls with unicode "symbol for null character", \u2400 - simple_data = simple_data.replace("\x00", "\u2400") + if options.get(self.OPTION_ESCAPE_XML_INVALID, False) and isinstance(simple_data, str): + # Replace control chars invalid in XML 1.0 with their Unicode + # control pictures (U+2400-U+241F) so lxml won't reject the string. + simple_data = _XML_INVALID_CTRL_RE.sub( + lambda m: chr(ord(m.group()) + 0x2400), simple_data + ) return simple_data def to_etree(self, data, options=None, name=None, depth=0): - # lxml does not escape nulls on its own, so ask to_simple() to do it. - # This is mostly (only?) an issue when generating errors responses for - # fuzzers. + # lxml rejects control characters that are invalid in XML 1.0. + # Ask to_simple() to escape them before they reach lxml. options = options or {} - options[self.OPTION_ESCAPE_NULLS] = True + options[self.OPTION_ESCAPE_XML_INVALID] = True return super().to_etree(data, options, name, depth) diff --git a/ietf/api/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py index 889b2cdfb51..8e2c6f4b5cf 100644 --- a/ietf/api/management/commands/makeresources.py +++ b/ietf/api/management/commands/makeresources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,7 +15,7 @@ from django.template import Template, Context from django.utils import timezone -from tastypie.resources import ModelResource +from ietf.api import ModelResource resource_head_template = """# Copyright The IETF Trust {{date}}, All Rights Reserved @@ -23,7 +23,7 @@ # Generated by the makeresources management command {{date}} -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 0e45ee3b391..ca605fe6441 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -805,7 +805,7 @@ class RfcFileSerializer(serializers.Serializer): # works. allowed_extensions = ( ".html", - ".json", + ".json", # deprecated - accepted but ignored, datatracker generates its own ".notprepped.xml", ".pdf", ".txt", diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 2a44791a5cf..87f7d684d32 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -12,6 +12,7 @@ from importlib import import_module from pathlib import Path from random import randrange +from urllib.parse import urljoin from django.apps import apps from django.conf import settings @@ -1542,20 +1543,57 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) - def test_serializer_to_etree_handles_nulls(self): - """Serializer to_etree() should handle a null character""" + def test_serializer_to_etree_handles_xml_invalid_control_chars(self): + """Serializer to_etree() must not raise ValueError for any XML-invalid control character.""" serializer = Serializer() + # Ordinary strings and strings with valid whitespace must pass through unchanged. try: - serializer.to_etree("string with no nulls in it") + serializer.to_etree("string with no special chars") + serializer.to_etree("tab\there lf\nhere cr\rhere") except ValueError: self.fail("serializer.to_etree raised ValueError on an ordinary string") - try: - serializer.to_etree("string with a \x00 in it") - except ValueError: - self.fail( - "serializer.to_etree raised ValueError on a string " - "containing a null character" + # Every control character that XML 1.0 forbids must be escaped rather than + # causing a ValueError. This is the class of characters that triggered the + # production exception (lxml.etree._utf8 rejects them all). + invalid_chars = [chr(c) for c in list(range(0x00, 0x09)) + [0x0b, 0x0c] + list(range(0x0e, 0x20))] + for ch in invalid_chars: + try: + serializer.to_etree(f"string with {ch!r} in it") + except ValueError: + self.fail( + f"serializer.to_etree raised ValueError on a string " + f"containing control character U+{ord(ch):04X}" + ) + + def test_post_detail_is_not_allowed(self): + """POST to a detail route returns 405 + + Added because default TastyPie behavior is a 500 due to a NotImplemented + exception. + """ + r = self.client.get("/api/v1", headers={"Accept": "application/json"}) + self.assertValidJSONResponse(r) + resource_list = r.json() + for name in self.apps: + r = self.client.get( + resource_list[name]["list_endpoint"], + headers={"Accept": "application/json"}, ) + self.assertValidJSONResponse(r) + app_resources = r.json() + model_list = apps.get_app_config(name).get_models() + for model in model_list: + model_name = model._meta.model_name + assert model_name + detail_url = urljoin( + app_resources[model_name]["list_endpoint"], "some-id/" + ) + r = self.client.post(detail_url) + self.assertEqual( + r.status_code, + 405, + f"POST to {name}.{model_name} detail should return 405 status", + ) class RfcdiffSupportTests(TestCase): diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 6d147c00b05..5748a7caa60 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -224,6 +224,7 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.update_rfc_json_task") @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") @@ -232,6 +233,7 @@ def test_upload_rfc_files( mock_trigger_red_task, mock_update_searchindex_task, mock_rebuild_relations, + mock_update_rfc_json, ): def _valid_post_data(): """Generate a valid post data dict @@ -292,7 +294,7 @@ def _valid_post_data(): ContentFile(b"", "myfile.txt"), ContentFile(b"", "myfile.html"), ContentFile(b"", "myfile.pdf"), - ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.json"), # deprecated! ContentFile(b"", "myfile.notprepped.xml"), ] }, @@ -346,18 +348,19 @@ def _valid_post_data(): # valid post mock_trigger_red_task.delay.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) self.assertEqual(r.status_code, 200) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), ) - for extension in ["xml", "txt", "html", "pdf", "json"]: + for extension in ["xml", "txt", "html", "pdf"]: # no "json"! filename = f"{rfc.name}.{extension}" self.assertEqual( (rfc_path / filename).read_text(), @@ -389,6 +392,12 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # special case for json (deprecated and should be ignored) + self.assertFalse((rfc_path / f"{rfc.name}.json").exists()) + self.assertFalse( + Blob.objects.filter(bucket="rfc", name=f"json/{rfc.name}.json").exists() + ) + # Confirm that the red precomputer was triggered correctly self.assertTrue(mock_trigger_red_task.delay.called) _, mock_kwargs = mock_trigger_red_task.delay.call_args @@ -404,6 +413,10 @@ def _valid_post_data(): _, mock_kwargs = mock_rebuild_relations.delay.call_args self.assertIn("doc_names", mock_kwargs) self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # Confirm rfc JSON creation was triggered correctly + self.assertTrue(mock_update_rfc_json.delay.called) + mock_args, _ = mock_update_rfc_json.delay.call_args + self.assertEqual(mock_args[0], [rfc.rfc_number]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() @@ -565,3 +578,77 @@ def test_process_rpc_queue(self, mock_task_delay): ) self.assertEqual(response.status_code, 202) mock_task_delay.assert_called_once_with(queue_entries) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task.delay") + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task.delay") + @mock.patch("ietf.api.views_rpc.update_rfc_json_task.delay") + def test_rfc_patch_triggers_json_update( + self, mock_delay, mock_precompute_delay, mock_searchindex_delay + ): + """PATCHing RFC metadata dispatches update_rfc_json_task for that RFC.""" + rfc = WgRfcFactory() + url = urlreverse( + "ietf.api.purple_api.rfc-detail", kwargs={"rfc_number": rfc.rfc_number} + ) + patch_data = {"title": "Updated Title"} + with self.captureOnCommitCallbacks(execute=True): + r = self.client.patch( + url, + data=patch_data, + format="json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + mock_delay.assert_called_once_with([rfc.rfc_number]) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + @mock.patch("ietf.api.views_rpc.update_rfc_json_task.delay") + def test_rfc_publish_triggers_related_json_update( + self, mock_json_delay, mock_signal_delay + ): + """Publishing an RFC that obsoletes/updates existing RFCs triggers JSON update for related RFCs only.""" + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") + draft = WgDraftFactory(group__parent=area, stream_id="ietf") + obsoletes = RfcFactory.create_batch(2) + updates = RfcFactory.create_batch(1) + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 20000 + ) + post_data = { + "published": "2025-06-01T00:00:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": "New RFC", + "authors": [], + "group": rfc_group.acronym, + "stream": "ietf", + "abstract": "Abstract.", + "pages": 10, + "std_level": "ps", + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [u.rfc_number for u in updates], + "subseries": [], + } + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + data=post_data, + format="json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + + # JSON update fired only for related RFCs, not for the new RFC itself + expected_related = sorted( + {o.rfc_number for o in obsoletes} | {u.rfc_number for u in updates} + ) + mock_json_delay.assert_called_once_with(expected_related) + self.assertNotIn(unused_rfc_number, mock_json_delay.call_args[0][0]) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 83d0abefb11..21d59e3f14c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,11 +1,12 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved import os import shutil +from functools import partial from pathlib import Path from tempfile import TemporaryDirectory from django.conf import settings -from django.db import IntegrityError +from django.db import IntegrityError, transaction from drf_spectacular.utils import OpenApiParameter from rest_framework import mixins, parsers, serializers, viewsets, status from rest_framework.decorators import action @@ -48,7 +49,7 @@ ) from ietf.person.models import Email, Person from ietf.sync.rfcindex import mark_rfcindex_as_dirty -from ietf.sync.tasks import process_rpc_queue_task +from ietf.sync.tasks import process_rpc_queue_task, update_rfc_json_task class Conflict(APIException): @@ -297,6 +298,8 @@ def perform_update(self, serializer): desc="Metadata update from RFC Editor", ) super().perform_update(serializer) + rfc_number = serializer.instance.rfc_number + transaction.on_commit(lambda: update_rfc_json_task.delay([rfc_number])) @action(detail=False, serializer_class=OriginalStreamSerializer) def rfc_original_stream(self, request): @@ -457,6 +460,13 @@ def post(self, request): ) rfc_number_list = sorted(set(rfc_number_list)) signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list) + related_numbers = sorted( + {d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))} + ) + if related_numbers: + # Only update json for related rfcs. We can't generate json for _this_ rfc + # until we receive the files and know what formats we have. + transaction.on_commit(partial(update_rfc_json_task.delay, related_numbers)) return Response(NotificationAckSerializer().data) @@ -516,6 +526,10 @@ def post(self, request): for upfile in uploaded_files: uploaded_filename = Path(upfile.name) # name supplied by request uploaded_ext = "".join(uploaded_filename.suffixes) + # Ignore json files, which are deprecated. Remove this when purple no + # longer tries to send them. + if uploaded_ext == ".json": + continue tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): @@ -558,9 +572,10 @@ def post(self, request): trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) - # Trigger reference relation srebuild + # Trigger reference relations rebuild rebuild_reference_relations_task.delay(doc_names=[rfc.name]) - + # Build rfc json (json for related rfcs was updated in RfcPubNotificationView) + transaction.on_commit(partial(update_rfc_json_task.delay, [rfc.rfc_number])) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 156bac4e779..ff879dd71c6 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -163,7 +163,7 @@ class DocumentInfo(models.Model): @property def doi(self) -> str | None: if self.type_id == "rfc" and self.rfc_number is not None: - return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number}" return None def file_extension(self): @@ -344,6 +344,11 @@ def _get_ref(self, meeting=None, versioned=True): setattr(self, cache_attr, href) return getattr(self, cache_attr) + def refresh_from_db(self, using=None, fields=None, **kwargs): + super().refresh_from_db(using=using, fields=fields, **kwargs) + self.state_cache = None + self._cached_state_slug = {} + def set_state(self, state): """Switch state type implicit in state to state. This just sets the state, doesn't log the change.""" @@ -1278,9 +1283,6 @@ def submission(self): def pub_datetime(self): """Get the publication datetime of this document""" if self.type_id == "rfc": - # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are - # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date - # assigned by the RFC editor. event = self.latest_event(type='published_rfc') else: event = self.latest_event(type='new_revision') diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index db9dbc2baff..7f075ee95ca 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2187,9 +2187,34 @@ def _assert_rejected(testcase, doc, person): chair = RoleFactory(name_id="chair", group=doc.group).person url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) login_testing_unauthorized(self, chair.user.username, url) + + # GET: check 200 and that the pre-populated body contains the archive URL r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) + prefilled_body = q("textarea#id_body").text() + expected_archive_url = f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt" + self.assertIn( + expected_archive_url, + prefilled_body, + "Pre-populated body must contain the I-D archive .txt URL", + ) + + # Invalid POST: end_date in the past must not change state + bad_postdict = dict( + end_date=date_today(DEADLINE_TZINFO), + to=q("input#id_to").attr("value"), + subject=q("input#id_subject").attr("value"), + body=prefilled_body, + ) + r = self.client.post(url, bad_postdict) + self.assertEqual(r.status_code, 200, "Invalid POST should re-render the form") + self.assertNotEqual( + doc.get_state_slug("draft-stream-ietf"), + "wg-lc", + "State must not change on invalid POST", + ) + postdict = dict() postdict["end_date"] = q("input#id_end_date").attr("value") postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" @@ -2199,15 +2224,36 @@ def _assert_rejected(testcase, doc, person): else: postdict["cc"] = "extracc@example.org" postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" - postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + postdict["body"] = prefilled_body + "FGgqbQ$UNeXs" empty_outbox() r = self.client.post( url, postdict, ) self.assertEqual(r.status_code, 302) + doc.refresh_from_db() self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") + + # A StateDocEvent recording the transition must exist + self.assertTrue( + doc.docevent_set.filter(type="changed_state").exists(), + "Expected a changed_state event after issuing WG LC", + ) + + # A stream-s reminder must be active for the doc + self.assertTrue( + DocReminder.objects.filter( + event__doc=doc, type_id="stream-s", active=True + ).exists(), + "Expected an active stream-s reminder after issuing WG LC", + ) + self.assertEqual(len(outbox), 2) + + # outbox[0] is the stream-state-changed notification + self.assertIn(doc.name, outbox[0]["Subject"]) + + # outbox[1] is the WGLC announcement sent to the WG list self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) self.assertIn("extrato@example.org", outbox[1]["To"]) self.assertIn("extracc@example.org", outbox[1]["Cc"]) @@ -2216,6 +2262,11 @@ def _assert_rejected(testcase, doc, person): body = get_payload_text(outbox[1]) self.assertIn("disclosure obligations", body) self.assertIn("FGgqbQ$UNeXs", body) + self.assertIn( + expected_archive_url, + body, + "WGLC email body must contain the I-D archive .txt URL", + ) def test_issue_wg_call_for_adoption_form(self): end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) @@ -2785,3 +2836,43 @@ def _post_json(self, url, json_to_post): ]: self.assertIn(snippet, response["text"]) + # Non-IETF stream: from address uses balloter.formatted_email(), not role_email("ad"). + # Give the balloter a chair role (in a different RG) with a distinct email to prove + # that chair role address is not used; formatted_email() uses the primary email. + irtf_doc = RgDraftFactory() + non_ad_balloter = PersonFactory(name="Some Irsgmember") + other_rg = GroupFactory(type_id="rg") + chair_role_email = EmailFactory( + person=non_ad_balloter, address="chair-role@example.com" + ) + RoleFactory( + name_id="chair", + group=other_rg, + person=non_ad_balloter, + email=chair_role_email, + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "", + "comment": "cccccc", + "position": "yes", + "balloter": non_ad_balloter.pk, + "docname": irtf_doc.name, + "cc_choices": [], + "additional_cc": "", + } + }, + ) + self.assertTrue(response["success"]) + from_line = next( + line for line in response["text"].split("\n") if line.startswith("From: ") + ) + # formatted_email() uses the primary email, not the chair role email + self.assertIn(non_ad_balloter.email().address, from_line) + self.assertNotIn(chair_role_email.address, from_line) + for snippet in ["cccccc", non_ad_balloter.plain_name(), irtf_doc.name]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py new file mode 100644 index 00000000000..ec3cd85893f --- /dev/null +++ b/ietf/doc/tests_utils_rfc_json.py @@ -0,0 +1,500 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import json +from unittest import mock + +from django.core.files.base import ContentFile +from django.core.files.storage import storages +from django.test.utils import override_settings +from django.utils import timezone + +from ietf.doc.factories import ( + PublishedRfcDocEventFactory, + RfcAuthorFactory, + RfcFactory, + RgRfcFactory, + WgDraftFactory, + WgRfcFactory, +) +from ietf.doc.models import RelatedDocument +from ietf.doc.utils_rfc_json import generate_rfc_json +from ietf.group.factories import GroupFactory +from ietf.name.models import StdLevelName +from ietf.utils.test_utils import TestCase + + +def _put_pub_levels(rfc_number, slug, path="input/"): + """Write a minimal publication-std-levels.json to the red bucket.""" + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{path}publication-std-levels.json", + ContentFile( + json.dumps([{"number": rfc_number, "publication_std_level": slug}]) + ), + ) + + +def _put_errata(rfc_number, path="other/errata.json"): + """Write an errata.json with one entry for the given RFC.""" + red_bucket = storages["red_bucket"] + red_bucket.save( + path, + ContentFile( + json.dumps( + [{"doc-id": f"RFC{rfc_number}", "errata_status_code": "Reported"}] + ) + ), + ) + + +def _put_empty_errata(path="other/errata.json"): + red_bucket = storages["red_bucket"] + red_bucket.save(path, ContentFile(json.dumps([]))) + + +def _put_april_first(rfc_number, path="input/"): + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{path}april-first-rfc-numbers.json", + ContentFile(json.dumps([rfc_number])), + ) + + +def _read_json(rfc_number): + from ietf.blobdb.models import Blob + + blob = Blob.objects.get(bucket="rfc", name=f"json/rfc{rfc_number}.json") + return json.loads(bytes(blob.content)) + + +@override_settings( + RFCINDEX_INPUT_PATH="input/", + ERRATA_JSON_BLOB_NAME="other/errata.json", + RFC_EDITOR_ERRATA_BASE_URL="https://www.rfc-editor.org/errata/", +) +class GenerateRfcJsonTests(TestCase): + def setUp(self): + super().setUp() + # Minimal red_bucket blobs required by all tests + red_bucket = storages["red_bucket"] + red_bucket.save( + "input/april-first-rfc-numbers.json", ContentFile(json.dumps([])) + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + for name in [ + "input/publication-std-levels.json", + "input/april-first-rfc-numbers.json", + "other/errata.json", + ]: + try: + red_bucket.delete(name) + except Exception: + pass + super().tearDown() + + def test_missing_rfc_logs_and_returns(self): + """Calling for a nonexistent RFC number logs and returns without raising.""" + # Should not raise; no blob should be written + generate_rfc_json(999999, pub_levels={}) + from ietf.blobdb.models import Blob + + self.assertFalse( + Blob.objects.filter(bucket="rfc", name="json/rfc999999.json").exists() + ) + + def test_all_fields(self): + """All 17 JSON fields are populated correctly from a fully-populated RFC.""" + area = GroupFactory(type_id="area") + wg = GroupFactory(type_id="wg", parent=area) + rfc = PublishedRfcDocEventFactory( + time="2021-05-01T00:00:00Z", + doc=WgRfcFactory( + group=wg, + stream_id="ietf", + std_level_id="ps", + pages=42, + abstract="Test abstract.", + keywords=["foo", "bar"], + ), + ).doc + author = RfcAuthorFactory(document=rfc, is_editor=False) + editor = RfcAuthorFactory(document=rfc, is_editor=True) + + obsoletes_rfc = RfcFactory() + updated_rfc = RfcFactory() + RelatedDocument.objects.create( + source=rfc, target=obsoletes_rfc, relationship_id="obs" + ) + RelatedDocument.objects.create( + source=rfc, target=updated_rfc, relationship_id="updates" + ) + obsoleted_by_rfc = RfcFactory() + updated_by_rfc = RfcFactory() + RelatedDocument.objects.create( + source=obsoleted_by_rfc, target=rfc, relationship_id="obs" + ) + RelatedDocument.objects.create( + source=updated_by_rfc, target=rfc, relationship_id="updates" + ) + + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["doc_id"], f"RFC{rfc.rfc_number}") + self.assertEqual(data["title"], rfc.title) + self.assertEqual(data["abstract"], "Test abstract.") + self.assertEqual(data["page_count"], "42") + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + self.assertEqual(data["status"], "PROPOSED STANDARD") + self.assertEqual(data["pub_date"], "May 2021") + self.assertEqual(data["keywords"], ["foo", "bar"]) + self.assertEqual(data["see_also"], []) + self.assertEqual(data["doi"], f"10.17487/RFC{rfc.rfc_number}") + self.assertIsNone(data["errata_url"]) + self.assertIsNone(data["draft"]) + + # authors — non-editor first (lower order), then editor + self.assertEqual( + data["authors"], + [author.titlepage_name, f"{editor.titlepage_name}, Ed."], + ) + + # relationships + self.assertIn(f"RFC{obsoletes_rfc.rfc_number}", data["obsoletes"]) + self.assertIn(f"RFC{updated_rfc.rfc_number}", data["updates"]) + self.assertIn(f"RFC{obsoleted_by_rfc.rfc_number}", data["obsoleted_by"]) + self.assertIn(f"RFC{updated_by_rfc.rfc_number}", data["updated_by"]) + + def test_pub_status_differs_from_status(self): + """pub_status reflects publication-std-levels.json; status reflects current std_level.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(std_level_id="hist"), + ).doc + # Record was published as "ps" but is now "hist" + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + self.assertEqual(data["status"], "HISTORIC") + + def test_errata_url_set_when_errata_exist(self): + """errata_url is populated when errata.json has any entry for the RFC.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_errata(rfc.rfc_number) + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual( + data["errata_url"], + f"https://www.rfc-editor.org/errata/rfc{rfc.rfc_number}", + ) + + def test_draft_field_includes_revision(self): + """draft field is '-' when the RFC originated from a draft.""" + draft = WgDraftFactory(rev="07") + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["draft"], f"{draft.name}-07") + + def test_errata_url_none_when_no_errata(self): + """errata_url is None when errata.json has no entries for the RFC.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertIsNone(data["errata_url"]) + + def test_errata_failure_yields_null_url(self): + """If reading errata.json fails, errata_url is null and no exception is raised.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + # Deliberately do not put errata blob — FileNotFoundError expected + + generate_rfc_json(rfc.rfc_number) # must not raise + data = _read_json(rfc.rfc_number) + self.assertIsNone(data["errata_url"]) + + def test_second_call_overwrites(self): + """Calling generate_rfc_json twice does not raise AlreadyExistsError.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + generate_rfc_json(rfc.rfc_number) # must not raise + + def test_april_first_date_format(self): + """April Fools RFCs get '1 April YYYY' date format.""" + rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=WgRfcFactory(), + ).doc + red_bucket = storages["red_bucket"] + red_bucket.delete("input/april-first-rfc-numbers.json") + _put_april_first(rfc.rfc_number) + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["pub_date"], "1 April 2020") + + def test_empty_keywords_filtered(self): + """Empty-string keywords are stripped from the keywords list.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(keywords=["foo", "", "bar"]), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["keywords"], ["foo", "bar"]) + + def test_non_april_first_april_date(self): + """An April publication that is NOT in the April Fools list gets 'April YYYY'.""" + rfc = PublishedRfcDocEventFactory( + time="2020-04-15T12:00:00Z", + doc=WgRfcFactory(), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["pub_date"], "April 2020") + + def test_source_ietf_wg(self): + """IETF-stream WG RFC: source is the group's full name.""" + area = GroupFactory(type_id="area") + wg = GroupFactory(type_id="wg", parent=area) + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(group=wg, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], wg.name) + + def test_source_ietf_no_wg(self): + """IETF-stream individual RFC (group acronym 'none'): source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory( + group=GroupFactory(acronym="none"), + stream_id="ietf", + ), + ).doc + rfc.group.parent = area + rfc.group.save() + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + + def test_source_ietf_area(self): + """IETF-stream RFC with area-type group: source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + area.parent = GroupFactory() + area.save() + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(group=area, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + + def test_source_iab(self): + """IAB-stream RFC: source is 'IAB'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="iab", group=GroupFactory(acronym="iab")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IAB") + + def test_source_ise(self): + """ISE-stream RFC: source is 'INDEPENDENT'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="ise", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "INDEPENDENT") + + def test_source_irtf_rg(self): + """IRTF-stream RG RFC: source is the group's full name.""" + rfc = PublishedRfcDocEventFactory(doc=RgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], rfc.group.name) + + def test_source_irtf_no_rg(self): + """IRTF-stream RFC with no specific RG (group acronym 'none'): source is 'IRTF'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="irtf", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IRTF") + + def test_pub_levels_passed_in(self): + """When pub_levels is passed in, get_publication_std_levels() is not called.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_empty_errata() + + ps_level = StdLevelName.objects.get(slug="ps") + pub_levels = {rfc.rfc_number: ps_level} + + with mock.patch( + "ietf.doc.utils_rfc_json.get_publication_std_levels" + ) as mock_get: + generate_rfc_json(rfc.rfc_number, pub_levels=pub_levels) + mock_get.assert_not_called() + + data = _read_json(rfc.rfc_number) + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + + def test_pub_levels_fetch_failure_returns_without_writing(self): + """If get_publication_std_levels() raises, function logs and returns without writing a blob.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_empty_errata() + + with mock.patch( + "ietf.doc.utils_rfc_json.get_publication_std_levels", + side_effect=FileNotFoundError("not found"), + ): + generate_rfc_json(rfc.rfc_number) # must not raise + + from ietf.blobdb.models import Blob + + self.assertFalse( + Blob.objects.filter( + bucket="rfc", name=f"json/rfc{rfc.rfc_number}.json" + ).exists() + ) + + def test_pub_status_fallback_to_status_for_recent_rfc(self): + """RFC missing from pub_levels but published within 2 days: pub_status falls back to current std_level.""" + now = timezone.now() + rfc = PublishedRfcDocEventFactory( + time=now - datetime.timedelta(hours=1), + doc=WgRfcFactory(std_level_id="ps"), + ).doc + _put_empty_errata() + + with mock.patch("ietf.doc.utils_rfc_json.timezone") as mock_tz: + mock_tz.now.return_value = now + generate_rfc_json(rfc.rfc_number, pub_levels={}) + + data = _read_json(rfc.rfc_number) + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + + def test_pub_status_unknown_for_old_rfc_missing_from_levels(self): + """RFC missing from pub_levels and published more than 2 days ago: pub_status is UNKNOWN.""" + rfc = PublishedRfcDocEventFactory( + time="2020-01-01T00:00:00Z", + doc=WgRfcFactory(std_level_id="ps"), + ).doc + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number, pub_levels={}) + + data = _read_json(rfc.rfc_number) + self.assertEqual(data["pub_status"], "UNKNOWN") + + def _assert_no_blob_written(self, rfc): + from ietf.blobdb.models import Blob + + self.assertFalse( + Blob.objects.filter( + bucket="rfc", name=f"json/rfc{rfc.rfc_number}.json" + ).exists() + ) + + def test_source_malformed_no_stream(self): + """RFC with stream=None triggers assertion (logged to admins in production) and no blob is written.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + rfc.stream = None + rfc.save() + ps_level = StdLevelName.objects.get(slug="ps") + + with self.assertRaises(AssertionError): + generate_rfc_json(rfc.rfc_number, pub_levels={rfc.rfc_number: ps_level}) + + self._assert_no_blob_written(rfc) + + def test_source_malformed_no_group(self): + """RFC with group=None triggers assertion (logged to admins in production) and no blob is written.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + rfc.group = None + rfc.save() + ps_level = StdLevelName.objects.get(slug="ps") + + with self.assertRaises(AssertionError): + generate_rfc_json(rfc.rfc_number, pub_levels={rfc.rfc_number: ps_level}) + + self._assert_no_blob_written(rfc) + + def test_source_ietf_malformed_no_area(self): + """IETF-stream RFC whose group has no parent triggers assertion (logged to admins in production) and no blob is written.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="ietf", group=GroupFactory()), + ).doc + rfc.group.parent = None + rfc.group.save() + ps_level = StdLevelName.objects.get(slug="ps") + + with self.assertRaises(AssertionError): + generate_rfc_json(rfc.rfc_number, pub_levels={rfc.rfc_number: ps_level}) + + self._assert_no_blob_written(rfc) diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py new file mode 100644 index 00000000000..1f134556869 --- /dev/null +++ b/ietf/doc/utils_rfc_json.py @@ -0,0 +1,231 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import json +from pathlib import Path + +from django.conf import settings +from django.utils import timezone + +from ietf.doc.models import Document, RelatedDocument +from ietf.name.models import StdLevelName +from ietf.doc.storage_utils import exists_in_storage, store_bytes +from ietf.sync.errata import errata_map_from_json, get_errata_data +from ietf.sync.rfcindex import get_april1_rfc_numbers, get_publication_std_levels +from ietf.utils.log import assertion, log + + +_FORMAT_CHECKS = [ + ("xml", "XML"), + ("txt", "TEXT"), + ("html", "HTML"), + ("pdf", "PDF"), +] + + +def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: + """Generate and store the JSON metadata file for a published RFC. + + Reads RFC metadata from the DB and errata data from the red bucket, combines + them, and writes json/rfc{N}.json to the "rfc" blob bucket (overwriting any + existing file). + + pub_levels, if provided, should be the defaultdict returned by + get_publication_std_levels(). Pass it when generating JSON for multiple RFCs + to avoid a redundant blob read per call. + """ + try: + rfc = ( + Document.objects.select_related("std_level", "stream", "group__parent") + .prefetch_related("rfcauthor_set") + .get(type_id="rfc", rfc_number=rfc_number) + ) + except Document.DoesNotExist: + log(f"generate_rfc_json: no RFC found for rfc_number={rfc_number}") + return + + if pub_levels is None: + try: + pub_levels = get_publication_std_levels() + except Exception as e: + log(f"generate_rfc_json: failed to get publication std levels: {e}") + return + + doc_id = f"RFC{rfc_number}" + + # draft name + draft_doc = rfc.came_from_draft() + draft = f"{draft_doc.name}-{draft_doc.rev}" if draft_doc else None + + # authors: ordered list of display strings + authors = [] + for author in rfc.rfcauthor_set.order_by("order"): + name = author.titlepage_name + if author.is_editor: + name = f"{name}, Ed." + authors.append(name) + + # format: check which file blobs are present + formats = [ + label + for ext, label in _FORMAT_CHECKS + if exists_in_storage(kind="rfc", name=f"{ext}/rfc{rfc_number}.{ext}") + ] + + # page_count + page_count = str(rfc.pages) if rfc.pages is not None else "" + + # status: current std_level + status = rfc.std_level.name.upper() if rfc.std_level else "" + + # pub_status from publication-std-levels.json in the red bucket + # but guard against recent publication not having updated the bucket yet + pub_event = rfc.latest_event(type="published_rfc") + if rfc_number in pub_levels: + pub_status = pub_levels[rfc_number].name.upper() + else: + if ( + pub_event is not None + and timezone.now() - pub_event.time < datetime.timedelta(days=2) + ): + pub_status = status + else: + log(f"Assuming an unknown publication status for rfc{rfc_number}") + pub_status = StdLevelName.objects.get(slug="unkn").name.upper() + + # source: adapted from errata system's display_source() logic + if rfc.stream is None or rfc.group is None: + # Basic expectations (should be constraints) on RFC Document objects + # have been violated. + assertion("rfc.stream is not None and rfc.group is not None") + log( + f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" + ) + return + stream_slug = rfc.stream.slug + group_acronym = rfc.group.acronym + + if stream_slug == "ietf": + if rfc.group.parent is None: + assertion("rfc.group.parent is not None") + log( + f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" + ) + return + + if stream_slug == "ise": + source = "INDEPENDENT" + elif stream_slug == "iab": + source = "IAB" + elif stream_slug == "ietf" and ( + group_acronym == "none" or rfc.group.type_id == "area" + ): + source = "IETF - NON WORKING GROUP" + elif stream_slug == "irtf": + if group_acronym == "none": + source = "IRTF" + else: + source = rfc.group.name + elif group_acronym not in ("none", "") and stream_slug in ["ietf", "editorial"]: + source = rfc.group.name + elif stream_slug: + source = "Legacy" if stream_slug == "legacy" else stream_slug.upper() + else: + source = "" + + # pub_date: month/year of publication, with April 1st special-casing + pub_date = None + if pub_event: + dt = pub_event.time + try: + april_first_numbers = get_april1_rfc_numbers() + except Exception: + april_first_numbers = [] + if dt.month == 4 and rfc_number in april_first_numbers: + pub_date = dt.strftime("1 %B %Y") + else: + pub_date = dt.strftime("%B %Y") + + # relationship lists — sorted by RFC number + def _rfc_list(qs, attr): + numbers = [ + getattr(rd, attr).rfc_number + for rd in qs + if getattr(rd, attr).rfc_number is not None + ] + return [f"RFC{n}" for n in sorted(numbers)] + + obsoletes = _rfc_list( + RelatedDocument.objects.filter( + source=rfc, relationship_id="obs" + ).select_related("target"), + "target", + ) + obsoleted_by = _rfc_list( + RelatedDocument.objects.filter( + target=rfc, relationship_id="obs" + ).select_related("source"), + "source", + ) + updates = _rfc_list( + RelatedDocument.objects.filter( + source=rfc, relationship_id="updates" + ).select_related("target"), + "target", + ) + updated_by = _rfc_list( + RelatedDocument.objects.filter( + target=rfc, relationship_id="updates" + ).select_related("source"), + "source", + ) + + # errata_url: non-None if any errata entry exists for this RFC (any status) + try: + errata_data = get_errata_data() + errata_map = errata_map_from_json(errata_data) + errata_url = ( + settings.RFC_EDITOR_ERRATA_BASE_URL + f"rfc{rfc_number}" + if rfc_number in errata_map + else None + ) + except Exception: + log(f"generate_rfc_json: could not load errata data for RFC {rfc_number}") + errata_url = None + + data = { + "draft": draft, + "doc_id": doc_id, + "title": rfc.title, + "authors": authors, + "format": formats, + "page_count": page_count, + "pub_status": pub_status, + "status": status, + "source": source, + "abstract": rfc.abstract, + "pub_date": pub_date, + "keywords": [kw for kw in rfc.keywords if kw], + "obsoletes": obsoletes, + "obsoleted_by": obsoleted_by, + "updates": updates, + "updated_by": updated_by, + "see_also": [], + "doi": f"10.17487/{doc_id}", + "errata_url": errata_url, + } + + content = json.dumps(data, indent=2).encode("utf-8") + store_bytes( + kind="rfc", + name=f"json/rfc{rfc_number}.json", + content=content, + allow_overwrite=True, + doc_name=f"rfc{rfc_number}", + doc_rev=None, + mtime=timezone.now(), + ) + fs_path = Path(settings.RFC_PATH) / f"rfc{rfc_number}.json" + if settings.SERVER_MODE != "production" and not fs_path.parent.exists(): + fs_path.parent.mkdir() + fs_path.write_bytes(content) diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 29aadfdb9b7..ec593934131 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -425,7 +425,10 @@ def build_position_email_from_dict(pos_dict): pos=pos_name, blocking_name=blocking_name, settings=settings)) - frm = balloter.role_email("ad").formatted_email() + if doc.stream_id == "ietf": + frm = balloter.role_email("ad").formatted_email() + else: + frm = balloter.formatted_email() if doc.stream_id == "irtf": addrs = gather_address_lists('irsg_ballot_saved',doc=doc) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 0f5ea49f5d4..a64d0a53fe9 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -1973,6 +1973,7 @@ def issue_wg_lc(request, name): doc=doc, end_date=end_date, wg_list=doc.group.list_email, + url=f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt", settings=settings, ), ) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 3f24e2e3d6b..97ec7ebdb15 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, WgRfcFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -396,6 +396,7 @@ def test_group_documents(self): draft7 = WgDraftFactory(group=group) draft7.set_state(State.objects.get(type='draft', slug='expired')) draft7.set_state(State.objects.get(type='draft-stream-%s' % draft7.stream_id, slug='dead')) # Expired WG draft, marked as dead + rfc = WgRfcFactory(group=group) clist = CommunityList.objects.get(group=group) related_docs_rule = clist.searchrule_set.get(rule_type='name_contains') @@ -426,6 +427,12 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # RFC rows must use the RFC number as the sort key so that numeric sort + # is not disrupted by the page-count text that precedes the name in the cell. + # Draft rows must use the document name. + self.assertTrue(q(f'td.doc[data-sort-number="{rfc.rfc_number}"]')) + self.assertTrue(q(f'td.doc[data-sort-number="{draft.name}"]')) + # Let's also check the IRTF stream rg = GroupFactory(type_id='rg') setup_default_community_list_for_group(rg) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 6777ed1933b..b1675de659e 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -241,6 +241,7 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) if group.acronym in ["iesg"]: entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) + entries.append(("Decisions", urlreverse("ietf.iesg.views.review_decisions"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index f03afb9fc1a..af04ad06fd5 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). @@ -59,6 +59,7 @@ from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES from ietf.doc.utils import update_telechat, augment_events_with_revision from ietf.group.models import GroupMilestone, Role +from ietf.group.utils import construct_group_menu_context, get_group_or_404 from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count @@ -89,13 +90,15 @@ def review_decisions(request, year=None): #doc_levels = ["exp", "inf"] timeframe = "%s" % year if year else "the past 6 months" + group_type = None + group = get_group_or_404("iesg", group_type) return render(request, 'iesg/review_decisions.html', - dict(events=events, - years=years, - year=year, - timeframe=timeframe), - ) + construct_group_menu_context(request, group, "decisions", group_type, { + "events": events, + "years": years, + "year": year, + "timeframe": timeframe})) def agenda_json(request, date=None): data = agenda_data(date) @@ -632,8 +635,17 @@ def working_groups(request): area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() - return render( - request, - "iesg/working_groups.html", - dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), - ) + group_type = None + group = get_group_or_404("iesg", group_type) + + return render(request, 'iesg/working_groups.html', + construct_group_menu_context(request, group, "working groups", group_type, { + "area_summary": area_summary, + "area_totals": area_totals, + "ad_summary": ad_summary, + "noad_summary": noad_summary, + "ad_totals": ad_totals, + "noad_totals": noad_totals, + "totals": totals, + "wg_summary": wg_summary, + })) diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 0df667fbd2f..30d51cddd09 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs): | Q(name="atlarge", group__acronym="irsg") ), "RSAB Member": Q(name="member", group__acronym="rsab"), + "LLC Staff": Q(name="member", group__acronym="llc-staff"), "Robot": Q(name="robot", group__acronym="secretariat"), } diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 7d9e318aabc..8cf386abf29 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -236,16 +236,24 @@ def get_proceedings_materials(self): document__states__slug='active', document__states__type_id='procmaterials' ).order_by('type__order') - def get_attendance(self): - """Get the meeting attendance from the Registrations + def get_attendees(self, sessions=None): + """Get the meeting attendees from the Registrations - Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable - for this meeting. + Returns a pair of sets containing Person objects for persons attending + the meeting either onsite or online (remote). Returns None if the record + is unavailable for this meeting. + + sessions: optional queryset of Sessions to restrict which Attended records + count toward attendance. Defaults to all sessions at this meeting. + Note: restricting sessions does not affect the registration-based path + (checkedin / attended flags), only the Attended-record path. """ number = self.get_number() if number is None or number < 110: return None - Attendance = namedtuple('Attendance', 'onsite remote') + + if sessions is None: + sessions = self.session_set.all() # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older @@ -254,11 +262,8 @@ def get_attendance(self): # Looking up by registration and attendance records separately and joining in # python is far faster than combining the Q objects in the query (~100x). # Further optimization may be possible, but the queries are tricky... - attended_per_meeting_registration = ( - Q(registration__meeting=self) & ( - Q(registration__attended=True) | - Q(registration__checkedin=True) - ) + attended_per_meeting_registration = Q(registration__meeting=self) & ( + Q(registration__attended=True) | Q(registration__checkedin=True) ) attendees_by_reg = set( Person.objects.filter(attended_per_meeting_registration).values_list( @@ -266,29 +271,81 @@ def get_attendance(self): ) ) - attended_per_meeting_attended = ( - Q(attended__session__meeting=self) - # Note that we are not filtering to plenary, wg, or rg sessions - # as we do for nomcom eligibility - if picking up a badge (see above) - # is good enough, just attending e.g. a training session is also good enough - ) attendees_by_att = set( - Person.objects.filter(attended_per_meeting_attended).values_list( + Person.objects.filter(attended__session__in=sessions).values_list( "pk", flat=True ) ) - - attendees = Person.objects.filter( - pk__in=attendees_by_att | attendees_by_reg + + attendees = Person.objects.filter(pk__in=attendees_by_att | attendees_by_reg) + onsite = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="onsite", + ) + ) + remote = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="remote", + ) ) - onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) - remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) - return Attendance( - onsite=len(onsite), - remote=len(remote) + return (onsite, remote) + + def get_attendance(self): + """Get the meeting attendance from the Registrations + + Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable + for this meeting. + """ + attendees = self.get_attendees() + if attendees is None: + return None + Attendance = namedtuple("Attendance", "onsite remote") + onsite, remote = attendees + return Attendance(onsite=len(onsite), remote=len(remote)) + + def nomcom_eligible_participants(self): + """Return (onsite_pks, remote_pks) as frozensets for nomcom eligibility. + + Onsite: has onsite ticket AND (checked in OR attended a qualifying session). + Remote: has remote ticket AND attended a qualifying plenary/wg/rg session. + The registration attended flag alone is not sufficient for remote. + """ + sessions = self.session_set.filter( + Q(type="plenary") | Q(group__type__in=["wg", "rg"]) + ) + attendees = self.get_attendees(sessions=sessions) + if attendees is None: + onsite_pks = frozenset( + self.registration_set.onsite() + .filter(checkedin=True) + .values_list("person", flat=True) + .distinct() + ) + remote_pks = ( + frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() + ) + - onsite_pks + ) + return (onsite_pks, remote_pks) + onsite, remote = attendees + onsite_pks = frozenset(p.pk for p in onsite) + # Remote requires actual qualifying session attendance; the registration + # attended flag alone (used as a proxy before Attended records existed) + # is not sufficient. + session_attended_pks = frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() ) + remote_pks = frozenset(p.pk for p in remote) & session_attended_pks + return (onsite_pks, remote_pks) @property def proceedings_format_version(self): diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 869d9ec814e..0b9ce3f607f 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2021-2024, All Rights Reserved # -*- coding: utf-8 -*- """Tests of models in the Meeting application""" + import datetime from unittest.mock import patch @@ -10,7 +11,12 @@ import ietf.meeting.models from ietf.group.factories import GroupFactory, GroupHistoryFactory -from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory +from ietf.meeting.factories import ( + MeetingFactory, + SessionFactory, + AttendedFactory, + SessionPresentationFactory, +) from ietf.meeting.factories import RegistrationFactory from ietf.meeting.models import Session from ietf.utils.test_utils import TestCase @@ -18,105 +24,200 @@ class MeetingTests(TestCase): - def test_get_attendance_pre110(self): - """Pre-110 meetings do not calculate attendance""" - meeting = MeetingFactory(type_id='ietf', number='109') - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}) - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}) - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}) - self.assertIsNone(meeting.get_attendance()) - - def test_get_attendance_110(self): - """Look at attendance as captured at 110""" - meeting = MeetingFactory(type_id='ietf', number='110') - - # start with attendees that should be ignored - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 0) - - # add online attendees with at least one who registered but did not attend - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 0) - - # and the same for onsite attendees - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 5) - - # and once more after removing all the online attendees + def test_get_attendees_pre110(self): + """Pre-110 meetings return None""" + meeting = MeetingFactory(type_id="ietf", number="109") + RegistrationFactory.create_batch( + 3, meeting=meeting, with_ticket={"attendance_type_id": "unknown"} + ) + RegistrationFactory.create_batch( + 4, meeting=meeting, with_ticket={"attendance_type_id": "remote"} + ) + RegistrationFactory.create_batch( + 5, meeting=meeting, with_ticket={"attendance_type_id": "onsite"} + ) + self.assertIsNone(meeting.get_attendees()) + + def test_get_attendees_110(self): + """Registration-based attendance (attended flag) used at IETF 110""" + meeting = MeetingFactory(type_id="ietf", number="110") + + # Unknown-ticket attendees are excluded regardless of attended flag + RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual(len(onsite), 0) + self.assertEqual(len(remote), 0) + + # Remote attendees: only those with attended=True are included + remote_regs = RegistrationFactory.create_batch( + 4, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, set()) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Onsite attendees: only those with attended=True are included + onsite_regs = RegistrationFactory.create_batch( + 5, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Deleting remote registrations empties the remote set meeting.registration_set.remote().delete() - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 5) - - def test_get_attendance_113(self): - """Simulate IETF 113 attendance gathering data""" - meeting = MeetingFactory(type_id='ietf', number='113') - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True, checkedin=False) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=True) - p1 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p1) - p2 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p2) - attendance = meeting.get_attendance() - self.assertEqual(attendance.onsite, 3) - self.assertEqual(attendance.remote, 1) - - def test_get_attendance_keeps_meetings_distinct(self): + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual(len(remote), 0) + + def test_get_attendees_113(self): + """Simulate IETF 113: mix of attended flag, checkedin flag, and Attended records""" + meeting = MeetingFactory(type_id="ietf", number="113") + p_attended = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + checkedin=False, + ).person + p_checkedin = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=True, + ).person + p_onsite_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_onsite_session) + p_remote_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_remote_session) + onsite, remote = meeting.get_attendees() + self.assertEqual( + {p.pk for p in onsite}, {p_attended.pk, p_checkedin.pk, p_onsite_session.pk} + ) + self.assertEqual({p.pk for p in remote}, {p_remote_session.pk}) + + def test_get_attendees_keeps_meetings_distinct(self): """No cross-talk between attendance for different meetings""" - # numbers are arbitrary here - first_mtg = MeetingFactory(type_id='ietf', number='114') - second_mtg = MeetingFactory(type_id='ietf', number='115') + first_mtg = MeetingFactory(type_id="ietf", number="114") + second_mtg = MeetingFactory(type_id="ietf", number="115") - # Create a person who attended a remote session for first_mtg and onsite for second_mtg without - # checking in for either. - p = RegistrationFactory(meeting=second_mtg, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person + # Person with remote ticket at first_mtg and onsite ticket at second_mtg, attended both via Attended records + p = RegistrationFactory( + meeting=second_mtg, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + RegistrationFactory( + meeting=first_mtg, + person=p, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ) AttendedFactory(session__meeting=first_mtg, person=p) - RegistrationFactory(meeting=first_mtg, person=p, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False) AttendedFactory(session__meeting=second_mtg, person=p) - att = first_mtg.get_attendance() - self.assertEqual(att.onsite, 0) - self.assertEqual(att.remote, 1) + first_onsite, first_remote = first_mtg.get_attendees() + self.assertEqual({q.pk for q in first_onsite}, set()) + self.assertEqual({q.pk for q in first_remote}, {p.pk}) + + second_onsite, second_remote = second_mtg.get_attendees() + self.assertEqual({q.pk for q in second_onsite}, {p.pk}) + self.assertEqual({q.pk for q in second_remote}, set()) + + def test_get_attendance(self): + """get_attendance delegates to get_attendees and returns counts as a NamedTuple""" + meeting = MeetingFactory(type_id="ietf", number="120") + onsite_regs = RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + remote_regs = RegistrationFactory.create_batch( + 2, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + att = meeting.get_attendance() + self.assertIsNotNone(att) + self.assertEqual(att.onsite, len(onsite_regs)) + self.assertEqual(att.remote, len(remote_regs)) - att = second_mtg.get_attendance() - self.assertEqual(att.onsite, 1) - self.assertEqual(att.remote, 0) + old_meeting = MeetingFactory(type_id="ietf", number="109") + self.assertIsNone(old_meeting.get_attendance()) def test_vtimezone(self): # normal time zone that should have a zoneinfo file - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNotNone(vtz) self.assertGreater(len(vtz), 0) # time zone that does not have a zoneinfo file should return None - meeting = MeetingFactory(type_id='ietf', time_zone='Fake/Time_Zone', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="Fake/Time_Zone", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNone(vtz) # ioerror trying to read zoneinfo should return None - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) - with patch('ietf.meeting.models.io.open', side_effect=IOError): + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) + with patch("ietf.meeting.models.io.open", side_effect=IOError): vtz = meeting.vtimezone() self.assertIsNone(vtz) def test_group_at_the_time(self): - m = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=10)) + m = MeetingFactory( + type_id="ietf", date=date_today() - datetime.timedelta(days=10) + ) cached_groups = GroupFactory.create_batch(2) m.cached_groups_at_the_time = {g.pk: g for g in cached_groups} # fake the cache - uncached_group_hist = GroupHistoryFactory(time=datetime_today() - datetime.timedelta(days=30)) - self.assertEqual(m.group_at_the_time(uncached_group_hist.group), uncached_group_hist) + uncached_group_hist = GroupHistoryFactory( + time=datetime_today() - datetime.timedelta(days=30) + ) + self.assertEqual( + m.group_at_the_time(uncached_group_hist.group), uncached_group_hist + ) self.assertIn(uncached_group_hist.group.pk, m.cached_groups_at_the_time) @@ -127,27 +228,36 @@ def test_chat_archive_url(self): meeting__number=120, # needs to use proceedings_format_version > 1 ) with override_settings(): - if hasattr(settings, 'CHAT_ARCHIVE_URL_PATTERN'): + if hasattr(settings, "CHAT_ARCHIVE_URL_PATTERN"): del settings.CHAT_ARCHIVE_URL_PATTERN self.assertEqual(session.chat_archive_url(), session.chat_room_url()) - settings.CHAT_ARCHIVE_URL_PATTERN = 'http://chat.example.com' - self.assertEqual(session.chat_archive_url(), 'http://chat.example.com') - chatlog = SessionPresentationFactory(session=session, document__type_id='chatlog').document + settings.CHAT_ARCHIVE_URL_PATTERN = "http://chat.example.com" + self.assertEqual(session.chat_archive_url(), "http://chat.example.com") + chatlog = SessionPresentationFactory( + session=session, document__type_id="chatlog" + ).document self.assertEqual(session.chat_archive_url(), chatlog.get_href()) # datatracker 8.8.0 rolled out on 2022-07-15. Before that, chat logs were jabber logs hosted at www.ietf.org. - session_with_jabber = SessionFactory(group__acronym='fakeacronym', meeting__date=datetime.date(2022,7,14)) - self.assertEqual(session_with_jabber.chat_archive_url(), 'https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D') - chatlog = SessionPresentationFactory(session=session_with_jabber, document__type_id='chatlog').document + session_with_jabber = SessionFactory( + group__acronym="fakeacronym", meeting__date=datetime.date(2022, 7, 14) + ) + self.assertEqual( + session_with_jabber.chat_archive_url(), + "https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D", + ) + chatlog = SessionPresentationFactory( + session=session_with_jabber, document__type_id="chatlog" + ).document self.assertEqual(session_with_jabber.chat_archive_url(), chatlog.get_href()) def test_chat_room_name(self): - session = SessionFactory(group__acronym='xyzzy') - self.assertEqual(session.chat_room_name(), 'xyzzy') - session.type_id = 'plenary' - self.assertEqual(session.chat_room_name(), 'plenary') - session.chat_room = 'fnord' - self.assertEqual(session.chat_room_name(), 'fnord') + session = SessionFactory(group__acronym="xyzzy") + self.assertEqual(session.chat_room_name(), "xyzzy") + session.type_id = "plenary" + self.assertEqual(session.chat_room_name(), "plenary") + session.chat_room = "fnord" + self.assertEqual(session.chat_room_name(), "fnord") def test_alpha_str(self): self.assertEqual(Session._alpha_str(0), "a") @@ -157,7 +267,11 @@ def test_alpha_str(self): self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz") self.assertEqual(Session._alpha_str(27 * 26), "aaa") - @patch.object(ietf.meeting.models.Session, "_session_recording_url_label", return_value="LABEL") + @patch.object( + ietf.meeting.models.Session, + "_session_recording_url_label", + return_value="LABEL", + ) def test_session_recording_url(self, mock): for session_type in ["ietf", "interim"]: session = SessionFactory(meeting__type_id=session_type) @@ -165,20 +279,29 @@ def test_session_recording_url(self, mock): if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): del settings.MEETECHO_SESSION_RECORDING_URL self.assertIsNone(session.session_recording_url()) - + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" - self.assertEqual(session.session_recording_url(), "http://player.example.com") - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" - self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL") + self.assertEqual( + session.session_recording_url(), "http://player.example.com" + ) + + settings.MEETECHO_SESSION_RECORDING_URL = ( + "http://player.example.com?{session_label}" + ) + self.assertEqual( + session.session_recording_url(), "http://player.example.com?LABEL" + ) - session.meetecho_recording_name="actualname" + session.meetecho_recording_name = "actualname" session.save() - self.assertEqual(session.session_recording_url(), "http://player.example.com?actualname") + self.assertEqual( + session.session_recording_url(), + "http://player.example.com?actualname", + ) def test_session_recording_url_label_ietf(self): session = SessionFactory( - meeting__type_id='ietf', + meeting__type_id="ietf", meeting__date=date_today(), meeting__number="123", group__acronym="acro", @@ -186,15 +309,17 @@ def test_session_recording_url_label_ietf(self): session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF123-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) def test_session_recording_url_label_interim(self): session = SessionFactory( - meeting__type_id='interim', + meeting__type_id="interim", meeting__date=date_today(), group__acronym="acro", ) session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 17988e50be4..eea08be8c77 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -55,7 +55,7 @@ generate_proceedings_content, diff_meeting_schedules, ) -from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting +from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data @@ -9440,20 +9440,32 @@ def test_get_next_sequence(self): sequence = get_next_sequence(group,meeting,'recording') self.assertEqual(sequence,1) - def test_participants_for_meeting(self): + def test_nomcom_eligible_participants_for_meeting(self): m = MeetingFactory.create(type_id='ietf') areg = RegistrationFactory(meeting=m, checkedin=True, with_ticket={'attendance_type_id': 'onsite'}) breg = RegistrationFactory(meeting=m, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) creg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) dreg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) AttendedFactory(session__meeting=m, session__type_id='plenary', person=creg.person) - checked_in, attended = participants_for_meeting(m) - self.assertIn(areg.person.pk, checked_in) - self.assertNotIn(breg.person.pk, checked_in) - self.assertNotIn(areg.person.pk, attended) - self.assertNotIn(breg.person.pk, attended) - self.assertIn(creg.person.pk, attended) - self.assertNotIn(dreg.person.pk, attended) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertIn(areg.person.pk, onsite_pks) + self.assertNotIn(breg.person.pk, onsite_pks) + self.assertNotIn(areg.person.pk, remote_pks) + self.assertNotIn(breg.person.pk, remote_pks) + self.assertIn(creg.person.pk, remote_pks) + self.assertNotIn(dreg.person.pk, remote_pks) + + def test_nomcom_eligible_participants_remote_requires_session(self): + """Remote registration.attended=True without a qualifying session does not count (>= 110 path)""" + m = MeetingFactory.create(type_id='ietf', number='119') + # Remote with attended=True but no Attended record: should not be eligible + no_session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + # Remote with attended=True AND a qualifying Attended record: should be eligible + session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=session_reg.person) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertNotIn(no_session_reg.person.pk, remote_pks) + self.assertIn(session_reg.person.pk, remote_pks) def test_session_attendance(self): meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number='118') diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 10ae0d3667d..ffd37fc363d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1362,17 +1362,6 @@ def post_process(doc): doc.save_with_history([e]) -def participants_for_meeting(meeting): - """ Return a tuple (checked_in, attended) - checked_in = queryset of onsite, checkedin participants values_list('person') - attended = queryset of remote participants who attended a session values_list('person') - """ - checked_in = meeting.registration_set.onsite().filter(checkedin=True).values_list('person', flat=True).distinct() - sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg'])) - attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct() - return (checked_in, attended) - - def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 67a81305b49..913c8bd3021 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -107,7 +107,7 @@ from ietf.meeting.utils import preprocess_meeting_important_dates from ietf.meeting.utils import new_doc_for_session, write_doc_for_session from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording -from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet +from ietf.meeting.utils import generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown @@ -4816,16 +4816,25 @@ def proceedings_attendees(request, num=None): chart_data = None if int(meeting.number) >= 118: - checked_in, attended = participants_for_meeting(meeting) - regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) - onsite_count = len(regs) - regs += [ + onsite, remote = meeting.get_attendees() + onsite_count = len(onsite) + remote_count = len(remote) + onsite_pks = frozenset(p.pk for p in onsite) + remote_pks = frozenset(p.pk for p in remote) + + regs = [ + reg + for reg in Registration.objects.onsite() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in onsite_pks + ] + [ reg - for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person') - if reg.person.pk in attended and reg.person.pk not in checked_in + for reg in Registration.objects.remote() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in remote_pks ] - remote_count = len(regs) - onsite_count - registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) country_codes = [r.country_code for r in registrations if r.country_code] diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 210788ce079..b406dd31528 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1190,8 +1190,8 @@ def setUp(self): today = datetime_today() t_minus_3 = today - datetime.timedelta(days=3) t_minus_4 = today - datetime.timedelta(days=4) - e1 = EmailFactory(address="nominee1@example.org", person=PersonFactory(name="Nominee 1"), origin='test') - e2 = EmailFactory(address="nominee2@example.org", person=PersonFactory(name="Nominee 2"), origin='test') + e1 = EmailFactory(address="nominee1@example.org", person__name="Nominee 1", origin='test', primary=True) + e2 = EmailFactory(address="nominee2@example.org", person__name="Nominee 2", origin='test', primary=True) n = make_nomineeposition(self.nomcom,e1.person,gen,None) np = n.nomineeposition_set.get(position=gen) np.time = t_minus_3 @@ -1640,6 +1640,20 @@ def test_feedback_topic_badges(self): q = PyQuery(response.content) self.assertEqual( len(q('.text-bg-success')), 0 ) + def test_feedback_index_sort_keys(self): + url = reverse('ietf.nomcom.views.view_feedback', kwargs={'year': self.nc.year()}) + login_testing_unauthorized(self, self.member.user.username, url) + provide_private_key_to_test_client(self) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + # Feedback count cells must carry a numeric data-sort-number so that + # a "New" badge appearing before the count doesn't corrupt the sort key. + sort_cells = q('td[data-sort-number]') + self.assertTrue(len(sort_cells) > 0) + for cell in sort_cells.items(): + self.assertRegex(cell.attr('data-sort-number'), r'^\d+$') + class NewActiveNomComTests(TestCase): def setUp(self): diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index a2ab680df6d..46418714880 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -32,7 +32,6 @@ from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists from ietf.meeting.models import Meeting -from ietf.meeting.utils import participants_for_meeting from ietf.utils.pipe import pipe from ietf.utils.mail import send_mail_text, send_mail, get_payload_text from ietf.utils.log import log @@ -743,8 +742,8 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts = defaultdict(lambda: 0) for meeting in previous_five: - checked_in, attended = participants_for_meeting(meeting) - for id in set(checked_in) | set(attended): + onsite_pks, remote_pks = meeting.nomcom_eligible_participants() + for id in onsite_pks | remote_pks: counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 98756f26c8a..655f25994b3 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -95,7 +95,7 @@ def default_emails(obj, create, extracted, **kwargs): # pylint: disable=no-self- extracted = True if create and extracted: make_email = getattr(EmailFactory, 'create' if create else 'build') - make_email(person=obj, address=obj.user.email) + make_email(person=obj, address=obj.user.email, primary=True, **kwargs) @factory.post_generation def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument @@ -151,7 +151,7 @@ class Meta: django_get_or_create = ('address',) address = factory.Sequence(fake_email_address) - person = factory.SubFactory(PersonFactory) + person = factory.SubFactory(PersonFactory, default_emails=False) active = True primary = False diff --git a/ietf/person/tests.py b/ietf/person/tests.py index f55d8b8a347..42c2c1547cb 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -78,7 +78,9 @@ def test_ajax_person_email_json(self): def test_default_email(self): person = PersonFactory() - primary = EmailFactory(person=person, primary=True, active=True) + primary = person.email_set.get() + self.assertEqual(primary.primary, True) + self.assertEqual(primary.active, True) EmailFactory(person=person, primary=False, active=True) EmailFactory(person=person, primary=False, active=False) self.assertTrue(primary.address in person.formatted_email()) @@ -358,7 +360,7 @@ def test_get_extra_primary(self): source = PersonFactory() target = PersonFactory() extra = get_extra_primary(source, target) - self.assertTrue(extra == list(source.email_set.filter(primary=True))) + self.assertEqual(set(extra), set(source.email_set.filter(primary=True))) def test_dedupe_aliases(self): person = PersonFactory() diff --git a/ietf/review/resources.py b/ietf/review/resources.py index f79d6dfc221..546ab50a5ca 100644 --- a/ietf/review/resources.py +++ b/ietf/review/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-14 04:21 PDT -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/settings.py b/ietf/settings.py index 95f2ffefd76..7af9c7ae5be 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -329,7 +329,8 @@ def skip_unreadable_post(record): "formatters": { "django.server": { "()": "django.utils.log.ServerFormatter", - "format": "[%(server_time)s] %(message)s", + "format": "[{server_time}] {message}", + "style": "{", }, "plain": { "style": "{", @@ -894,7 +895,6 @@ def skip_unreadable_post(record): IANA_SYNC_PROTOCOLS_URL = "https://www.iana.org/protocols/" RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org" -RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" @@ -976,6 +976,10 @@ def skip_unreadable_post(record): ) RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES +# Paths in the red bucket +RFCINDEX_INPUT_PATH = "other/" +RFCINDEX_OUTPUT_PATH = "other/" + IDSUBMIT_MAX_DRAFT_SIZE = { 'txt': 2*1024*1024, # Max size of txt draft file in bytes 'xml': 3*1024*1024, # Max size of xml draft file in bytes diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 59722c505ef..21e2c171acc 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2017-02-15 10:10 PST -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index dc5b5d6ae88..57e3b76b39b 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,8 +1,9 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved -import calendar -import json +import csv import datetime +import io +import json from django.http import Http404 from pyquery import PyQuery @@ -18,11 +19,12 @@ import ietf.stats.views -from ietf.group.factories import RoleFactory -from ietf.person.factories import PersonFactory +from ietf.doc.factories import NewRevisionDocEventFactory +from ietf.group.factories import GroupFactory, RoleFactory +from ietf.person.factories import EmailFactory, PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory -from ietf.utils.timezone import date_today +from ietf.submit.factories import SubmissionFactory class StatisticsTests(TestCase): @@ -154,16 +156,19 @@ def test_review_stats(self): self.assertTrue(q('.review-stats td:contains("1")')) # check stacked chart - expected_date = date_today().replace(day=1) - expected_js_timestamp = calendar.timegm(expected_date.timetuple()) * 1000 url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "time" }) url += "?team={}".format(review_req.team.acronym) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [ - {"label": "in time", "color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}, - {"label": "late", "color": "#b42222", "data": [[expected_js_timestamp, 0]]} - ]) + stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(stacked_data), 2) + self.assertEqual(stacked_data[0]['label'], 'in time') + self.assertEqual(stacked_data[0]['color'], '#3d22b3') + self.assertEqual(stacked_data[0]['data'], [[stacked_data[0]['data'][0][0], 0]]) + self.assertEqual(stacked_data[1]['label'], 'late') + self.assertEqual(stacked_data[1]['color'], '#b42222') + self.assertEqual(stacked_data[1]['data'], [[stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) @@ -173,7 +178,11 @@ def test_review_stats(self): url += "&completion=not_completed" r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [{"color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}]) + non_stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(non_stacked_data), 1) + self.assertEqual(non_stacked_data[0]['color'], '#3d22b3') + self.assertEqual(non_stacked_data[0]['data'], [[non_stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) @@ -183,3 +192,139 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + +class AnnualReportInputsTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + self.non_member = PersonFactory() + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + + def test_access_unauthenticated(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + def test_access_non_member_forbidden(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + self.client.login( + username=self.non_member.user.username, + password=f"{self.non_member.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + + def test_access_member_allowed(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + + def test_default_year(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], datetime.date.today().year - 1) + + def test_year_param_redirects_to_year_url(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url, {"year": "2022"}) + self.assertRedirects( + r, + urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2022"}), + ) + + def test_summary_counts(self): + self._member_login() + year = 2021 + # author1 has a matching Person record; author2 does not + EmailFactory(address="author1@example.com") + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 6, 1), + submitter_email="submitter@example.com", + ) + sub.authors = [ + {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []}, + {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + # Same draft, second revision — should count once + extra = NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 15, tzinfo=datetime.timezone.utc), + doc=extra.doc, + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], year) + self.assertEqual(r.context["author_count"], 2) + self.assertEqual(r.context["author_person_count"], 1) + self.assertEqual(r.context["author_noperson_count"], 1) + self.assertEqual(r.context["submitter_count"], 1) + self.assertEqual(r.context["submitter_person_count"], 0) + self.assertEqual(r.context["submitter_noperson_count"], 1) + self.assertEqual(r.context["draft_count"], 3) + + def test_download_authors_csv(self): + self._member_login() + year = 2020 + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + ) + sub.authors = [ + {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "authors"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"authors-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvauthor@example.com", rows[0]) + + def test_download_submitters_csv(self): + self._member_login() + year = 2020 + SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + submitter_email="csvsubmitter@example.com", + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "submitters"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"submitters-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvsubmitter@example.com", rows[0]) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 01b8758c840..3bb107813a6 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,4 +15,5 @@ url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), + url(r"^annual_report_inputs/(?:(?P\d{4})/)?$", views.annual_report_inputs), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index d61c9cab64d..fe2fa82f554 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,8 +1,9 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- import calendar +import csv import datetime import itertools import json @@ -12,7 +13,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.cache import cache -from django.http import HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render, get_object_or_404 from django.urls import reverse as urlreverse from django.db.models import Count @@ -28,7 +29,7 @@ from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName from ietf.meeting.models import Registration, Meeting -from ietf.ietfauth.utils import has_role +from ietf.ietfauth.utils import has_role, role_required from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO from ietf.meeting.helpers import get_current_ietf_meeting_num @@ -957,3 +958,46 @@ def time_key_fn(t): "possible_states": possible_states, "selected_state": selected_state, }) + + +@role_required("LLC Staff") +def annual_report_inputs(request, year=None): + if year is None and "year" in request.GET: + return HttpResponseRedirect( + urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}) + ) + year = int(year) if year else datetime.date.today().year - 1 + + from ietf.doc.models import NewRevisionDocEvent + from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people + + download = request.GET.get("download") + if download in ("authors", "submitters"): + addresses = authors_by_year(year) if download == "authors" else submitters_by_year(year) + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="{download}-{year}.csv"' + writer = csv.writer(response) + writer.writerow(sorted(addresses)) + return response + + authors = authors_by_year(year) + submitters = submitters_by_year(year) + author_persons, author_nopersons = unique_people(authors) + submitter_persons, submitter_nopersons = unique_people(submitters) + + draft_count = len(set( + NewRevisionDocEvent.objects.filter( + doc__type_id="draft", time__year=year + ).values_list("doc__name", flat=True) + )) + + return render(request, "stats/annual_report_inputs.html", { + "year": year, + "author_count": len(authors), + "submitter_count": len(submitters), + "author_person_count": author_persons.count(), + "author_noperson_count": len(author_nopersons), + "submitter_person_count": submitter_persons.count(), + "submitter_noperson_count": len(submitter_nopersons), + "draft_count": draft_count, + }) diff --git a/ietf/submit/factories.py b/ietf/submit/factories.py index 1076434b822..69888a055dd 100644 --- a/ietf/submit/factories.py +++ b/ietf/submit/factories.py @@ -21,6 +21,9 @@ class Meta: class SubmissionFactory(factory.django.DjangoModelFactory): state_id = 'uploaded' + submitter_name = factory.Faker("name") + submitter_email = factory.Faker("email") + submitter = factory.LazyAttribute(lambda o: f"{o.submitter_name} <{o.submitter_email}>") @factory.lazy_attribute_sequence def name(self, n): @@ -32,3 +35,4 @@ def auth_key(self): class Meta: model = Submission + exclude = ("submitter_name", "submitter_email") diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py index 113d9872917..bc1a1cbd283 100644 --- a/ietf/sync/errata.py +++ b/ietf/sync/errata.py @@ -20,6 +20,7 @@ type ErrataJsonEntry = dict[str, str] + def get_errata_last_updated() -> datetime.datetime: """Get timestamp of the last errata.json update @@ -123,10 +124,12 @@ def update_errata_tags(errata_data: list[ErrataJsonEntry]): desc=summary, ) + return {rfc.rfc_number for rfc in changes} + -def update_errata_from_rfceditor(): +def update_errata_from_rfceditor() -> set[int]: errata_data = get_errata_data() - update_errata_tags(errata_data) + return update_errata_tags(errata_data) ## DirtyBits management for the errata tags diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py deleted file mode 100644 index 347b58efbb2..00000000000 --- a/ietf/sync/rfceditor.py +++ /dev/null @@ -1,845 +0,0 @@ -# Copyright The IETF Trust 2012-2025, All Rights Reserved -# -*- coding: utf-8 -*- - - -import datetime -import re - -from typing import Iterator, Optional, Union -from xml.dom import pulldom, Node - -from django.db import transaction -from django.db.models import Subquery, OuterRef, F, Q -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.doc.models import ( Document, State, StateType, DocEvent, DocRelationshipName, - DocTagName, RelatedDocument, RelatedDocHistory ) -from ietf.doc.expire import move_draft_files_to_archive -from ietf.doc.utils import add_state_change_event, new_state_change_event, prettify_std_name, update_action_holders -from ietf.group.models import Group -from ietf.ipr.models import IprDocRel -from ietf.name.models import StdLevelName, StreamName -from ietf.person.models import Person -from ietf.utils.log import log -from ietf.utils.mail import send_mail_text -from ietf.utils.timezone import datetime_from_date, RPC_TZINFO - -# QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" -# INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" -# POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" - -MIN_ERRATA_RESULTS = 5000 -MIN_INDEX_RESULTS = 8000 -MIN_QUEUE_RESULTS = 10 - -def get_child_text(parent_node, tag_name): - text = [] - for node in parent_node.childNodes: - if node.nodeType == Node.ELEMENT_NODE and node.localName == tag_name: - text.append(node.firstChild.data) - return '\n\n'.join(text) - - -def parse_queue(response): - """Parse RFC Editor queue XML into a bunch of tuples + warnings.""" - - events = pulldom.parse(response) - drafts = [] - warnings = [] - stream = None - - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName == "entry": - events.expandNode(node) - node.normalize() - draft_name = get_child_text(node, "draft").strip() - draft_name = re.sub(r"(-\d\d)?(.txt){1,2}$", "", draft_name) - date_received = get_child_text(node, "date-received") - - state = "" - tags = [] - missref_generation = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "state": - state = child.firstChild.data - # state has some extra annotations encoded, parse - # them out - if '*R' in state: - tags.append("ref") - state = state.replace("*R", "") - if '*A' in state: - tags.append("iana") - state = state.replace("*A", "") - m = re.search(r"\(([0-9]+)G\)", state) - if m: - missref_generation = m.group(1) - state = state.replace("(%sG)" % missref_generation, "") - - # AUTH48 link - auth48 = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "auth48-url": - auth48 = child.firstChild.data - - # cluster link (if it ever gets implemented) - cluster = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "cluster-url": - cluster = child.firstChild.data - - refs = [] - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "normRef": - ref_name = get_child_text(child, "ref-name") - ref_state = get_child_text(child, "ref-state") - in_queue = ref_state.startswith("IN-QUEUE") - refs.append((ref_name, ref_state, in_queue)) - - drafts.append((draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs)) - - elif event == pulldom.START_ELEMENT and node.tagName == "section": - name = node.getAttribute('name') - if name.startswith("IETF"): - stream = "ietf" - elif name.startswith("IAB"): - stream = "iab" - elif name.startswith("IRTF"): - stream = "irtf" - elif name.startswith("INDEPENDENT"): - stream = "ise" - elif name.startswith("Editorial Stream"): - stream = "editorial" - else: - stream = None - warnings.append("unrecognized section " + name) - except Exception as e: - log("Exception when processing an RFC queue entry: %s" % e) - log("node: %s" % node) - raise - - return drafts, warnings - -def update_drafts_from_queue(drafts): - """Given a list of parsed drafts from the RFC Editor queue, update the - documents in the database. Return those that were changed.""" - - tag_mapping = { - 'IANA': DocTagName.objects.get(slug='iana'), - 'REF': DocTagName.objects.get(slug='ref') - } - - slookup = dict((s.slug, s) - for s in State.objects.filter(used=True, type=StateType.objects.get(slug="draft-rfceditor"))) - state_mapping = { - 'AUTH': slookup['auth'], - 'AUTH48': slookup['auth48'], - 'AUTH48-DONE': slookup['auth48-done'], - 'EDIT': slookup['edit'], - 'IANA': slookup['iana'], - 'IESG': slookup['iesg'], - 'ISR': slookup['isr'], - 'ISR-AUTH': slookup['isr-auth'], - 'REF': slookup['ref'], - 'RFC-EDITOR': slookup['rfc-edit'], - 'TI': slookup['tooling-issue'], - 'TO': slookup['timeout'], - 'MISSREF': slookup['missref'], - } - - system = Person.objects.get(name="(System)") - - warnings = [] - - names = [t[0] for t in drafts] - - drafts_in_db = dict((d.name, d) - for d in Document.objects.filter(type="draft", name__in=names)) - - changed = set() - - for name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs in drafts: - if name not in drafts_in_db: - warnings.append("unknown document %s" % name) - continue - - if not state or state not in state_mapping: - warnings.append("unknown state '%s' for %s" % (state, name)) - continue - - d = drafts_in_db[name] - - prev_state = d.get_state("draft-rfceditor") - next_state = state_mapping[state] - events = [] - - # check if we've noted it's been received - if d.get_state_slug("draft-iesg") == "ann" and not prev_state and not d.latest_event(DocEvent, type="rfc_editor_received_announcement"): - e = DocEvent(doc=d, rev=d.rev, by=system, type="rfc_editor_received_announcement") - e.desc = "Announcement was received by RFC Editor" - e.save() - send_mail_text(None, "iesg-secretary@ietf.org", None, - '%s in RFC Editor queue' % d.name, - 'The announcement for %s has been received by the RFC Editor.' % d.name) - # change draft-iesg state to RFC Ed Queue - prev_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="ann") - next_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="rfcqueue") - - d.set_state(next_iesg_state) - e = add_state_change_event(d, system, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - e = update_action_holders(d, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - changed.add(name) - - # check draft-rfceditor state - if prev_state != next_state: - d.set_state(next_state) - - e = new_state_change_event(d, system, prev_state, next_state) # unsaved - if e: - if auth48: - e.desc = re.sub(r"(.*)", "\\1" % auth48, e.desc) - e.save() - events.append(e) - - if auth48: - # Create or update the auth48 URL whether or not this is a state expected to have one. - d.documenturl_set.update_or_create( - tag_id='auth48', # look up existing based on this field - defaults=dict(url=auth48) # create or update with this field - ) - else: - # Remove any existing auth48 URL when an update does not have one. - d.documenturl_set.filter(tag_id='auth48').delete() - - changed.add(name) - - t = DocTagName.objects.filter(slug__in=tags) - if set(t) != set(d.tags.all()): - d.tags.clear() - d.tags.set(t) - changed.add(name) - - if events: - d.save_with_history(events) - - - # remove tags and states for those not in the queue anymore - for d in Document.objects.exclude(name__in=names).filter(states__type="draft-rfceditor").distinct(): - d.tags.remove(*list(tag_mapping.values())) - d.unset_state("draft-rfceditor") - # we do not add a history entry here - most likely we already - # have something that explains what happened - changed.add(name) - - return changed, warnings - - -def parse_index(response): - """Parse RFC Editor index XML into a bunch of tuples.""" - - def normalize_std_name(std_name): - # remove zero padding - prefix = std_name[:3] - if prefix in ("RFC", "FYI", "BCP", "STD"): - try: - return prefix + str(int(std_name[3:])) - except ValueError: - pass - return std_name - - def extract_doc_list(parentNode, tagName): - l = [] - for u in parentNode.getElementsByTagName(tagName): - for d in u.getElementsByTagName("doc-id"): - l.append(normalize_std_name(d.firstChild.data)) - return l - - also_list = {} - data = [] - events = pulldom.parse(response) - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName in ["bcp-entry", "fyi-entry", "std-entry"]: - events.expandNode(node) - node.normalize() - bcpid = normalize_std_name(get_child_text(node, "doc-id")) - doclist = extract_doc_list(node, "is-also") - for docid in doclist: - if docid in also_list: - also_list[docid].append(bcpid) - else: - also_list[docid] = [bcpid] - - elif event == pulldom.START_ELEMENT and node.tagName == "rfc-entry": - events.expandNode(node) - node.normalize() - rfc_number = int(get_child_text(node, "doc-id")[3:]) - title = get_child_text(node, "title") - - authors = [] - for author in node.getElementsByTagName("author"): - authors.append(get_child_text(author, "name")) - - d = node.getElementsByTagName("date")[0] - year = int(get_child_text(d, "year")) - month = get_child_text(d, "month") - month = ["January","February","March","April","May","June","July","August","September","October","November","December"].index(month)+1 - rfc_published_date = datetime.date(year, month, 1) - - current_status = get_child_text(node, "current-status").title() - - updates = extract_doc_list(node, "updates") - updated_by = extract_doc_list(node, "updated-by") - obsoletes = extract_doc_list(node, "obsoletes") - obsoleted_by = extract_doc_list(node, "obsoleted-by") - pages = get_child_text(node, "page-count") - stream = get_child_text(node, "stream") - wg = get_child_text(node, "wg_acronym") - if wg and ((wg == "NON WORKING GROUP") or len(wg) > 15): - wg = None - - l = [] - for fmt in node.getElementsByTagName("format"): - l.append(get_child_text(fmt, "file-format")) - file_formats = (",".join(l)).lower() - - abstract = "" - for abstract in node.getElementsByTagName("abstract"): - abstract = get_child_text(abstract, "p") - - draft = get_child_text(node, "draft") - if draft and re.search(r"-\d\d$", draft): - draft = draft[0:-3] - - if len(node.getElementsByTagName("errata-url")) > 0: - has_errata = 1 - else: - has_errata = 0 - - data.append((rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,[],draft,has_errata,stream,wg,file_formats,pages,abstract)) - except Exception as e: - log("Exception when processing an RFC index entry: %s" % e) - log("node: %s" % node) - raise - for d in data: - k = "RFC%d" % d[0] - if k in also_list: - d[9].extend(also_list[k]) - return data - - -def update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date: Optional[datetime.date] = None -) -> Iterator[tuple[int, list[str], Document, bool]]: - """Given parsed data from the RFC Editor index, update the documents in the database - - Returns an iterator that yields (rfc_number, change_list, doc, rfc_published) for the - RFC document and, if applicable, the I-D that it came from. - - The skip_older_than_date is a bare date, not a datetime. - """ - # Create dict mapping doc-id to list of errata records that apply to it - errata: dict[str, list[dict]] = {} - for item in errata_data: - name = item["doc-id"] - if name.upper().startswith("RFC"): - name = f"RFC{int(name[3:])}" # removes leading 0s on the rfc number - if not name in errata: - errata[name] = [] - errata[name].append(item) - - std_level_mapping = { - "Standard": StdLevelName.objects.get(slug="std"), - "Internet Standard": StdLevelName.objects.get(slug="std"), - "Draft Standard": StdLevelName.objects.get(slug="ds"), - "Proposed Standard": StdLevelName.objects.get(slug="ps"), - "Informational": StdLevelName.objects.get(slug="inf"), - "Experimental": StdLevelName.objects.get(slug="exp"), - "Best Current Practice": StdLevelName.objects.get(slug="bcp"), - "Historic": StdLevelName.objects.get(slug="hist"), - "Unknown": StdLevelName.objects.get(slug="unkn"), - } - - stream_mapping = { - "IETF": StreamName.objects.get(slug="ietf"), - "INDEPENDENT": StreamName.objects.get(slug="ise"), - "IRTF": StreamName.objects.get(slug="irtf"), - "IAB": StreamName.objects.get(slug="iab"), - "Editorial": StreamName.objects.get(slug="editorial"), - "Legacy": StreamName.objects.get(slug="legacy"), - } - - tag_has_errata = DocTagName.objects.get(slug="errata") - tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") - relationship_obsoletes = DocRelationshipName.objects.get(slug="obs") - relationship_updates = DocRelationshipName.objects.get(slug="updates") - rfc_published_state = State.objects.get(type_id="rfc", slug="published") - - system = Person.objects.get(name="(System)") - - first_sync_creating_subseries = not Document.objects.filter(type_id__in=["bcp","std","fyi"]).exists() - - for ( - rfc_number, - title, - authors, - rfc_published_date, - current_status, - updates, - updated_by, - obsoletes, - obsoleted_by, - also, - draft_name, - has_errata, - stream, - wg, - file_formats, - pages, - abstract, - ) in index_data: - if skip_older_than_date and rfc_published_date < skip_older_than_date: - # speed up the process by skipping old entries (n.b., the comparison above is a - # lexical comparison between "YYYY-MM-DD"-formatted dates) - continue - - # we assume two things can happen: we get a new RFC, or an - # attribute has been updated at the RFC Editor (RFC Editor - # attributes take precedence over our local attributes) - rfc_events = [] - rfc_changes = [] - rfc_published = False - - # Find the draft, if any - draft = None - if draft_name: - try: - draft = Document.objects.get(name=draft_name, type_id="draft") - except Document.DoesNotExist: - pass - # Logging below warning turns out to be unhelpful - there are many references - # to such things in the index: - # * all april-1 RFCs have an internal name that looks like a draft name, but there - # was never such a draft. More of these will exist in the future - # * Several documents were created with out-of-band input to the RFC-editor, for a - # variety of reasons. - # - # What this exposes is that the rfc-index needs to stop talking about these things. - # If there is no draft to point to, don't point to one, even if there was an RPC - # internal name in use (and in the RPC database). This will be a requirement on the - # reimplementation of the creation of the rfc-index. - # - # log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}") - - # Find or create the RFC document - creation_args: dict[str, Optional[Union[str, int]]] = {"name": f"rfc{rfc_number}"} - if draft: - creation_args.update( - { - "title": draft.title, - "stream": draft.stream, - "group": draft.group, - "abstract": draft.abstract, - "pages": draft.pages, - "words": draft.words, - "std_level": draft.std_level, - "ad": draft.ad, - "external_url": draft.external_url, - "uploaded_filename": draft.uploaded_filename, - "note": draft.note, - } - ) - doc, created_rfc = Document.objects.get_or_create( - rfc_number=rfc_number, type_id="rfc", defaults=creation_args - ) - if created_rfc: - rfc_changes.append(f"created document {prettify_std_name(doc.name)}") - doc.set_state(rfc_published_state) - if draft: - doc.formal_languages.set(draft.formal_languages.all()) - # Create authors based on the last draft in the datatracker. This - # path will go away when we publish via the modernized RPC workflow - # but until then, these are the only data we have for authors that - # are easily connected to Person records. - for documentauthor in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - documentauthor.pk = None - documentauthor.id = None - documentauthor._state.adding = True - documentauthor.document = doc - documentauthor.save() - - if draft: - draft_events = [] - draft_changes = [] - - # Ensure the draft is in the "rfc" state and move its files to the archive - # if necessary. - if draft.get_state_slug() != "rfc": - draft.set_state( - State.objects.get(used=True, type="draft", slug="rfc") - ) - move_draft_files_to_archive(draft, draft.rev) - draft_changes.append(f"changed state to {draft.get_state()}") - - # Ensure the draft and rfc are linked with a "became_rfc" relationship - r, created_relateddoc = RelatedDocument.objects.get_or_create( - source=draft, target=doc, relationship_id="became_rfc" - ) - if created_relateddoc: - change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( - rel_name=r.relationship.name.lower(), - pretty_draft_name=prettify_std_name(draft_name), - pretty_rfc_name=prettify_std_name(doc.name), - ) - draft_changes.append(change) - rfc_changes.append(change) - - # Always set the "draft-iesg" state. This state should be set for all drafts, so - # log a warning if it is not set. What should happen here is that ietf stream - # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. - # Other stream documents should normally be "idexists" and be left that way. The - # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", - # and changes any other state to "pub". If unset, it changes it to "idexists". - # This reflects historical behavior and should probably be updated, but a migration - # of existing drafts (and validation of the change) is needed before we change the - # handling. - prev_iesg_state = draft.get_state("draft-iesg") - if prev_iesg_state is None: - log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state') - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") - elif prev_iesg_state.slug not in ("pub", "idexists"): - if prev_iesg_state.slug != "rfcqueue": - log( - 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( - doc.name, draft.name, prev_iesg_state.slug - ) - ) - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") - else: - new_iesg_state = prev_iesg_state - - if new_iesg_state != prev_iesg_state: - draft.set_state(new_iesg_state) - draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") - e = update_action_holders(draft, prev_iesg_state, new_iesg_state) - if e: - draft_events.append(e) - - # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. - if draft.stream != doc.stream: - log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( - doc.name, draft.name, draft.stream, doc.stream - )) - elif draft.stream.slug in ["iab", "irtf", "ise"]: - stream_slug = f"draft-stream-{draft.stream.slug}" - prev_state = draft.get_state(stream_slug) - if prev_state is not None and prev_state.slug != "pub": - new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") - draft.set_state(new_state) - draft_changes.append( - f"changed {new_state.type.label} to {new_state}" - ) - e = update_action_holders(draft, prev_state, new_state) - if e: - draft_events.append(e) - if draft_changes: - draft_events.append( - DocEvent.objects.create( - doc=draft, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})", - ) - ) - draft.save_with_history(draft_events) - yield rfc_number, draft_changes, draft, False # yield changes to the draft - - # check attributes - verbed = "set" if created_rfc else "changed" - if title != doc.title: - doc.title = title - rfc_changes.append(f"{verbed} title to '{doc.title}'") - - if abstract and abstract != doc.abstract: - doc.abstract = abstract - rfc_changes.append(f"{verbed} abstract to '{doc.abstract}'") - - if pages and int(pages) != doc.pages: - doc.pages = int(pages) - rfc_changes.append(f"{verbed} pages to {doc.pages}") - - if std_level_mapping[current_status] != doc.std_level: - doc.std_level = std_level_mapping[current_status] - rfc_changes.append(f"{verbed} standardization level to {doc.std_level}") - - if doc.stream != stream_mapping[stream]: - doc.stream = stream_mapping[stream] - rfc_changes.append(f"{verbed} stream to {doc.stream}") - - if doc.get_state() != rfc_published_state: - doc.set_state(rfc_published_state) - rfc_changes.append(f"{verbed} {rfc_published_state.type.label} to {rfc_published_state}") - - # if we have no group assigned, check if RFC Editor has a suggestion - if not doc.group: - if wg: - doc.group = Group.objects.get(acronym=wg) - rfc_changes.append(f"set group to {doc.group}") - else: - doc.group = Group.objects.get( - type="individ" - ) # fallback for newly created doc - - if not doc.latest_event(type="published_rfc"): - e = DocEvent(doc=doc, rev=doc.rev, type="published_rfc") - # unfortunately, rfc_published_date doesn't include the correct day - # at the moment because the data only has month/year, so - # try to deduce it - # - # Note: This is in done PST8PDT to preserve compatibility with events created when - # USE_TZ was False. The published_rfc event was created with a timestamp whose - # server-local datetime (PST8PDT) matched the publication date from the RFC index. - # When switching to USE_TZ=True, the timestamps were migrated so they still - # matched the publication date in PST8PDT. When interpreting the event timestamp - # as a publication date, you must treat it in the PST8PDT time zone. The - # RPC_TZINFO constant in ietf.utils.timezone is defined for this purpose. - d = datetime_from_date(rfc_published_date, RPC_TZINFO) - synthesized = timezone.now().astimezone(RPC_TZINFO) - if abs(d - synthesized) > datetime.timedelta(days=60): - synthesized = d - else: - direction = -1 if (d - synthesized).total_seconds() < 0 else +1 - while synthesized.month != d.month or synthesized.year != d.year: - synthesized += datetime.timedelta(days=direction) - e.time = synthesized - e.by = system - e.desc = "RFC published" - e.save() - rfc_events.append(e) - - rfc_changes.append( - f"added RFC published event at {e.time.strftime('%Y-%m-%d')}" - ) - rfc_published = True - - def parse_relation_list(rel_list: list[str]) -> list[Document]: - return list( - Document.objects.filter( - name__in=[name.strip().lower() for name in rel_list], - type_id="rfc" - ) - ) - - # Create missing obsoletes relations - docs_this_obsoletes = parse_relation_list(obsoletes) - for obs_doc in docs_this_obsoletes: - if not doc.relateddocument_set.filter( - target=obs_doc, relationship=relationship_obsoletes - ): - r = doc.relateddocument_set.create( - target=obs_doc, relationship=relationship_obsoletes - ) - rfc_changes.append( - "created {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - # Remove stale obsoletes relations - for r in doc.relateddocument_set.filter( - relationship=relationship_obsoletes - ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): - r.delete() - rfc_changes.append( - "removed {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - - docs_this_updates = parse_relation_list(updates) - for upd_doc in docs_this_updates: - if not RelatedDocument.objects.filter( - source=doc, target=upd_doc, relationship=relationship_updates - ): - r = doc.relateddocument_set.create( - target=upd_doc, relationship=relationship_updates - ) - rfc_changes.append( - "created {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - # Remove stale updates relations - for r in doc.relateddocument_set.filter( - relationship=relationship_updates - ).exclude(target_id__in=[d.pk for d in docs_this_updates]): - r.delete() - rfc_changes.append( - "removed {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - - if also: - # recondition also to have proper subseries document names: - conditioned_also = [] - for a in also: - a = a.lower() - subseries_slug = a[:3] - if subseries_slug not in ["bcp", "std", "fyi"]: - log(f"Unexpected 'also' relationship of {a} encountered for {doc}") - continue - maybe_number = a[3:].strip() - if not maybe_number.isdigit(): - log(f"Unexpected 'also' subseries element identifier {a} encountered for {doc}") - continue - else: - subseries_number = int(maybe_number) - conditioned_also.append(f"{subseries_slug}{subseries_number}") # Note the lack of leading zeros - also = conditioned_also - - for a in also: - subseries_doc_name = a - subseries_slug=a[:3] - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=subseries_slug, name=subseries_doc_name) - if created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Imported {subseries_doc_name} into the datatracker via sync to the rfc-index") - else: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Created {subseries_doc_name} via sync to the rfc-index") - _, relationship_created = subseries_doc.relateddocument_set.get_or_create(relationship_id="contains", target=doc) - if relationship_created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index") - rfc_events.append(doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of {subseries_doc.name.upper()} is currently available in the datatracker before this point")) - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index")) - else: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}") - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}")) - - # Delete subseries relations that are no longer current. Use a transaction - # so we are sure we iterate over the same relations that we delete! - with transaction.atomic(): - stale_subseries_relations = doc.relations_that("contains").exclude( - source__name__in=also - ) - for stale_relation in stale_subseries_relations: - stale_subseries_doc = stale_relation.source - rfc_events.append( - doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - ) - stale_subseries_doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - stale_subseries_relations.delete() - - doc_errata = errata.get(f"RFC{rfc_number}", []) - all_rejected = doc_errata and all( - er["errata_status_code"] == "Rejected" for er in doc_errata - ) - if has_errata and not all_rejected: - if not doc.tags.filter(pk=tag_has_errata.pk).exists(): - doc.tags.add(tag_has_errata) - rfc_changes.append("added Errata tag") - has_verified_errata = any( - [er["errata_status_code"] == "Verified" for er in doc_errata] - ) - if ( - has_verified_errata - and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists() - ): - doc.tags.add(tag_has_verified_errata) - rfc_changes.append("added Verified Errata tag") - else: - if doc.tags.filter(pk=tag_has_errata.pk): - doc.tags.remove(tag_has_errata) - if all_rejected: - rfc_changes.append("removed Errata tag (all errata rejected)") - else: - rfc_changes.append("removed Errata tag") - if doc.tags.filter(pk=tag_has_verified_errata.pk): - doc.tags.remove(tag_has_verified_errata) - rfc_changes.append("removed Verified Errata tag") - - if rfc_changes: - rfc_events.append( - DocEvent.objects.create( - doc=doc, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(rfc_changes)})", - ) - ) - doc.save_with_history(rfc_events) - yield rfc_number, rfc_changes, doc, rfc_published # yield changes to the RFC - - if first_sync_creating_subseries: - # First - create the known subseries documents that have ghosted. - # The RFC editor (as of 31 Oct 2023) claims these subseries docs do not exist. - # The datatracker, on the other hand, will say that the series doc currently contains no RFCs. - for name in ["fyi17", "std1", "bcp12", "bcp113", "bcp66"]: - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=name[:3], name=name) - if not created: - log(f"Warning: {name} unexpectedly already exists") - else: - subseries_slug = name[:3] - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - - RelatedDocument.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - RelatedDocHistory.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - IprDocRel.objects.filter( - Q(originaldocumentaliasname__startswith="bcp") | - Q(originaldocumentaliasname__startswith="std") | - Q(originaldocumentaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaldocumentaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(document=F("subseries_target")) - - - diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index be55a6866e3..f47974f9002 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -48,9 +48,17 @@ def errata_url(rfc: Document): return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") +def red_bucket_input_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_INPUT_PATH) / filename) + + +def red_bucket_output_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_OUTPUT_PATH) / filename) + + def save_to_red_bucket(filename: str, content: str | bytes): red_bucket = storages["red_bucket"] - bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + bucket_path = red_bucket_output_path(filename) if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): # Django 4.2's FileSystemStorage does not support allow_overwrite. red_bucket.delete(bucket_path) @@ -87,8 +95,7 @@ class UnusableRfcNumber: def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: - FILENAME = "unusable-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("unusable-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -115,8 +122,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: def get_april1_rfc_numbers() -> Container[int]: - FILENAME = "april-first-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("april-first-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -139,8 +145,7 @@ def get_april1_rfc_numbers() -> Container[int]: def get_publication_std_levels() -> dict[int, StdLevelName]: - FILENAME = "publication-std-levels.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("publication-std-levels.json") values: dict[int, StdLevelName] = {} try: with storages["red_bucket"].open(bucket_path) as urn_file: diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 3808fb1db59..3af5cb8984d 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -3,7 +3,6 @@ # Celery task definitions # import datetime -import io from pathlib import Path from tempfile import NamedTemporaryFile import requests @@ -19,13 +18,11 @@ from ietf.person.models import Person from ietf.utils.mail import send_mail_text from ietf.sync import iana -from ietf.sync import rfceditor from ietf.sync.errata import ( errata_are_dirty, mark_errata_as_processed, update_errata_from_rfceditor, ) -from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue from ietf.sync.rfcindex import ( create_bcp_txt_index, create_fyi_txt_index, @@ -34,94 +31,13 @@ create_std_txt_index, rfcindex_is_dirty, mark_rfcindex_as_processed, mark_rfcindex_as_dirty, ) -from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils import log -from ietf.utils.timezone import date_today - - -@shared_task -def rfc_editor_index_update_task(full_index=False): - """Update metadata from the RFC index - - Default is to examine only changes in the past 365 days. Call with full_index=True to update - the full RFC index. - - According to comments on the original script, a year's worth took about 20s on production as of - August 2022 - - The original rfc-editor-index-update script had a long-disabled provision for running the - rebuild_reference_relations scripts after the update. That has not been brought over - at all because it should be implemented as its own task if it is needed. - """ - skip_date = None if full_index else date_today() - datetime.timedelta(days=365) - log.log( - "Updating document metadata from RFC index going back to {since}, from {url}".format( - since=skip_date if skip_date is not None else "the beginning", - url=settings.RFC_EDITOR_INDEX_URL, - ) - ) - try: - response = requests.get( - settings.RFC_EDITOR_INDEX_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor index: {exc}") - return # failed - rfc_index_xml = response.text - index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml)) - try: - response = requests.get( - settings.RFC_EDITOR_ERRATA_JSON_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor errata: {exc}") - return # failed - errata_data = response.json() - if len(index_data) < rfceditor.MIN_INDEX_RESULTS: - log.log("Not enough index entries, only %s" % len(index_data)) - return # failed - if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: - log.log("Not enough errata entries, only %s" % len(errata_data)) - return # failed - newly_published = set() - for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date=skip_date - ): - for c in changes: - log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) - if rfc_published: - newly_published.add(rfc_number) - if len(newly_published) > 0: - rsync_rfcs_from_rfceditor_task.delay(list(newly_published)) - - -@shared_task -def rfc_editor_queue_updates_task(): - log.log(f"Updating RFC Editor queue states from {settings.RFC_EDITOR_QUEUE_URL}") - try: - response = requests.get( - settings.RFC_EDITOR_QUEUE_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor queue: {exc}") - return # failed - drafts, warnings = parse_queue(io.StringIO(response.text)) - for w in warnings: - log.log(f"Warning: {w}") - - if len(drafts) < MIN_QUEUE_RESULTS: - log.log("Not enough results, only %s" % len(drafts)) - return # failed - - changed, warnings = update_drafts_from_queue(drafts) - for w in warnings: - log.log(f"Warning: {w}") - - for c in changed: - log.log(f"Updated {c}") @shared_task @@ -302,9 +218,45 @@ def update_errata_from_rfceditor_task(): # new_processed_time is the *start* of processing so that any changes after # this point will trigger another refresh new_processed_time = timezone.now() - update_errata_from_rfceditor() + changed_numbers = update_errata_from_rfceditor() mark_errata_as_processed(new_processed_time) mark_rfcindex_as_dirty() # ensure any changes are reflected in the indexes + if changed_numbers: + update_rfc_json_task.delay(list(changed_numbers)) + +@shared_task +def update_rfc_json_by_range_list_task(ranges: str) -> None: + """Regenerate RFC JSON for the RFCs described by a range-list string + + See expand_rfc_number_range_list() for the accepted format (e.g. + "[1,100,1000-1004]"). Invalid input is logged and otherwise ignored. + """ + try: + rfc_numbers = expand_rfc_number_range_list(ranges) + except ValueError as e: + log.log( + f"update_rfc_json_by_range_list_task: ignoring invalid input " + f"'{ranges}': {e}" + ) + return + if rfc_numbers: + update_rfc_json_task(rfc_numbers) + +@shared_task +def update_rfc_json_task(rfc_numbers: list[int]) -> None: + from ietf.doc.utils_rfc_json import generate_rfc_json + from ietf.sync.rfcindex import get_publication_std_levels + + try: + pub_levels = get_publication_std_levels() + except Exception as e: + log.log(f"update_rfc_json_task: failed to get publication std levels: {e}") + return + for rfc_number in rfc_numbers: + try: + generate_rfc_json(rfc_number, pub_levels=pub_levels) + except Exception as e: + log.log(f"update_rfc_json_task: failed for RFC {rfc_number}: {e}") @shared_task diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 207c78cf6ae..e13dcd19fd7 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,17 +1,11 @@ # Copyright The IETF Trust 2012-2026, All Rights Reserved - -import os -import io import json import datetime from unittest import mock import quopri import requests -from dataclasses import dataclass - -from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages from django.urls import reverse as urlreverse @@ -24,24 +18,21 @@ from ietf.doc.factories import ( WgDraftFactory, RfcFactory, - DocumentAuthorFactory, DocEventFactory, - BcpFactory, WgRfcFactory, + WgRfcFactory, ) from ietf.doc.models import ( Document, DocEvent, DeletedEvent, DocTagName, - RelatedDocument, State, StateDocEvent, ) from ietf.doc.utils import add_state_change_event -from ietf.group.factories import GroupFactory from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor, tasks +from ietf.sync import iana, tasks from ietf.sync.errata import ( update_errata_from_rfceditor, get_errata_last_updated, @@ -56,7 +47,6 @@ from ietf.utils.models import DirtyBits from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase -from ietf.utils.timezone import date_today, RPC_TZINFO class IANASyncTests(TestCase): @@ -306,494 +296,6 @@ def test_notify_page(self): self.assertContains(r, "new changes at") # we don't actually try posting as that would trigger a real run - - -class RFCSyncTests(TestCase): - def write_draft_file(self, name, size): - with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), 'w') as f: - f.write("a" * size) - - def test_rfc_index(self): - area = GroupFactory(type_id='area') - draft_doc = WgDraftFactory( - group__parent=area, - states=[('draft-iesg','rfcqueue')], - ad=Person.objects.get(user__username='ad'), - external_url="http://my-external-url.example.com", - note="this is a note", - pages=54, # make sure this is not 42 - ) - DocumentAuthorFactory.create_batch(2, document=draft_doc) - draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared - - RfcFactory(rfc_number=123) - - today = date_today() - - t = ''' - - - BCP0001 - - RFC1234 - RFC2345 - - - - FYI0001 - - RFC1234 - - - - STD0002 - Test - - RFC1234 - - - - RFC1234 - A Testing RFC - - A. Irector - - - %(month)s - %(year)s - - - ASCII - - 42 - - test - -

This is some interesting text.

 - %(name)s-%(rev)s - - RFC123 - - - BCP0001 - - PROPOSED STANDARD - PROPOSED STANDARD - IETF - %(area)s - %(group)s - http://www.rfc-editor.org/errata_search.php?rfc=1234 - -''' % dict(year=today.strftime("%Y"), - month=today.strftime("%B"), - name=draft_doc.name, - rev=draft_doc.rev, - area=draft_doc.group.parent.acronym, - group=draft_doc.group.acronym) - - errata = [{ - "errata_id":1, - "doc-id":"RFC123", # n.b. this is not the same RFC as in the above index XML! - "errata_status_code":"Verified", - "errata_type_code":"Editorial", - "section": "4.1", - "orig_text":" S: 220-smtp.example.com ESMTP Server", - "correct_text":" S: 220 smtp.example.com ESMTP Server", - "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", - "submit_date":"2007-07-19", - "submitter_name":"Rob Siemborski", - "verifier_id":99, - "verifier_name":None, - "update_date":"2019-09-10 09:09:03"}, - ] - - data = rfceditor.parse_index(io.StringIO(t)) - self.assertEqual(len(data), 1) - rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0] - - # currently, we only check what we actually use - self.assertEqual(rfc_number, 1234) - self.assertEqual(title, "A Testing RFC") - self.assertEqual(rfc_published_date.year, today.year) - self.assertEqual(rfc_published_date.month, today.month) - self.assertEqual(current_status, "Proposed Standard") - self.assertEqual(updates, ["RFC123"]) - self.assertEqual(set(also), set(["BCP1", "FYI1", "STD2"])) - self.assertEqual(draft, draft_doc.name) - self.assertEqual(wg, draft_doc.group.acronym) - self.assertEqual(has_errata, True) - self.assertEqual(stream, "IETF") - self.assertEqual(pages, "42") - self.assertEqual(abstract, "This is some interesting text.") - - draft_filename = "%s-%s.txt" % (draft_doc.name, draft_doc.rev) - self.write_draft_file(draft_filename, 5000) - - event_count_before = draft_doc.docevent_set.count() - draft_title_before = draft_doc.title - draft_abstract_before = draft_doc.abstract - draft_pages_before = draft_doc.pages - - changes = [] - with mock.patch("ietf.sync.rfceditor.log") as mock_log: - for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): - changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list - self.assertEqual(rfc_number, 1234) - if rfc_published: - self.assertEqual(d.type_id, "rfc") - self.assertEqual(d.rfc_number, rfc_number) - else: - self.assertEqual(d.type_id, "draft") - self.assertIsNone(d.rfc_number) - - self.assertFalse(mock_log.called, "No log messages expected") - - draft_doc = Document.objects.get(name=draft_doc.name) - draft_events = draft_doc.docevent_set.all() - self.assertEqual(len(draft_events) - event_count_before, 2) - self.assertEqual(draft_events[0].type, "sync_from_rfc_editor") - self.assertEqual(draft_events[1].type, "changed_action_holders") - self.assertEqual(draft_doc.get_state_slug(), "rfc") - self.assertEqual(draft_doc.get_state_slug("draft-iesg"), "pub") - self.assertCountEqual(draft_doc.action_holders.all(), []) - self.assertEqual(draft_doc.title, draft_title_before) - self.assertEqual(draft_doc.abstract, draft_abstract_before) - self.assertEqual(draft_doc.pages, draft_pages_before) - self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename))) - self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename))) - - rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() - self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names()) - rfc_events = rfc_doc.docevent_set.all() - self.assertEqual(len(rfc_events), 8) - expected_events = [ - ["sync_from_rfc_editor", ""], # Not looking for exact desc match here - see detailed tests below - ["sync_from_rfc_editor", "Imported membership of rfc1234 in std2 via sync to the rfc-index"], - ["std_history_marker", "No history of STD2 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in fyi1 via sync to the rfc-index"], - ["fyi_history_marker", "No history of FYI1 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in bcp1 via sync to the rfc-index"], - ["bcp_history_marker", "No history of BCP1 is currently available in the datatracker before this point"], - ["published_rfc", "RFC published"] - ] - for index, [event_type, desc] in enumerate(expected_events): - self.assertEqual(rfc_events[index].type, event_type) - if index == 0: - self.assertIn("Received changes through RFC Editor sync (created document RFC 1234,", rfc_events[0].desc) - self.assertIn(f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", rfc_events[0].desc) - self.assertIn("set title to 'A Testing RFC'", rfc_events[0].desc) - self.assertIn("set abstract to 'This is some interesting text.'", rfc_events[0].desc) - self.assertIn("set pages to 42", rfc_events[0].desc) - self.assertIn("set standardization level to Proposed Standard", rfc_events[0].desc) - self.assertIn(f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", rfc_events[0].desc) - self.assertIn("created updates relation between RFC 1234 and RFC 123", rfc_events[0].desc) - self.assertIn("added Errata tag", rfc_events[0].desc) - else: - self.assertEqual(rfc_events[index].desc, desc) - self.assertEqual(rfc_events[7].time.astimezone(RPC_TZINFO).date(), today) - for subseries_name in ["bcp1", "fyi1", "std2"]: - sub = Document.objects.filter(type_id=subseries_name[:3],name=subseries_name).first() - self.assertIsNotNone(sub, f"{subseries_name} not created") - self.assertTrue(rfc_doc in sub.contains()) - self.assertTrue(sub in rfc_doc.part_of()) - self.assertEqual(rfc_doc.get_state_slug(), "published") - # Should have an "errata" tag because there is an errata-url in the index XML, but no "verified-errata" tag - # because there is no verified item in the errata JSON with doc-id matching the RFC document. - tag_slugs = rfc_doc.tags.values_list("slug", flat=True) - self.assertTrue("errata" in tag_slugs) - self.assertFalse("verified-errata" in tag_slugs) - # TODO: adjust these when we have subseries document types - # self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc)) - self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists()) - self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists()) - self.assertEqual(rfc_doc.title, "A Testing RFC") - self.assertEqual(rfc_doc.abstract, "This is some interesting text.") - self.assertEqual(rfc_doc.std_level_id, "ps") - self.assertEqual(rfc_doc.pages, 42) - self.assertEqual(rfc_doc.stream, draft_doc.stream) - self.assertEqual(rfc_doc.group, draft_doc.group) - self.assertEqual(rfc_doc.words, draft_doc.words) - self.assertEqual(rfc_doc.ad, draft_doc.ad) - self.assertEqual(rfc_doc.external_url, draft_doc.external_url) - self.assertEqual(rfc_doc.note, draft_doc.note) - - # check that we got the expected changes - self.assertEqual(len(changes), 2) - self.assertEqual(changes[0]["doc_pk"], draft_doc.pk) - self.assertEqual(changes[0]["rfc_published"], False) - self.assertEqual(changes[1]["doc_pk"], rfc_doc.pk) - self.assertEqual(changes[1]["rfc_published"], True) - - # make sure we can apply it again with no changes - changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) - self.assertEqual(len(changed), 0) - - def test_rfc_index_subseries_replacement(self): - today = date_today() - author = PersonFactory(name="Some Bozo") - - # Start with two BCPs, each containing an rfc - rfc1, rfc2, rfc3 = RfcFactory.create_batch(3, authors=[author]) - bcp1 = BcpFactory(contains=[rfc1]) - bcp2 = BcpFactory(contains=[rfc2]) - - def _nameify(doc): - """Convert a name like 'rfc1' to 'RFC0001""" - return f"{doc.name[:3].upper()}{int(doc.name[3:]):04d}" - - # RFC index that replaces rfc2 with rfc3 in bcp2 - index_xml = f""" - - - {_nameify(bcp1)} - - {_nameify(rfc1)} - - - - {_nameify(bcp2)} - - {_nameify(rfc3)} - - - - {_nameify(rfc1)} - {rfc1.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

This is some interesting text.

 - - {_nameify(bcp1)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF - - - {_nameify(rfc2)} - {rfc2.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

This is some interesting text.

 - PROPOSED STANDARD - PROPOSED STANDARD - IETF - - - {_nameify(rfc3)} - {rfc3.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

This is some interesting text.

 - - {_nameify(bcp2)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF - -""" - data = rfceditor.parse_index(io.StringIO(index_xml)) # parse index - self.assertEqual(len(data), 3) # check that we parsed 3 RFCs - # Process the data by consuming the generator - for _ in rfceditor.update_docs_from_rfc_index(data, []): - pass - # Confirm that the expected changes were made - self.assertCountEqual(rfc1.related_that("contains"), [bcp1]) - self.assertCountEqual(rfc2.related_that("contains"), []) - self.assertCountEqual(rfc3.related_that("contains"), [bcp2]) - - def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): - """Generate an RFC queue xml string for a draft""" - t = ''' -
- -%(name)s-%(rev)s.txt -2010-09-08 -%(state)s -%(auth48_url)s - -%(ref)s -IN-QUEUE - -A. Author - -%(title)s - -10000000 -%(group)s - -
-''' % dict(name=draft.name, - rev=draft.rev, - title=draft.title, - group=draft.group.name, - ref="draft-ietf-test", - state=state, - auth48_url=(auth48_url or '')) - t = t.replace('\n', '') # strip empty auth48-url tags - return t - - def test_rfc_queue(self): - draft = WgDraftFactory(states=[('draft-iesg','ann')], ad=Person.objects.get(user__username='ad')) - draft.action_holders.add(draft.ad) # add an action holder so we can test that it's removed later - - expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234" - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url=expected_auth48_url) - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - # rfceditor.parse_queue() is tested independently; just sanity check here - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - mailbox_before = len(outbox) - - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - - draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.get_state_slug("draft-rfceditor"), "edit") - self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") - self.assertCountEqual(draft.action_holders.all(), []) - self.assertEqual(set(draft.tags.all()), set(DocTagName.objects.filter(slug__in=("iana", "ref")))) - events = draft.docevent_set.all() - self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state - self.assertEqual(events[1].type, "changed_action_holders") - self.assertEqual(events[2].type, "changed_state") # changed draft-rfceditor state - self.assertEqual(events[3].type, "rfc_editor_received_announcement") - - self.assertEqual(len(outbox), mailbox_before + 1) - self.assertTrue("RFC Editor queue" in outbox[-1]["Subject"]) - - # make sure we can apply it again with no changes - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 0) - self.assertEqual(len(warnings), 0) - - def test_rfceditor_parse_queue(self): - """Test that rfceditor.parse_queue() behaves as expected. - - Currently does a limited test - old comment was - "currently, we only check what we actually use". - """ - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0] - self.assertEqual(draft_name, draft.name) - self.assertEqual(state, "EDIT") - self.assertEqual(set(tags), set(["iana", "ref"])) - self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234") - - def test_rfceditor_parse_queue_TI_state(self): - # Test with TI state introduced 11 Sep 2019 - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='TI', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - __, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(warnings), 0) - - def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None): - """Helper to generate fake output from rfceditor.parse_queue()""" - return [[ - draft.name, # draft_name - '2020-06-03', # date_received - state, - tags or [], - '1', # missref_generation - 'ietf', # stream - auth48_url or '', - '', # cluster - ['draft-ietf-test'], # refs - ]] - - def test_update_draft_auth48_url(self): - """Test that auth48 URLs are handled correctly.""" - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - - # Step 1 setup: update to a state with no auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='EDIT') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) - - # Step 2: update to auth48 state with auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNotNone(auth48_docurl) - self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234') - - # Step 3: update to auth48-done state without auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48-DONE') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) class DiscrepanciesTests(TestCase): def test_discrepancies(self): @@ -922,11 +424,14 @@ def test_errata_map_from_json(self): @mock.patch("ietf.sync.errata.get_errata_data") def test_update_errata_from_rfceditor(self, mock_get_data, mock_update): fake_data = object() + fake_changed = {1234, 5678} mock_get_data.return_value = fake_data - update_errata_from_rfceditor() + mock_update.return_value = fake_changed + result = update_errata_from_rfceditor() self.assertTrue(mock_get_data.called) self.assertTrue(mock_update.called) self.assertEqual(mock_update.call_args, mock.call(fake_data)) + self.assertEqual(result, fake_changed) def test_update_errata_tags(self): tag_has_errata = DocTagName.objects.get(slug="errata") @@ -964,7 +469,7 @@ def test_update_errata_tags(self): {"doc-id": rfcs[8].name, "errata_status_code": "Rejected"}, # rfcs[9] had none and it should stay that way (no entry at all) ] - update_errata_tags(errata_data) + changed = update_errata_tags(errata_data) self.assertCountEqual(rfcs[0].tags.all(), [tag_has_errata]) self.assertIsNone(rfcs[0].docevent_set.first()) # no change @@ -1018,6 +523,14 @@ def test_update_errata_tags(self): self.assertCountEqual(rfcs[9].tags.all(), []) self.assertIsNone(rfcs[9].docevent_set.first()) # no change + # return value: only RFCs whose tags actually changed + # rfcs[0], rfcs[1], rfcs[8], rfcs[9] had no tag changes + for unchanged_rfc in (rfcs[0], rfcs[1], rfcs[8], rfcs[9]): + self.assertNotIn(unchanged_rfc.rfc_number, changed) + # rfcs[2..7] had tag changes + for changed_rfc in rfcs[2:8]: + self.assertIn(changed_rfc.rfc_number, changed) + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") @mock.patch("ietf.sync.errata.get_errata_last_updated") def test_update_errata_dirty_time(self, mock_last_updated): @@ -1064,192 +577,7 @@ def test_mark_errata_as_processed(self): class TaskTests(TestCase): - @override_settings( - RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", - RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", - ) - @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") - @mock.patch("ietf.sync.tasks.rfceditor.parse_index") - @mock.patch("ietf.sync.tasks.requests.get") - @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay") - def test_rfc_editor_index_update_task( - self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock - ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked - """rfc_editor_index_update_task calls helpers correctly - - This tests that data flow is as expected. Assumes the individual helpers are - separately tested to function correctly. - """ - @dataclass - class MockIndexData: - """Mock index item that claims to be a specified length""" - length: int - - def __len__(self): - return self.length - - @dataclass - class MockResponse: - """Mock object that contains text and json() that claims to be a specified length""" - text: str - json_length: int = 0 - - def json(self): - return MockIndexData(length=self.json_length) - - # Response objects - index_response = MockResponse(text="this is the index") - errata_response = MockResponse( - text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS - ) - rfc = RfcFactory() - # Test with full_index = False - rsync_task_mock.return_value = None - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, False), - ) - - tasks.rfc_editor_index_update_task(full_index=False) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) - - self.assertFalse(rsync_task_mock.called) - - # Test again with full_index = True - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - tasks.rfc_editor_index_update_task(full_index=True) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) - - self.assertFalse(rsync_task_mock.called) - - # Test again where the index would cause a new RFC to come into existance - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = ( - index_response, - errata_response, - ) # will step through these - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, True), - ) - tasks.rfc_editor_index_update_task(full_index=True) - self.assertTrue(rsync_task_mock.called) - rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args - self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs)) - - # Test error handling - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = requests.Timeout # timeout on every get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - # feed in an index that is too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertTrue(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") - @mock.patch("ietf.sync.tasks.update_drafts_from_queue") - @mock.patch("ietf.sync.tasks.parse_queue") - def test_rfc_editor_queue_updates_task(self, mock_parse, mock_update): - # test a request timeout - self.requests_mock.get("https://rfc-editor.example.com/queue/", exc=requests.exceptions.Timeout) - tasks.rfc_editor_queue_updates_task() - self.assertFalse(mock_parse.called) - self.assertFalse(mock_update.called) - - # now return a value rather than an exception - self.requests_mock.get("https://rfc-editor.example.com/queue/", text="the response") - - # mock returning < MIN_QUEUE_RESULTS values - treated as an error, so no update takes place - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS - 1)], ["a warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertFalse(mock_update.called) - mock_parse.reset_mock() - - # mock returning +. MIN_QUEUE_RESULTS - should succeed - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS)], ["a warning"]) - mock_update.return_value = ([1,2,3], ["another warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertEqual(mock_update.call_count, 1) - self.assertEqual(mock_update.call_args, mock.call([n for n in range(rfceditor.MIN_QUEUE_RESULTS)])) - @override_settings(IANA_SYNC_CHANGES_URL="https://iana.example.com/sync/") @mock.patch("ietf.sync.tasks.iana.update_history_with_changes") @mock.patch("ietf.sync.tasks.iana.parse_changes_json") @@ -1395,7 +723,7 @@ def test_load_rfcs_into_blobdb_task( self.assertEqual(mock_args, ([3261, 3262, 3263],)) self.assertEqual(mock_kwargs, {}) - + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") @mock.patch("ietf.sync.tasks.mark_errata_as_processed") @@ -1406,6 +734,7 @@ def test_update_errata_from_rfceditor_task( mock_mark_errata_processed, mock_mark_rfcindex_dirty, mock_update, + mock_rfc_json_delay, ): mock_errata_are_dirty.return_value = False update_errata_from_rfceditor_task() diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 2b70924db39..226b41af946 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -28,9 +28,11 @@ get_april1_rfc_numbers, get_publication_std_levels, get_unusable_rfc_numbers, + red_bucket_input_path, + red_bucket_output_path, + save_to_filesystem, save_to_red_bucket, subseries_text_line, - save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -139,7 +141,10 @@ def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", stripped_contents, ) - self.assertIn("1 April 2020", contents) # from the April 1 RFC + # "1 April 2020" may be split across a line wrap (e.g. "1 April\n 2020") + # when the randomly-generated title is long enough to push the date off the line. + # assertRegex handles both wrapped and non-wrapped cases explicitly. + self.assertRegex(contents, r"1\s+April\s+2020") # from the April 1 RFC self.assertIn( f"{self.rfc.rfc_number} {self.rfc.title}", stripped_contents, @@ -398,25 +403,47 @@ def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): ) +@override_settings(RFCINDEX_INPUT_PATH="input/", RFCINDEX_OUTPUT_PATH="output/") class HelperTests(TestCase): + INPUT_PATH = "input" + OUTPUT_PATH = "output" + def test_format_rfc_number(self): self.assertEqual(format_rfc_number(10), "10") with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): self.assertEqual(format_rfc_number(10), "0010") + def test_red_bucket_input_path(self): + with override_settings(RFCINDEX_INPUT_PATH="bar"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + with override_settings(RFCINDEX_INPUT_PATH="bar/"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + + def test_red_bucket_output_path(self): + self.assertEqual(red_bucket_input_path("foo"), f"{self.INPUT_PATH}/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar/"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + def test_save_to_red_bucket(self): red_bucket = storages["red_bucket"] with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): save_to_red_bucket("test", "contents \U0001f600") # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") - red_bucket.delete("test") # clean up like a good child + red_bucket.delete(f"{self.OUTPUT_PATH}/test") # clean up like a good child + # check that we can override the path + with override_settings(RFCINDEX_OUTPUT_PATH="fruit"): + save_to_red_bucket("test", "content") + self.assertTrue(red_bucket.exists("fruit/test")) + red_bucket.delete("fruit/test") # clean up like a good child def test_save_to_filesystem(self): rfc_path = Path(settings.RFC_PATH) @@ -442,30 +469,36 @@ def test_get_unusable_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_unusable_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/unusable-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_unusable_rfc_numbers() - red_bucket.delete("unusable-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/unusable-rfc-numbers.json") def test_get_april1_rfc_numbers_raises(self): """get_april1_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError): get_april1_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/april-first-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_april1_rfc_numbers() - red_bucket.delete("april-first-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/april-first-rfc-numbers.json") def test_get_publication_std_levels_raises(self): """get_publication_std_levels should bail on errors""" with self.assertRaises(FileNotFoundError): get_publication_std_levels() red_bucket = storages["red_bucket"] - red_bucket.save("publication-std-levels.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/publication-std-levels.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() - red_bucket.delete("publication-std-levels.json") + red_bucket.delete(f"{self.INPUT_PATH}/publication-std-levels.json") def test_subseries_text_line(self): text = "foobar" diff --git a/ietf/sync/tests_tasks.py b/ietf/sync/tests_tasks.py index 6657dd617a6..57284b72980 100644 --- a/ietf/sync/tests_tasks.py +++ b/ietf/sync/tests_tasks.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2026, All Rights Reserved +import mock from django.test.utils import override_settings from ietf.doc.factories import WgDraftFactory @@ -474,3 +475,81 @@ def test_docs_in_queue_retain_rfceditor_state(self): draft = Document.objects.get(pk=draft.pk) self.assertIsNotNone(draft.get_state("draft-rfceditor")) + + +class UpdateErrataFromRfcEditorTaskTests(TestCase): + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_no_update_when_not_dirty( + self, + mock_dirty, + mock_mark_processed, + mock_mark_index, + mock_update, + mock_json_delay, + ): + """When errata are not dirty nothing runs.""" + mock_dirty.return_value = False + tasks.update_errata_from_rfceditor_task() + mock_update.assert_not_called() + mock_json_delay.assert_not_called() + + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_json_task_called_for_changed_rfcs( + self, + mock_dirty, + mock_mark_processed, + mock_mark_index, + mock_update, + mock_json_delay, + ): + """update_rfc_json_task is dispatched with the changed RFC numbers.""" + mock_dirty.return_value = True + mock_update.return_value = {3261, 9000} + tasks.update_errata_from_rfceditor_task() + mock_json_delay.assert_called_once() + called_numbers = mock_json_delay.call_args[0][0] + self.assertCountEqual(called_numbers, [3261, 9000]) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_json_task_not_called_when_no_changes( + self, + mock_dirty, + mock_mark_processed, + mock_mark_index, + mock_update, + mock_json_delay, + ): + """update_rfc_json_task is not dispatched when no errata tags changed.""" + mock_dirty.return_value = True + mock_update.return_value = set() + tasks.update_errata_from_rfceditor_task() + mock_json_delay.assert_not_called() + + +class UpdateRfcJsonByRangeListTaskTests(TestCase): + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_expands_range_list_and_calls_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[1,100,1000-1004]") + mock_update.assert_called_once_with([1, 100, 1000, 1001, 1002, 1003]) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_invalid_input_is_ignored(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[5-3]") + mock_update.assert_not_called() + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_empty_input_does_not_call_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[]") + mock_update.assert_not_called() diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py index bb4a859e301..ed1dd4a62aa 100644 --- a/ietf/sync/tests_utils.py +++ b/ietf/sync/tests_utils.py @@ -7,10 +7,55 @@ from ietf import settings from ietf.doc.factories import RfcFactory from ietf.doc.storage_utils import exists_in_storage, retrieve_str -from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils.test_utils import TestCase +class ExpandRfcNumberRangeListTests(TestCase): + def test_expands_bare_numbers_and_ranges(self): + # ranges follow Python range() conventions: right value is excluded + self.assertEqual( + expand_rfc_number_range_list("[1,100,1000-1004]"), + [1, 100, 1000, 1001, 1002, 1003], + ) + + def test_accepts_input_without_brackets_and_whitespace(self): + self.assertEqual( + expand_rfc_number_range_list(" 1 , 3 - 5 , 9 "), + [1, 3, 4, 9], + ) + + def test_overlapping_ranges_are_sorted_and_deduplicated(self): + self.assertEqual( + expand_rfc_number_range_list("[5,1-4,3-6,2]"), + [1, 2, 3, 4, 5], + ) + + def test_empty_input_yields_empty_list(self): + self.assertEqual(expand_rfc_number_range_list("[]"), []) + self.assertEqual(expand_rfc_number_range_list(""), []) + + def test_rejects_invalid_input(self): + for bad in [ + "[0]", # zero is not positive + "[-5]", # negatives are not accepted + "[abc]", # non-numeric + "[5-3]", # reversed range + "[5-5]", # empty range (start not less than end) + "[1-]", # missing end + "[-1]", # missing start + "[1-2-3]", # malformed range + "[1.5]", # not an integer + ]: + with self.assertRaises(ValueError, msg=f"expected ValueError for {bad!r}"): + expand_rfc_number_range_list(bad) + + class RsyncHelperTests(TestCase): def test_rsync_helper(self): with ( diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py index b3bdd8d2068..b75d80e76bc 100644 --- a/ietf/sync/utils.py +++ b/ietf/sync/utils.py @@ -14,6 +14,57 @@ def rsync_helper(subprocess_arg_array: list[str]): subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + +def _parse_positive_int(value: str) -> int: + """Parse a string as a positive (>= 1) integer, raising ValueError otherwise""" + token = value.strip() + # str.isdigit() rejects signs, whitespace, and non-digits, so anything that + # passes is a non-negative integer literal; guard against zero separately. + if not token.isdigit(): + raise ValueError(f"'{value}' is not a positive integer") + number = int(token) + if number < 1: + raise ValueError(f"'{value}' is not a positive integer") + return number + + +def expand_rfc_number_range_list(ranges: str) -> list[int]: + """Expand a range-list string into a list of RFC numbers + + The string is a comma-separated list of tokens, optionally surrounded by a + pair of square brackets. Each token is either a bare positive integer or a + pair of positive integers separated by a hyphen. A hyphenated pair is + expanded following the convention of Python's range(): the left value is + included and the right value is excluded. For example, "[1,100,1000-1004]" + expands to [1, 100, 1000, 1001, 1002, 1003]. + + The returned list is sorted and deduplicated, so overlapping ranges do not + produce repeated numbers. + + Raises ValueError if the input contains anything other than positive + integers and well-formed (non-reversed) ranges. + """ + numbers: set[int] = set() + stripped = ranges.strip() + if stripped.startswith("[") and stripped.endswith("]"): + stripped = stripped[1:-1] + for raw_token in stripped.split(","): + token = raw_token.strip() + if not token: + continue + if "-" in token: + start_str, _, end_str = token.partition("-") + start = _parse_positive_int(start_str) + end = _parse_positive_int(end_str) + if start >= end: + raise ValueError( + f"'{token}' is not a valid range (start must be less than end)" + ) + numbers.update(range(start, end)) + else: + numbers.add(_parse_positive_int(token)) + return sorted(numbers) + def build_from_file_content(rfc_numbers: list[int]) -> str: types_to_sync = settings.RFC_FILE_TYPES + ("json",) lines = [] diff --git a/ietf/sync/views.py b/ietf/sync/views.py index bd539d4e186..58209c2c6ee 100644 --- a/ietf/sync/views.py +++ b/ietf/sync/views.py @@ -35,7 +35,6 @@ def notify(request, org, notification): known_orgs = { "iana": "IANA", - "rfceditor": "RFC Editor", } if org not in known_orgs: @@ -67,29 +66,13 @@ def notify(request, org, notification): known_notifications = { "protocols": "an added reference to an RFC at the IANA protocols page" % settings.IANA_SYNC_PROTOCOLS_URL, "changes": "new changes at the changes JSON dump" % settings.IANA_SYNC_CHANGES_URL, - "queue": "new changes to queue2.xml" % settings.RFC_EDITOR_QUEUE_URL, - "index": "new changes to rfc-index.xml" % settings.RFC_EDITOR_INDEX_URL, } if notification not in known_notifications: raise Http404 if request.method == "POST": - if notification == "index": - log("Queuing RFC Editor index sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_index_update_task.delay() - ) - elif notification == "queue": - log("Queuing RFC Editor queue sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_queue_updates_task.delay() - ) - elif notification == "changes": + if notification == "changes": log("Queuing IANA changes sync from notify view POST") # Wrap in on_commit in case a transaction is open # (As of 2024-11-08, this only runs in a transaction during tests) diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 43ca025e28b..905c337850e 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} {% origin %} {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %} @@ -132,6 +132,12 @@ IESG dashboard +
  • + + IESG decisions + +
  • @@ -453,6 +459,14 @@
    • {% endif %} + {% if user|has_role:"LLC Staff" %} +
  • + + Annual report inputs + +
    • + {% endif %}
  • diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index eab1d779fba..57194144989 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -601,7 +601,7 @@ @@ -666,17 +666,14 @@ Nits - {% if user|has_role:"Area Director" %} - {# IDNITS3 is an experimental service, so only show it to Area Directors #} - Nits-v3 (Experimental) + Nits v3 - {% endif %} {% endfor %} - + {% if doc.pages %}{{ doc.pages }} page{{ doc.pages|pluralize }}{% endif %}
    diff --git a/ietf/templates/doc/search/status_columns.html b/ietf/templates/doc/search/status_columns.html index 5ba41bb9c4a..50d70fcb379 100644 --- a/ietf/templates/doc/search/status_columns.html +++ b/ietf/templates/doc/search/status_columns.html @@ -11,7 +11,7 @@ {{ doc.friendly_state|safe }} {% endif %} {% if doc|state:"draft-rfceditor" %} - : {{ doc|state:"draft-rfceditor" }} + : {{ doc|state:"draft-rfceditor" }} {% endif %} {{ doc|auth48_alert_badge }} {{ doc|state_age_colored }} diff --git a/ietf/templates/iesg/review_decisions.html b/ietf/templates/iesg/review_decisions.html index 68a9d7fb953..2f69c1809b4 100644 --- a/ietf/templates/iesg/review_decisions.html +++ b/ietf/templates/iesg/review_decisions.html @@ -1,8 +1,8 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} -{% block title %}IESG review decisions in {{ timeframe }}{% endblock %} -{% block content %} +{% block group_subtitle %}IESG review decisions in {{ timeframe }}{% endblock %} +{% block group_content %} {% origin %}

    IESG review decisions in {{ timeframe }}

    diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html index b799636857f..1bde1362c50 100644 --- a/ietf/templates/iesg/working_groups.html +++ b/ietf/templates/iesg/working_groups.html @@ -1,11 +1,11 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2025-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} {% endblock %} -{% block title %}IESG view of working groups{% endblock %} -{% block content %} +{% block group_subtitle %}IESG view of working groups{% endblock %} +{% block group_content %} {% origin %}

    IESG view of working groups

    Area Size and Load

    diff --git a/ietf/templates/nomcom/view_feedback.html b/ietf/templates/nomcom/view_feedback.html index 93d61b7f420..0758329d606 100644 --- a/ietf/templates/nomcom/view_feedback.html +++ b/ietf/templates/nomcom/view_feedback.html @@ -43,7 +43,7 @@

    Declined each nominated position

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} @@ -82,7 +82,7 @@

    Feedback related to topics

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 00000000000..15add7ece3c --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    +
    +

    Summary

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 2d34f8361c1..c878ba49f10 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -83,34 +83,6 @@ def create_default_tasks(self): ), ) - PeriodicTask.objects.get_or_create( - name="Partial sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=False)), - defaults=dict( - enabled=False, - crontab=self.crontabs["every_15m_except_midnight"], # don't collide with full sync - description=( - "Reparse the last _year_ of RFC index entries until " - "https://github.com/ietf-tools/datatracker/issues/3734 is addressed. " - "This takes about 20s on production as of 2022-08-11." - ) - ), - ) - - PeriodicTask.objects.get_or_create( - name="Full sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=True)), - defaults=dict( - enabled=False, - crontab=self.crontabs["daily"], - description=( - "Run an extended version of the rfc editor update to catch changes with backdated timestamps" - ), - ), - ) - PeriodicTask.objects.get_or_create( name="Fetch meeting attendance", task="ietf.stats.tasks.fetch_meeting_attendance_task", diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 00000000000..9a969a52178 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved + +from typing import List, Set, Tuple +from django.db.models import QuerySet + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" + addresses = set() + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): + addresses.update([a["email"] for a in submission.authors]) + return addresses + + +def submitters_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year, state="posted" + ).values_list("submitter", flat=True) + ] + ) + + +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lengths of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (persons, set(addresses) - set(known_email)) + + diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index a9e02ca740b..3c6e1b8f26f 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -64,8 +64,11 @@ def get_collection_name() -> str: return collection_name -def _sanitize_text(content): - """Sanitize content or abstract text for search""" +def _sanitize_text(content: str): + """Sanitize content text for search + + Aggressively simplifies whitespace, removes most punctuation + """ # REs (with approximate names) RE_DOT_OR_BANG_SPACE = r"\. |! " # -> " " (space) RE_COMMENT_OR_TOC_CRUD = r"<--|-->|--+|\+|\.\.+" # -> "" @@ -84,6 +87,18 @@ def _sanitize_text(content): return content.strip() +def _sanitize_abstract(abstract: str): + """Sanitize abstract text for search + + Simplifies whitespace but mostly leaves text intact. Abstract text will be + displayed in search results, so a light touch is needed. + """ + abstract = abstract.strip() + abstract = re.sub("\r\n|\n\r|\r", "\n", abstract) # normalize on \n + abstract = "\n".join(line.strip() for line in abstract.split("\n")) # strip by line + return abstract + + def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None @@ -123,7 +138,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "rfc": str(rfc.rfc_number), "filename": rfc.name, "title": rfc.title, - "abstract": _sanitize_text(rfc.abstract), + "abstract": _sanitize_abstract(rfc.abstract), "pages": rfc.pages, "keywords": keywords, "type": "rfc", @@ -156,6 +171,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "acronym": rfc.group.acronym, "name": rfc.group.name, "full": f"{rfc.group.acronym} - {rfc.group.name}", + "type": rfc.group.type.slug, } if ( rfc.group.parent is not None @@ -367,9 +383,9 @@ def update_or_create_rfc_entries( }, "red-content": { "collection": "docs", - "infix": "off,always,off,off", - "query_by": "rfc,filename,authors,content", - "query_by_weights": "127,50,5,1" + "infix": "off,always,off,off,off", + "query_by": "rfc,filename,keywords,authors,content", + "query_by_weights": "127,50,20,5,1" }, } diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 00000000000..83daa15cc14 --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,189 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + authors2020, + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + authors2021, + set( + [ + "author0@example.net", + "author0@example.com", + "author2@example.net", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + sub2020, + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 7eb4e8acea6..b97bc2e266c 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -46,6 +46,39 @@ def test_sanitize_text(self): sanitized = "This is text It is full of problems Fix it." self.assertEqual(searchindex._sanitize_text(dirty_text), sanitized) + def test_sanitize_abstract(self): + dirty_abstract = ( + "Mixed\n" + "Newlines\r" + "And\r\n" + "Things\n\r" + " Sometimes\n" + "\n" + "With\r\n" + "\r\n" + "Double \n\r" + "\n\r" + " Newlines\r" + "\r" + "Whee!" + ) + sanitized = ( + "Mixed\n" + "Newlines\n" + "And\n" + "Things\n" + "Sometimes\n" + "\n" + "With\n" + "\n" + "Double\n" + "\n" + "Newlines\n" + "\n" + "Whee!" + ) + self.assertEqual(searchindex._sanitize_abstract(dirty_abstract), sanitized) + def test_typesense_doc_from_rfc(self): not_rfc = WgDraftFactory() assert isinstance(not_rfc, Document) @@ -63,7 +96,7 @@ def test_typesense_doc_from_rfc(self): # Check a few values, not exhaustive self.assertEqual(result["id"], f"doc-{rfc.pk}") self.assertEqual(result["rfcNumber"], rfc.rfc_number) - self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertEqual(result["abstract"], searchindex._sanitize_abstract(rfc.abstract)) self.assertEqual(result["pages"], rfc.pages) self.assertNotIn("adName", result) self.assertNotIn("content", result) # no blob diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py index e08dfa02f2c..5e82c225dc6 100644 --- a/ietf/utils/timezone.py +++ b/ietf/utils/timezone.py @@ -15,7 +15,7 @@ DEADLINE_TZINFO = ZoneInfo('PST8PDT') # Time zone for dates from the RPC. This value is baked into the timestamps on DocEvents -# of type="published_rfc" - see Document.pub_date() and ietf.sync.refceditor.update_docs_from_rfc_index() +# of type="published_rfc" - see Document.pub_date() and the rfc publication workflow # for more information about how that works. RPC_TZINFO = ZoneInfo('PST8PDT') diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 992c90557a1..ef8c259933b 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: auth - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: @@ -69,7 +67,7 @@ spec: # Nginx Container # ----------------------------------------------------- - name: nginx - image: "ghcr.io/nginxinc/nginx-unprivileged:1.27" + image: "ghcr.io/nginx/nginx-unprivileged:1.30" imagePullPolicy: IfNotPresent ports: - containerPort: 8080 diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index ff89fb37228..5183893bc8c 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: datatracker - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: @@ -69,7 +67,7 @@ spec: # Nginx Container # ----------------------------------------------------- - name: nginx - image: "ghcr.io/nginxinc/nginx-unprivileged:1.27" + image: "ghcr.io/nginx/nginx-unprivileged:1.30" imagePullPolicy: IfNotPresent ports: - containerPort: 8080 diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index 1dc2f4fae0d..cdb5c665e22 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -1,5 +1,6 @@ upstream datatracker_backend { server 127.0.0.1:8000; + keepalive 0; # default = 32 since nginx 1.29.7 } server { diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index a7d5936a193..77ce902b17a 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -1,5 +1,6 @@ upstream datatracker_backend { server 127.0.0.1:8000; + keepalive 0; # default = 32 since nginx 1.29.7 } server { diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 402f89787b1..5dc31bac0e5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -248,8 +248,19 @@ def _multiline_to_list(s): EMAIL_COPY_TO = "" -# Until we teach the datatracker to look beyond cloudflare for this check -IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 +# I-D Submission settings + +# Until we teach the datatracker to look beyond cloudflare for this check, it needs +# to be very large. 5000 has been working without complaint. +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = int( + os.environ.get("DATATRACKER_IDSUBMIT_MAX_DAILY_SAME_SUBMITTER", "5000") +) + +# Default is 20 minutes. Allow override via environment. +if "DATATRACKER_IDSUBMIT_MAX_VALIDATION_TIME" in os.environ: + IDSUBMIT_MAX_VALIDATION_TIME = datetime.timedelta( + minutes=int(os.environ.get("DATATRACKER_IDSUBMIT_MAX_VALIDATION_TIME")) + ) # Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting if "DATATRACKER_MATOMO_SITE_ID" in os.environ: @@ -461,8 +472,10 @@ def _multiline_to_list(s): ), } RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default -RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") -RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") +if "DATATRACKER_RFCINDEX_OUTPUT_PATH" in os.environ: + RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH") +if "DATATRACKER_RFCINDEX_INPUT_PATH" in os.environ: + RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH") # Configure the blobdb app for artifact storage _blobdb_replication_enabled = (