diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bf285500840..e4964e8909c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -32,8 +32,6 @@ "mutantdino.resourcemonitor", "oderwat.indent-rainbow", "redhat.vscode-yaml", - "spmeesseman.vscode-taskexplorer", - "visualstudioexptteam.vscodeintellicode", "ms-python.pylint", "charliermarsh.ruff" ], diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index a92f42bc6d4..ce1ce259fdb 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,8 @@ services: network_mode: service:db blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' volumes: datatracker-vscode-ext: diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 4a4394fca0c..a2abe089ce6 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: token: ${{ secrets.GH_COMMON_TOKEN }} @@ -28,20 +28,20 @@ jobs: echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -60,7 +60,7 @@ jobs: echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - name: Commit CHANGELOG.md - uses: stefanzweifel/git-auto-commit-action@v6 + uses: stefanzweifel/git-auto-commit-action@v7 with: branch: ${{ github.ref_name }} commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index f49a11af19c..429aa08b006 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,20 +20,20 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 4de861dbcd3..7832e65a3a3 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,23 +24,32 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set rabbitmq version + id: rabbitmq-version + run: | + if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then + echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT + else + echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT + fi + - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -48,7 +57,7 @@ jobs: file: dev/mq/Dockerfile platforms: linux/amd64,linux/arm64 push: true - build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }} + build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} tags: | - ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }} + ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} ghcr.io/ietf-tools/datatracker-mq:latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7eac7b1c64f..5088d763e92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,7 @@ -name: Build and Release -run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} +name: Build Dev and Deploy +run-name: Dev Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} on: - push: - branches: [release] - workflow_dispatch: inputs: deploy: @@ -14,24 +11,28 @@ on: type: choice options: - Skip - - Staging Only - - Staging + Prod - dev: - description: 'Deploy to Dev' - default: true + - Dev + - Staging + deployStrategy: + description: 'Deploy Strategy (Staging Only)' + default: 'Recreate' required: true - type: boolean + type: choice + options: + - Recreate + - RollingUpdate + - Current devNoDbRefresh: - description: 'Dev Disable Daily DB Refresh' + description: 'Disable Daily DB Refresh (Dev Only)' default: false required: true type: boolean - skiptests: + skipTests: description: 'Skip Tests' default: false required: true type: boolean - skiparm: + skipArmBuild: description: 'Skip ARM64 Build' default: false required: true @@ -56,34 +57,21 @@ jobs: # PREPARE # ----------------------------------------------------------------- prepare: - name: Prepare Release + name: Prepare runs-on: ubuntu-latest outputs: - should_deploy: ${{ steps.buildvars.outputs.should_deploy }} - pkg_version: ${{ steps.buildvars.outputs.pkg_version }} - from_tag: ${{ steps.semver.outputs.nextStrict }} - to_tag: ${{ steps.semver.outputs.current }} - base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false - - name: Get Next Version (Prod) - if: ${{ github.ref_name == 'release' }} - id: semver - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - patchList: fix, bugfix, perf, refactor, test, tests, chore - - name: Get Dev Version - if: ${{ github.ref_name != 'release' }} - id: semverdev + id: semver uses: ietf-tools/semver-action@v1 with: token: ${{ github.token }} @@ -91,260 +79,54 @@ jobs: skipInvalidTags: true noVersionBumpBehavior: 'current' noNewCommitBehavior: 'current' - - - name: Set Release Flag - if: ${{ github.ref_name == 'release' }} - run: | - echo "IS_RELEASE=true" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1.18.0 - if: ${{ github.ref_name == 'release' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ steps.semver.outputs.nextStrict }} - name: ${{ steps.semver.outputs.nextStrict }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} - name: Set Build Variables id: buildvars run: | - if [[ $IS_RELEASE ]]; then - echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}" - echo "should_deploy=true" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT - echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" - else - echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" - echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" - fi + SHORT_SHA="${GITHUB_SHA:0:7}" + echo "Will set build version to: ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" + echo "buildVersion=${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" >> $GITHUB_OUTPUT + echo "::notice::Non-production build ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA created using branch $GITHUB_REF_NAME" - name: Get Base Image Target Version id: baseimgversion run: | - echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- - tests: name: Run Tests - uses: ./.github/workflows/tests.yml - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: ./.github/workflows/reusable-tests.yml + if: ${{ inputs.skipTests == 'false' }} needs: [prepare] secrets: inherit with: - ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + ignoreLowerCoverage: ${{ inputs.ignoreLowerCoverage == 'true' }} skipSelenium: true - targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} # ----------------------------------------------------------------- - # RELEASE + # BUILD IMAGE # ----------------------------------------------------------------- - release: - name: Make Release + build: + name: Build Image + uses: ./.github/workflows/reusable-build.yml if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: - group: hperf-8c32r permissions: contents: write packages: write - env: - SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - FROM_TAG: ${{needs.prepare.outputs.from_tag}} - TO_TAG: ${{needs.prepare.outputs.to_tag}} - TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Setup Node.js environment - uses: actions/setup-node@v4 - with: - node-version: 18.x - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Setup AWS CLI - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2.22.35 - - - name: Download a Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 - with: - name: coverage - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - run: | - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - - name: Set Production Flags - if: ${{ env.SHOULD_DEPLOY == 'true' }} - run: | - echo "Setting production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - - - name: Make Release Tarball - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . - - - name: Collect + Push Statics - env: - DEBIAN_FRONTEND: noninteractive - AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - run: | - echo "Collecting statics..." - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - - - name: Augment dockerignore for docker image build - env: - DEBIAN_FRONTEND: noninteractive - run: | - cat >> .dockerignore <> $GITHUB_ENV - - - name: Build Images - uses: docker/build-push-action@v6 - env: - DOCKER_BUILD_SUMMARY: false - with: - context: . - file: dev/build/Dockerfile - platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} - push: true - tags: | - ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - - - name: Update CHANGELOG - id: changelog - uses: Requarks/changelog-action@v1 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - token: ${{ github.token }} - fromTag: ${{ env.FROM_TAG }} - toTag: ${{ env.TO_TAG }} - writeToFile: false - - - name: Download Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 - with: - name: coverage - - - name: Prepare Coverage Action - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - working-directory: ./dev/coverage-action - run: npm install - - - name: Process Coverage Stats + Chart - id: covprocess - uses: ./dev/coverage-action/ - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - with: - token: ${{ github.token }} - tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} - repoCommon: common - version: ${{needs.prepare.outputs.pkg_version}} - changelog: ${{ steps.changelog.outputs.changes }} - summary: '' - coverageResultsPath: coverage.json - histCoveragePath: historical-coverage.json - - - name: Create Release - uses: ncipollo/release-action@v1.18.0 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - allowUpdates: true - makeLatest: true - draft: false - tag: ${{ env.PKG_VERSION }} - name: ${{ env.PKG_VERSION }} - body: ${{ steps.covprocess.outputs.changelog }} - artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.18.0 - if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} - with: - allowUpdates: true - tag: baseline - omitBodyDuringUpdate: true - omitNameDuringUpdate: true - omitPrereleaseDuringUpdate: true - replacesArtifacts: true - artifacts: "coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 - with: - name: release-${{ env.PKG_VERSION }} - path: /home/runner/work/release/release.tar.gz + secrets: inherit + with: + buildVersion: ${{ needs.prepare.outputs.buildVersion }} + isReleaseBuild: false + previousVersion: ${{ needs.prepare.outputs.previousVersion }} + handleCoverageResults: ${{ inputs.skipTests == 'false' }} + updateCoverageBaseline: ${{ inputs.updateCoverage }} + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + skipArmBuild: ${{ inputs.skipArmBuild }} # ----------------------------------------------------------------- # NOTIFY @@ -352,21 +134,21 @@ jobs: notify: name: Notify if: ${{ always() }} - needs: [prepare, tests, release] + needs: [prepare, tests, build] runs-on: ubuntu-latest env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "28a745" fields: @@ -375,35 +157,35 @@ jobs: value: "Completed" - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "a82929" fields: - title: "Status" short: true - value: "Failed" + value: "Failed 😱" # ----------------------------------------------------------------- # DEV # ----------------------------------------------------------------- dev: name: Deploy to Dev - if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Dev' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: dev env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: ref: main @@ -418,81 +200,45 @@ jobs: echo "DEPLOY_NAMESPACE=$(node cli.js --branch ${{ github.ref_name }})" >> "$GITHUB_ENV" - name: Deploy to dev - uses: the-actions-org/workflow-dispatch@v4 + uses: ietf-tools/workflow-dispatch-action@v1 with: workflow: deploy-dev.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' - wait-for-completion: true - wait-for-completion-timeout: 30m - wait-for-completion-interval: 30s - display-workflow-run-url: false + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + waitForCompletionTimeout: 60m # ----------------------------------------------------------------- # STAGING # ----------------------------------------------------------------- staging: name: Deploy to Staging - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Staging' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: staging env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Refresh Staging DB - uses: the-actions-org/workflow-dispatch@v4 + uses: ietf-tools/workflow-dispatch-action@v1 with: workflow: deploy-db.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }' - wait-for-completion: true - wait-for-completion-timeout: 30m - wait-for-completion-interval: 20s - display-workflow-run-url: false + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m - name: Deploy to staging - uses: the-actions-org/workflow-dispatch@v4 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - wait-for-completion: true - wait-for-completion-timeout: 10m - wait-for-completion-interval: 30s - display-workflow-run-url: false - - # ----------------------------------------------------------------- - # PROD - # ----------------------------------------------------------------- - prod: - name: Deploy to Production - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, staging] - runs-on: ubuntu-latest - environment: - name: production - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to production - uses: the-actions-org/workflow-dispatch@v4 + uses: ietf-tools/workflow-dispatch-action@v1 with: workflow: deploy.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - wait-for-completion: true - wait-for-completion-timeout: 10m - wait-for-completion-interval: 30s - display-workflow-run-url: false + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 278bd8af2f3..50ec4c1943b 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false @@ -38,7 +38,7 @@ jobs: # ----------------------------------------------------------------- tests: name: Run Tests - uses: ./.github/workflows/tests.yml + uses: ./.github/workflows/reusable-tests.yml needs: [prepare] with: ignoreLowerCoverage: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3444c03b5e0..15b2231ce39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,12 +26,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 6d0683c471c..5bb26e4f6fa 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,8 +15,8 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@v5 with: vulnerability-check: false diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 4cfbf6365b4..83121354e64 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,17 +29,17 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index 22652dab889..b90300c09b2 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -16,9 +16,10 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: ietf-tools/lock-threads@v3.1.1 + - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} + process-only: 'issues, prs' issue-inactive-days: 7 pr-inactive-days: 3 log-output: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..2de02eb2852 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,209 @@ +name: Make Release and Deploy +run-name: RELEASE ${{ github.run_number }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} + + steps: + - name: Ensure release branch + if: github.ref != 'refs/heads/release' + run: | + echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" + exit 1 + + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1 + with: + token: ${{ github.token }} + branch: release + skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore + + - name: Create Draft Release + uses: ncipollo/release-action@v1.21.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Build Variables + id: buildvars + run: | + echo "Will set build version to: ${{ steps.semver.outputs.nextStrict }}" + echo "buildVersion=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT + echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD RELEASE + # ----------------------------------------------------------------- + build: + name: Build Release Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{needs.prepare.outputs.buildVersion}} + isReleaseBuild: true + previousVersion: ${{needs.prepare.outputs.previousVersion}} + handleCoverageResults: true + updateCoverageBaseline: true + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() }} + needs: [prepare, staging] + runs-on: ubuntu-latest + environment: + name: production + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Deploy to production + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 00000000000..4d9a3b4bf11 --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,226 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + buildVersion: + required: true + type: string + isReleaseBuild: + default: false + required: false + type: boolean + previousVersion: + required: false + type: string + handleCoverageResults: + default: true + required: false + type: boolean + updateCoverageBaseline: + default: false + required: false + type: boolean + baseImageVersion: + default: latest + required: false + type: string + skipArmBuild: + default: false + required: false + type: boolean + +jobs: + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + runs-on: + group: hperf-8c32r + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 18.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Setup AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2.22.35 + + - name: Download a Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Make Build + env: + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "BUILD_VERSION: ${{ inputs.buildVersion }}" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/${{ inputs.buildVersion }}/ + yarn legacy:build + echo "Setting version ${{ inputs.buildVersion }}..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '${{ inputs.buildVersion }}'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ inputs.isReleaseBuild }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/${{ inputs.buildVersion }} --only-show-errors + + - name: Augment dockerignore for docker image build + env: + DEBIAN_FRONTEND: noninteractive + run: | + cat >> .dockerignore <> $GITHUB_ENV + + - name: Build Images + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/build/Dockerfile + platforms: ${{ inputs.skipArmBuild && 'linux/amd64' || 'linux/amd64,linux/arm64' }} + push: true + tags: | + ghcr.io/ietf-tools/datatracker:${{ inputs.buildVersion }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} + + - name: Update CHANGELOG + id: changelog + uses: Requarks/changelog-action@v1 + if: ${{ inputs.isReleaseBuild }} + with: + token: ${{ github.token }} + fromTag: ${{ inputs.buildVersion }} + toTag: ${{ inputs.previousVersion }} + writeToFile: false + + - name: Download Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Prepare Coverage Action + if: ${{ inputs.handleCoverageResults }} + working-directory: ./dev/coverage-action + run: npm install + + - name: Process Coverage Stats + Chart + id: covprocess + uses: ./dev/coverage-action/ + if: ${{ inputs.handleCoverageResults }} + with: + token: ${{ github.token }} + tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} + repoCommon: common + version: ${{ inputs.buildVersion }} + changelog: ${{ steps.changelog.outputs.changes }} + summary: '' + coverageResultsPath: coverage.json + histCoveragePath: historical-coverage.json + + - name: Create Release + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.isReleaseBuild }} + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ inputs.buildVersion }} + name: ${{ inputs.buildVersion }} + body: ${{ steps.covprocess.outputs.changelog }} + artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Baseline Coverage + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.updateCoverageBaseline }} + with: + allowUpdates: true + tag: baseline + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + replacesArtifacts: true + artifacts: "coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ inputs.buildVersion }} + path: /home/runner/work/release/release.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/reusable-tests.yml similarity index 92% rename from .github/workflows/tests.yml rename to .github/workflows/reusable-tests.yml index f10c1db9a30..852e2d6fedd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Prepare for tests run: | @@ -68,14 +68,14 @@ jobs: coverage xml - name: Upload geckodriver.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ failure() }} with: name: geckodriverlog path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v7 with: disable_search: true files: coverage.xml @@ -87,7 +87,7 @@ jobs: mv latest-coverage.json coverage.json - name: Upload Coverage Results as Build Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} with: name: coverage @@ -102,9 +102,9 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: '18' @@ -121,7 +121,7 @@ jobs: npx playwright test --project=${{ matrix.project }} - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: @@ -130,6 +130,7 @@ jobs: if-no-files-found: ignore tests-playwright-legacy: + if: ${{ false }} # disable until we sort out suspected test runner issue name: Playwright Legacy Tests runs-on: ubuntu-latest container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} @@ -143,7 +144,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Prepare for tests run: | @@ -180,7 +181,7 @@ jobs: npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml deleted file mode 100644 index 8553563a192..00000000000 --- a/.github/workflows/tests-az.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Tests (Azure Test) - -on: - workflow_dispatch: - -jobs: - main: - name: Run Tests on Azure temp VM - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - name: Launch VM on Azure - id: azlaunch - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_TESTS_APP_ID }} -p ${{ secrets.AZ_TESTS_PWD }} --tenant ${{ secrets.AZ_TESTS_TENANT_ID }} - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatrackerTests \ - --name tmpGhaVM2 \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D4as_v5 \ - --max-price -1 \ - --os-disk-size-gb 30 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - - - name: Remote SSH into VM - uses: appleboy/ssh-action@2ead5e36573f08b82fbfce1504f1a4b05a647c6f - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - host: ${{ steps.azlaunch.outputs.ipaddr }} - port: 22 - username: azureuser - command_timeout: 60m - key_path: ${{ github.workspace }}/prvkey.key - envs: GITHUB_TOKEN - script_stop: true - script: | - export DEBIAN_FRONTEND=noninteractive - lsb_release -a - sudo apt-get update - sudo apt-get upgrade -y - - echo "Installing Docker..." - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh - - echo "Starting Containers..." - sudo docker network create dtnet - sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & - wait - - echo "Cloning datatracker repo..." - sudo docker exec app git clone --depth=1 https://github.com/ietf-tools/datatracker.git . - echo "Prepare tests..." - sudo docker exec app chmod +x ./dev/tests/prepare.sh - sudo docker exec app sh ./dev/tests/prepare.sh - echo "Running checks..." - sudo docker exec app ietf/manage.py check - sudo docker exec app ietf/manage.py migrate --fake-initial - echo "Running tests..." - sudo docker exec app ietf/manage.py test -v2 --validate-html-harder --settings=settings_test - - - name: Destroy VM + resources - if: always() - shell: pwsh - run: | - echo "Destroying VM..." - az vm delete -g ghaDatatrackerTests -n tmpGhaVM2 --yes --force-deletion true - - $resourceOrderRemovalOrder = [ordered]@{ - "Microsoft.Compute/virtualMachines" = 0 - "Microsoft.Compute/disks" = 1 - "Microsoft.Network/networkInterfaces" = 2 - "Microsoft.Network/publicIpAddresses" = 3 - "Microsoft.Network/networkSecurityGroups" = 4 - "Microsoft.Network/virtualNetworks" = 5 - } - echo "Fetching remaining resources..." - $resources = az resource list --resource-group ghaDatatrackerTests | ConvertFrom-Json - - $orderedResources = $resources - | Sort-Object @{ - Expression = {$resourceOrderRemovalOrder[$_.type]} - Descending = $False - } - - echo "Deleting remaining resources..." - $orderedResources | ForEach-Object { - az resource delete --resource-group ghaDatatrackerTests --ids $_.id --verbose - } - - echo "Logout from Azure..." - az logout diff --git a/.gitignore b/.gitignore index 84bc800e3b8..ccc7a46b08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_store datatracker.sublime-project datatracker.sublime-workspace +/.claude /.coverage /.factoryboy_random_state /.mypy_cache diff --git a/.pnp.cjs b/.pnp.cjs index 5fcce34d2f2..6c76263c7ea 100644 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -42,6 +42,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@fullcalendar/luxon3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/timegrid", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/vue3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ + ["@kurkle/color", "npm:0.3.1"],\ ["@parcel/optimizer-data-url", "npm:2.12.0"],\ ["@parcel/transformer-inline-string", "npm:2.12.0"],\ ["@parcel/transformer-sass", "npm:2.12.0"],\ @@ -56,6 +57,9 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["browserlist", "npm:1.0.1"],\ ["c8", "npm:9.1.0"],\ ["caniuse-lite", "npm:1.0.30001603"],\ + ["chart.js", "npm:4.5.1"],\ + ["chartjs-plugin-autocolors", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1"],\ + ["chartjs-plugin-zoom", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0"],\ ["d3", "npm:7.9.0"],\ ["eslint", "npm:8.57.0"],\ ["eslint-config-standard", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:17.1.0"],\ @@ -883,6 +887,22 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@kurkle/color", [\ + ["npm:0.3.1", {\ + "packageLocation": "./.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip/node_modules/@kurkle/color/",\ + "packageDependencies": [\ + ["@kurkle/color", "npm:0.3.1"]\ + ],\ + "linkType": "HARD"\ + }],\ + ["npm:0.3.4", {\ + "packageLocation": "./.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip/node_modules/@kurkle/color/",\ + "packageDependencies": [\ + ["@kurkle/color", "npm:0.3.4"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@lezer/common", [\ ["npm:0.15.12", {\ "packageLocation": "./.yarn/cache/@lezer-common-npm-0.15.12-62017272b0-dae6581618.zip/node_modules/@lezer/common/",\ @@ -2616,6 +2636,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@types/hammerjs", [\ + ["npm:2.0.46", {\ + "packageLocation": "./.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip/node_modules/@types/hammerjs/",\ + "packageDependencies": [\ + ["@types/hammerjs", "npm:2.0.46"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@types/istanbul-lib-coverage", [\ ["npm:2.0.4", {\ "packageLocation": "./.yarn/cache/@types-istanbul-lib-coverage-npm-2.0.4-734954bb56-a25d7589ee.zip/node_modules/@types/istanbul-lib-coverage/",\ @@ -3545,6 +3574,66 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["chart.js", [\ + ["npm:4.5.1", {\ + "packageLocation": "./.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip/node_modules/chart.js/",\ + "packageDependencies": [\ + ["chart.js", "npm:4.5.1"],\ + ["@kurkle/color", "npm:0.3.4"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ + ["chartjs-plugin-autocolors", [\ + ["npm:0.3.1", {\ + "packageLocation": "./.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip/node_modules/chartjs-plugin-autocolors/",\ + "packageDependencies": [\ + ["chartjs-plugin-autocolors", "npm:0.3.1"]\ + ],\ + "linkType": "SOFT"\ + }],\ + ["virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1", {\ + "packageLocation": "./.yarn/__virtual__/chartjs-plugin-autocolors-virtual-6e228c1a1e/0/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip/node_modules/chartjs-plugin-autocolors/",\ + "packageDependencies": [\ + ["chartjs-plugin-autocolors", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1"],\ + ["@kurkle/color", "npm:0.3.1"],\ + ["@types/chart.js", null],\ + ["@types/kurkle__color", null],\ + ["chart.js", "npm:4.5.1"]\ + ],\ + "packagePeers": [\ + "@kurkle/color",\ + "@types/chart.js",\ + "@types/kurkle__color",\ + "chart.js"\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ + ["chartjs-plugin-zoom", [\ + ["npm:2.2.0", {\ + "packageLocation": "./.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip/node_modules/chartjs-plugin-zoom/",\ + "packageDependencies": [\ + ["chartjs-plugin-zoom", "npm:2.2.0"]\ + ],\ + "linkType": "SOFT"\ + }],\ + ["virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0", {\ + "packageLocation": "./.yarn/__virtual__/chartjs-plugin-zoom-virtual-45332d2c47/0/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip/node_modules/chartjs-plugin-zoom/",\ + "packageDependencies": [\ + ["chartjs-plugin-zoom", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0"],\ + ["@types/chart.js", null],\ + ["@types/hammerjs", "npm:2.0.46"],\ + ["chart.js", "npm:4.5.1"],\ + ["hammerjs", "npm:2.0.8"]\ + ],\ + "packagePeers": [\ + "@types/chart.js",\ + "chart.js"\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["chokidar", [\ ["npm:3.5.3", {\ "packageLocation": "./.yarn/cache/chokidar-npm-3.5.3-c5f9b0a56a-b49fcde401.zip/node_modules/chokidar/",\ @@ -5709,6 +5798,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["hammerjs", [\ + ["npm:2.0.8", {\ + "packageLocation": "./.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip/node_modules/hammerjs/",\ + "packageDependencies": [\ + ["hammerjs", "npm:2.0.8"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["has", [\ ["npm:1.0.3", {\ "packageLocation": "./.yarn/cache/has-npm-1.0.3-b7f00631c1-b9ad53d53b.zip/node_modules/has/",\ @@ -8326,6 +8424,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@fullcalendar/luxon3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/timegrid", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/vue3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ + ["@kurkle/color", "npm:0.3.1"],\ ["@parcel/optimizer-data-url", "npm:2.12.0"],\ ["@parcel/transformer-inline-string", "npm:2.12.0"],\ ["@parcel/transformer-sass", "npm:2.12.0"],\ @@ -8340,6 +8439,9 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["browserlist", "npm:1.0.1"],\ ["c8", "npm:9.1.0"],\ ["caniuse-lite", "npm:1.0.30001603"],\ + ["chart.js", "npm:4.5.1"],\ + ["chartjs-plugin-autocolors", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1"],\ + ["chartjs-plugin-zoom", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0"],\ ["d3", "npm:7.9.0"],\ ["eslint", "npm:8.57.0"],\ ["eslint-config-standard", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:17.1.0"],\ diff --git a/.vscode/settings.json b/.vscode/settings.json index b0ceba5c9d8..ad6b0adc849 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -56,5 +56,9 @@ "python.linting.pylintArgs": ["--load-plugins", "pylint_django"], "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": false, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.terminal.shellIntegration.enabled": false, + "vs-kubernetes": { + "disable-linters": ["resource-limits"] + } } diff --git a/.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip b/.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip new file mode 100644 index 00000000000..65b1d3941c4 Binary files /dev/null and b/.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip differ diff --git a/.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip b/.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip new file mode 100644 index 00000000000..e4d1e3fd1df Binary files /dev/null and b/.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip differ diff --git a/.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip b/.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip new file mode 100644 index 00000000000..1babeca0a97 Binary files /dev/null and b/.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip differ diff --git a/.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip b/.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip new file mode 100644 index 00000000000..2dba0d6d9f9 Binary files /dev/null and b/.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip differ diff --git a/.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip b/.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip new file mode 100644 index 00000000000..16cdf883994 Binary files /dev/null and b/.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip differ diff --git a/.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip b/.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip new file mode 100644 index 00000000000..edf50232bc1 Binary files /dev/null and b/.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip differ diff --git a/.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip b/.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip new file mode 100644 index 00000000000..36d13b6aaf3 Binary files /dev/null and b/.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip differ diff --git a/client/agenda/AgendaDetailsModal.vue b/client/agenda/AgendaDetailsModal.vue index 2582bf21593..69c8ef8b532 100644 --- a/client/agenda/AgendaDetailsModal.vue +++ b/client/agenda/AgendaDetailsModal.vue @@ -274,6 +274,7 @@ async function fetchSessionMaterials () { diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue index fc8b5fd30f7..bbe5dfee8b7 100644 --- a/client/agenda/AgendaScheduleList.vue +++ b/client/agenda/AgendaScheduleList.vue @@ -398,16 +398,6 @@ const meetingEvents = computed(() => { color: 'teal' }) } - // -> Calendar item - if (item.links.calendar) { - links.push({ - id: `lnk-${item.id}-calendar`, - label: 'Calendar (.ics) entry for this session', - icon: 'calendar-check', - href: item.links.calendar, - color: 'pink' - }) - } } else { // -> Post event if (meetingNumberInt >= 60) { @@ -484,6 +474,16 @@ const meetingEvents = computed(() => { } } } + // Add Calendar item for all events that has a calendar link + if (item.adjustedEnd > current && item.links.calendar) { + links.push({ + id: `lnk-${item.id}-calendar`, + label: 'Calendar (.ics) entry for this session', + icon: 'calendar-check', + href: item.links.calendar, + color: 'pink' + }) + } // Event icon let icon = null diff --git a/client/components/ChatLog.vue b/client/components/ChatLog.vue index f9dc382bfea..b3a4f7b40f4 100644 --- a/client/components/ChatLog.vue +++ b/client/components/ChatLog.vue @@ -159,4 +159,18 @@ onMounted(() => { } } } + +[data-bs-theme="dark"] .chatlog { + .n-timeline-item-content__title { + color: #d63384 !important; + } + + .n-timeline-item-content__content { + color: #fff !important; + } + + .n-timeline-item-content__meta { + color: #0569ffd9 !important; + } +} diff --git a/client/components/Polls.vue b/client/components/Polls.vue index 30cc9e8f364..0846d4ed160 100644 --- a/client/components/Polls.vue +++ b/client/components/Polls.vue @@ -90,3 +90,21 @@ onMounted(() => { }) + diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d3b186e1f58..aa1ab4a55cf 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250903T2216 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0341 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 9d8427efdb3..2fe337efb27 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250903T2216 +20260711T0341 diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index a676415a261..012a563412e 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -45,16 +45,6 @@ cleanup () { trap 'trap "" TERM; cleanup' TERM # start gunicorn in the background so we can trap the TERM signal -gunicorn \ - -c /workspace/gunicorn.conf.py \ - --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ - --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ - --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ - --bind :8000 \ - --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ - --capture-output \ - --access-logfile -\ - ${DATATRACKER_GUNICORN_EXTRA_ARGS} \ - ietf.wsgi:application & +gunicorn -c /workspace/gunicorn.conf.py ${DATATRACKER_GUNICORN_EXTRA_ARGS} ietf.wsgi:application & gunicorn_pid=$! wait "${gunicorn_pid}" diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 032d95ee0d8..03e81eac5e2 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -1,4 +1,33 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved + +import os +import ietf +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor +from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor + +# Bind all ipv4 interfaces (nginx uses loopback, but k8s health checks don't) +_BIND_PORT = os.environ.get("DATATRACKER_GUNICORN_BIND_PORT", "8000") +bind = [f"0.0.0.0:{_BIND_PORT}"] + +# Disable control socket +control_socket_disable = True + +# Settings configurable via environment +workers = int(os.environ.get("DATATRACKER_GUNICORN_WORKERS", "9")) +max_requests = int(os.environ.get("DATATRACKER_GUNICORN_MAX_REQUESTS", "32768")) +timeout = int(os.environ.get("DATATRACKER_GUNICORN_TIMEOUT", "180")) +loglevel = os.environ.get("DATATRACKER_GUNICORN_LOG_LEVEL", "info") + +# Logging / stdout capture +capture_output = True +accesslog = "-" # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto # for us. Don't trust any of the other similar headers. Only trust the header if it's coming @@ -119,3 +148,34 @@ def post_request(worker, req, environ, resp): in_flight = in_flight_by_pid.get(worker.pid, []) if request_description in in_flight: in_flight.remove(request_description) + +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + + # Setting DATATRACKER_OPENTELEMETRY_ENABLE=all in the environment will enable all + # opentelemetry instrumentations. Individual instrumentations can be selected by + # using a space-separated list. See the code below for available instrumentations. + telemetry_env = os.environ.get("DATATRACKER_OPENTELEMETRY_ENABLE", "").strip() + if telemetry_env != "": + enabled_telemetry = [tok.strip().lower() for tok in telemetry_env.split()] + resource = Resource.create(attributes={ + "service.name": "datatracker", + "service.version": ietf.__version__, + "service.instance.id": worker.pid, + "service.namespace": "datatracker", + "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev") + }) + trace.set_tracer_provider(TracerProvider(resource=resource)) + otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces") + + trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) + + # Instrumentations + if "all" in enabled_telemetry or "django" in enabled_telemetry: + DjangoInstrumentor().instrument() + if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: + Psycopg2Instrumentor().instrument() + if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: + PymemcacheInstrumentor().instrument() + if "all" in enabled_telemetry or "requests" in enabled_telemetry: + RequestsInstrumentor().instrument() diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh index 901026e53b5..578daf5ceff 100644 --- a/dev/build/migration-start.sh +++ b/dev/build/migration-start.sh @@ -3,7 +3,11 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local -echo "Running Blobdb migrations ..." -./ietf/manage.py migrate --settings=settings_local --database=blobdb +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ./ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + echo "Running Blobdb migrations ..." + ./ietf/manage.py migrate --settings=settings_local --database=blobdb +fi echo "Done!" diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index 0954ec9af47..5d5bef56040 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.6", - "fs-extra": "^11.3.0", - "nanoid": "5.1.5", + "dockerode": "^4.0.10", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", - "slugify": "1.6.6", - "tar": "^7.4.3", + "slugify": "1.6.9", + "tar": "^7.5.13", "yargs": "^17.7.2" }, "engines": { @@ -52,95 +52,6 @@ "node": ">=6" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -161,15 +72,6 @@ "url": "https://opencollective.com/js-sdsl" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, - "engines": { - "node": ">=14" - } - }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -258,16 +160,10 @@ "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", "dependencies": { "safer-buffer": "~2.1.0" } }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -285,14 +181,12 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "MIT" + ] }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", "dependencies": { "tweetnacl": "^0.14.3" } @@ -301,21 +195,12 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -334,16 +219,15 @@ "url": "https://feross.org/support" } ], - "license": "MIT", "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "node_modules/buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true, "engines": { "node": ">=10.0.0" @@ -352,8 +236,7 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "node_modules/cliui": { "version": "8.0.1", @@ -398,24 +281,10 @@ "node": ">=10.0.0" } }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dependencies": { "ms": "^2.1.3" }, @@ -429,10 +298,9 @@ } }, "node_modules/docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", - "license": "Apache-2.0", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "dependencies": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -444,38 +312,31 @@ } }, "node_modules/dockerode": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", - "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", - "license": "Apache-2.0", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", - "tar-fs": "~2.1.2", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" }, "engines": { "node": ">= 8.0" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "license": "MIT", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "dependencies": { "once": "^1.4.0" } @@ -488,32 +349,15 @@ "node": ">=6" } }, - "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "license": "MIT" + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", - "license": "MIT", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -531,27 +375,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -574,8 +397,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "BSD-3-Clause" + ] }, "node_modules/inherits": { "version": "2.0.4", @@ -590,28 +412,6 @@ "node": ">=8" } }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -633,28 +433,6 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "node_modules/lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", - "engines": { - "node": "14 || >=16.14" - } - }, - "node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -664,61 +442,42 @@ } }, "node_modules/minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "dependencies": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" }, "engines": { "node": ">= 18" } }, - "node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT" + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", - "license": "MIT", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "node_modules/nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==", + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], - "license": "MIT", "bin": { "nanoid": "bin/nanoid.js" }, @@ -736,34 +495,10 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", "dependencies": { "wrappy": "1" } }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -788,10 +523,9 @@ } }, "node_modules/pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", - "license": "MIT", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -818,23 +552,6 @@ "node": ">=0.10.0" } }, - "node_modules/rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -857,43 +574,12 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "version": "1.6.9", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz", + "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==", "engines": { "node": ">=8.0.0" } @@ -901,13 +587,12 @@ "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", - "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", - "license": "ISC" + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "node_modules/ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "hasInstallScript": true, "dependencies": { "asn1": "^0.2.6", @@ -918,7 +603,7 @@ }, "optionalDependencies": { "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "node_modules/string_decoder": { @@ -942,20 +627,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -967,28 +638,15 @@ "node": ">=8" } }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", + "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "engines": { @@ -996,10 +654,9 @@ } }, "node_modules/tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", - "license": "MIT", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -1011,7 +668,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "license": "MIT", "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -1034,8 +690,7 @@ "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "node_modules/undici-types": { "version": "6.20.0", @@ -1067,20 +722,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1097,28 +738,10 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/y18n": { "version": "5.0.8", @@ -1188,64 +811,6 @@ "yargs": "^17.7.2" } }, - "@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "requires": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" - }, - "ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" - }, - "emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "requires": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - } - }, - "strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "requires": { - "ansi-regex": "^6.0.1" - } - }, - "wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "requires": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - } - } - } - }, "@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -1259,12 +824,6 @@ "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" }, - "@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true - }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1348,11 +907,6 @@ "safer-buffer": "~2.1.0" } }, - "balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1376,14 +930,6 @@ "readable-stream": "^3.4.0" } }, - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "requires": { - "balanced-match": "^1.0.0" - } - }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -1394,9 +940,9 @@ } }, "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true }, "chownr": { @@ -1437,28 +983,18 @@ "nan": "^2.19.0" } }, - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, "debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { "ms": "^2.1.3" } }, "docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "requires": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -1467,33 +1003,28 @@ } }, "dockerode": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", - "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", - "tar-fs": "~2.1.2", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" } }, - "eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "requires": { "once": "^1.4.0" } @@ -1503,24 +1034,15 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, - "foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "requires": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - } - }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1532,18 +1054,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "requires": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - } - }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -1564,20 +1074,6 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "requires": { - "@isaacs/cliui": "^8.0.2", - "@pkgjs/parseargs": "^0.11.0" - } - }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -1597,38 +1093,19 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "requires": { - "brace-expansion": "^2.0.1" - } - }, "minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" }, "minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "requires": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" } }, - "mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" - }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -1640,15 +1117,15 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==" + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==" }, "nanoid-dictionary": { "version": "5.0.0", @@ -1663,20 +1140,6 @@ "wrappy": "1" } }, - "path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" - }, - "path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "requires": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - } - }, "protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -1697,9 +1160,9 @@ } }, "pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -1720,14 +1183,6 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, - "rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "requires": { - "glob": "^10.3.7" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1738,28 +1193,10 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" - }, - "signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" - }, "slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + "version": "1.6.9", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz", + "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==" }, "split-ca": { "version": "1.0.1", @@ -1767,14 +1204,14 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "requires": { "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2", "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "string_decoder": { @@ -1795,16 +1232,6 @@ "strip-ansi": "^6.0.1" } }, - "string-width-cjs": { - "version": "npm:string-width@4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1813,24 +1240,15 @@ "ansi-regex": "^5.0.1" } }, - "strip-ansi-cjs": { - "version": "npm:strip-ansi@6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", + "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "dependencies": { @@ -1842,9 +1260,9 @@ } }, "tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -1889,14 +1307,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "requires": { - "isexe": "^2.0.0" - } - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1907,16 +1317,6 @@ "strip-ansi": "^6.0.0" } }, - "wrap-ansi-cjs": { - "version": "npm:wrap-ansi@7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 09716c3094a..ccc78fc63b1 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.6", - "fs-extra": "^11.3.0", - "nanoid": "5.1.5", + "dockerode": "^4.0.10", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", - "slugify": "1.6.6", - "tar": "^7.4.3", + "slugify": "1.6.9", + "tar": "^7.5.13", "yargs": "^17.7.2" }, "engines": { diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index aacf000093f..055b48d0f5f 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -71,11 +71,11 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' -# No real secrets here, these are public testing values _only_ APP_API_TOKENS = { - "ietf.api.views.ingest_email_test": ["ingestion-test-token"] + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views.ingest_email_test": ["ingestion-test-token"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret } - # OIDC configuration SITE_URL = 'https://__HOSTNAME__' diff --git a/docker-compose.yml b/docker-compose.yml index 2440faf1214..073d04b896a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,10 @@ services: # network_mode: service:db depends_on: + - blobdb + - blobstore - db - mq - - blobstore ipc: host @@ -79,7 +80,10 @@ services: command: - '--loglevel=INFO' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -102,7 +106,10 @@ services: - '--concurrency=1' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -125,11 +132,23 @@ services: volumes: - blobdb-data:/var/lib/postgresql/data +# typesense: +# image: typesense/typesense:30.1 +# restart: on-failure +# ports: +# - "8108:8108" +# volumes: +# - ./typesense-data:/data +# command: +# - '--data-dir=/data' +# - '--api-key=typesense-api-key' +# - '--enable-cors' + # Celery Beat is a periodic task runner. It is not normally needed for development, # but can be enabled by uncommenting the following. # # beat: -# image: ghcr.io/ietf-tools/datatracker-celery:latest +# image: "${COMPOSE_PROJECT_NAME}-celery" # init: true # environment: # CELERY_APP: ietf diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index fee38337330..dd4cf72ffd9 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index c1fe5b093e9..25016360494 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -11,21 +11,22 @@ RUN apt-get update \ # Add Node.js Source RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ - && mkdir -p /etc/apt/keyrings\ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list -RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences && \ - echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences && \ - echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences \ + && echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences \ + && echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences # Add Docker Source -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Add PostgreSQL Source -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list + +# Add PostgreSQL Source +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/keyrings/apt.postgresql.org.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/apt.postgresql.org.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list # Install the packages we need RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index e7c7b9cc3fd..e93ca3cf77c 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index 3068cc71d73..0a9dde04eb4 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -1,9 +1,16 @@ +upstream datatracker_backend { + server 127.0.0.1:8001; +# Uncomment when changing to nginx 1.29.7 or later. +# keepalive 0; # default = 32 since nginx 1.29.7 +} + server { listen 8000 default_server; listen [::]:8000 default_server; proxy_read_timeout 1d; proxy_send_timeout 1d; + client_max_body_size 0; # disable checking root /var/www/html; index index.html index.htm index.nginx-debian.html; @@ -23,7 +30,7 @@ server { location / { error_page 502 /502.html; - proxy_pass http://localhost:8001/; + proxy_pass http://datatracker_backend/; proxy_set_header Host localhost:8000; } diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 3ee7a4295d2..94adc516a47 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -100,3 +100,23 @@ bucket_name=f"{storagename}", ), } + +# For dev on rfc-index generation, create a red_bucket/ directory in the project root +# and uncomment these settings. Generated files will appear in this directory. To +# generate an accurate index, put up-to-date copies of unusable-rfc-numbers.json, +# april-first-rfc-numbers.json, and publication-std-levels.json in this directory +# before generating the index. +# +# STORAGES["red_bucket"] = { +# "BACKEND": "django.core.files.storage.FileSystemStorage", +# "OPTIONS": {"location": "red_bucket"}, +# } + +APP_API_TOKENS = { + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret +} + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_URL = "http://host.docker.internal:8808/api/rfc_metadata_update/" +ERRATA_METADATA_NOTIFICATION_API_KEY = "not a real secret" diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index a69a453110c..12ebe447d5c 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -18,8 +18,8 @@ services: - '5433' blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' celery: volumes: - .:/workspace diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index 3140e393064..9ae64e00418 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -24,10 +24,13 @@ def init_blobstore(): ), ) for bucketname in ARTIFACT_STORAGE_NAMES: + adjusted_bucket_name = ( + os.environ.get("BLOB_STORE_BUCKET_PREFIX", "") + + bucketname + + os.environ.get("BLOB_STORE_BUCKET_SUFFIX", "") + ).strip() try: - blobstore.create_bucket( - Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() - ) + blobstore.create_bucket(Bucket=adjusted_bucket_name) except botocore.exceptions.ClientError as err: if err.response["Error"]["Code"] == "BucketAlreadyExists": print(f"Bucket {bucketname} already exists") @@ -36,5 +39,6 @@ def init_blobstore(): else: print(f"Bucket {bucketname} created") + if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index d4562f97ddf..f4bfe8330ee 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -10,6 +10,7 @@ from django.apps import apps as django_apps from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponseNotAllowed from django.utils.module_loading import autodiscover_modules @@ -50,6 +51,9 @@ def autodiscover(): class ModelResource(tastypie.resources.ModelResource): + def post_detail(self, request, **kwargs): + return HttpResponseNotAllowed(["GET"]) + def generate_cache_key(self, *args, **kwargs): """ Creates a unique-enough cache key. @@ -177,8 +181,15 @@ def dehydrate(self, bundle, for_list=True): return dehydrated + +# XML 1.0 forbids all control characters except tab (#x9), LF (#xA), and CR (#xD). +# Replace each with its Unicode control picture (U+2400 + codepoint) so the +# substitution is lossless and the result is valid XML. +_XML_INVALID_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") + + class Serializer(tastypie.serializers.Serializer): - OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + OPTION_ESCAPE_XML_INVALID = "datatracker-escape-xml-invalid" def format_datetime(self, data): return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" @@ -186,18 +197,17 @@ def format_datetime(self, data): def to_simple(self, data, options): options = options or {} simple_data = super().to_simple(data, options) - if ( - options.get(self.OPTION_ESCAPE_NULLS, False) - and isinstance(simple_data, str) - ): - # replace nulls with unicode "symbol for null character", \u2400 - simple_data = simple_data.replace("\x00", "\u2400") + if options.get(self.OPTION_ESCAPE_XML_INVALID, False) and isinstance(simple_data, str): + # Replace control chars invalid in XML 1.0 with their Unicode + # control pictures (U+2400-U+241F) so lxml won't reject the string. + simple_data = _XML_INVALID_CTRL_RE.sub( + lambda m: chr(ord(m.group()) + 0x2400), simple_data + ) return simple_data def to_etree(self, data, options=None, name=None, depth=0): - # lxml does not escape nulls on its own, so ask to_simple() to do it. - # This is mostly (only?) an issue when generating errors responses for - # fuzzers. + # lxml rejects control characters that are invalid in XML 1.0. + # Ask to_simple() to escape them before they reach lxml. options = options or {} - options[self.OPTION_ESCAPE_NULLS] = True + options[self.OPTION_ESCAPE_XML_INVALID] = True return super().to_etree(data, options, name, depth) diff --git a/ietf/api/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py index 889b2cdfb51..8e2c6f4b5cf 100644 --- a/ietf/api/management/commands/makeresources.py +++ b/ietf/api/management/commands/makeresources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,7 +15,7 @@ from django.template import Template, Context from django.utils import timezone -from tastypie.resources import ModelResource +from ietf.api import ModelResource resource_head_template = """# Copyright The IETF Trust {{date}}, All Rights Reserved @@ -23,7 +23,7 @@ # Generated by the makeresources management command {{date}} -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/api/routers.py b/ietf/api/routers.py index 745ddaa811a..99afdb242af 100644 --- a/ietf/api/routers.py +++ b/ietf/api/routers.py @@ -3,14 +3,29 @@ from django.core.exceptions import ImproperlyConfigured from rest_framework import routers -class PrefixedSimpleRouter(routers.SimpleRouter): - """SimpleRouter that adds a dot-separated prefix to its basename""" + +class PrefixedBasenameMixin: + """Mixin to add a prefix to the basename of a rest_framework BaseRouter""" def __init__(self, name_prefix="", *args, **kwargs): self.name_prefix = name_prefix if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".": raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'") super().__init__(*args, **kwargs) - def get_default_basename(self, viewset): - basename = super().get_default_basename(viewset) - return f"{self.name_prefix}.{basename}" + def register(self, prefix, viewset, basename=None): + # Get the superclass "register" method from the class this is mixed-in with. + # This avoids typing issues with calling super().register() directly in a + # mixin class. + super_register = getattr(super(), "register") + if not super_register or not callable(super_register): + raise TypeError("Must mixin with superclass that has register() method") + super_register(prefix, viewset, basename=f"{self.name_prefix}.{basename}") + + +class PrefixedSimpleRouter(PrefixedBasenameMixin, routers.SimpleRouter): + """SimpleRouter that adds a dot-separated prefix to its basename""" + + +class PrefixedDefaultRouter(PrefixedBasenameMixin, routers.DefaultRouter): + """DefaultRouter that adds a dot-separated prefix to its basename""" + diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py new file mode 100644 index 00000000000..ca605fe6441 --- /dev/null +++ b/ietf/api/serializers_rpc.py @@ -0,0 +1,871 @@ +# Copyright The IETF Trust 2025-2026, All Rights Reserved +import datetime +from pathlib import Path +from typing import Literal, Optional + +from django.db import transaction +from django.urls import reverse as urlreverse +from django.utils import timezone +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field +from rest_framework import fields, serializers + +from ietf.doc.expire import move_draft_files_to_archive +from ietf.doc.models import ( + DocumentAuthor, + Document, + RelatedDocument, + State, + DocEvent, + RfcAuthor, +) +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.tasks import trigger_red_precomputer_task, update_rfc_searchindex_task +from ietf.doc.utils import ( + default_consensus, + prettify_std_name, + update_action_holders, + update_rfcauthors, +) +from ietf.group.models import Group, Role +from ietf.group.serializers import AreaSerializer +from ietf.name.models import StreamName, StdLevelName +from ietf.person.models import Person +from ietf.utils import log + + +class PersonSerializer(serializers.ModelSerializer): + email = serializers.EmailField(read_only=True) + picture = serializers.URLField(source="cdn_photo_url", read_only=True) + url = serializers.SerializerMethodField( + help_text="relative URL for datatracker person page" + ) + + class Meta: + model = Person + fields = ["id", "plain_name", "email", "picture", "url"] + read_only_fields = ["id", "plain_name", "email", "picture", "url"] + + @extend_schema_field(OpenApiTypes.URI) + def get_url(self, object: Person): + return urlreverse( + "ietf.person.views.profile", + kwargs={"email_or_name": object.email_address() or object.name}, + ) + + +class EmailPersonSerializer(serializers.Serializer): + email = serializers.EmailField(source="address") + person_pk = serializers.IntegerField(source="person.pk") + name = serializers.CharField(source="person.name") + last_name = serializers.CharField(source="person.last_name") + initials = serializers.CharField(source="person.initials") + + +class LowerCaseEmailField(serializers.EmailField): + def to_representation(self, value): + return super().to_representation(value).lower() + + +class AuthorPersonSerializer(serializers.ModelSerializer): + person_pk = serializers.IntegerField(source="pk", read_only=True) + last_name = serializers.CharField() + initials = serializers.CharField() + email_addresses = serializers.ListField( + source="email_set.all", child=LowerCaseEmailField() + ) + + class Meta: + model = Person + fields = ["person_pk", "name", "last_name", "initials", "email_addresses"] + + +class RfcWithAuthorsSerializer(serializers.ModelSerializer): + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["rfc_number", "authors"] + + +class DraftWithAuthorsSerializer(serializers.ModelSerializer): + draft_name = serializers.CharField(source="name") + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["draft_name", "authors"] + + +class WgChairSerializer(serializers.Serializer): + """Serialize a WG chair's name and email from a Role""" + + name = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_name(self, role: Role) -> str: + return role.person.plain_name() + + @extend_schema_field(serializers.EmailField) + def get_email(self, role: Role) -> str: + return role.email.email_address() + + +class DocumentAuthorSerializer(serializers.ModelSerializer): + """Serializer for a Person in a response""" + + plain_name = serializers.SerializerMethodField() + + class Meta: + model = DocumentAuthor + fields = ["person", "plain_name", "affiliation"] + + def get_plain_name(self, document_author: DocumentAuthor) -> str: + return document_author.person.plain_name() + + +class FullDraftSerializer(serializers.ModelSerializer): + # Redefine these fields so they don't pick up the regex validator patterns. + # There seem to be some non-compliant drafts in the system! If this serializer + # is used for a writeable view, the validation will need to be added back. + name = serializers.CharField(max_length=255) + title = serializers.CharField(max_length=255) + group = serializers.SlugRelatedField(slug_field="acronym", read_only=True) + area = AreaSerializer(read_only=True) + + # Other fields we need to add / adjust + source_format = serializers.SerializerMethodField() + authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") + shepherd = serializers.PrimaryKeyRelatedField( + source="shepherd.person", read_only=True + ) + consensus = serializers.SerializerMethodField() + wg_chairs = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "group", + "area", + "abstract", + "pages", + "source_format", + "authors", + "intended_std_level", + "consensus", + "shepherd", + "ad", + "wg_chairs", + ] + + def get_consensus(self, doc: Document) -> Optional[bool]: + return default_consensus(doc) + + @extend_schema_field(WgChairSerializer(many=True)) + def get_wg_chairs(self, doc: Document): + if doc.group is None: + return [] + chairs = doc.group.role_set.filter(name_id="chair").select_related( + "person", "email" + ) + return WgChairSerializer(chairs, many=True).data + + def get_source_format( + self, doc: Document + ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: + submission = doc.submission() + if submission is None: + return "unknown" + if ".xml" in submission.file_types: + if submission.xml_version == "3": + return "xml-v3" + else: + return "xml-v2" + elif ".txt" in submission.file_types: + return "txt" + return "unknown" + + +class DraftSerializer(FullDraftSerializer): + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "group", + "pages", + "source_format", + "authors", + "consensus", + ] + + +class SubmittedToQueueSerializer(FullDraftSerializer): + submitted = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "stream", + "submitted", + "consensus", + ] + + def get_submitted(self, doc) -> Optional[datetime.datetime]: + event = doc.sent_to_rfc_editor_event() + return None if event is None else event.time + + def get_consensus(self, doc) -> Optional[bool]: + return default_consensus(doc) + + +class OriginalStreamSerializer(serializers.ModelSerializer): + stream = serializers.CharField(read_only=True, source="orig_stream_id") + + class Meta: + model = Document + fields = ["rfc_number", "stream"] + + +class ReferenceSerializer(serializers.ModelSerializer): + class Meta: + model = Document + fields = ["id", "name"] + read_only_fields = ["id", "name"] + + +def _update_authors(rfc, authors_data): + # Construct unsaved instances from validated author data + new_authors = [RfcAuthor(**authdata) for authdata in authors_data] + # Update the RFC with the new author set + with transaction.atomic(): + change_events = update_rfcauthors(rfc, new_authors) + for event in change_events: + event.save() + return change_events + + +class SubseriesNameField(serializers.RegexField): + + def __init__(self, **kwargs): + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex = r"^(bcp|std|fyi)[1-9][0-9]{0,4}$" + super().__init__(regex, **kwargs) + + +class RfcGroupRelatedField(serializers.SlugRelatedField): + """SlugRelatedField that translates None / "" to the acronym "none" """ + + def __init__(self, **kwargs): + super().__init__( + slug_field="acronym", + queryset=Group.objects.all(), + allow_null=True, + required=False, + ) + + def run_validation(self, data=fields.empty): + # Use the Group with acronym "none" when group is not specified + if data is fields.empty or data is None or data == "": + data = "none" + return super().run_validation(data) + + +class RfcPubSerializer(serializers.ModelSerializer): + """Write-only serializer for RFC publication""" + # publication-related fields + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + + # fields on the RFC Document that need tweaking from ModelSerializer defaults + rfc_number = serializers.IntegerField(min_value=1, required=True) + group = RfcGroupRelatedField() + stream = serializers.PrimaryKeyRelatedField( + queryset=StreamName.objects.filter(used=True) + ) + std_level = serializers.PrimaryKeyRelatedField( + queryset=StdLevelName.objects.filter(used=True), + ) + ad = serializers.PrimaryKeyRelatedField( + queryset=Person.objects.all(), + allow_null=True, + required=False, + ) + obsoletes = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + updates = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + subseries = serializers.ListField(child=SubseriesNameField(required=False)) + # N.b., authors is _not_ a field on Document! + authors = RfcAuthorSerializer(many=True) + + class Meta: + model = Document + fields = [ + "published", + "draft_name", + "draft_rev", + "rfc_number", + "title", + "authors", + "group", + "stream", + "abstract", + "pages", + "std_level", + "ad", + "obsoletes", + "updates", + "subseries", + "keywords", + ] + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def update(self, instance, validated_data): + raise RuntimeError("Cannot update with this serializer") + + def create(self, validated_data): + """Publish an RFC""" + published = validated_data.pop("published") + draft_name = validated_data.pop("draft_name", None) + draft_rev = validated_data.pop("draft_rev", None) + obsoletes = validated_data.pop("obsoletes", []) + updates = validated_data.pop("updates", []) + subseries = validated_data.pop("subseries", []) + + system_person = Person.objects.get(name="(System)") + + # If specified, retrieve draft and extract RFC default values from it + if draft_name is None: + draft = None + else: + # validation enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", + name=draft_name, + rev=draft_rev, + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft", + "draft_rev": "No such draft", + }, + code="invalid-draft" + ) + elif draft.get_state_slug() == "rfc": + raise serializers.ValidationError( + { + "draft_name": "Draft already published as RFC", + }, + code="already-published-draft", + ) + + # Transaction to clean up if something fails + with transaction.atomic(): + # create rfc, letting validated request data override draft defaults + rfc = self._create_rfc(validated_data) + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.get_or_create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + ) + for updated_rfc_pk in updates: + RelatedDocument.objects.get_or_create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" + ) + + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + + + # create relation with draft and update draft state + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") + ) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) + if e: + draft_events.append(e) + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) + ) + draft.save_with_history(draft_events) + + return rfc + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + return rfc + + +class EditableRfcSerializer(serializers.ModelSerializer): + # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. + # The purposes of that serializer (representing data for Red) and this one + # (accepting updates from Purple) are different enough that separate formats + # may be needed, but if not it'd be nice to have a single RfcSerializer that + # can serve both. + # + # Should also consider whether this and RfcPubSerializer should merge. + # + # Treats published and subseries fields as write-only. This isn't quite correct, + # but makes it easier and we don't currently use the serialized value except for + # debugging. + published = serializers.DateTimeField( + default_timezone=datetime.timezone.utc, + write_only=True, + ) + authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") + subseries = serializers.ListField( + child=SubseriesNameField(required=False), + write_only=True, + ) + updates = serializers.ListField( + child=serializers.IntegerField(), + required=False, + write_only=True, + help_text="List of RFC numbers this document updates." + ) + obsoletes = serializers.ListField( + child=serializers.IntegerField(), + required=False, + write_only=True, + help_text="List of RFC numbers this document obsoletes." + ) + + class Meta: + model = Document + fields = [ + "published", + "title", + "authors", + "stream", + "abstract", + "pages", + "std_level", + "subseries", + "keywords", + "updates", + "obsoletes", + ] + + def _validate_rfc_number_list(self, field_name, rfc_numbers): + """Raise ValidationError if any RFC numbers in the list don't exist.""" + unknown = [ + n for n in rfc_numbers + if not Document.objects.filter(rfc_number=n, type_id="rfc").exists() + ] + if unknown: + raise serializers.ValidationError( + {field_name: [f"Unknown RFC number: {n}" for n in unknown]} + ) + return rfc_numbers + + def validate_updates(self, value): + return self._validate_rfc_number_list("updates", value) + + def validate_obsoletes(self, value): + return self._validate_rfc_number_list("obsoletes", value) + + def create(self, validated_data): + raise RuntimeError("Cannot create with this serializer") + + def update(self, instance, validated_data): + assert isinstance(instance, Document) + assert instance.type_id == "rfc" + rfc = instance # get better name + + system_person = Person.objects.get(name="(System)") + + # Remove data that needs special handling. Use a singleton object to detect + # missing values in case we ever support a value that needs None as an option. + omitted = object() + published = validated_data.pop("published", omitted) + subseries = validated_data.pop("subseries", omitted) + authors_data = validated_data.pop("rfcauthor_set", omitted) + updates = validated_data.pop("updates", omitted) + obsoletes = validated_data.pop("obsoletes", omitted) + + # Transaction to clean up if something fails + with transaction.atomic(): + # update the rfc Document itself + rfc_changes = [] + rfc_events = [] + + for attr, new_value in validated_data.items(): + old_value = getattr(rfc, attr) + if new_value != old_value: + rfc_changes.append( + f"changed {attr} to '{new_value}' from '{old_value}'" + ) + setattr(rfc, attr, new_value) + if len(rfc_changes) > 0: + rfc_change_summary = f"{', '.join(rfc_changes)}" + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Changed metadata: {rfc_change_summary}", + ) + ) + if authors_data is not omitted: + rfc_events.extend(_update_authors(instance, authors_data)) + + if published is not omitted: + published_event = rfc.latest_event(type="published_rfc") + if published_event is None: + # unexpected, but possible in theory + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + ) + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Set publication timestamp to {published.isoformat()}" + ), + ) + ) + else: + original_pub_time = published_event.time + if published != original_pub_time: + published_event.time = published + published_event.save() + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Changed publication time to " + f"{published.isoformat()} from " + f"{original_pub_time.isoformat()}" + ) + ) + ) + if updates is not omitted: + RelatedDocument.objects.filter( + source=rfc, relationship_id="updates" + ).exclude(target__rfc_number__in=updates).delete() + for rfc_num in updates: + target = Document.objects.get(rfc_number=rfc_num, type_id="rfc") + RelatedDocument.objects.get_or_create( + source=rfc, relationship_id="updates", target=target + ) + if obsoletes is not omitted: + RelatedDocument.objects.filter( + source=rfc, relationship_id="obs" + ).exclude(target__rfc_number__in=obsoletes).delete() + for rfc_num in obsoletes: + target = Document.objects.get(rfc_number=rfc_num, type_id="rfc") + RelatedDocument.objects.get_or_create( + source=rfc, relationship_id="obs", target=target + ) + + # update subseries relations + if subseries is not omitted: + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via update of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + ) + # Delete subseries relations that are no longer current + stale_subseries_relations = rfc.relations_that("contains").exclude( + source__name__in=subseries + ) + for stale_relation in stale_subseries_relations: + stale_subseries_doc = stale_relation.source + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + ) + stale_subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + stale_subseries_relations.delete() + if len(rfc_events) > 0: + rfc.save_with_history(rfc_events) + # Gather obs and updates in both directions as a title/author change to + # this doc affects the info rendering of all of the other RFCs + needs_updating = sorted( + [ + d.rfc_number + for d in [rfc] + + rfc.related_that_doc(("obs", "updates")) + + rfc.related_that(("obs", "updates")) + ] + ) + trigger_red_precomputer_task.delay(rfc_number_list=needs_updating) + # Update the search index also + update_rfc_searchindex_task.delay(rfc.rfc_number) + return rfc + + +class RfcFileSerializer(serializers.Serializer): + # The structure of this serializer is constrained by what openapi-generator-cli's + # python generator can correctly serialize as multipart/form-data. It does not + # handle nested serializers well (or perhaps at all). ListFields with child + # ChoiceField or RegexField do not serialize correctly. DictFields don't seem + # to work. + # + # It does seem to correctly send filenames along with FileFields, even as a child + # in a ListField, so we use that to convey the file format of each item. There + # are other options we could consider (e.g., a structured CharField) but this + # works. + allowed_extensions = ( + ".html", + ".json", # deprecated - accepted but ignored, datatracker generates its own + ".notprepped.xml", + ".pdf", + ".txt", + ".xml", + ) + + rfc = serializers.SlugRelatedField( + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + help_text="RFC number to which the contents belong", + ) + contents = serializers.ListField( + child=serializers.FileField( + allow_empty_file=False, + use_url=False, + ), + help_text=( + "List of content files. Filename extensions are used to identify " + "file types, but filenames are otherwise ignored." + ), + ) + mtime = serializers.DateTimeField( + required=False, + default=timezone.now, + default_timezone=datetime.UTC, + help_text="Modification timestamp to apply to uploaded files", + ) + replace = serializers.BooleanField( + required=False, + default=False, + help_text=( + "Replace existing files for this RFC. Defaults to false. When false, " + "if _any_ files already exist for the specified RFC the upload will be " + "rejected regardless of which files are being uploaded. When true," + "existing files will be removed and new ones will be put in place. BE" + "VERY CAREFUL WITH THIS OPTION IN PRODUCTION." + ), + ) + + def validate_contents(self, data): + found_extensions = [] + for uploaded_file in data: + if not hasattr(uploaded_file, "name"): + raise serializers.ValidationError( + "filename not specified for uploaded file", + code="missing-filename", + ) + ext = "".join(Path(uploaded_file.name).suffixes) + if ext not in self.allowed_extensions: + raise serializers.ValidationError( + f"File uploaded with invalid extension '{ext}'", + code="invalid-filename-ext", + ) + if ext in found_extensions: + raise serializers.ValidationError( + f"More than one file uploaded with extension '{ext}'", + code="duplicate-filename-ext", + ) + return data + + +class NotificationAckSerializer(serializers.Serializer): + message = serializers.CharField(default="ack") diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 2a44791a5cf..87f7d684d32 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -12,6 +12,7 @@ from importlib import import_module from pathlib import Path from random import randrange +from urllib.parse import urljoin from django.apps import apps from django.conf import settings @@ -1542,20 +1543,57 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) - def test_serializer_to_etree_handles_nulls(self): - """Serializer to_etree() should handle a null character""" + def test_serializer_to_etree_handles_xml_invalid_control_chars(self): + """Serializer to_etree() must not raise ValueError for any XML-invalid control character.""" serializer = Serializer() + # Ordinary strings and strings with valid whitespace must pass through unchanged. try: - serializer.to_etree("string with no nulls in it") + serializer.to_etree("string with no special chars") + serializer.to_etree("tab\there lf\nhere cr\rhere") except ValueError: self.fail("serializer.to_etree raised ValueError on an ordinary string") - try: - serializer.to_etree("string with a \x00 in it") - except ValueError: - self.fail( - "serializer.to_etree raised ValueError on a string " - "containing a null character" + # Every control character that XML 1.0 forbids must be escaped rather than + # causing a ValueError. This is the class of characters that triggered the + # production exception (lxml.etree._utf8 rejects them all). + invalid_chars = [chr(c) for c in list(range(0x00, 0x09)) + [0x0b, 0x0c] + list(range(0x0e, 0x20))] + for ch in invalid_chars: + try: + serializer.to_etree(f"string with {ch!r} in it") + except ValueError: + self.fail( + f"serializer.to_etree raised ValueError on a string " + f"containing control character U+{ord(ch):04X}" + ) + + def test_post_detail_is_not_allowed(self): + """POST to a detail route returns 405 + + Added because default TastyPie behavior is a 500 due to a NotImplemented + exception. + """ + r = self.client.get("/api/v1", headers={"Accept": "application/json"}) + self.assertValidJSONResponse(r) + resource_list = r.json() + for name in self.apps: + r = self.client.get( + resource_list[name]["list_endpoint"], + headers={"Accept": "application/json"}, ) + self.assertValidJSONResponse(r) + app_resources = r.json() + model_list = apps.get_app_config(name).get_models() + for model in model_list: + model_name = model._meta.model_name + assert model_name + detail_url = urljoin( + app_resources[model_name]["list_endpoint"], "some-id/" + ) + r = self.client.post(detail_url) + self.assertEqual( + r.status_code, + 405, + f"POST to {name}.{model_name} detail should return 405 status", + ) class RfcdiffSupportTests(TestCase): diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py new file mode 100644 index 00000000000..5151f337d55 --- /dev/null +++ b/ietf/api/tests_serializers_rpc.py @@ -0,0 +1,241 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from unittest import mock + +from django.utils import timezone + +from ietf.utils.test_utils import TestCase +from ietf.doc.models import Document +from ietf.doc.factories import WgRfcFactory +from .serializers_rpc import EditableRfcSerializer + + +class EditableRfcSerializerTests(TestCase): + def test_create(self): + serializer = EditableRfcSerializer( + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ietf", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + } + ) + self.assertTrue(serializer.is_valid()) + with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): + serializer.save() + + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_update(self, mock_trigger_red_task, mock_update_searchindex_task): + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) + rfc = WgRfcFactory(pages=10) + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + serializer = EditableRfcSerializer( + instance=rfc, + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ise", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(result.title, "Yadda yadda yadda") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ise") + self.assertEqual( + result.abstract, "A long time ago in a galaxy far, far away..." + ) + self.assertEqual(result.pages, 3) + self.assertEqual(result.std_level_id, "inf") + self.assertEqual( + result.part_of(), + [Document.objects.get(name="fyi999")], + ) + # Confirm that red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was triggered correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_partial_update(self, mock_trigger_red_task, mock_update_searchindex_task): + # We could test other permutations of fields, but authors is a partial update + # we know we are going to use, so verifying that one in particular. + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) + rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={ + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "padawan") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ietf") + self.assertEqual(result.abstract, "do or do not") + self.assertEqual(result.pages, 10) + self.assertEqual(result.std_level_id, "ps") + self.assertEqual(result.part_of(), []) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + # Test only a field on the Document itself to be sure that it works + mock_trigger_red_task.delay.reset_mock() + mock_update_searchindex_task.delay.reset_mock() + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={"title": "jedi master"}, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "jedi master") + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + def test_unknown_rfc_number_rejected(self): + """Unknown RFC numbers in updates/obsoletes should cause validation failure.""" + from django.db.models import Max + + rfc = WgRfcFactory() + unknown_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + m=Max("rfc_number") + 1 + )["m"] + or 10000 + ) + + for field in ("updates", "obsoletes"): + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={field: [unknown_rfc_number]}, + ) + self.assertFalse( + serializer.is_valid(), + msg=f"{field} with unknown RFC number should be invalid", + ) + self.assertIn(field, serializer.errors) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py new file mode 100644 index 00000000000..5748a7caa60 --- /dev/null +++ b/ietf/api/tests_views_rpc.py @@ -0,0 +1,654 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime +from io import StringIO +from pathlib import Path + +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Max +from django.db.models.functions import Coalesce +from django.test.utils import override_settings +from django.urls import reverse as urlreverse +import mock +from django.utils import timezone + +from ietf.api.views_rpc import DestinationHelperMixin +from ietf.blobdb.models import Blob +from ietf.doc.factories import ( + IndividualDraftFactory, + RfcFactory, + WgDraftFactory, + WgRfcFactory, +) +from ietf.doc.models import RelatedDocument, Document +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.person.factories import PersonFactory +from ietf.sync.rfcindex import rfcindex_is_dirty +from ietf.utils.models import DirtyBits +from ietf.utils.test_utils import APITestCase, reload_db_objects + + +class RpcApiTests(APITestCase): + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_draftviewset_references(self): + viewname = "ietf.api.purple_api.draft-references" + + # non-existent draft + bad_id = Document.objects.aggregate(unused_id=Coalesce(Max("id"), 0) + 100)[ + "unused_id" + ] + url = urlreverse(viewname, kwargs={"doc_id": bad_id}) + # Without credentials + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + # Add credentials + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404) + + # draft without any normative references + draft = IndividualDraftFactory() + draft = reload_db_objects(draft) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft without any normative references but with an informative reference + draft_foo = IndividualDraftFactory() + draft_foo = reload_db_objects(draft_foo) + RelatedDocument.objects.create( + source=draft, target=draft_foo, relationship_id="refinfo" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft with a normative reference + draft_bar = IndividualDraftFactory() + draft_bar = reload_db_objects(draft_bar) + RelatedDocument.objects.create( + source=draft, target=draft_bar, relationship_id="refnorm" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]["id"], draft_bar.id) + self.assertEqual(refs[0]["name"], draft_bar.name) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + def test_notify_rfc_published(self, mock_task_delay): + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") + draft_ad = RoleFactory(group=area, name_id="ad").person + rfc_ad = PersonFactory() + draft_authors = PersonFactory.create_batch(2) + rfc_authors = PersonFactory.create_batch(3) + draft = WgDraftFactory( + group__parent=area, authors=draft_authors, ad=draft_ad, stream_id="ietf" + ) + rfc_stream_id = "ise" + assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + post_data = { + "published": "2025-12-17T20:29:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": "RFC " + draft.title, + "authors": [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in rfc_authors + ], + "group": rfc_group.acronym, + "stream": rfc_stream_id, + "abstract": "RFC version of " + draft.abstract, + "pages": draft.pages + 10, + "std_level": "ps", + "ad": rfc_ad.pk, + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [o.rfc_number for o in updates], + "subseries": [], + } + r = self.client.post(url, data=post_data, format="json") + self.assertEqual(r.status_code, 403) + + # Put a file in the way. Post should fail because files exists + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + file_in_the_way = rfc_path / f"rfc{unused_rfc_number}.txt" + file_in_the_way.touch() + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 200) + rfc = Document.objects.filter(rfc_number=unused_rfc_number).first() + self.assertIsNotNone(rfc) + self.assertEqual(rfc.came_from_draft(), draft) + self.assertEqual( + rfc.docevent_set.filter( + type="published_rfc", time="2025-12-17T20:29:00Z" + ).count(), + 1, + ) + self.assertEqual(rfc.title, "RFC " + draft.title) + self.assertEqual(rfc.documentauthor_set.count(), 0) + self.assertEqual( + [ + { + "titlepage_name": ra.titlepage_name, + "is_editor": ra.is_editor, + "person": ra.person, + "email": ra.email, + "affiliation": ra.affiliation, + "country": ra.country, + } + for ra in rfc.rfcauthor_set.all() + ], + [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author, + "email": author.email(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in rfc_authors + ], + ) + self.assertEqual(rfc.group, rfc_group) + self.assertEqual(rfc.stream_id, rfc_stream_id) + self.assertEqual(rfc.abstract, "RFC version of " + draft.abstract) + self.assertEqual(rfc.pages, draft.pages + 10) + self.assertEqual(rfc.std_level_id, "ps") + self.assertEqual(rfc.ad, rfc_ad) + self.assertEqual(set(rfc.related_that_doc("obs")), set([o for o in obsoletes])) + self.assertEqual( + set(rfc.related_that_doc("updates")), set([o for o in updates]) + ) + self.assertEqual(rfc.part_of(), []) + self.assertEqual(draft.get_state().slug, "rfc") + # todo test non-empty relationships + # todo test references (when updating that is part of the handling) + + self.assertTrue(mock_task_delay.called) + mock_args, mock_kwargs = mock_task_delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.update_rfc_json_task") + @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") + @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") + @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") + def test_upload_rfc_files( + self, + mock_trigger_red_task, + mock_update_searchindex_task, + mock_rebuild_relations, + mock_update_rfc_json, + ): + def _valid_post_data(): + """Generate a valid post data dict + + Each API call needs a fresh set of files, so don't reuse the return + value from this for multiple calls! + """ + return { + "rfc": rfc.rfc_number, + "contents": [ + ContentFile(b"This is .xml", "myfile.xml"), + ContentFile(b"This is .txt", "myfile.txt"), + ContentFile(b"This is .html", "myfile.html"), + ContentFile(b"This is .pdf", "myfile.pdf"), + ContentFile(b"This is .json", "myfile.json"), + ContentFile(b"This is .notprepped.xml", "myfile.notprepped.xml"), + ], + "replace": False, + } + + url = urlreverse("ietf.api.purple_api.upload_rfc_files") + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + + rfc = WgRfcFactory() + for r in obsoletes: + rfc.relateddocument_set.create(relationship_id="obs", target=r) + for r in updates: + rfc.relateddocument_set.create(relationship_id="updates", target=r) + assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" + + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": rfc.rfc_number + 10}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # empty files + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), # deprecated! + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # bad file type + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # Put a file in the way. Post should fail because replace = False + file_in_the_way = rfc_path / f"{rfc.name}.txt" + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + blob_in_the_way.delete() + + # valid post + mock_trigger_red_task.delay.reset_mock() + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + for extension in ["xml", "txt", "html", "pdf"]: # no "json"! + filename = f"{rfc.name}.{extension}" + self.assertEqual( + (rfc_path / filename).read_text(), + f"This is .{extension}", + f"{extension} file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"{extension}/{filename}" + ).content + ), + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", + ) + # special case for notprepped + notprepped_fn = f"{rfc.name}.notprepped.xml" + self.assertEqual( + (rfc_path / "prerelease" / notprepped_fn).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) + # special case for json (deprecated and should be ignored) + self.assertFalse((rfc_path / f"{rfc.name}.json").exists()) + self.assertFalse( + Blob.objects.filter(bucket="rfc", name=f"json/{rfc.name}.json").exists() + ) + + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # Confirm rfc JSON creation was triggered correctly + self.assertTrue(mock_update_rfc_json.delay.called) + mock_args, _ = mock_update_rfc_json.delay.call_args + self.assertEqual(mock_args[0], [rfc.rfc_number]) + + # re-post with replace = False should now fail + mock_update_searchindex_task.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_refresh_rfc_index(self): + DirtyBits.objects.create( + slug=DirtyBits.Slugs.RFCINDEX, + dirty_time=timezone.now() - datetime.timedelta(days=1), + processed_time=timezone.now() - datetime.timedelta(hours=12), + ) + self.assertFalse(rfcindex_is_dirty()) + url = urlreverse("ietf.api.purple_api.refresh_rfc_index") + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + self.assertFalse(rfcindex_is_dirty()) + response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 202) + self.assertTrue(rfcindex_is_dirty()) + + def test_destination_helper_mixin_fs_destination(self): + file_list = [f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]] + for filename in file_list: + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}") / filename, + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}/prerelease") / filename, + ) + + def test_destination_helper_mixin_blob_destination(self): + file_list = {ext: f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]} + for file_type, filename in file_list.items(): + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"{file_type}/{filename}", + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"notprepped/{filename}", + ) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.process_rpc_queue_task.delay") + def test_process_rpc_queue(self, mock_task_delay): + url = urlreverse("ietf.api.purple_api.process_rpc_queue") + queue_entries = [ + { + "id": 9850, + "name": "draft-ietf-netmod-system-config", + "title": "System-defined Configuration", + "draft_url": "http://localhost:8000/doc/draft-ietf-netmod-system-config-20", + "disposition": "in_progress", + "external_deadline": None, + "labels": [], + "cluster": None, + "assignment_set": [ + { + "id": 434, + "rfc_to_be": 9850, + "role": "first_editor", + "state": "in_progress", + } + ], + "actionholder_set": [], + "pending_activities": [], + "rfc_number": None, + "pages": 33, + "enqueued_at": "2026-01-26T12:00:00Z", + "final_approval": [], + "iana_status": { + "slug": "completed", + "name": "completed", + "desc": "IANA has completed actions in draft", + }, + "blocking_reasons": [], + "authors": [{"titlepage_name": "Q. Ma", "is_editor": True}], + "approval_log_message": [], + "stream": "ietf", + "group": "netmod", + "group_name": "Network Modeling", + "std_level": "ps", + "references": [], + "rev": "20", + } + ] + queue_data = {"data": queue_entries} + + # no credentials + response = self.client.post( + url, data=queue_data, content_type="application/json" + ) + self.assertEqual(response.status_code, 403) + mock_task_delay.assert_not_called() + + # invalid token + response = self.client.post( + url, + data=queue_data, + content_type="application/json", + headers={"X-Api-Key": "invalid-token"}, + ) + self.assertEqual(response.status_code, 403) + mock_task_delay.assert_not_called() + + # valid token, wrong method + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + mock_task_delay.assert_not_called() + + # valid token, missing "data" field + response = self.client.post( + url, + data={}, + content_type="application/json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(response.status_code, 400) + mock_task_delay.assert_not_called() + + # valid token, POST with data + response = self.client.post( + url, + data=queue_data, + content_type="application/json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(response.status_code, 202) + mock_task_delay.assert_called_once_with(queue_entries) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task.delay") + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task.delay") + @mock.patch("ietf.api.views_rpc.update_rfc_json_task.delay") + def test_rfc_patch_triggers_json_update( + self, mock_delay, mock_precompute_delay, mock_searchindex_delay + ): + """PATCHing RFC metadata dispatches update_rfc_json_task for that RFC.""" + rfc = WgRfcFactory() + url = urlreverse( + "ietf.api.purple_api.rfc-detail", kwargs={"rfc_number": rfc.rfc_number} + ) + patch_data = {"title": "Updated Title"} + with self.captureOnCommitCallbacks(execute=True): + r = self.client.patch( + url, + data=patch_data, + format="json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + mock_delay.assert_called_once_with([rfc.rfc_number]) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + @mock.patch("ietf.api.views_rpc.update_rfc_json_task.delay") + def test_rfc_publish_triggers_related_json_update( + self, mock_json_delay, mock_signal_delay + ): + """Publishing an RFC that obsoletes/updates existing RFCs triggers JSON update for related RFCs only.""" + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") + draft = WgDraftFactory(group__parent=area, stream_id="ietf") + obsoletes = RfcFactory.create_batch(2) + updates = RfcFactory.create_batch(1) + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 20000 + ) + post_data = { + "published": "2025-06-01T00:00:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": "New RFC", + "authors": [], + "group": rfc_group.acronym, + "stream": "ietf", + "abstract": "Abstract.", + "pages": 10, + "std_level": "ps", + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [u.rfc_number for u in updates], + "subseries": [], + } + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + data=post_data, + format="json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + + # JSON update fired only for related RFCs, not for the new RFC itself + expected_related = sorted( + {o.rfc_number for o in obsoletes} | {u.rfc_number for u in updates} + ) + mock_json_delay.assert_called_once_with(expected_related) + self.assertNotIn(unused_rfc_number, mock_json_delay.call_args[0][0]) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 04575b34cb9..7a082567b8a 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -1,26 +1,31 @@ # Copyright The IETF Trust 2017-2024, All Rights Reserved +from drf_spectacular.views import SpectacularAPIView + from django.conf import settings -from django.urls import include +from django.urls import include, path from django.views.generic import TemplateView from ietf import api -from ietf.doc import views_ballot +from ietf.doc import views_ballot, api as doc_api from ietf.meeting import views as meeting_views from ietf.submit import views as submit_views from ietf.utils.urls import url from . import views as api_views +from .routers import PrefixedSimpleRouter # DRF API routing - disabled until we plan to use it -# from drf_spectacular.views import SpectacularAPIView -# from django.urls import path # from ietf.person import api as person_api -# from .routers import PrefixedSimpleRouter # core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api") # core api router # core_router.register("email", person_api.EmailViewSet) # core_router.register("person", person_api.PersonViewSet) +# todo more general name for this API? +red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router +red_router.register("doc", doc_api.RfcViewSet) +red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries") + api.autodiscover() urlpatterns = [ @@ -32,7 +37,9 @@ url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()), # --- DRF API --- # path("core/", include(core_router.urls)), - # path("schema/", SpectacularAPIView.as_view()), + path("purple/", include("ietf.api.urls_rpc")), + path("red/", include(red_router.urls)), + path("schema/", SpectacularAPIView.as_view()), # # --- Custom API endpoints, sorted alphabetically --- # Email alias information for drafts diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py new file mode 100644 index 00000000000..07f2cf8751e --- /dev/null +++ b/ietf/api/urls_rpc.py @@ -0,0 +1,52 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +from django.urls import include, path + +from ietf.api import views_rpc +from ietf.api.routers import PrefixedDefaultRouter +from ietf.utils.urls import url + +router = PrefixedDefaultRouter(use_regex_path=False, name_prefix="ietf.api.purple_api") +router.include_format_suffixes = False +router.register(r"draft", views_rpc.DraftViewSet, basename="draft") +router.register(r"person", views_rpc.PersonViewSet) +router.register(r"rfc", views_rpc.RfcViewSet, basename="rfc") + +router.register( + r"rfc//authors", + views_rpc.RfcAuthorViewSet, + basename="rfc-authors", +) + +urlpatterns = [ + url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), + url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), + path( + r"rfc/publish/", + views_rpc.RfcPubNotificationView.as_view(), + name="ietf.api.purple_api.notify_rfc_published", + ), + path( + r"rfc/publish/files/", + views_rpc.RfcPubFilesView.as_view(), + name="ietf.api.purple_api.upload_rfc_files", + ), + path( + r"rfc_index/refresh/", + views_rpc.RfcIndexView.as_view(), + name="ietf.api.purple_api.refresh_rfc_index", + ), + path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), + path( + r"queue/process/", + views_rpc.ProcessRpcQueueView.as_view(), + name="ietf.api.purple_api.process_rpc_queue", + ), +] + +# add routers at the end so individual routes can steal parts of their address +# space (e.g., ^rfc/publish/ superseding the ^rfc/ routes of RfcViewSet) +urlpatterns.extend( + [ + path("", include(router.urls)), + ] +) diff --git a/ietf/api/views.py b/ietf/api/views.py index 22523b2f172..420bc396934 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -97,7 +97,7 @@ class PersonalInformationExportView(DetailView, JsonExportMixin): def get(self, request): person = get_object_or_404(self.model, user=request.user) - expand = ['searchrule', 'documentauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', + expand = ['searchrule', 'documentauthor', 'rfcauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', 'ballotpositiondocevent', 'deletedevent', 'email_set', 'groupevent', 'role', 'rolehistory', 'iprdisclosurebase', 'iprevent', 'liaisonstatementevent', 'allowlisted', 'schedule', 'constraint', 'schedulingevent', 'message', 'sendqueue', 'nominee', 'topicfeedbacklastseen', 'alias', 'email', 'apikeys', 'personevent', diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py new file mode 100644 index 00000000000..21d59e3f14c --- /dev/null +++ b/ietf/api/views_rpc.py @@ -0,0 +1,615 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +import os +import shutil +from functools import partial +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.conf import settings +from django.db import IntegrityError, transaction +from drf_spectacular.utils import OpenApiParameter +from rest_framework import mixins, parsers, serializers, viewsets, status +from rest_framework.decorators import action +from rest_framework.exceptions import APIException +from rest_framework.views import APIView +from rest_framework.response import Response + +from django.db.models import CharField as ModelCharField, OuterRef, Subquery, Q +from django.db.models.functions import Coalesce +from django.http import Http404 +from drf_spectacular.utils import extend_schema_view, extend_schema +from rest_framework import generics +from rest_framework.fields import CharField as DrfCharField +from rest_framework.filters import SearchFilter +from rest_framework.pagination import LimitOffsetPagination + +from ietf.api.serializers_rpc import ( + PersonSerializer, + FullDraftSerializer, + DraftSerializer, + SubmittedToQueueSerializer, + OriginalStreamSerializer, + ReferenceSerializer, + EmailPersonSerializer, + RfcWithAuthorsSerializer, + DraftWithAuthorsSerializer, + NotificationAckSerializer, + RfcPubSerializer, + RfcFileSerializer, + EditableRfcSerializer, +) +from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage +from ietf.doc.tasks import ( + signal_update_rfc_metadata_task, + rebuild_reference_relations_task, + trigger_red_precomputer_task, + update_rfc_searchindex_task, +) +from ietf.person.models import Email, Person +from ietf.sync.rfcindex import mark_rfcindex_as_dirty +from ietf.sync.tasks import process_rpc_queue_task, update_rfc_json_task + + +class Conflict(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Conflict." + default_code = "conflict" + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_person_by_id", + summary="Find person by ID", + description="Returns a single person", + parameters=[ + OpenApiParameter( + name="person_id", + type=int, + location="path", + description="Person ID identifying this person.", + ), + ], + ), +) +class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Person.objects.all() + serializer_class = PersonSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "person_id" + + @extend_schema( + operation_id="get_persons", + summary="Get a batch of persons", + description="Returns a list of persons matching requested ids. Omits any that are missing.", + request=list[int], + responses=PersonSerializer(many=True), + ) + @action(detail=False, methods=["post"]) + def batch(self, request): + """Get a batch of rpc person names""" + pks = request.data + return Response( + self.get_serializer(Person.objects.filter(pk__in=pks), many=True).data + ) + + @extend_schema( + operation_id="persons_by_email", + summary="Get a batch of persons by email addresses", + description=( + "Returns a list of persons matching requested ids. " + "Omits any that are missing." + ), + request=list[str], + responses=EmailPersonSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=EmailPersonSerializer) + def batch_by_email(self, request): + emails = Email.objects.filter(address__in=request.data, person__isnull=False) + serializer = self.get_serializer(emails, many=True) + return Response(serializer.data) + + +class SubjectPersonView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_subject_person_by_id", + summary="Find person for OIDC subject by ID", + description="Returns a single person", + responses=PersonSerializer, + parameters=[ + OpenApiParameter( + name="subject_id", + type=str, + description="subject ID of person to return", + location="path", + ), + ], + ) + def get(self, request, subject_id: str): + try: + user_id = int(subject_id) + except ValueError: + raise serializers.ValidationError( + {"subject_id": "This field must be an integer value."} + ) + person = Person.objects.filter(user__pk=user_id).first() + if person: + return Response(PersonSerializer(person).data) + raise Http404 + + +class RpcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 100 + + +class SingleTermSearchFilter(SearchFilter): + """SearchFilter backend that does not split terms + + The default SearchFilter treats comma or whitespace-separated terms as individual + search terms. This backend instead searches for the exact term. + """ + + def get_search_terms(self, request): + value = request.query_params.get(self.search_param, "") + field = DrfCharField(trim_whitespace=False, allow_blank=True) + cleaned_value = field.run_validation(value) + return [cleaned_value] + + +@extend_schema_view( + get=extend_schema( + operation_id="search_person", + description="Get a list of persons, matching by partial name or email", + ), +) +class RpcPersonSearch(generics.ListAPIView): + # n.b. the OpenAPI schema for this can be generated by running + # ietf/manage.py spectacular --file spectacular.yaml + # and extracting / touching up the rpc_person_search_list operation + api_key_endpoint = "ietf.api.views_rpc" + queryset = Person.objects.all() + serializer_class = PersonSerializer + pagination_class = RpcLimitOffsetPagination + + # Searchable on all name-like fields or email addresses + filter_backends = [SingleTermSearchFilter] + search_fields = ["name", "plain", "email__address"] + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_draft_by_id", + summary="Get a draft", + description="Returns the draft for the requested ID", + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + ), + submitted_to_rpc=extend_schema( + operation_id="submitted_to_rpc", + summary="List documents ready to enter the RFC Editor Queue", + description="List documents ready to enter the RFC Editor Queue", + responses=SubmittedToQueueSerializer(many=True), + ), +) +class DraftViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="draft") + serializer_class = FullDraftSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "doc_id" + + @action(detail=False, serializer_class=SubmittedToQueueSerializer) + def submitted_to_rpc(self, request): + """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) + irtf_iab_ise_editorial_docs = Q( + states__type_id__in=[ + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ], + states__slug__in=["rfc-edit"], + ) + docs = ( + self.get_queryset() + .filter(type_id="draft") + .filter(ietf_docs | irtf_iab_ise_editorial_docs) + ) + serializer = self.get_serializer(docs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_references", + summary="Get normative references to I-Ds", + description=( + "Returns the id and name of each normatively " + "referenced Internet-Draft for the given docId" + ), + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + responses=ReferenceSerializer(many=True), + ) + @action(detail=True, serializer_class=ReferenceSerializer) + def references(self, request, doc_id=None): + doc = self.get_object() + serializer = self.get_serializer( + [ + reference + for reference in doc.related_that_doc("refnorm") + if reference.type_id == "draft" + ], + many=True, + ) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_authors", + summary="Gather authors of the drafts with the given names", + description="returns a list mapping draft names to objects describing authors", + request=list[str], + responses=DraftWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=DraftWithAuthorsSerializer) + def bulk_authors(self, request): + drafts = self.get_queryset().filter(name__in=request.data) + serializer = self.get_serializer(drafts, many=True) + return Response(serializer.data) + + +@extend_schema_view( + rfc_original_stream=extend_schema( + operation_id="get_rfc_original_streams", + summary="Get the streams RFCs were originally published into", + description="returns a list of dicts associating an RFC with its originally published stream", + responses=OriginalStreamSerializer(many=True), + ) +) +class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="rfc") + api_key_endpoint = "ietf.api.views_rpc" + lookup_field = "rfc_number" + serializer_class = EditableRfcSerializer + + def perform_update(self, serializer): + DocEvent.objects.create( + doc=serializer.instance, + rev=serializer.instance.rev, + by=Person.objects.get(name="(System)"), + type="sync_from_rfc_editor", + desc="Metadata update from RFC Editor", + ) + super().perform_update(serializer) + rfc_number = serializer.instance.rfc_number + transaction.on_commit(lambda: update_rfc_json_task.delay([rfc_number])) + + @action(detail=False, serializer_class=OriginalStreamSerializer) + def rfc_original_stream(self, request): + rfcs = self.get_queryset().annotate( + orig_stream_id=Coalesce( + Subquery( + DocHistory.objects.filter(doc=OuterRef("pk")) + .exclude(stream__isnull=True) + .order_by("time") + .values_list("stream_id", flat=True)[:1] + ), + "stream_id", + output_field=ModelCharField(), + ), + ) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_rfc_authors", + summary="Gather authors of the RFCs with the given numbers", + description="returns a list mapping rfc numbers to objects describing authors", + request=list[int], + responses=RfcWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=RfcWithAuthorsSerializer) + def bulk_authors(self, request): + rfcs = self.get_queryset().filter(rfc_number__in=request.data) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + +class DraftsByNamesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_drafts_by_names", + summary="Get a batch of drafts by draft names", + description="returns a list of drafts with matching names", + request=list[str], + responses=DraftSerializer(many=True), + ) + def post(self, request): + names = request.data + docs = Document.objects.filter(type_id="draft", name__in=names) + return Response(DraftSerializer(docs, many=True).data) + + +class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet): + """ViewSet for RfcAuthor model + + Router needs to provide rfc_number as a kwarg + """ + + api_key_endpoint = "ietf.api.views_rpc" + + queryset = RfcAuthor.objects.all() + serializer_class = RfcAuthorSerializer + lookup_url_kwarg = "author_id" + rfc_number_param = "rfc_number" + + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + document__type_id="rfc", + document__rfc_number=self.kwargs[self.rfc_number_param], + ) + ) + + +class DestinationHelperMixin: + def fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + + def blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + + +class RfcPubNotificationView(DestinationHelperMixin, APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="notify_rfc_published", + summary="Notify datatracker of RFC publication", + request=RfcPubSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcPubSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + # Check blobstore & filesystem for conflicts + rfc_number = serializer.validated_data["rfc_number"] + dest_stem = f"rfc{rfc_number}" + blob_kind = "rfc" + possible_rfc_files = [ + self.fs_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + possible_rfc_blobs = [ + self.blob_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) + # Create RFC + try: + rfc = serializer.save() + except IntegrityError as err: + if Document.objects.filter( + rfc_number=serializer.validated_data["rfc_number"] + ): + raise serializers.ValidationError( + "RFC with that number already exists", + code="rfc-number-in-use", + ) + raise serializers.ValidationError( + f"Unable to publish: {err}", + code="unknown-integrity-error", + ) + rfc_number_list = [rfc.rfc_number] + rfc_number_list.extend( + [d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))] + ) + rfc_number_list = sorted(set(rfc_number_list)) + signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list) + related_numbers = sorted( + {d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))} + ) + if related_numbers: + # Only update json for related rfcs. We can't generate json for _this_ rfc + # until we receive the files and know what formats we have. + transaction.on_commit(partial(update_rfc_json_task.delay, related_numbers)) + return Response(NotificationAckSerializer().data) + + +class RfcPubFilesView(DestinationHelperMixin, APIView): + api_key_endpoint = "ietf.api.views_rpc" + parser_classes = [parsers.MultiPartParser] + + @extend_schema( + operation_id="upload_rfc_files", + summary="Upload files for a published RFC", + request=RfcFileSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcFileSerializer( + # many=True, + data=request.data, + ) + serializer.is_valid(raise_exception=True) + rfc = serializer.validated_data["rfc"] + uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] + replace = serializer.validated_data["replace"] + dest_stem = f"rfc{rfc.rfc_number}" + mtime = serializer.validated_data["mtime"] + mtimestamp = mtime.timestamp() + blob_kind = "rfc" + + # List of files that might exist for an RFC + possible_rfc_files = [ + self.fs_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + possible_rfc_blobs = [ + self.blob_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + if not replace: + # this is the default: refuse to overwrite anything if not replacing + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) + + with TemporaryDirectory() as tempdir: + # Save files in a temporary directory. Use the uploaded filename + # extensions to identify files, but ignore the stems and generate our own. + files_to_move = [] # list[Path] + tmpfile_stem = Path(tempdir) / dest_stem + for upfile in uploaded_files: + uploaded_filename = Path(upfile.name) # name supplied by request + uploaded_ext = "".join(uploaded_filename.suffixes) + # Ignore json files, which are deprecated. Remove this when purple no + # longer tries to send them. + if uploaded_ext == ".json": + continue + tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) + with tempfile_path.open("wb") as dest: + for chunk in upfile.chunks(): + dest.write(chunk) + os.utime(tempfile_path, (mtimestamp, mtimestamp)) + files_to_move.append(tempfile_path) + # copy files to final location, removing any existing ones first if the + # remove flag was set + if replace: + for possible_existing_file in possible_rfc_files: + possible_existing_file.unlink(missing_ok=True) + for possible_existing_blob in possible_rfc_blobs: + remove_from_storage( + blob_kind, possible_existing_blob, warn_if_missing=False + ) + for ftm in files_to_move: + with ftm.open("rb") as f: + store_file( + kind=blob_kind, + name=self.blob_destination(ftm), + file=f, + doc_name=rfc.name, + doc_rev=rfc.rev, # expect blank, but match whatever it is + mtime=mtime, + ) + destination = self.fs_destination(ftm) + if ( + settings.SERVER_MODE != "production" + and not destination.parent.exists() + ): + destination.parent.mkdir() + shutil.move(ftm, destination) + + # Trigger red precomputer + needs_updating = [rfc.rfc_number] + for rel in rfc.relateddocument_set.filter( + relationship_id__in=["obs", "updates"] + ): + needs_updating.append(rel.target.rfc_number) + trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) + # Trigger search index update + update_rfc_searchindex_task.delay(rfc.rfc_number) + # Trigger reference relations rebuild + rebuild_reference_relations_task.delay(doc_names=[rfc.name]) + # Build rfc json (json for related rfcs was updated in RfcPubNotificationView) + transaction.on_commit(partial(update_rfc_json_task.delay, [rfc.rfc_number])) + return Response(NotificationAckSerializer().data) + + +class RfcIndexView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="refresh_rfc_index", + summary="Refresh rfc-index files", + description="Requests creation of various index files.", + responses={202: None}, + request=None, + ) + def post(self, request): + mark_rfcindex_as_dirty() + return Response(status=202) + + +class RpcQueueDataSerializer(serializers.Serializer): + data = serializers.JSONField() + + +class ProcessRpcQueueView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="process_rpc_queue", + summary="Process the provided RPC queue", + description="Schedules parsing the provided queue to update documents with change dqueue data", + responses={202: None}, + request=RpcQueueDataSerializer, + ) + def post(self, request): + serializer = RpcQueueDataSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + process_rpc_queue_task.delay(serializer.validated_data["data"]) + return Response(status=202) diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py index 3e1a2a311f7..44a30d1d7f2 100644 --- a/ietf/blobdb/admin.py +++ b/ietf/blobdb/admin.py @@ -1,9 +1,12 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved from django.contrib import admin +from django.db.models import QuerySet from django.db.models.functions import Length from rangefilter.filters import DateRangeQuickSelectListFilterBuilder +from .apps import get_blobdb from .models import Blob, ResolvedMaterial +from .utils import queue_for_replication @admin.register(Blob) @@ -17,6 +20,7 @@ class BlobAdmin(admin.ModelAdmin): ] search_fields = ["name"] list_display_links = ["name"] + actions = ["replicate_blob"] def get_queryset(self, request): return ( @@ -30,6 +34,20 @@ def object_size(self, instance): """Get the size of the object""" return instance.object_size # annotation added in get_queryset() + @admin.action(description="Replicate blobs") + def replicate_blob(self, request, queryset: QuerySet[Blob]): + blob_count = 0 + for blob in queryset.all(): + if isinstance(blob, Blob): + queue_for_replication( + bucket=blob.bucket, name=blob.name, using=get_blobdb() + ) + blob_count += 1 + self.message_user( + request, + f"Queued replication of a total of {blob_count} Blob(s)", + ) + @admin.register(ResolvedMaterial) class ResolvedMaterialAdmin(admin.ModelAdmin): diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py index fa7831f2032..6dbb615fa04 100644 --- a/ietf/blobdb/models.py +++ b/ietf/blobdb/models.py @@ -1,14 +1,11 @@ -# Copyright The IETF Trust 2025, All Rights Reserved -import json -from functools import partial +# Copyright The IETF Trust 2025-2026, All Rights Reserved from hashlib import sha384 from django.db import models, transaction from django.utils import timezone from .apps import get_blobdb -from .replication import replication_enabled -from .tasks import pybob_the_blob_replicator_task +from .utils import queue_for_replication class BlobQuerySet(models.QuerySet): @@ -64,6 +61,9 @@ class Meta: ), ] + def __str__(self): + return f"{self.bucket}:{self.name}" + def save(self, **kwargs): db = get_blobdb() with transaction.atomic(using=db): @@ -78,24 +78,8 @@ def delete(self, **kwargs): self._emit_blob_change_event(using=db) return retval - def _emit_blob_change_event(self, using=None): - if not replication_enabled(self.bucket): - return - - # For now, fire a celery task we've arranged to guarantee in-order processing. - # Later becomes pushing an event onto a queue to a dedicated worker. - transaction.on_commit( - partial( - pybob_the_blob_replicator_task.delay, - json.dumps( - { - "name": self.name, - "bucket": self.bucket, - } - ) - ), - using=using, - ) + def _emit_blob_change_event(self, using: str | None=None): + queue_for_replication(self.bucket, self.name, using=using) class ResolvedMaterial(models.Model): diff --git a/ietf/blobdb/replication.py b/ietf/blobdb/replication.py index b9d55c94982..d251d3b95c1 100644 --- a/ietf/blobdb/replication.py +++ b/ietf/blobdb/replication.py @@ -146,11 +146,11 @@ def replicate_blob(bucket, name): blob = fetch_blob_via_sql(bucket, name) if blob is None: if verbose_logging_enabled(): - log.log("Deleting {bucket}:{name} from replica") + log.log(f"Deleting {bucket}:{name} from replica") try: destination_storage.delete(name) except Exception as e: - log.log("Failed to delete {bucket}:{name} from replica: {e}") + log.log(f"Failed to delete {bucket}:{name} from replica: {e}") raise ReplicationError from e else: # Add metadata expected by the MetadataS3Storage @@ -170,7 +170,7 @@ def replicate_blob(bucket, name): try: destination_storage.save(name, file_with_metadata) except Exception as e: - log.log("Failed to save {bucket}:{name} to replica: {e}") + log.log(f"Failed to save {bucket}:{name} to replica: {e}") raise ReplicationError from e diff --git a/ietf/blobdb/storage.py b/ietf/blobdb/storage.py index 4213ec801df..e304dabc5dd 100644 --- a/ietf/blobdb/storage.py +++ b/ietf/blobdb/storage.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved from typing import Optional from django.core.exceptions import SuspiciousFileOperation @@ -10,6 +10,7 @@ from ietf.utils.storage import MetadataFile from .models import Blob +from .utils import queue_for_replication class BlobFile(MetadataFile): @@ -94,3 +95,12 @@ def get_available_name(self, name, max_length=None): f"asked to store the name '{name[:5]}...{name[-5:]} of length {len(name)}" ) return name # overwrite is permitted + + def force_replication(self, name: str): + """Force replication of a blob by name + + Be careful with this - replication includes replicating deletion of blobs, so + if you call it with a name that does not exist in blobdb, it will be removed + from R2 if it exists there! + """ + queue_for_replication(bucket=self.bucket_name, name=name) diff --git a/ietf/blobdb/utils.py b/ietf/blobdb/utils.py new file mode 100644 index 00000000000..93f8f2f5217 --- /dev/null +++ b/ietf/blobdb/utils.py @@ -0,0 +1,32 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from functools import partial + +from django.db import transaction + +from ietf.blobdb.replication import replication_enabled +from ietf.blobdb.tasks import pybob_the_blob_replicator_task + + +def queue_for_replication(bucket: str, name: str, using: str | None=None): + """Queue a blob for replication + + This is private to the blobdb app. Do not call it directly from other apps. + """ + if not replication_enabled(bucket): + return + + # For now, fire a celery task we've arranged to guarantee in-order processing. + # Later becomes pushing an event onto a queue to a dedicated worker. + transaction.on_commit( + partial( + pybob_the_blob_replicator_task.delay, + json.dumps( + { + "name": name, + "bucket": bucket, + } + ) + ), + using=using, + ) diff --git a/ietf/community/utils.py b/ietf/community/utils.py index f23e8d26abd..b6137095ef5 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -72,8 +72,10 @@ def docs_matching_community_list_rule(rule): return docs.filter(group=rule.group_id) elif rule.rule_type.startswith("state_"): return docs - elif rule.rule_type in ["author", "author_rfc"]: + elif rule.rule_type == "author": return docs.filter(documentauthor__person=rule.person) + elif rule.rule_type == "author_rfc": + return docs.filter(Q(rfcauthor__person=rule.person)|Q(rfcauthor__isnull=True,documentauthor__person=rule.person)) elif rule.rule_type == "ad": return docs.filter(ad=rule.person) elif rule.rule_type == "shepherd": @@ -122,9 +124,16 @@ def community_list_rules_matching_doc(doc): # author rules if doc.type_id == "rfc": + has_rfcauthors = doc.rfcauthor_set.exists() rules |= SearchRule.objects.filter( rule_type="author_rfc", - person__in=list(Person.objects.filter(documentauthor__document=doc)), + person__in=list( + Person.objects.filter( + Q(rfcauthor__document=doc) + if has_rfcauthors + else Q(documentauthor__document=doc) + ) + ), ) else: rules |= SearchRule.objects.filter( diff --git a/ietf/context_processors.py b/ietf/context_processors.py index baa8d7a5d26..5aaa4ab2567 100644 --- a/ietf/context_processors.py +++ b/ietf/context_processors.py @@ -5,6 +5,7 @@ from django.conf import settings from django.utils import timezone from ietf import __version__, __patch__, __release_branch__, __release_hash__ +from opentelemetry.propagate import inject def server_mode(request): return {'server_mode': settings.SERVER_MODE} @@ -51,3 +52,8 @@ def timezone_now(request): return { 'timezone_now': timezone.now(), } + +def traceparent_id(request): + context_extras = {} + inject(context_extras) + return { "otel": context_extras } diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 745536f9a1e..86f5ac5fda1 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -5,6 +5,7 @@ from django.contrib import admin from django.db import models from django import forms +from django.db.models import QuerySet from rangefilter.filters import DateRangeQuickSelectListFilterBuilder from .models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory, @@ -13,9 +14,14 @@ TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent, AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL, ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject ) + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent) +from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value +from .storage_utils import force_replication +from .utils import replicate_stored_objects_for_document + class StateTypeAdmin(admin.ModelAdmin): list_display = ["slug", "label"] @@ -28,17 +34,17 @@ class StateAdmin(admin.ModelAdmin): filter_horizontal = ["next_states"] admin.site.register(State, StateAdmin) -class DocAuthorInline(admin.TabularInline): +class DocAuthorInline(SaferTabularInline): model = DocumentAuthor raw_id_fields = ['person', 'email'] extra = 1 -class DocActionHolderInline(admin.TabularInline): +class DocActionHolderInline(SaferTabularInline): model = DocumentActionHolder raw_id_fields = ['person'] extra = 1 -class RelatedDocumentInline(admin.TabularInline): +class RelatedDocumentInline(SaferTabularInline): model = RelatedDocument fk_name= 'source' def this(self, instance): @@ -48,7 +54,7 @@ def this(self, instance): raw_id_fields = ['target'] extra = 1 -class AdditionalUrlInLine(admin.TabularInline): +class AdditionalUrlInLine(SaferTabularInline): model = DocumentURL fields = ['tag','desc','url',] extra = 1 @@ -71,7 +77,9 @@ class DocumentAuthorAdmin(admin.ModelAdmin): search_fields = ['document__name', 'person__name', 'email__address', 'affiliation', 'country'] raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) - + + + class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] @@ -79,6 +87,7 @@ class DocumentAdmin(admin.ModelAdmin): raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAuthorInline, DocActionHolderInline, RelatedDocumentInline, AdditionalUrlInLine] form = DocumentForm + actions = ["replicate_stored_objects"] def save_model(self, request, obj, form, change): e = DocEvent.objects.create( @@ -93,6 +102,22 @@ def save_model(self, request, obj, form, change): def state(self, instance): return self.get_state() + @admin.action(description="Replicate related blobs") + def replicate_stored_objects(self, request, queryset: QuerySet[Document]): + doc_count = 0 + stored_obj_count = 0 + for doc in queryset.all(): + doc_count += 1 + if isinstance(doc, Document): + stored_obj_count += replicate_stored_objects_for_document(doc) + self.message_user( + request, + ( + f"Queued replication of a total of {stored_obj_count} StoredObject(s) " + f"for {doc_count} Document(s)" + ) + ) + admin.site.register(Document, DocumentAdmin) class DocHistoryAdmin(admin.ModelAdmin): @@ -173,6 +198,7 @@ def short_desc(self, obj): admin.site.register(TelechatDocEvent, DocEventAdmin) admin.site.register(InitialReviewDocEvent, DocEventAdmin) admin.site.register(EditedAuthorsDocEvent, DocEventAdmin) +admin.site.register(EditedRfcAuthorsDocEvent, DocEventAdmin) admin.site.register(IanaExpertDocEvent, DocEventAdmin) class BallotPositionDocEventAdmin(DocEventAdmin): @@ -203,6 +229,10 @@ class SubmissionDocEventAdmin(DocEventAdmin): raw_id_fields = DocEventAdmin.raw_id_fields + ["submission"] admin.site.register(SubmissionDocEvent, SubmissionDocEventAdmin) +class RpcAssignmentDocEventAdmin(DocEventAdmin): + search_fields = DocEventAdmin.search_fields + ["assignments"] +admin.site.register(RpcAssignmentDocEvent, RpcAssignmentDocEventAdmin) + class DocumentUrlAdmin(admin.ModelAdmin): list_display = ['id', 'doc', 'tag', 'url', 'desc', ] search_fields = ['doc__name', 'url', ] @@ -229,10 +259,31 @@ class StoredObjectAdmin(admin.ModelAdmin): ] search_fields = ['name', 'doc_name', 'doc_rev'] list_display_links = ['name'] + actions = ["replicate_stored_object"] @admin.display(boolean=True, description="Deleted?", ordering="deleted") def is_deleted(self, instance): return instance.deleted is not None - + + @admin.action(description="Replicate related blobs") + def replicate_stored_object(self, request, queryset: QuerySet[StoredObject]): + stored_obj_count = 0 + for stored_object in queryset.all(): + if isinstance(stored_object, StoredObject): + force_replication(kind=stored_object.store, name=stored_object.name) + stored_obj_count += 1 + self.message_user( + request, + f"Queued replication of a total of {stored_obj_count} StoredObject(s)", + ) + admin.site.register(StoredObject, StoredObjectAdmin) + +class RfcAuthorAdmin(admin.ModelAdmin): + # the email field in the list_display/readonly_fields works through a @property + list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'person__email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person"] + readonly_fields = ["email"] +admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/api.py b/ietf/doc/api.py new file mode 100644 index 00000000000..73fff6b27ff --- /dev/null +++ b/ietf/doc/api.py @@ -0,0 +1,213 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""Doc API implementations""" + +from django.db.models import ( + BooleanField, + Count, + OuterRef, + Prefetch, + Q, + QuerySet, + Subquery, +) +from django.db.models.functions import TruncDate +from django_filters import rest_framework as filters +from rest_framework import filters as drf_filters +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.viewsets import GenericViewSet + +from ietf.group.models import Group +from ietf.name.models import StreamName, DocTypeName +from ietf.utils.timezone import RPC_TZINFO +from .models import ( + Document, + DocEvent, + RelatedDocument, + DocumentAuthor, + SUBSERIES_DOC_TYPE_IDS, +) +from .serializers import ( + RfcMetadataSerializer, + RfcStatus, + RfcSerializer, + SubseriesDocSerializer, +) + + +class RfcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 500 + + +class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): + """Filter against a comma-separated list of numbers""" + pass + + +class RfcFilter(filters.FilterSet): + published = filters.DateFromToRangeFilter() + stream = filters.ModelMultipleChoiceFilter( + queryset=StreamName.objects.filter(used=True) + ) + number = NumberInFilter( + field_name="rfc_number" + ) + group = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.all(), + field_name="group__acronym", + to_field_name="acronym", + ) + area = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.areas(), + field_name="group__parent__acronym", + to_field_name="acronym", + ) + status = filters.MultipleChoiceFilter( + choices=[(slug, slug) for slug in RfcStatus.status_slugs], + method=RfcStatus.filter, + ) + sort = filters.OrderingFilter( + fields=( + ("rfc_number", "number"), # ?sort=number / ?sort=-number + ("published", "published"), # ?sort=published / ?sort=-published + ), + ) + + +class PrefetchRelatedDocument(Prefetch): + """Prefetch via a RelatedDocument + + Prefetches following RelatedDocument relationships to other docs. By default, includes + those for which the current RFC is the `source`. If `reverse` is True, includes those + for which it is the `target` instead. Defaults to only "rfc" documents. + """ + + @staticmethod + def _get_queryset(relationship_id, reverse, doc_type_ids): + """Get queryset to use for the prefetch""" + if isinstance(doc_type_ids, str): + doc_type_ids = (doc_type_ids,) + + return RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id__in": doc_type_ids, + } + ).select_related("source" if reverse else "target") + + def __init__(self, to_attr, relationship_id, reverse=False, doc_type_ids="rfc"): + super().__init__( + lookup="targets_related" if reverse else "relateddocument_set", + queryset=self._get_queryset(relationship_id, reverse, doc_type_ids), + to_attr=to_attr, + ) + + +def augment_rfc_queryset(queryset: QuerySet[Document]): + return ( + queryset.select_related("std_level", "stream") + .prefetch_related( + Prefetch( + "group", + Group.objects.select_related("parent"), + ), + Prefetch( + "documentauthor_set", + DocumentAuthor.objects.select_related("email", "person"), + ), + PrefetchRelatedDocument( + to_attr="drafts", + relationship_id="became_rfc", + doc_type_ids="draft", + reverse=True, + ), + PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"), + PrefetchRelatedDocument( + to_attr="obsoleted_by", relationship_id="obs", reverse=True + ), + PrefetchRelatedDocument(to_attr="updates", relationship_id="updates"), + PrefetchRelatedDocument( + to_attr="updated_by", relationship_id="updates", reverse=True + ), + PrefetchRelatedDocument( + to_attr="subseries", + relationship_id="contains", + reverse=True, + doc_type_ids=SUBSERIES_DOC_TYPE_IDS, + ), + ) + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) + .annotate( + # Count of "verified-errata" tags will be 1 or 0, convert to Boolean + has_errata=Count( + "tags", + filter=Q( + tags__slug="verified-errata", + ), + output_field=BooleanField(), + ) + ) + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "rfc_number" + queryset = augment_rfc_queryset( + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) + ).order_by("-rfc_number") + + pagination_class = RfcLimitOffsetPagination + filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] + filterset_class = RfcFilter + search_fields = ["title", "abstract"] + + def get_serializer_class(self): + if self.action == "retrieve": + return RfcSerializer + return RfcMetadataSerializer + + +class PrefetchSubseriesContents(Prefetch): + def __init__(self, to_attr): + super().__init__( + lookup="relateddocument_set", + queryset=RelatedDocument.objects.filter( + relationship_id="contains", + target__type_id="rfc", + ).prefetch_related( + Prefetch( + "target", + queryset=augment_rfc_queryset(Document.objects.all()), + ) + ), + to_attr=to_attr, + ) + + +class SubseriesFilter(filters.FilterSet): + type = filters.ModelMultipleChoiceFilter( + queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS) + ) + + +class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "name" + serializer_class = SubseriesDocSerializer + queryset = Document.objects.subseries_docs().prefetch_related( + PrefetchSubseriesContents(to_attr="contents") + ) + filter_backends = [filters.DjangoFilterBackend] + filterset_class = SubseriesFilter diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index bf8523aa98f..d42af628f80 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -38,22 +38,46 @@ def expirable_drafts(queryset=None): # Populate this first time through (but after django has been set up) if nonexpirable_states is None: # all IESG states except I-D Exists and Dead block expiry - nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead"))) + nonexpirable_states = list( + State.objects.filter(used=True, type="draft-iesg").exclude( + slug__in=("idexists", "dead") + ) + ) # sent to RFC Editor and RFC Published block expiry (the latter # shouldn't be possible for an active draft, though) - nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub"))) + nonexpirable_states += list( + State.objects.filter( + used=True, + type__in=( + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ), + slug__in=("rfc-edit", "pub"), + ) + ) # other IRTF states that block expiration - nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",))) - - return queryset.filter( - states__type="draft", states__slug="active" - ).exclude( - expires=None - ).exclude( - states__in=nonexpirable_states - ).exclude( - tags="rfc-rev" # under review by the RFC Editor blocks expiry - ).distinct() + nonexpirable_states += list( + State.objects.filter( + used=True, + type_id="draft-stream-irtf", + slug__in=( + "irsgpoll", + "iesg-rev", + ), + ) + ) + + return ( + queryset.filter(states__type="draft", states__slug="active") + .exclude(expires=None) + .exclude(states__in=nonexpirable_states) + .exclude( + tags="rfc-rev" # under review by the RFC Editor blocks expiry + ) + .distinct() + ) def get_soon_to_expire_drafts(days_of_warning): diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 19aa9ecc9cc..0cf83e4525f 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -14,7 +14,7 @@ from ietf.doc.models import ( Document, DocEvent, NewRevisionDocEvent, State, DocumentAuthor, StateDocEvent, BallotPositionDocEvent, BallotDocEvent, BallotType, IRSGBallotDocEvent, TelechatDocEvent, - DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource ) + DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor ) from ietf.group.models import Group from ietf.person.factories import PersonFactory from ietf.group.factories import RoleFactory @@ -128,6 +128,13 @@ def states(obj, create, extracted, **kwargs): else: obj.set_state(State.objects.get(type_id='rfc',slug='published')) + @factory.post_generation + def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument + # Override base class, creating RfcAuthor instead of DocumentAuthor + if create and extracted: + for person in extracted: + RfcAuthorFactory(document=obj, person=person) + class IndividualDraftFactory(BaseDocumentFactory): @@ -311,6 +318,12 @@ class Meta: def desc(self): return 'New version available %s-%s'%(self.doc.name,self.rev) +class PublishedRfcDocEventFactory(DocEventFactory): + class Meta: + model = DocEvent + type = "published_rfc" + doc = factory.SubFactory(WgRfcFactory) + class StateDocEventFactory(DocEventFactory): class Meta: model = StateDocEvent @@ -382,6 +395,18 @@ class Meta: country = factory.Faker('country') order = factory.LazyAttribute(lambda o: o.document.documentauthor_set.count() + 1) +class RfcAuthorFactory(factory.django.DjangoModelFactory): + class Meta: + model = RfcAuthor + + document = factory.SubFactory(DocumentFactory) + titlepage_name = factory.LazyAttribute( + lambda obj: " ".join([obj.person.initials(), obj.person.last_name()]) + ) + person = factory.SubFactory('ietf.person.factories.PersonFactory') + affiliation = factory.Faker('company') + order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1) + class WgDocumentAuthorFactory(DocumentAuthorFactory): document = factory.SubFactory(WgDraftFactory) diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 500ed3cb182..7472b14c187 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,11 +1,11 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2026, All Rights Reserved import debug # pyflakes:ignore import datetime import unicodedata +from django.conf import settings from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed from django.urls import reverse as urlreverse @@ -224,7 +224,7 @@ def item_extra_kwargs(self, item): extra.update({"dcterms_accessRights": "gratis"}) extra.update({"dcterms_format": "text/html"}) media_contents = [] - if item.rfc_number < 8650: + if item.rfc_number < settings.FIRST_V3_RFC: if item.rfc_number not in [8, 9, 51, 418, 500, 530, 589]: for fmt, media_type in [("txt", "text/plain"), ("html", "text/html")]: media_contents.append( @@ -234,14 +234,6 @@ def item_extra_kwargs(self, item): "is_format_of": self.item_link(item), } ) - if item.rfc_number not in [571, 587]: - media_contents.append( - { - "url": f"https://www.rfc-editor.org/rfc/pdfrfc/{item.name}.txt.pdf", - "media_type": "application/pdf", - "is_format_of": self.item_link(item), - } - ) else: media_contents.append( { @@ -263,9 +255,11 @@ def item_extra_kwargs(self, item): ) extra.update({"media_contents": media_contents}) - extra.update({"doi": "10.17487/%s" % item.name.upper()}) extra.update( - {"doiuri": "http://dx.doi.org/10.17487/%s" % item.name.upper()} + { + "doi": item.doi, + "doiuri": f"https://doi.org/{item.doi}", + } ) # R104 Publisher (Mandatory - but we need a string from them first) diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index f20d398c3cb..ddecbb6b54b 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -103,61 +103,6 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""): dict(text=text, url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()), cc=cc) - -def email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=None): - if cfa_duration_weeks is None: - cfa_duration_weeks=2 - (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) - frm = request.user.person.formatted_email() - - end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * cfa_duration_weeks) - - subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" - - send_mail( - request, - to, - frm, - subject, - "doc/mail/wg_call_for_adoption_issued.txt", - dict( - doc=doc, - subject=subject, - url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), - end_date=end_date, - cfa_duration_weeks=cfa_duration_weeks, - wg_list=doc.group.list_email, - ), - cc=cc, - ) - - -def email_wg_last_call_issued(request, doc, wglc_duration_weeks=None): - if wglc_duration_weeks is None: - wglc_duration_weeks = 2 - (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) - frm = request.user.person.formatted_email() - - - end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * wglc_duration_weeks) - subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" - - send_mail( - request, - to, - frm, - subject, - "doc/mail/wg_last_call_issued.txt", - dict( - doc=doc, - subject=subject, - url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), - end_date=end_date, - wglc_duration_weeks=wglc_duration_weeks, - wg_list=doc.group.list_email, - ), - cc=cc, - ) def email_pulled_from_rfc_queue(request, doc, comment, prev_state, next_state): extra=extra_automation_headers(doc) diff --git a/ietf/doc/management/commands/reset_rfc_authors.py b/ietf/doc/management/commands/reset_rfc_authors.py deleted file mode 100644 index e2ab5f12081..00000000000 --- a/ietf/doc/management/commands/reset_rfc_authors.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved - -# Reset an RFC's authors to those of the draft it came from -from django.core.management.base import BaseCommand, CommandError - -from ietf.doc.models import Document, DocEvent -from ietf.person.models import Person - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument("rfcnum", type=int, help="RFC number to modify") - parser.add_argument( - "--force", - action="store_true", - help="reset even if RFC already has authors", - ) - - def handle(self, *args, **options): - try: - rfc = Document.objects.get(type="rfc", rfc_number=options["rfcnum"]) - except Document.DoesNotExist: - raise CommandError( - f"rfc{options['rfcnum']} does not exist in the Datatracker." - ) - - draft = rfc.came_from_draft() - if draft is None: - raise CommandError(f"{rfc.name} did not come from a draft. Can't reset.") - - orig_authors = rfc.documentauthor_set.all() - if orig_authors.exists(): - # Potentially dangerous, so refuse unless "--force" is specified - if not options["force"]: - raise CommandError( - f"{rfc.name} already has authors. Not resetting. Use '--force' to reset anyway." - ) - removed_auth_names = list(orig_authors.values_list("person__name", flat=True)) - rfc.documentauthor_set.all().delete() - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Removed all authors: {', '.join(removed_auth_names)}", - ) - self.stdout.write( - self.style.SUCCESS( - f"Removed author(s): {', '.join(removed_auth_names)}" - ) - ) - - for author in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = rfc - author.save() - self.stdout.write( - self.style.SUCCESS(f"Added author {author.person.name} <{author.email}>") - ) - auth_names = draft.documentauthor_set.values_list("person__name", flat=True) - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Set authors from rev {draft.rev} of {draft.name}: {', '.join(auth_names)}", - ) diff --git a/ietf/doc/management/commands/tests.py b/ietf/doc/management/commands/tests.py deleted file mode 100644 index 8244d872663..00000000000 --- a/ietf/doc/management/commands/tests.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved -# -*- coding: utf-8 -*- - -from io import StringIO - -from django.core.management import call_command, CommandError - -from ietf.doc.factories import DocumentAuthorFactory, WgDraftFactory, WgRfcFactory -from ietf.doc.models import Document, DocumentAuthor -from ietf.utils.test_utils import TestCase - - -class CommandTests(TestCase): - @staticmethod - def _call_command(command_name, *args, **options): - """Call command, capturing (and suppressing) output""" - out = StringIO() - err = StringIO() - options["stdout"] = out - options["stderr"] = err - call_command(command_name, *args, **options) - return out.getvalue(), err.getvalue() - - def test_reset_rfc_authors(self): - command_name = "reset_rfc_authors" - - draft = WgDraftFactory() - DocumentAuthorFactory.create_batch(3, document=draft) - rfc = WgRfcFactory() # rfc does not yet have a draft - DocumentAuthorFactory.create_batch(3, document=rfc) - bad_rfc_num = ( - 1 - + Document.objects.filter(rfc_number__isnull=False) - .order_by("-rfc_number") - .first() - .rfc_number - ) - docauthor_fields = [ - field.name - for field in DocumentAuthor._meta.get_fields() - if field.name not in ["document", "id"] - ] - - with self.assertRaises(CommandError, msg="Cannot reset a bad RFC number"): - self._call_command(command_name, bad_rfc_num) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number) - - with self.assertRaises(CommandError, msg="Cannot force-reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number, "--force") - - # Link the draft to the rfc - rfc.targets_related.create(relationship_id="became_rfc", source=draft) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with authors"): - self._call_command(command_name, rfc.rfc_number) - - # Calling with force should work - self._call_command(command_name, rfc.rfc_number, "--force") - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) - - # Calling on an RFC with no authors should also work - rfc.documentauthor_set.all().delete() - self._call_command(command_name, rfc.rfc_number) - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) diff --git a/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py new file mode 100644 index 00000000000..e0d8560e6f9 --- /dev/null +++ b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0026_change_wg_state_descriptions"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0028_rfcauthor.py b/ietf/doc/migrations/0028_rfcauthor.py new file mode 100644 index 00000000000..776dc22eb10 --- /dev/null +++ b/ietf/doc/migrations/0028_rfcauthor.py @@ -0,0 +1,84 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + dependencies = [ + ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"), + ("doc", "0027_alter_dochistory_title_alter_document_title"), + ] + + operations = [ + migrations.CreateModel( + name="RfcAuthor", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("titlepage_name", models.CharField(max_length=128)), + ("is_editor", models.BooleanField(default=False)), + ( + "affiliation", + models.CharField( + blank=True, + help_text="Organization/company used by author for submission", + max_length=100, + ), + ), + ( + "country", + models.CharField( + blank=True, + help_text="Country used by author for submission", + max_length=255, + ), + ), + ("order", models.IntegerField(default=1)), + ( + "document", + ietf.utils.models.ForeignKey( + limit_choices_to={"type_id": "rfc"}, + on_delete=django.db.models.deletion.CASCADE, + to="doc.document", + ), + ), + ( + "email", + ietf.utils.models.ForeignKey( + blank=True, + help_text="Email address used by author for submission", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.email", + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.person", + ), + ), + ], + options={ + "ordering": ["document", "order"], + "indexes": [ + models.Index( + fields=["document", "order"], + name="doc_rfcauth_documen_6b5dc4_idx", + ) + ], + }, + ), + ] diff --git a/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py new file mode 100644 index 00000000000..60837c5cb2d --- /dev/null +++ b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py @@ -0,0 +1,30 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0028_rfcauthor"), + ] + + operations = [ + migrations.CreateModel( + name="EditedRfcAuthorsDocEvent", + fields=[ + ( + "docevent_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="doc.docevent", + ), + ), + ], + bases=("doc.docevent",), + ), + ] diff --git a/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py new file mode 100644 index 00000000000..9ee858b2e8d --- /dev/null +++ b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0029_editedrfcauthorsdocevent"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py new file mode 100644 index 00000000000..c664126da32 --- /dev/null +++ b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py @@ -0,0 +1,57 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", + ), + ( + "WG Document", + "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", + ), + ( + "WG Document", + "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/doc/migrations/0032_remove_rfcauthor_email.py b/ietf/doc/migrations/0032_remove_rfcauthor_email.py new file mode 100644 index 00000000000..a0e147da591 --- /dev/null +++ b/ietf/doc/migrations/0032_remove_rfcauthor_email.py @@ -0,0 +1,16 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0031_change_draft_stream_ietf_state_descriptions"), + ] + + operations = [ + migrations.RemoveField( + model_name="rfcauthor", + name="email", + ), + ] diff --git a/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py new file mode 100644 index 00000000000..5e2513e15a9 --- /dev/null +++ b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0032_remove_rfcauthor_email"), + ] + + operations = [ + migrations.AddField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AddField( + model_name="document", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/migrations/0034_alter_dochistory_keywords_alter_document_keywords.py b/ietf/doc/migrations/0034_alter_dochistory_keywords_alter_document_keywords.py new file mode 100644 index 00000000000..2b89b67e88c --- /dev/null +++ b/ietf/doc/migrations/0034_alter_dochistory_keywords_alter_document_keywords.py @@ -0,0 +1,33 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0033_dochistory_keywords_document_keywords"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + blank=True, + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AlterField( + model_name="document", + name="keywords", + field=models.JSONField( + blank=True, + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/migrations/0035_add_rpc_queue_draft_rfceditor_states.py b/ietf/doc/migrations/0035_add_rpc_queue_draft_rfceditor_states.py new file mode 100644 index 00000000000..9805970ef0d --- /dev/null +++ b/ietf/doc/migrations/0035_add_rpc_queue_draft_rfceditor_states.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for slug, name in [("in_progress", "In Progress"), ("blocked", "Blocked")]: + State.objects.get_or_create( + type_id="draft-rfceditor", + slug=slug, + defaults={"name": name, "used": True, "desc": "", "order": 0}, + ) + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + Document = apps.get_model("doc", "Document") + for slug in ("in_progress", "blocked"): + assert not Document.objects.filter( + states__type="draft-rfceditor", states__slug=slug + ).exists() + State.objects.filter(type_id="draft-rfceditor", slug=slug).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0034_alter_dochistory_keywords_alter_document_keywords"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/doc/migrations/0036_alter_docevent_type.py b/ietf/doc/migrations/0036_alter_docevent_type.py new file mode 100644 index 00000000000..1cc11d4ee96 --- /dev/null +++ b/ietf/doc/migrations/0036_alter_docevent_type.py @@ -0,0 +1,92 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0035_add_rpc_queue_draft_rfceditor_states"), + ] + + operations = [ + migrations.AlterField( + model_name="docevent", + name="type", + field=models.CharField( + choices=[ + ("new_revision", "Added new revision"), + ("new_submission", "Uploaded new revision"), + ("changed_document", "Changed document metadata"), + ("added_comment", "Added comment"), + ("added_message", "Added message"), + ("edited_authors", "Edited the documents author list"), + ("deleted", "Deleted document"), + ("changed_state", "Changed state"), + ("changed_stream", "Changed document stream"), + ("expired_document", "Expired document"), + ("extended_expiry", "Extended expiry of document"), + ("requested_resurrect", "Requested resurrect"), + ("completed_resurrect", "Completed resurrect"), + ("changed_consensus", "Changed consensus"), + ("published_rfc", "Published RFC"), + ( + "added_suggested_replaces", + "Added suggested replacement relationships", + ), + ( + "reviewed_suggested_replaces", + "Reviewed suggested replacement relationships", + ), + ("changed_action_holders", "Changed action holders for document"), + ("changed_group", "Changed group"), + ("changed_protocol_writeup", "Changed protocol writeup"), + ("changed_charter_milestone", "Changed charter milestone"), + ("initial_review", "Set initial review time"), + ("changed_review_announcement", "Changed WG Review text"), + ("changed_action_announcement", "Changed WG Action text"), + ("started_iesg_process", "Started IESG process on document"), + ("created_ballot", "Created ballot"), + ("closed_ballot", "Closed ballot"), + ("sent_ballot_announcement", "Sent ballot announcement"), + ("changed_ballot_position", "Changed ballot position"), + ("changed_ballot_approval_text", "Changed ballot approval text"), + ("changed_ballot_writeup_text", "Changed ballot writeup text"), + ("changed_rfc_editor_note_text", "Changed RFC Editor Note text"), + ("changed_last_call_text", "Changed last call text"), + ("requested_last_call", "Requested last call"), + ("sent_last_call", "Sent last call"), + ("scheduled_for_telechat", "Scheduled for telechat"), + ("iesg_approved", "IESG approved document (no problem)"), + ("iesg_disapproved", "IESG disapproved document (do not publish)"), + ("approved_in_minute", "Approved in minute"), + ("iana_review", "IANA review comment"), + ("rfc_in_iana_registry", "RFC is in IANA registry"), + ( + "rfc_editor_received_announcement", + "Announcement was received by RFC Editor", + ), + ("requested_publication", "Publication at RFC Editor requested"), + ( + "sync_from_rfc_editor", + "Received updated information from RFC Editor", + ), + ("changed_rpc_assignments", "Changed RPC queue assignments"), + ("requested_review", "Requested review"), + ("assigned_review_request", "Assigned review request"), + ("closed_review_request", "Closed review request"), + ("closed_review_assignment", "Closed review assignment"), + ("downref_approved", "Downref approved"), + ("posted_related_ipr", "Posted related IPR"), + ("removed_related_ipr", "Removed related IPR"), + ( + "removed_objfalse_related_ipr", + "Removed Objectively False related IPR", + ), + ("changed_editors", "Changed BOF Request editors"), + ("published_statement", "Published statement"), + ("approved_slides", "Slides approved"), + ], + max_length=50, + ), + ), + ] diff --git a/ietf/doc/migrations/0037_rpcassignmentdocevent.py b/ietf/doc/migrations/0037_rpcassignmentdocevent.py new file mode 100644 index 00000000000..648376e1188 --- /dev/null +++ b/ietf/doc/migrations/0037_rpcassignmentdocevent.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0036_alter_docevent_type"), + ] + + operations = [ + migrations.CreateModel( + name="RpcAssignmentDocEvent", + fields=[ + ( + "docevent_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="doc.docevent", + ), + ), + ("assignments", models.TextField(blank=True)), + ], + bases=("doc.docevent",), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 8bb79b64ed3..ff879dd71c6 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1,7 +1,8 @@ -# Copyright The IETF Trust 2010-2025, All Rights Reserved +# Copyright The IETF Trust 2010-2026, All Rights Reserved # -*- coding: utf-8 -*- +from collections import namedtuple import datetime import logging import os @@ -11,6 +12,9 @@ from io import BufferedReader from pathlib import Path + +from django.core.exceptions import ValidationError +from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML @@ -20,7 +24,11 @@ from django.core import checks from django.core.files.base import File from django.core.cache import caches -from django.core.validators import URLValidator, RegexValidator +from django.core.validators import ( + URLValidator, + RegexValidator, + ProhibitNullCharactersValidator, +) from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings @@ -44,6 +52,7 @@ from ietf.person.utils import get_active_balloters from ietf.utils import log from ietf.utils.decorators import memoize +from ietf.utils.text import decode_document_content from ietf.utils.validators import validate_no_control_chars from ietf.utils.mail import formataddr from ietf.utils.models import ForeignKey @@ -102,12 +111,27 @@ class Meta: IESG_STATCHG_CONFLREV_ACTIVE_STATES = ("iesgeval", "defer") IESG_SUBSTATE_TAGS = ('ad-f-up', 'need-rev', 'extpty') + +def validate_doc_keywords(value): + if ( + not isinstance(value, list | tuple | set) + or not all(isinstance(elt, str) for elt in value) + ): + raise ValidationError("Value must be an array of strings") + + class DocumentInfo(models.Model): """Any kind of document. Draft, RFC, Charter, IPR Statement, Liaison Statement""" time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True type = ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ... - title = models.CharField(max_length=255, validators=[validate_no_control_chars, ]) + title = models.CharField( + max_length=255, + validators=[ + ProhibitNullCharactersValidator(), + validate_no_control_chars, + ], + ) states = models.ManyToManyField(State, blank=True) # plain state (Active/Expired/...), IESG state, stream state tags = models.ManyToManyField(DocTagName, blank=True) # Revised ID Needed, ExternalParty, AD Followup, ... @@ -129,6 +153,18 @@ class DocumentInfo(models.Model): uploaded_filename = models.TextField(blank=True) note = models.TextField(blank=True) rfc_number = models.PositiveIntegerField(blank=True, null=True) # only valid for type="rfc" + keywords = models.JSONField( + default=list, + max_length=1000, + validators=[validate_doc_keywords], + blank=True, + ) + + @property + def doi(self) -> str | None: + if self.type_id == "rfc" and self.rfc_number is not None: + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number}" + return None def file_extension(self): if not hasattr(self, '_cached_extension'): @@ -228,14 +264,14 @@ def revisions_by_newrevisionevent(self): return revisions def get_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_HREFS) + return self._get_ref(meeting=meeting, versioned=True) def get_versionless_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_GREFS) + return self._get_ref(meeting=meeting, versioned=False) - def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): + def _get_ref(self, meeting=None, versioned=True): """ Returns an url to the document text. This differs from .get_absolute_url(), which returns an url to the datatracker page for the document. @@ -244,12 +280,16 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): # the earlier resolution order, but there's at the moment one single # instance which matches this (with correct results), so we won't # break things all over the place. - if not hasattr(self, '_cached_href'): + cache_attr = "_cached_href" if versioned else "_cached_versionless_href" + if not hasattr(self, cache_attr): validator = URLValidator() if self.external_url and self.external_url.split(':')[0] in validator.schemes: validator(self.external_url) return self.external_url + meeting_doc_refs = ( + settings.MEETING_DOC_HREFS if versioned else settings.MEETING_DOC_GREFS + ) if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs: if self.meeting_related(): self.is_meeting_related = True @@ -301,8 +341,13 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): if href.startswith('/'): href = settings.IDTRACKER_BASE_URL + href - self._cached_href = href - return self._cached_href + setattr(self, cache_attr, href) + return getattr(self, cache_attr) + + def refresh_from_db(self, using=None, fields=None, **kwargs): + super().refresh_from_db(using=using, fields=fields, **kwargs) + self.state_cache = None + self._cached_state_slug = {} def set_state(self, state): """Switch state type implicit in state to state. This just @@ -407,9 +452,56 @@ def friendly_state(self): else: return state.name + def author_names(self): + """Author names as a list of strings""" + names = [] + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + if author.person: + names.append(author.person.name) + else: + # titlepage_name cannot be blank + names.append(author.titlepage_name) + else: + names = [ + author.person.name + for author in self.documentauthor_set.select_related("person") + ] + return names + + def author_persons_or_names(self): + """Authors as a list of named tuples with person and/or titlepage_name""" + Author = namedtuple("Author", "person titlepage_name") + persons_or_names = [] + if self.type_id=="rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name=author.titlepage_name)) + else: + for author in self.documentauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name="")) + return persons_or_names + + def author_persons(self): + """Authors as a list of Persons + + Omits any RfcAuthors with a null person field. + """ + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + authors_qs = self.rfcauthor_set.filter(person__isnull=False) + else: + authors_qs = self.documentauthor_set.all() + return [a.person for a in authors_qs.select_related("person")] + def author_list(self): + """List of author emails""" + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + author_qs = self.rfcauthor_set.select_related("person").order_by("order") + else: + author_qs = self.documentauthor_set.select_related("email").order_by( + "order" + ) best_addresses = [] - for author in self.documentauthor_set.all(): + for author in author_qs: if author.email: if author.email.active or not author.email.person: best_addresses.append(author.email.address) @@ -417,9 +509,6 @@ def author_list(self): best_addresses.append(author.email.person.email_address()) return ", ".join(best_addresses) - def authors(self): - return [ a.person for a in self.documentauthor_set.all() ] - # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot def ballot_open(self, ballot_type_slug): @@ -558,19 +647,7 @@ def text(self, size = -1): except IOError as e: log.log(f"Error reading text for {path}: {e}") return None - text = None - try: - text = raw.decode('utf-8') - except UnicodeDecodeError: - for back in range(1,4): - try: - text = raw[:-back].decode('utf-8') - break - except UnicodeDecodeError: - pass - if text is None: - text = raw.decode('latin-1') - return text + return decode_document_content(raw) def text_or_error(self): return self.text() or "Error; cannot read '%s'"%self.get_base_name() @@ -721,7 +798,14 @@ def referenced_by_rfcs_as_rfc_or_draft(self): if self.type_id == "rfc" and self.came_from_draft(): refs_to |= self.came_from_draft().referenced_by_rfcs() return refs_to - + + def sent_to_rfc_editor_event(self): + if self.stream_id == "ietf": + return self.docevent_set.filter(type="iesg_approved").order_by("-time").first() + elif self.stream_id in ["editorial", "iab", "irtf", "ise"]: + return self.docevent_set.filter(type="requested_publication").order_by("-time").first() + else: + return None class Meta: abstract = True @@ -845,6 +929,54 @@ def is_approved_downref(self): return False +class RfcAuthor(models.Model): + """Captures the authors of an RFC as represented on the RFC title page. + + This deviates from DocumentAuthor in that it does not get moved into the DocHistory + hierarchy as documents are saved. It will attempt to preserve email, country, and affiliation + from the DocumentAuthor objects associated with the draft leading to this RFC (which + may be wrong if the author moves or changes affiliation while the document is in the + queue). + + It does not, at this time, attempt to capture the authors from anything _but_ the title + page. The datatracker may know more about such authors based on information from the draft + leading to the RFC, and future work may take that into account. + + Once doc.rfcauthor_set.exists() for a doc of type `rfc`, doc.documentauthor_set should be + ignored. + """ + + document = ForeignKey( + "Document", + on_delete=models.CASCADE, + limit_choices_to={"type_id": "rfc"}, # only affects ModelForms (e.g., admin) + ) + titlepage_name = models.CharField(max_length=128, blank=False) + is_editor = models.BooleanField(default=False) + person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") + order = models.IntegerField(default=1) + + def __str__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + + class Meta: + ordering=["document", "order"] + indexes=[ + models.Index(fields=["document", "order"]) + ] + + @property + def email(self) -> Email | None: + return self.person.email() if self.person else None + + def format_for_titlepage(self): + if self.is_editor: + return f"{self.titlepage_name}, Ed." + return self.titlepage_name + + class DocumentAuthorInfo(models.Model): person = ForeignKey(Person) # email should only be null for some historic documents @@ -894,7 +1026,7 @@ class Meta: def role_for_doc(self): """Brief string description of this person's relationship to the doc""" roles = [] - if self.person in self.document.authors(): + if self.person in self.document.author_persons(): roles.append('Author') if self.person == self.document.ad: roles.append('Responsible AD') @@ -920,7 +1052,18 @@ def role_for_doc(self): 'invalid' ) + +SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std") + + +class DocumentQuerySet(models.QuerySet): + def subseries_docs(self): + return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS) + + class Document(StorableMixin, DocumentInfo): + objects = DocumentQuerySet.as_manager() + name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) @@ -1026,6 +1169,22 @@ def request_closed_time(self, review_req): e = self.latest_event(ReviewRequestDocEvent, type="closed_review_request", review_request=review_req) return e.time if e and e.time else None + @property + def area(self) -> Group | None: + """Get area for document, if one exists + + None for non-IETF-stream documents. N.b., this is stricter than Group.area() and + uses different logic from Document.area_acronym(). + """ + if self.stream_id != "ietf": + return None + if self.group is None: + return None + parent = self.group.parent + if parent.type_id == "area": + return parent + return None + def area_acronym(self): g = self.group if g: @@ -1121,19 +1280,21 @@ def submission(self): s = s.first() return s + def pub_datetime(self): + """Get the publication datetime of this document""" + if self.type_id == "rfc": + event = self.latest_event(type='published_rfc') + else: + event = self.latest_event(type='new_revision') + return event.time.astimezone(RPC_TZINFO) if event else None + def pub_date(self): """Get the publication date for this document This is the rfc publication date for RFCs, and the new-revision date for other documents. """ - if self.type_id == "rfc": - # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are - # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date - # assigned by the RFC editor. - event = self.latest_event(type='published_rfc') - else: - event = self.latest_event(type='new_revision') - return event.time.astimezone(RPC_TZINFO).date() if event else None + pub_datetime = self.pub_datetime() + return None if pub_datetime is None else pub_datetime.date() def is_dochistory(self): return False @@ -1169,6 +1330,32 @@ def action_holders_enabled(self): iesg_state = self.get_state('draft-iesg') return iesg_state and iesg_state.slug != 'idexists' + def formats(self): + """List of file formats available + + Only implemented for RFCs. Relies on StoredObject. + """ + if self.type_id != "rfc": + raise RuntimeError("Only allowed for type=rfc") + + # StoredObject.doc_rev can be null or "" to represent no rev. Match either + # of these when self.rev is "" (always expected to be the case for RFCs) + rev_q = Q(doc_rev=self.rev) + if self.rev == "": + rev_q |= Q(doc_rev__isnull=True) + return [ + { + "fmt": Path(object_name).parts[0], + "name": object_name, + } + for object_name in StoredObject.objects.filter( + rev_q, + store="rfc", + doc_name=self.name, + ).values_list("name", flat=True) + ] + + class DocumentURL(models.Model): doc = ForeignKey(Document) tag = ForeignKey(DocUrlTagName) @@ -1361,6 +1548,7 @@ class DocReminder(models.Model): ("rfc_editor_received_announcement", "Announcement was received by RFC Editor"), ("requested_publication", "Publication at RFC Editor requested"), ("sync_from_rfc_editor", "Received updated information from RFC Editor"), + ("changed_rpc_assignments", "Changed RPC queue assignments"), # review ("requested_review", "Requested review"), @@ -1426,6 +1614,9 @@ class StateDocEvent(DocEvent): class ConsensusDocEvent(DocEvent): consensus = models.BooleanField(null=True, default=None) +class RpcAssignmentDocEvent(DocEvent): + assignments = models.TextField(blank=True) + # IESG events class BallotType(models.Model): doc_type = ForeignKey(DocTypeName, blank=True, null=True) @@ -1581,6 +1772,11 @@ class EditedAuthorsDocEvent(DocEvent): """ basis = models.CharField(help_text="What is the source or reasoning for the changes to the author list",max_length=255) + +class EditedRfcAuthorsDocEvent(DocEvent): + """Change to the RfcAuthor list for a document""" + + class BofreqEditorDocEvent(DocEvent): """ Capture the proponents of a BOF Request.""" editors = models.ManyToManyField('person.Person', blank=True) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 157a3ad5568..9da7cb57d80 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -17,8 +17,9 @@ InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument, RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL, - IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject) + IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent) from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource class BallotTypeResource(ModelResource): @@ -650,6 +651,31 @@ class Meta: api.doc.register(EditedAuthorsDocEventResource()) + +from ietf.person.resources import PersonResource +class EditedRfcAuthorsDocEventResource(ModelResource): + by = ToOneField(PersonResource, 'by') + doc = ToOneField(DocumentResource, 'doc') + docevent_ptr = ToOneField(DocEventResource, 'docevent_ptr') + class Meta: + queryset = EditedRfcAuthorsDocEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'editedrfcauthorsdocevent' + ordering = ['id', ] + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "rev": ALL, + "desc": ALL, + "by": ALL_WITH_RELATIONS, + "doc": ALL_WITH_RELATIONS, + "docevent_ptr": ALL_WITH_RELATIONS, + } +api.doc.register(EditedRfcAuthorsDocEventResource()) + + from ietf.name.resources import DocUrlTagNameResource class DocumentURLResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') @@ -865,3 +891,53 @@ class Meta: "deleted": ALL, } api.doc.register(StoredObjectResource()) + + +from ietf.person.resources import EmailResource, PersonResource +class RfcAuthorResource(ModelResource): + document = ToOneField(DocumentResource, 'document') + person = ToOneField(PersonResource, 'person', null=True) + email = ToOneField(EmailResource, 'email', null=True, readonly=True) + class Meta: + queryset = RfcAuthor.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rfcauthor' + ordering = ['id', ] + filtering = { + "id": ALL, + "titlepage_name": ALL, + "is_editor": ALL, + "affiliation": ALL, + "country": ALL, + "order": ALL, + "document": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, + } +api.doc.register(RfcAuthorResource()) + + +from ietf.person.resources import PersonResource +class RpcAssignmentDocEventResource(ModelResource): + by = ToOneField(PersonResource, 'by') + doc = ToOneField(DocumentResource, 'doc') + docevent_ptr = ToOneField(DocEventResource, 'docevent_ptr') + class Meta: + queryset = RpcAssignmentDocEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rpcassignmentdocevent' + ordering = ['docevent_ptr', ] + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "rev": ALL, + "desc": ALL, + "assignments": ALL, + "by": ALL_WITH_RELATIONS, + "doc": ALL_WITH_RELATIONS, + "docevent_ptr": ALL_WITH_RELATIONS, + } +api.doc.register(RpcAssignmentDocEventResource()) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py new file mode 100644 index 00000000000..3651670962b --- /dev/null +++ b/ietf/doc/serializers.py @@ -0,0 +1,360 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from dataclasses import dataclass +from typing import Literal, ClassVar + +from django.db.models.manager import BaseManager +from django.db.models.query import QuerySet +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.name.serializers import StreamNameSerializer +from ietf.utils import log +from .models import Document, DocumentAuthor, RfcAuthor + + +class RfcAuthorSerializer(serializers.ModelSerializer): + """Serializer for an RfcAuthor / DocumentAuthor in a response""" + + email = serializers.EmailField(source="email.address", read_only=True) + datatracker_person_path = serializers.URLField( + source="person.get_absolute_url", + required=False, + help_text="URL for person link (relative to datatracker base URL)", + read_only=True, + ) + + class Meta: + model = RfcAuthor + fields = [ + "titlepage_name", + "is_editor", + "person", + "email", + "affiliation", + "country", + "datatracker_person_path", + ] + + def to_representation(self, instance): + """instance -> primitive data types + + Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same + serializer for either type. + """ + if isinstance(instance, DocumentAuthor): + # create a non-persisted RfcAuthor as a shim - do not save it! + document_author = instance + instance = RfcAuthor( + titlepage_name=document_author.person.plain_name(), + is_editor=False, + person=document_author.person, + affiliation=document_author.affiliation, + country=document_author.country, + order=document_author.order, + ) + return super().to_representation(instance) + + def validate(self, data): + email = data.get("email") + if email is not None: + person = data.get("person") + if person is None: + raise serializers.ValidationError( + { + "email": "cannot have an email without a person", + }, + code="email-without-person", + ) + if email.person_id != person.pk: + raise serializers.ValidationError( + { + "email": "email must belong to person", + }, + code="email-person-mismatch", + ) + return data + + +@dataclass +class DocIdentifier: + type: Literal["doi", "issn"] + value: str + + +class DocIdentifierSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=["doi", "issn"]) + value = serializers.CharField() + + +type RfcStatusSlugT = Literal[ + "std", + "ps", + "ds", + "bcp", + "inf", + "exp", + "hist", + "unkn", + "not-issued", +] + + +@dataclass +class RfcStatus: + """Helper to extract the 'Status' from an RFC document for serialization""" + + slug: RfcStatusSlugT + + # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field. + fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = { + "std": "internet standard", + "ps": "proposed standard", + "ds": "draft standard", + "bcp": "best current practice", + "inf": "informational", + "exp": "experimental", + "hist": "historic", + "unkn": "unknown", + } + + # ClassVar annotation prevents dataclass from treating this as a field + stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = { + "bcp": "bcp", + "ds": "ds", + "exp": "exp", + "hist": "hist", + "inf": "inf", + "std": "std", + "ps": "ps", + "unkn": "unkn", + } + + # ClassVar annotation prevents dataclass from treating this as a field + status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted( + # TODO implement "not-issued" RFCs + set(stdlevelname_slug_map.values()) | {"not-issued"} + ) + + @property + def name(self): + return RfcStatus.fancy_names.get(self.slug, self.slug) + + @classmethod + def from_document(cls, doc: Document): + """Decide the status that applies to a document""" + return cls( + slug=(cls.stdlevelname_slug_map.get(doc.std_level.slug, "unkn")), + ) + + @classmethod + def filter(cls, queryset, name, value: list[RfcStatusSlugT]): + """Filter a queryset by status + + This is basically the inverse of the from_document() method. Given a status name, filter + the queryset to those in that status. The queryset should be a Document queryset. + """ + interesting_slugs = [ + stdlevelname_slug + for stdlevelname_slug, status_slug in cls.stdlevelname_slug_map.items() + if status_slug in value + ] + if len(interesting_slugs) == 0: + return queryset.none() + return queryset.filter(std_level__slug__in=interesting_slugs) + + +class RfcStatusSerializer(serializers.Serializer): + """Status serializer for a Document instance""" + + slug = serializers.ChoiceField(choices=RfcStatus.status_slugs) + name = serializers.CharField() + + def to_representation(self, instance: Document): + return super().to_representation(instance=RfcStatus.from_document(instance)) + + +class ShepherdSerializer(serializers.Serializer): + email = serializers.EmailField(source="email_address") + + +class RelatedDraftSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + name = serializers.CharField(source="source.name") + title = serializers.CharField(source="source.title") + shepherd = ShepherdSerializer(source="source.shepherd", allow_null=True) + ad = AreaDirectorSerializer(source="source.ad", allow_null=True) + + +class RelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="target.id") + number = serializers.IntegerField(source="target.rfc_number") + title = serializers.CharField(source="target.title") + + +class ReverseRelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + number = serializers.IntegerField(source="source.rfc_number") + title = serializers.CharField(source="source.title") + + +class ContainingSubseriesSerializer(serializers.Serializer): + name = serializers.CharField(source="source.name") + type = serializers.CharField(source="source.type_id") + + +class RfcFormatSerializer(serializers.Serializer): + RFC_FORMATS = ("xml", "txt", "html", "pdf", "ps", "json", "notprepped") + + fmt = serializers.ChoiceField(choices=RFC_FORMATS) + name = serializers.CharField(help_text="Name of blob in the blob store") + + +class RfcMetadataSerializer(serializers.ModelSerializer): + """Serialize metadata of an RFC + + This needs to be called with a Document queryset that has been processed with + api.augment_rfc_queryset() or it very likely will not work. Some of the typing + refers to Document, but this should really be WithAnnotations[Document, ...]. + However, have not been able to make that work yet. + """ + + number = serializers.IntegerField(source="rfc_number") + published = serializers.DateField() + status = RfcStatusSerializer(source="*") + authors = serializers.SerializerMethodField() + group = GroupSerializer() + area = AreaSerializer(read_only=True) + stream = StreamNameSerializer() + ad = AreaDirectorSerializer(read_only=True, allow_null=True) + group_list_email = serializers.EmailField(source="group.list_email", read_only=True) + identifiers = serializers.SerializerMethodField() + draft = serializers.SerializerMethodField() + obsoletes = RelatedRfcSerializer(many=True, read_only=True) + obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + updates = RelatedRfcSerializer(many=True, read_only=True) + updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + subseries = ContainingSubseriesSerializer(many=True, read_only=True) + formats = RfcFormatSerializer( + many=True, read_only=True, help_text="Available formats" + ) + keywords = serializers.ListField(child=serializers.CharField(), read_only=True) + has_errata = serializers.BooleanField(read_only=True) + + class Meta: + model = Document + fields = [ + "number", + "title", + "published", + "status", + "pages", + "authors", + "group", + "area", + "stream", + "ad", + "group_list_email", + "identifiers", + "obsoletes", + "obsoleted_by", + "updates", + "updated_by", + "subseries", + "draft", + "abstract", + "formats", + "keywords", + "has_errata", + ] + + @extend_schema_field(RfcAuthorSerializer(many=True)) + def get_authors(self, doc: Document): + # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors + author_queryset: QuerySet[RfcAuthor] | QuerySet[DocumentAuthor] = ( + doc.rfcauthor_set.all() + if doc.rfcauthor_set.exists() + else doc.documentauthor_set.all() + ) + # RfcAuthorSerializer can deal with DocumentAuthor instances + return RfcAuthorSerializer( + instance=author_queryset, + many=True, + ).data + + @extend_schema_field(DocIdentifierSerializer(many=True)) + def get_identifiers(self, doc: Document): + identifiers = [] + if doc.doi: + identifiers.append( + DocIdentifier(type="doi", value=doc.doi) + ) + return DocIdentifierSerializer(instance=identifiers, many=True).data + + @extend_schema_field(RelatedDraftSerializer) + def get_draft(self, doc: Document): + if hasattr(doc, "drafts"): + # This is the expected case - drafts is added by a Prefetch in + # the augment_rfc_queryset() method. + try: + related_doc = doc.drafts[0] + except IndexError: + return None + else: + # Fallback in case augment_rfc_queryset() was not called + log.log( + f"Warning: {self.__class__}.get_draft() called without prefetched draft" + ) + related_doc = doc.came_from_draft() + return RelatedDraftSerializer(related_doc).data + + +class RfcSerializer(RfcMetadataSerializer): + """Serialize an RFC, including its metadata and text content if available""" + + text = serializers.CharField(allow_null=True) + + class Meta: + model = RfcMetadataSerializer.Meta.model + fields = RfcMetadataSerializer.Meta.fields + ["text"] + + +class SubseriesContentListSerializer(serializers.ListSerializer): + """ListSerializer that gets its object from item.target""" + + def to_representation(self, data): + """ + List of object instances -> List of dicts of primitive datatypes. + """ + # Dealing with nested relationships, data can be a Manager, + # so, first get a queryset from the Manager if needed + iterable = data.all() if isinstance(data, BaseManager) else data + # Serialize item.target instead of item itself + return [self.child.to_representation(item.target) for item in iterable] + + +class SubseriesContentSerializer(RfcMetadataSerializer): + """Serialize RFC contained in a subseries doc""" + + class Meta(RfcMetadataSerializer.Meta): + list_serializer_class = SubseriesContentListSerializer + + +class SubseriesDocSerializer(serializers.ModelSerializer): + """Serialize a subseries document (e.g., a BCP or STD)""" + + contents = SubseriesContentSerializer(many=True) + + class Meta: + model = Document + fields = [ + "name", + "type", + "contents", + ] diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py index 375620ccaf3..ee1e76c4fac 100644 --- a/ietf/doc/storage.py +++ b/ietf/doc/storage.py @@ -114,7 +114,6 @@ def _get_write_parameters(self, name, content=None): class StoredObjectBlobdbStorage(BlobdbStorage): - ietf_log_blob_timing = True warn_if_missing = True # TODO-BLOBSTORE make this configurable (or remove it) def _save_stored_object(self, name, content) -> StoredObject: diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 81588c83ec3..c7cc6989cd0 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -10,6 +10,7 @@ from django.core.files.storage import storages, Storage from ietf.utils.log import log +from ietf.utils.text import decode_document_content class StorageUtilsError(Exception): @@ -164,34 +165,39 @@ def store_str( def retrieve_bytes(kind: str, name: str) -> bytes: from ietf.doc.storage import maybe_log_timing - content = b"" - if settings.ENABLE_BLOBSTORAGE: - try: - store = _get_storage(kind) - with store.open(name) as f: - with maybe_log_timing( - hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, - "read", - bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", - name=name, - ): - content = f.read() - except Exception as err: - log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") - if settings.SERVER_MODE == "development": - raise + if not settings.ENABLE_BLOBSTORAGE: + return b"" + try: + store = _get_storage(kind) + with store.open(name) as f: + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() + except Exception as err: + log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") + raise return content def retrieve_str(kind: str, name: str) -> str: - content = "" - if settings.ENABLE_BLOBSTORAGE: - try: - content_bytes = retrieve_bytes(kind, name) - # TODO-BLOBSTORE: try to decode all the different ways doc.text() does - content = content_bytes.decode("utf-8") - except Exception as err: - log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") - if settings.SERVER_MODE == "development": - raise + if not settings.ENABLE_BLOBSTORAGE: + return "" + try: + content = decode_document_content(retrieve_bytes(kind, name)) + except Exception as err: + log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") + raise return content + + +def force_replication(kind: str, name: str): + if not settings.ENABLE_BLOBSTORAGE: + return + storage = _get_storage(kind) + from ietf.blobdb.storage import BlobdbStorage + if isinstance(storage, BlobdbStorage): + storage.force_replication(name) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 4f7fe377823..37c235b9115 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -1,17 +1,21 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime + import debug # pyflakes:ignore from celery import shared_task +from celery.exceptions import MaxRetriesExceededError from pathlib import Path from django.conf import settings from django.utils import timezone -from ietf.utils import log +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer +from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today from .expire import ( @@ -29,10 +33,13 @@ from .utils import ( generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted, + rebuild_reference_relations, update_or_create_draft_bibxml_file, ensure_draft_bibxml_path_exists, investigate_fragment, ) +from .utils_bofreq import fixup_bofreq_timestamps +from .utils_errata import signal_update_rfc_metadata @shared_task @@ -74,17 +81,19 @@ def expire_last_calls_task(): try: expire_last_call(doc) except Exception: - log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})") + log.log( + f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})" + ) else: log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})") -@shared_task +@shared_task def generate_idnits2_rfc_status_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" blob = generate_idnits2_rfc_status() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfc-status: {e}") @@ -94,7 +103,7 @@ def generate_idnits2_rfcs_obsoleted_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" blob = generate_idnits2_rfcs_obsoleted() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") @@ -102,7 +111,7 @@ def generate_idnits2_rfcs_obsoleted_task(): @shared_task def generate_draft_bibxml_files_task(days=7, process_all=False): """Generate bibxml files for recently updated docs - + If process_all is False (the default), processes only docs with new revisions in the last specified number of days. """ @@ -114,7 +123,9 @@ def generate_draft_bibxml_files_task(days=7, process_all=False): doc__type_id="draft", ).order_by("time") if not process_all: - doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days)) + doc_events = doc_events.filter( + time__gte=timezone.now() - datetime.timedelta(days=days) + ) for event in doc_events: try: update_or_create_draft_bibxml_file(event.doc, event.rev) @@ -128,3 +139,88 @@ def investigate_fragment_task(name_fragment: str): "name_fragment": name_fragment, "results": investigate_fragment(name_fragment), } + + +@shared_task +def rebuild_reference_relations_task(doc_names: list[str]): + log.log(f"Task: Rebuilding reference relations for {doc_names}") + for doc in Document.objects.filter(name__in=doc_names, type__in=["rfc", "draft"]): + filenames = dict() + base = ( + settings.RFC_PATH + if doc.type_id == "rfc" + else settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR + ) + stem = doc.name if doc.type_id == "rfc" else f"{doc.name}-{doc.rev}" + for ext in ["xml", "txt"]: + path = Path(base) / f"{stem}.{ext}" + if path.is_file(): + filenames[ext] = str(path) + if len(filenames) > 0: + rebuild_reference_relations(doc, filenames) + else: + log.log(f"Found no content for {stem}") + + +@shared_task +def fixup_bofreq_timestamps_task(): # pragma: nocover + fixup_bofreq_timestamps() + + +@shared_task +def signal_update_rfc_metadata_task(rfc_number_list=()): + signal_update_rfc_metadata(rfc_number_list) + + +@shared_task(bind=True) +def trigger_red_precomputer_task(self, rfc_number_list=()): + if not rfcs_are_in_r2(rfc_number_list): + log.log(f"Objects are not yet in R2 for RFCs {rfc_number_list}") + try: + countdown = getattr(settings, "RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", 10) + max_retries = getattr(settings, "RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", 12) + self.retry(countdown=countdown, max_retries=max_retries) + except MaxRetriesExceededError: + log.log(f"Gave up waiting for objects in R2 for RFCs {rfc_number_list}") + else: + trigger_red_precomputer(rfc_number_list) + + +@shared_task(bind=True) +def update_rfc_searchindex_task(self, rfc_number: int): + """Update the search index for one RFC""" + if not searchindex.enabled(): + log.log("Search indexing is not enabled, skipping") + return + + rfc = Document.objects.filter(type_id="rfc", rfc_number=rfc_number).first() + if rfc is None: + log.log( + f"ERROR: Document for rfc{rfc_number} not found, not updating search index" + ) + return + try: + searchindex.update_or_create_rfc_entry(rfc) + except Exception as err: + log.log(f"Search index update for {rfc.name} failed ({err})") + if isinstance(err, searchindex.RETRYABLE_ERROR_CLASSES): + searchindex_settings = searchindex.get_settings() + self.retry( + countdown=searchindex_settings["TASK_RETRY_DELAY"], + max_retries=searchindex_settings["TASK_MAX_RETRIES"], + ) + + +@shared_task +def rebuild_searchindex_task( + *, batchsize=40, drop_collection=False, upsert_presets=True +): + if drop_collection: + searchindex.delete_collection() + searchindex.create_collection() + if upsert_presets: + searchindex.upsert_presets() # ok if they already exist + searchindex.update_or_create_rfc_entries( + Document.objects.filter(type_id="rfc").order_by("-rfc_number"), + batchsize=batchsize, + ) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 5cabe1728d0..e72cc04ff34 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -138,7 +138,7 @@ def prettystdname(string, space=" "): @register.filter def rfceditor_info_url(rfcnum : str): """Link to the RFC editor info page for an RFC""" - return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}') + return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}/') def doc_name(name): @@ -1017,3 +1017,61 @@ def is_in_stream(doc): elif stream == "editorial": return True return False + + +@register.filter +def is_doc_ietf_adoptable(doc): + return doc.stream_id is None or all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in [ + "c-adopt", + "adopt-wg", + "info", + "wg-doc", + "parked", + "dead", + "wg-lc", + "waiting-for-implementation", + "chair-w", + "writeupw", + "sub-pub", + ], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_issue_ietf_wg_lc(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in ["wg-cand", "c-adopt", "wg-lc"], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_submit_to_iesg(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-iesg") == "idexists", + doc.get_state_slug("draft-stream-ietf") not in ["wg-cand", "c-adopt"], + ] + ) + + +@register.filter +def has_had_ietf_wg_lc(doc): + return ( + doc.stream_id == "ietf" + and doc.docevent_set.filter(statedocevent__state__slug="wg-lc").exists() + ) + diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 16dcfb7754d..86731000730 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -36,14 +36,18 @@ import debug # pyflakes:ignore -from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, - DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, - EditedAuthorsDocEvent, StateType) -from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory, - ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, - IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, - BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, - StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) +from ietf.doc.models import (Document, DocRelationshipName, RelatedDocument, State, + DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, + EditedAuthorsDocEvent, StateType, RfcAuthor) +from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, + ConflictReviewFactory, WgDraftFactory, + IndividualDraftFactory, WgRfcFactory, + IndividualRfcFactory, StateDocEventFactory, + BallotPositionDocEventFactory, + BallotDocEventFactory, DocumentAuthorFactory, + NewRevisionDocEventFactory, + StatusChangeFactory, DocExtResourceFactory, + RgDraftFactory, BcpFactory, RfcAuthorFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import ( @@ -69,7 +73,7 @@ from ietf.utils.mail import get_payload_text, outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized, unicontent from ietf.utils.test_utils import TestCase -from ietf.utils.text import normalize_text +from ietf.utils.text import normalize_text, texescape from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO from ietf.doc.utils_search import AD_WORKLOAD @@ -979,7 +983,7 @@ def test_edit_authors_permissions(self): # Relevant users not authorized to edit authors unauthorized_usernames = [ 'plain', - *[author.user.username for author in draft.authors()], + *[author.user.username for author in draft.author_persons()], draft.group.get_chair().person.user.username, 'ad' ] @@ -994,7 +998,7 @@ def test_edit_authors_permissions(self): self.client.logout() # Try to add an author via POST - still only the secretary should be able to do this. - orig_authors = draft.authors() + orig_authors = draft.author_persons() post_data = self.make_edit_authors_post_data( basis='permission test', authors=draft.documentauthor_set.all(), @@ -1012,12 +1016,40 @@ def test_edit_authors_permissions(self): for username in unauthorized_usernames: login_testing_unauthorized(self, username, url, method='post', request_kwargs=dict(data=post_data)) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors) # ensure draft author list was not modified + self.assertEqual(draft.author_persons(), orig_authors) # ensure draft author list was not modified login_testing_unauthorized(self, 'secretary', url, method='post', request_kwargs=dict(data=post_data)) r = self.client.post(url, post_data) self.assertEqual(r.status_code, 302) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors + [new_auth_person]) + self.assertEqual(draft.author_persons(), orig_authors + [new_auth_person]) + + def test_edit_authors_blocked_when_rfcauthors_exist(self): + """edit_authors returns 403 for all users when RfcAuthors exist""" + rfc = WgRfcFactory() + RfcAuthorFactory(document=rfc) + url = urlreverse('ietf.doc.views_doc.edit_authors', kwargs=dict(name=rfc.name)) + + self.client.login(username='secretary', password='secretary+password') + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.post(url, {}) + self.assertEqual(r.status_code, 403) + + def test_document_main_hides_edit_authors_when_rfcauthors_exist(self): + """document_main does not offer edit link for authors when RfcAuthors exist""" + rfc = WgRfcFactory() + edit_authors_url = urlreverse('ietf.doc.views_doc.edit_authors', kwargs=dict(name=rfc.name)) + + self.client.login(username='secretary', password='secretary+password') + + r = self.client.get(urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=rfc.name))) + self.assertEqual(r.status_code, 200) + self.assertContains(r, edit_authors_url) + + RfcAuthorFactory(document=rfc) + r = self.client.get(urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=rfc.name))) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r, edit_authors_url) def make_edit_authors_post_data(self, basis, authors): """Helper to generate edit_authors POST data for a set of authors""" @@ -1365,8 +1397,8 @@ def test_edit_authors_edit_fields(self): basis=change_reason ) - old_address = draft.authors()[0].email() - new_email = EmailFactory(person=draft.authors()[0], address=f'changed-{old_address}') + old_address = draft.author_persons()[0].email() + new_email = EmailFactory(person=draft.author_persons()[0], address=f'changed-{old_address}') post_data['author-0-email'] = new_email.address post_data['author-1-affiliation'] = 'University of Nowhere' post_data['author-2-country'] = 'Chile' @@ -1399,17 +1431,17 @@ def test_edit_authors_edit_fields(self): country_event = change_events.filter(desc__icontains='changed country').first() self.assertIsNotNone(email_event) - self.assertIn(draft.authors()[0].name, email_event.desc) + self.assertIn(draft.author_persons()[0].name, email_event.desc) self.assertIn(before[0]['email'], email_event.desc) self.assertIn(after[0]['email'], email_event.desc) self.assertIsNotNone(affiliation_event) - self.assertIn(draft.authors()[1].name, affiliation_event.desc) + self.assertIn(draft.author_persons()[1].name, affiliation_event.desc) self.assertIn(before[1]['affiliation'], affiliation_event.desc) self.assertIn(after[1]['affiliation'], affiliation_event.desc) self.assertIsNotNone(country_event) - self.assertIn(draft.authors()[2].name, country_event.desc) + self.assertIn(draft.author_persons()[2].name, country_event.desc) self.assertIn(before[2]['country'], country_event.desc) self.assertIn(after[2]['country'], country_event.desc) @@ -1863,13 +1895,63 @@ def test_document_ballot_needed_positions(self): def test_document_json(self): doc = IndividualDraftFactory() - + author = DocumentAuthorFactory(document=doc) + r = self.client.get(urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) data = r.json() - self.assertEqual(doc.name, data['name']) - self.assertEqual(doc.pages,data['pages']) + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": author.person.name, + "email": author.email.address, + "affiliation": author.affiliation, + } + ] + ) + + def test_document_json_rfc(self): + doc = IndividualRfcFactory() + old_style_author = DocumentAuthorFactory(document=doc) + url = urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name)) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": old_style_author.person.name, + "email": old_style_author.email.address, + "affiliation": old_style_author.affiliation, + } + ] + ) + + new_style_author = RfcAuthorFactory(document=doc) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": new_style_author.titlepage_name, + "email": new_style_author.email.address, + "affiliation": new_style_author.affiliation, + } + ] + ) + def test_writeup(self): doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')],) @@ -2011,9 +2093,9 @@ def test_rfc_feed(self): self.assertEqual(len(q("item")), 3) item = q("item")[0] media_content = item.findall("{http://search.yahoo.com/mrss/}content") - self.assertEqual(len(media_content), 3) + self.assertEqual(len(media_content), 2) types = set([m.attrib["type"] for m in media_content]) - self.assertEqual(types, set(["text/plain", "text/html", "application/pdf"])) + self.assertEqual(types, set(["text/plain", "text/html"])) def test_state_help(self): url = urlreverse('ietf.doc.views_help.state_help', kwargs=dict(type="draft-iesg")) @@ -2051,9 +2133,11 @@ def test_document_bibtex(self): doc = factory() url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name)) r = self.client.get(url) - self.assertEqual(r.status_code, 404) + self.assertEqual(r.status_code, 404) + authors = PersonFactory.create_batch(2) rfc = WgRfcFactory.create( - time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)) + time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)), + authors=authors, ) num = rfc.rfc_number DocEventFactory.create( @@ -2070,7 +2154,12 @@ def test_document_bibtex(self): self.assertEqual(entry["doi"], "10.17487/RFC%s" % num) self.assertEqual(entry["year"], "2010") self.assertEqual(entry["month"].lower()[0:3], "oct") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") + escaped_author_names = [ + texescape(ra.titlepage_name) + for ra in RfcAuthor.objects.filter(document=rfc) + ] + self.assertEqual(entry["author"], " and ".join(escaped_author_names)) # self.assertNotIn("day", entry) @@ -2106,7 +2195,7 @@ def test_document_bibtex(self): self.assertEqual(entry["year"], "1990") self.assertEqual(entry["month"].lower()[0:3], "apr") self.assertEqual(entry["day"], "1") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") draft = IndividualDraftFactory.create() docname = "%s-%s" % (draft.name, draft.rev) diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index 8420e411e29..517a2ce0563 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -3,7 +3,6 @@ import datetime -from unittest import mock from pyquery import PyQuery @@ -716,11 +715,8 @@ def verify_can_see(username, url): verify_can_see(username, url) class ApproveBallotTests(TestCase): - @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True) - def test_approve_ballot(self, mock_urlopen): - mock_urlopen.return_value.text = b'OK' - mock_urlopen.return_value.status_code = 200 - # + def test_approve_ballot(self): + ad = Person.objects.get(name="Areað Irector") draft = IndividualDraftFactory(ad=ad, intended_std_level_id='ps') draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="iesg-eva")) # make sure it's approvable diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py index 6a7c9393ef8..6b142149be0 100644 --- a/ietf/doc/tests_bofreq.py +++ b/ietf/doc/tests_bofreq.py @@ -307,17 +307,20 @@ def test_submit(self): url = urlreverse('ietf.doc.views_bofreq.submit', kwargs=dict(name=doc.name)) rev = doc.rev + doc_time = doc.time r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) nobody = PersonFactory() self.client.login(username=nobody.user.username, password=nobody.user.username+'+password') r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 403) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) self.client.logout() editor = bofreq_editors(doc).first() @@ -339,12 +342,14 @@ def test_submit(self): r = self.client.post(url, postdict) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual('%02d'%(int(rev)+1) ,doc.rev) - self.assertEqual(f'# {username}', doc.text()) - self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name())) - self.assertEqual(docevent_count+1, doc.docevent_set.count()) - self.assertEqual(1, len(outbox)) + self.assertEqual(doc.rev, '%02d'%(int(rev)+1)) + self.assertGreater(doc.time, doc_time) + self.assertEqual(doc.text(), f'# {username}') + self.assertEqual(retrieve_str('bofreq', doc.get_base_name()), f'# {username}') + self.assertEqual(doc.docevent_set.count(), docevent_count+1) + self.assertEqual(len(outbox), 1) rev = doc.rev + doc_time = doc.time finally: os.unlink(file.name) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 4d262c5a2fb..7f075ee95ca 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -6,7 +6,6 @@ import os import datetime import io -from unittest import mock from collections import Counter from pathlib import Path @@ -21,13 +20,14 @@ import debug # pyflakes:ignore from ietf.doc.expire import expirable_drafts, get_expired_drafts, send_expire_notice_for_draft, expire_draft -from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory +from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory, WgRfcFactory from ietf.doc.models import ( Document, DocReminder, DocEvent, ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent ) from ietf.doc.storage_utils import exists_in_storage, store_str from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open -from ietf.doc.views_draft import AdoptDraftForm +from ietf.doc.views_draft import AdoptDraftForm, IssueCallForAdoptionForm, IssueWorkingGroupLastCallForm +from ietf.ietfauth.utils import has_role from ietf.name.models import DocTagName, RoleName from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group, Role @@ -86,7 +86,7 @@ def test_ad_approved(self): self.assertTrue("Approved: " in outbox[-1]['Subject']) self.assertTrue(draft.name in outbox[-1]['Subject']) self.assertTrue('iesg@' in outbox[-1]['To']) - + def test_change_state(self): ad = Person.objects.get(user__username="ad") draft = WgDraftFactory( @@ -139,7 +139,7 @@ def test_change_state(self): self.assertEqual(draft.get_state_slug("draft-iesg"), "review-e") self.assertTrue(not draft.tags.filter(slug="ad-f-up")) self.assertTrue(draft.tags.filter(slug="need-rev")) - self.assertCountEqual(draft.action_holders.all(), [ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.count(), events_before + 3) self.assertTrue("Test comment" in draft.docevent_set.all()[0].desc) self.assertTrue("Changed action holders" in draft.docevent_set.all()[1].desc) @@ -178,7 +178,7 @@ def test_pull_from_rfc_queue(self): states=[('draft-iesg','rfcqueue')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) login_testing_unauthorized(self, "secretary", url) @@ -278,7 +278,7 @@ def test_request_last_call(self): states=[('draft-iesg','ad-eval')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) self.client.login(username="secretary", password="secretary+password") url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) @@ -1368,7 +1368,7 @@ def _test_changing_ah(action_holders, reason): _test_changing_ah([doc.ad, doc.shepherd.person], 'this is a first test') _test_changing_ah([doc.ad], 'this is a second test') - _test_changing_ah(doc.authors(), 'authors can do it, too') + _test_changing_ah(doc.author_persons(), 'authors can do it, too') _test_changing_ah([], 'clear it back out') def test_doc_change_action_holders_as_doc_manager(self): @@ -1548,11 +1548,8 @@ def test_confirm_submission_no_doc_ad(self): class RequestPublicationTests(TestCase): - @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True) - def test_request_publication(self, mockobj): - mockobj.return_value.text = b'OK' - mockobj.return_value.status_code = 200 - # + def test_request_publication(self): + draft = IndividualDraftFactory(stream_id='iab',group__acronym='iab',intended_std_level_id='inf',states=[('draft-stream-iab','approved')]) url = urlreverse('ietf.doc.views_draft.request_publication', kwargs=dict(name=draft.name)) @@ -1708,11 +1705,7 @@ def test_adopt_document(self): self.assertEqual(draft.group, chair_role.group) self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-" self.assertEqual(draft.docevent_set.count() - events_before, 5) - self.assertEqual(len(outbox), 2) - self.assertTrue("Call For Adoption" in outbox[0]["Subject"]) - self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[0]['To']) - self.assertTrue(f"{draft.name}@" in outbox[0]['To']) - self.assertTrue(f"{chair_role.group.acronym}@" in outbox[0]['To']) + self.assertEqual(len(outbox), 1) # contents of outbox[1] are tested elsewhere # adopt @@ -2003,6 +1996,40 @@ def test_set_state(self): self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) + def test_set_stream_state_to_wglc(self): + def _form_presents_state_option(response, state): + q = PyQuery(response.content) + option = q(f"select#id_new_state option[value='{state.pk}']") + return len(option) != 0 + + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + StateDocEventFactory( + doc=doc, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(doc.docevent_set.count(), 2) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + other_doc = WgDraftFactory() + self.client.logout() + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=other_doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + def test_wg_call_for_adoption_issued(self): role = RoleFactory( name_id="chair", @@ -2029,12 +2056,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) + self.assertEqual(len(outbox), 1) # Test not entering a duration on the form draft = IndividualDraftFactory() url = urlreverse( @@ -2051,12 +2073,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertEqual(len(outbox), 1) # Test the less usual workflow of issuing a call for adoption # of a document that's already in the ietf stream @@ -2086,12 +2103,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) + self.assertEqual(len(outbox), 1) draft = WgDraftFactory(group=role.group) url = urlreverse( "ietf.doc.views_draft.change_stream_state", @@ -2117,85 +2129,261 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertEqual(len(outbox), 1) - def test_wg_last_call_issued(self): - role = RoleFactory( - name_id="chair", - group__acronym="mars", - group__list_email="mars-wg@ietf.org", - person__user__username="marschairman", - person__name="WG Cháir Man", + def test_issue_wg_lc_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", ) - draft = WgDraftFactory(group=role.group) - url = urlreverse( - "ietf.doc.views_draft.change_stream_state", - kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + form = IssueWorkingGroupLastCallForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", ) - login_testing_unauthorized(self, "marschairman", url) - old_state = draft.get_state("draft-stream-%s" % draft.stream_id) - new_state = State.objects.get( - used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", ) - self.assertNotEqual(old_state, new_state) - empty_outbox() - r = self.client.post( - url, - dict( - new_state=new_state.pk, - comment="some comment", - weeks="10", - tags=[ - t.pk - for t in draft.tags.filter( - slug__in=get_tags_for_stream_id(draft.stream_id) - ) - ], - ), + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", ) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("WG Last Call", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) - draft = WgDraftFactory(group=role.group) - url = urlreverse( - "ietf.doc.views_draft.change_stream_state", - kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + + def test_issue_wg_lc(self): + def _assert_rejected(testcase, doc, person): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name) + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 404) + testcase.client.logout() + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) + login_testing_unauthorized(self, chair.user.username, url) + + # GET: check 200 and that the pre-populated body contains the archive URL + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + prefilled_body = q("textarea#id_body").text() + expected_archive_url = f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt" + self.assertIn( + expected_archive_url, + prefilled_body, + "Pre-populated body must contain the I-D archive .txt URL", ) - old_state = draft.get_state("draft-stream-%s" % draft.stream_id) - new_state = State.objects.get( - used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + + # Invalid POST: end_date in the past must not change state + bad_postdict = dict( + end_date=date_today(DEADLINE_TZINFO), + to=q("input#id_to").attr("value"), + subject=q("input#id_subject").attr("value"), + body=prefilled_body, ) - self.assertNotEqual(old_state, new_state) + r = self.client.post(url, bad_postdict) + self.assertEqual(r.status_code, 200, "Invalid POST should re-render the form") + self.assertNotEqual( + doc.get_state_slug("draft-stream-ietf"), + "wg-lc", + "State must not change on invalid POST", + ) + + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.org" + else: + postdict["cc"] = "extracc@example.org" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = prefilled_body + "FGgqbQ$UNeXs" empty_outbox() r = self.client.post( url, - dict( - new_state=new_state.pk, - comment="some comment", - tags=[ - t.pk - for t in draft.tags.filter( - slug__in=get_tags_for_stream_id(draft.stream_id) - ) - ], - ), + postdict, ) self.assertEqual(r.status_code, 302) + doc.refresh_from_db() + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") + + # A StateDocEvent recording the transition must exist + self.assertTrue( + doc.docevent_set.filter(type="changed_state").exists(), + "Expected a changed_state event after issuing WG LC", + ) + + # A stream-s reminder must be active for the doc + self.assertTrue( + DocReminder.objects.filter( + event__doc=doc, type_id="stream-s", active=True + ).exists(), + "Expected an active stream-s reminder after issuing WG LC", + ) + self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + + # outbox[0] is the stream-state-changed notification + self.assertIn(doc.name, outbox[0]["Subject"]) + + # outbox[1] is the WGLC announcement sent to the WG list + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.org", outbox[1]["To"]) + self.assertIn("extracc@example.org", outbox[1]["Cc"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) self.assertIn("WG Last Call", outbox[1]["Subject"]) body = get_payload_text(outbox[1]) self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertIn("FGgqbQ$UNeXs", body) + self.assertIn( + expected_archive_url, + body, + "WGLC email body must contain the I-D archive .txt URL", + ) + + def test_issue_wg_call_for_adoption_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", + ) + form = IssueCallForAdoptionForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", + ) + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", + ) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_call_for_adoption(self): + def _assert_rejected(testcase, doc, person, group=None): + target_acronym = group.acronym if group is not None else doc.group.acronym + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=target_acronym), + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 403) + testcase.client.logout() + + def _verify_call_issued(testcase, doc, chair_role): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=chair_role.group.acronym), + ) + login_testing_unauthorized(testcase, chair_role.person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.com" + self.assertIn(chair_role.group.list_email, postdict["to"]) + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.com" + else: + postdict["cc"] = "extracc@example.com" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + empty_outbox() + r = testcase.client.post( + url, + postdict, + ) + testcase.assertEqual(r.status_code, 302) + doc.refresh_from_db() + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertEqual(len(outbox), 2) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.com", outbox[1]["To"]) + self.assertIn("extracc@example.com", outbox[1]["Cc"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("FGgqbQ$UNeXs", body) + self.client.logout() + return doc + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc = WgRfcFactory(group=already_rfc.group) + already_rfc.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + ind_doc = IndividualDraftFactory() + _assert_rejected(self, ind_doc, rg_chair, rg_doc.group) + + # Successful call issued for doc already in WG + doc = WgDraftFactory(states=[("draft-stream-ietf","wg-cand")]) + chair_role = RoleFactory(name_id="chair",group=doc.group) + _ = _verify_call_issued(self, doc, chair_role) + + # Successful call issued for doc not yet in WG + doc = IndividualDraftFactory() + chair_role = RoleFactory(name_id="chair",group__type_id="wg") + doc = _verify_call_issued(self, doc, chair_role) + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.stream_id, "ietf") + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertCountEqual( + doc.docevent_set.values_list("type", flat=True), + ["changed_state", "changed_group", "changed_stream", "new_revision"] + ) def test_pubreq_validation(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') @@ -2393,6 +2581,188 @@ def test_editorial_metadata(self): self.assertNotIn("IESG", top_level_metadata_headings) self.assertNotIn("IANA", top_level_metadata_headings) +class IetfGroupActionHelperTests(TestCase): + def test_manage_adoption_routing(self): + draft = IndividualDraftFactory() + nobody = PersonFactory() + rgchair = RoleFactory(group__type_id="rg", name_id="chair").person + wgchair = RoleFactory(group__type_id="wg", name_id="chair").person + multichair = RoleFactory(group__type_id="rg", name_id="chair").person + RoleFactory(group__type_id="wg", person=multichair, name_id="chair") + ad = RoleFactory(group__type_id="area", name_id="ad").person + secretary = Role.objects.filter( + name_id="secr", group__acronym="secretariat" + ).first() + self.assertIsNotNone(secretary) + secretary = secretary.person + self.assertFalse( + has_role(rgchair.user, ["Secretariat", "Area Director", "WG Chair"]) + ) + url = urlreverse( + "ietf.doc.views_doc.document_main", kwargs={"name": draft.name} + ) + ask_about_ietf_link = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": draft.name}, + ) + non_ietf_adoption_link = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs={"name": draft.name} + ) + for person in (None, nobody, rgchair, wgchair, multichair, ad, secretary): + if person is not None: + self.client.login( + username=person.user.username, + password=f"{person.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + has_ask_about_ietf_link = len(q(f'a[href="{ask_about_ietf_link}"]')) != 0 + has_non_ietf_adoption_link = ( + len(q(f'a[href="{non_ietf_adoption_link}"]')) != 0 + ) + ask_about_r = self.client.get(ask_about_ietf_link) + ask_about_link_return_code = ask_about_r.status_code + if person == rgchair: + self.assertFalse(has_ask_about_ietf_link) + self.assertTrue(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 403) + elif person in (ad, nobody, None): + self.assertFalse(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual( + ask_about_link_return_code, 302 if person is None else 403 + ) + else: + self.assertTrue(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 200) + self.client.logout() + + def test_ask_about_ietf_adoption_call(self): + # Basic permission tests above + doc = IndividualDraftFactory() + self.assertEqual(doc.docevent_set.count(), 1) + chair_role = RoleFactory(group__type_id="wg", name_id="chair") + chair = chair_role.person + group = chair_role.group + othergroup = GroupFactory(type_id="wg") + url = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": doc.name}, + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.post(url, {"group": othergroup.pk}) + self.assertEqual(r.status_code, 200) + r = self.client.post(url, {"group": group.pk}) + self.assertEqual(r.status_code, 302) + + def test_offer_wg_action_helpers(self): + def _assert_view_presents_buttons(testcase, response, expected): + q = PyQuery(response.content) + for id, expect in expected: + button = q(f"#{id}") + testcase.assertEqual( + len(button) != 0, + expect + ) + + # View rejects access + came_from_draft = WgDraftFactory(states=[("draft","rfc")]) + rfc = WgRfcFactory(group=came_from_draft.group) + came_from_draft.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=rfc.group).person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=came_from_draft.name)) + login_testing_unauthorized(self, rfc_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + self.client.logout() + rg_draft = RgDraftFactory() + rg_chair = RoleFactory(group=rg_draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=rg_draft.name)) + login_testing_unauthorized(self, rg_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,404) + self.client.logout() + + # View offers access + draft = WgDraftFactory() + chair = RoleFactory(group=draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=draft.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-cand")) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", True), + ("id_wglc_button", False), + ("id_pubreq_button", False), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-lc")) + StateDocEventFactory( + doc=draft, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(draft.docevent_set.count(), 2) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", False), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf",slug="chair-w")) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains(response=r,text="Issue Another Working Group Last Call", status_code=200) + other_draft = WgDraftFactory() + self.client.logout() + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=other_draft.name)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains( + response=r, text="Issue Working Group Last Call", status_code=200 + ) + class BallotEmailAjaxTests(TestCase): def test_ajax_build_position_email(self): def _post_json(self, url, json_to_post): @@ -2466,3 +2836,43 @@ def _post_json(self, url, json_to_post): ]: self.assertIn(snippet, response["text"]) + # Non-IETF stream: from address uses balloter.formatted_email(), not role_email("ad"). + # Give the balloter a chair role (in a different RG) with a distinct email to prove + # that chair role address is not used; formatted_email() uses the primary email. + irtf_doc = RgDraftFactory() + non_ad_balloter = PersonFactory(name="Some Irsgmember") + other_rg = GroupFactory(type_id="rg") + chair_role_email = EmailFactory( + person=non_ad_balloter, address="chair-role@example.com" + ) + RoleFactory( + name_id="chair", + group=other_rg, + person=non_ad_balloter, + email=chair_role_email, + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "", + "comment": "cccccc", + "position": "yes", + "balloter": non_ad_balloter.pk, + "docname": irtf_doc.name, + "cc_choices": [], + "additional_cc": "", + } + }, + ) + self.assertTrue(response["success"]) + from_line = next( + line for line in response["text"].split("\n") if line.startswith("From: ") + ) + # formatted_email() uses the primary email, not the chair role email + self.assertIn(non_ad_balloter.email().address, from_line) + self.assertNotIn(chair_role_email.address, from_line) + for snippet in ["cccccc", non_ad_balloter.plain_name(), irtf_doc.name]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/tests_notprepped.py b/ietf/doc/tests_notprepped.py new file mode 100644 index 00000000000..f417aa79310 --- /dev/null +++ b/ietf/doc/tests_notprepped.py @@ -0,0 +1,122 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.conf import settings +from django.utils import timezone +from django.urls import reverse as urlreverse + +from pyquery import PyQuery + +from ietf.doc.factories import WgRfcFactory +from ietf.doc.models import StoredObject +from ietf.doc.storage_utils import store_bytes +from ietf.utils.test_utils import TestCase + + +class NotpreppedRfcXmlTests(TestCase): + def test_editor_source_button_visibility(self): + pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1) + first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC) + post_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC + 1) + + for rfc, expect_button in [(pre_v3, False), (first_v3, True), (post_v3, True)]: + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.document_main", kwargs=dict(name=rfc.name) + ) + ) + self.assertEqual(r.status_code, 200) + buttons = PyQuery(r.content)('a.btn:contains("Get editor source")') + if expect_button: + self.assertEqual(len(buttons), 1, msg=f"rfc_number={rfc.rfc_number}") + expected_href = urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=rfc.rfc_number), + ) + self.assertEqual( + buttons.attr("href"), + expected_href, + msg=f"rfc_number={rfc.rfc_number}", + ) + else: + self.assertEqual(len(buttons), 0, msg=f"rfc_number={rfc.rfc_number}") + + def test_rfcxml_notprepped(self): + number = settings.FIRST_V3_RFC + stored_name = f"notprepped/rfc{number}.notprepped.xml" + url = f"/doc/rfc{number}/notprepped/" + + # 404 for pre-v3 RFC numbers (no document needed) + r = self.client.get(f"/doc/rfc{number - 1}/notprepped/") + self.assertEqual(r.status_code, 404) + + # 404 when no RFC document exists in the database + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 404 when RFC document exists but has no StoredObject + WgRfcFactory(rfc_number=number) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 404 when StoredObject exists but backing storage is missing (FileNotFoundError) + now = timezone.now() + StoredObject.objects.create( + store="rfc", + name=stored_name, + sha384="a" * 96, + len=0, + store_created=now, + created=now, + modified=now, + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 200 with correct content-type, attachment disposition, and body when object is fully stored + xml_content = b"test" + store_bytes("rfc", stored_name, xml_content, allow_overwrite=True) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "application/xml") + self.assertEqual( + r["Content-Disposition"], + f'attachment; filename="rfc{number}.notprepped.xml"', + ) + self.assertEqual(b"".join(r.streaming_content), xml_content) + + def test_rfcxml_notprepped_wrapper(self): + number = settings.FIRST_V3_RFC + + # 404 for pre-v3 RFC numbers (no document needed) + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number - 1), + ) + ) + self.assertEqual(r.status_code, 404) + + # 404 when no RFC document exists in the database + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number), + ) + ) + self.assertEqual(r.status_code, 404) + + # 200 with rendered template when RFC document exists + rfc = WgRfcFactory(rfc_number=number) + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number), + ) + ) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertIn(str(rfc.rfc_number), q("h1").text()) + download_url = urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped", kwargs=dict(number=number) + ) + self.assertEqual(len(q(f'a.btn[href="{download_url}"]')), 1) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 8c1fc99ffe2..82d1b5c232b 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -822,7 +822,7 @@ def test_complete_review_upload_content(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, assignment.review_request.team.list_email) - for author in assignment.review_request.doc.authors(): + for author in assignment.review_request.doc.author_persons(): self.assertContains(r, author.formatted_email()) # faulty post diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 29689cd5964..48db95d0474 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,18 +1,20 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved -import debug # pyflakes:ignore import datetime from unittest import mock from pathlib import Path +from celery.exceptions import Retry from django.conf import settings +from django.test.utils import override_settings from django.utils import timezone +from typesense import exceptions as typesense_exceptions from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from .factories import DocumentFactory, NewRevisionDocEventFactory +from .factories import DocumentFactory, NewRevisionDocEventFactory, WgRfcFactory from .models import Document, NewRevisionDocEvent from .tasks import ( expire_ids_task, @@ -22,8 +24,11 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + rebuild_searchindex_task, + update_rfc_searchindex_task, ) + class TaskTests(TestCase): @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -87,7 +92,7 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire): self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) - + # Check that it runs even if exceptions occur mock_get_expired.reset_mock() mock_expire.reset_mock() @@ -111,9 +116,92 @@ def test_investigate_fragment_task(self): retval, {"name_fragment": "some fragment", "results": investigation_results} ) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entry") + @mock.patch("ietf.doc.tasks.searchindex.enabled") + def test_update_rfc_searchindex_task( + self, mock_searchindex_enabled, mock_create_entry + ): + mock_searchindex_enabled.return_value = False + + self.assertFalse(Document.objects.filter(rfc_number=5073).exists()) + rfc = WgRfcFactory() + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertFalse(mock_create_entry.called) + + mock_searchindex_enabled.return_value = True + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertTrue(mock_create_entry.called) + + with override_settings(SEARCHINDEX_CONFIG={"TASK_MAX_RETRIES": 0}): + # Try a non-retryable error (there are others) + mock_create_entry.side_effect = typesense_exceptions.RequestMalformed + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) # no retry + # Now what should be a retryable error + mock_create_entry.side_effect = typesense_exceptions.Timeout + with self.assertRaises(Retry): + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entries") + @mock.patch("ietf.doc.tasks.searchindex.upsert_presets") + @mock.patch("ietf.doc.tasks.searchindex.create_collection") + @mock.patch("ietf.doc.tasks.searchindex.delete_collection") + def test_rebuild_searchindex_task( + self, mock_delete, mock_create, mock_presets, mock_update + ): + rfcs = WgRfcFactory.create_batch(10) + rebuild_searchindex_task() + self.assertFalse(mock_delete.called) + self.assertFalse(mock_create.called) + self.assertTrue(mock_presets.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_presets.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_presets.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_presets.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task( + drop_collection=True, batchsize=3, upsert_presets=False + ) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertFalse(mock_presets.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + self.assertEqual(mock_update.call_args.kwargs["batchsize"], 3) + class Idnits2SupportTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ + "DERIVED_DIR" + ] @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): @@ -151,7 +239,9 @@ def setUp(self): ) # a couple that should always be ignored NewRevisionDocEventFactory( - time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc" # not a draft + time=now - datetime.timedelta(days=6), + rev="09", + doc__type_id="rfc", # not a draft ) NewRevisionDocEventFactory( type="changed_document", # not a "new_revision" type @@ -164,7 +254,9 @@ def setUp(self): @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_all_drafts_task( + self, mock_create, mock_ensure_path + ): generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( @@ -193,12 +285,15 @@ def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensur @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task( + self, mock_create, mock_ensure_path + ): # default args - look back 7 days generate_draft_bibxml_files_task() self.assertTrue(mock_ensure_path.called) self.assertCountEqual( - mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)] + mock_create.call_args_list, + [mock.call(self.young_event.doc, self.young_event.rev)], ) mock_create.reset_mock() mock_ensure_path.reset_mock() @@ -223,7 +318,9 @@ def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_en @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value( + self, mock_create, mock_ensure_path + ): with self.assertRaises(ValueError): generate_draft_bibxml_files_task(days=0) self.assertFalse(mock_create.called) diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index 7db59819da8..ba672cd8479 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -1,25 +1,36 @@ # Copyright The IETF Trust 2020, All Rights Reserved import datetime +from io import BytesIO + +import mock import debug # pyflakes:ignore +import requests from pathlib import Path from unittest.mock import call, patch from django.conf import settings +from django.core.files.storage import storages from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone + +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory +from ietf.doc.factories import BallotPositionDocEventFactory from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory -from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor +from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor, StoredObject from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, - ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file) + ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, + last_ballot_doc_revision) +from ietf.doc.storage_utils import store_str from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -387,13 +398,13 @@ def test_requires_txt_or_xml(self): result = rebuild_reference_relations(self.doc, {}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no Internet-Draft file is given') result = rebuild_reference_relations(self.doc, {'md': 'cant-do-this.md'}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no XML or plaintext file is given') @patch.object(XMLDraft, 'get_refs') @@ -533,3 +544,156 @@ def test_update_draft_bibxml_file(self, mock): self.assertEqual(mock.call_count, 1) self.assertEqual(mock.call_args, call(doc, "26")) self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + +class LastBallotDocRevisionTests(TestCase): + def test_last_ballot_doc_revision(self): + now = timezone.now() + ad = Person.objects.get(user__username="ad") + bpde_with_null_send_email = BallotPositionDocEventFactory( + time=now - datetime.timedelta(minutes=30), + send_email=None, + ) + ballot = bpde_with_null_send_email.ballot + BallotPositionDocEventFactory( + ballot=ballot, + balloter=ad, + pos_id='noobj', + comment='Commentary', + comment_time=timezone.now(), + send_email=None, + ) + doc = bpde_with_null_send_email.doc + rev = bpde_with_null_send_email.rev + nobody = PersonFactory() + self.assertIsNone(last_ballot_doc_revision(doc, nobody)) + self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) + + +class UtilsRedTests(TestCase): + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer_not_configured(self, mock_post, mock_log): + with override_settings(): + try: + del settings.CUSTOM_SETTING_NAME + except AttributeError: + pass + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + mock_log.reset_mock() + with override_settings(TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL=None): + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertFalse(mock_post.called) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=requests.Timeout()) + def test_trigger_red_precomputer_swallows_timeout_exception( + self, mock_post, mock_log + ): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertFalse(exception_raised) + self.assertEqual(mock_log.call_count, 2) + # only checking the last log call + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + self.assertIn("POST request timed out", mock_args[0]) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=Exception()) + def test_trigger_red_precomputer_does_not_swallow_too_much(self, mock_post): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertTrue(exception_raised) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + DEFAULT_REQUESTS_TIMEOUT=314159265, + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer(self, mock_post, mock_log): + mock_post.return_value = mock.Mock(status_code=200) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertTrue(mock_post.called) + _, mock_kwargs = mock_post.call_args + self.assertIn("url", mock_kwargs) + self.assertEqual(mock_kwargs["url"], "urlbits") + self.assertIn("json", mock_kwargs) + self.assertEqual(mock_kwargs["json"], {"rfcs": "1,2,3"}) + self.assertIn("timeout", mock_kwargs) + self.assertEqual(mock_kwargs["timeout"], 314159265) + self.assertEqual(mock_log.call_count, 1) # Not testing the first info log value + mock_log.reset_mock() + mock_post.reset_mock() + mock_post.return_value = mock.Mock( + status_code=500, + ) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 2) + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + expected = f"POST request failed for {settings.TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL} : status_code=500" + self.assertEqual(mock_args[0], expected) + + +class UtilsR2TestCase(TestCase): + def test_rfcs_are_in_r2(self): + rfcs = WgRfcFactory.create_batch(2) + rfc_name_list = [rfc.name for rfc in rfcs] + rfc_number_list = [rfc.rfc_number for rfc in rfcs] + r2_rfc_bucket = storages["r2-rfc"] + # Right now the various doc Factories do not populate any content + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 0, + ) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + for rfc in rfcs: + store_str( + kind="rfc", + name=f"testartifact/{rfc.name}.testartifact", + content="", + doc_name=rfc.name, + doc_rev=None, + ) + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 2, + ) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[0].name}.testartifact", BytesIO(b"")) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[1].name}.testartifact", BytesIO(b"")) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + + + diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py new file mode 100644 index 00000000000..ec3cd85893f --- /dev/null +++ b/ietf/doc/tests_utils_rfc_json.py @@ -0,0 +1,500 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import json +from unittest import mock + +from django.core.files.base import ContentFile +from django.core.files.storage import storages +from django.test.utils import override_settings +from django.utils import timezone + +from ietf.doc.factories import ( + PublishedRfcDocEventFactory, + RfcAuthorFactory, + RfcFactory, + RgRfcFactory, + WgDraftFactory, + WgRfcFactory, +) +from ietf.doc.models import RelatedDocument +from ietf.doc.utils_rfc_json import generate_rfc_json +from ietf.group.factories import GroupFactory +from ietf.name.models import StdLevelName +from ietf.utils.test_utils import TestCase + + +def _put_pub_levels(rfc_number, slug, path="input/"): + """Write a minimal publication-std-levels.json to the red bucket.""" + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{path}publication-std-levels.json", + ContentFile( + json.dumps([{"number": rfc_number, "publication_std_level": slug}]) + ), + ) + + +def _put_errata(rfc_number, path="other/errata.json"): + """Write an errata.json with one entry for the given RFC.""" + red_bucket = storages["red_bucket"] + red_bucket.save( + path, + ContentFile( + json.dumps( + [{"doc-id": f"RFC{rfc_number}", "errata_status_code": "Reported"}] + ) + ), + ) + + +def _put_empty_errata(path="other/errata.json"): + red_bucket = storages["red_bucket"] + red_bucket.save(path, ContentFile(json.dumps([]))) + + +def _put_april_first(rfc_number, path="input/"): + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{path}april-first-rfc-numbers.json", + ContentFile(json.dumps([rfc_number])), + ) + + +def _read_json(rfc_number): + from ietf.blobdb.models import Blob + + blob = Blob.objects.get(bucket="rfc", name=f"json/rfc{rfc_number}.json") + return json.loads(bytes(blob.content)) + + +@override_settings( + RFCINDEX_INPUT_PATH="input/", + ERRATA_JSON_BLOB_NAME="other/errata.json", + RFC_EDITOR_ERRATA_BASE_URL="https://www.rfc-editor.org/errata/", +) +class GenerateRfcJsonTests(TestCase): + def setUp(self): + super().setUp() + # Minimal red_bucket blobs required by all tests + red_bucket = storages["red_bucket"] + red_bucket.save( + "input/april-first-rfc-numbers.json", ContentFile(json.dumps([])) + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + for name in [ + "input/publication-std-levels.json", + "input/april-first-rfc-numbers.json", + "other/errata.json", + ]: + try: + red_bucket.delete(name) + except Exception: + pass + super().tearDown() + + def test_missing_rfc_logs_and_returns(self): + """Calling for a nonexistent RFC number logs and returns without raising.""" + # Should not raise; no blob should be written + generate_rfc_json(999999, pub_levels={}) + from ietf.blobdb.models import Blob + + self.assertFalse( + Blob.objects.filter(bucket="rfc", name="json/rfc999999.json").exists() + ) + + def test_all_fields(self): + """All 17 JSON fields are populated correctly from a fully-populated RFC.""" + area = GroupFactory(type_id="area") + wg = GroupFactory(type_id="wg", parent=area) + rfc = PublishedRfcDocEventFactory( + time="2021-05-01T00:00:00Z", + doc=WgRfcFactory( + group=wg, + stream_id="ietf", + std_level_id="ps", + pages=42, + abstract="Test abstract.", + keywords=["foo", "bar"], + ), + ).doc + author = RfcAuthorFactory(document=rfc, is_editor=False) + editor = RfcAuthorFactory(document=rfc, is_editor=True) + + obsoletes_rfc = RfcFactory() + updated_rfc = RfcFactory() + RelatedDocument.objects.create( + source=rfc, target=obsoletes_rfc, relationship_id="obs" + ) + RelatedDocument.objects.create( + source=rfc, target=updated_rfc, relationship_id="updates" + ) + obsoleted_by_rfc = RfcFactory() + updated_by_rfc = RfcFactory() + RelatedDocument.objects.create( + source=obsoleted_by_rfc, target=rfc, relationship_id="obs" + ) + RelatedDocument.objects.create( + source=updated_by_rfc, target=rfc, relationship_id="updates" + ) + + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["doc_id"], f"RFC{rfc.rfc_number}") + self.assertEqual(data["title"], rfc.title) + self.assertEqual(data["abstract"], "Test abstract.") + self.assertEqual(data["page_count"], "42") + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + self.assertEqual(data["status"], "PROPOSED STANDARD") + self.assertEqual(data["pub_date"], "May 2021") + self.assertEqual(data["keywords"], ["foo", "bar"]) + self.assertEqual(data["see_also"], []) + self.assertEqual(data["doi"], f"10.17487/RFC{rfc.rfc_number}") + self.assertIsNone(data["errata_url"]) + self.assertIsNone(data["draft"]) + + # authors — non-editor first (lower order), then editor + self.assertEqual( + data["authors"], + [author.titlepage_name, f"{editor.titlepage_name}, Ed."], + ) + + # relationships + self.assertIn(f"RFC{obsoletes_rfc.rfc_number}", data["obsoletes"]) + self.assertIn(f"RFC{updated_rfc.rfc_number}", data["updates"]) + self.assertIn(f"RFC{obsoleted_by_rfc.rfc_number}", data["obsoleted_by"]) + self.assertIn(f"RFC{updated_by_rfc.rfc_number}", data["updated_by"]) + + def test_pub_status_differs_from_status(self): + """pub_status reflects publication-std-levels.json; status reflects current std_level.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(std_level_id="hist"), + ).doc + # Record was published as "ps" but is now "hist" + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + self.assertEqual(data["status"], "HISTORIC") + + def test_errata_url_set_when_errata_exist(self): + """errata_url is populated when errata.json has any entry for the RFC.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_errata(rfc.rfc_number) + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual( + data["errata_url"], + f"https://www.rfc-editor.org/errata/rfc{rfc.rfc_number}", + ) + + def test_draft_field_includes_revision(self): + """draft field is '-' when the RFC originated from a draft.""" + draft = WgDraftFactory(rev="07") + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["draft"], f"{draft.name}-07") + + def test_errata_url_none_when_no_errata(self): + """errata_url is None when errata.json has no entries for the RFC.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertIsNone(data["errata_url"]) + + def test_errata_failure_yields_null_url(self): + """If reading errata.json fails, errata_url is null and no exception is raised.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + # Deliberately do not put errata blob — FileNotFoundError expected + + generate_rfc_json(rfc.rfc_number) # must not raise + data = _read_json(rfc.rfc_number) + self.assertIsNone(data["errata_url"]) + + def test_second_call_overwrites(self): + """Calling generate_rfc_json twice does not raise AlreadyExistsError.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + generate_rfc_json(rfc.rfc_number) # must not raise + + def test_april_first_date_format(self): + """April Fools RFCs get '1 April YYYY' date format.""" + rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=WgRfcFactory(), + ).doc + red_bucket = storages["red_bucket"] + red_bucket.delete("input/april-first-rfc-numbers.json") + _put_april_first(rfc.rfc_number) + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["pub_date"], "1 April 2020") + + def test_empty_keywords_filtered(self): + """Empty-string keywords are stripped from the keywords list.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(keywords=["foo", "", "bar"]), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["keywords"], ["foo", "bar"]) + + def test_non_april_first_april_date(self): + """An April publication that is NOT in the April Fools list gets 'April YYYY'.""" + rfc = PublishedRfcDocEventFactory( + time="2020-04-15T12:00:00Z", + doc=WgRfcFactory(), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["pub_date"], "April 2020") + + def test_source_ietf_wg(self): + """IETF-stream WG RFC: source is the group's full name.""" + area = GroupFactory(type_id="area") + wg = GroupFactory(type_id="wg", parent=area) + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(group=wg, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], wg.name) + + def test_source_ietf_no_wg(self): + """IETF-stream individual RFC (group acronym 'none'): source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory( + group=GroupFactory(acronym="none"), + stream_id="ietf", + ), + ).doc + rfc.group.parent = area + rfc.group.save() + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + + def test_source_ietf_area(self): + """IETF-stream RFC with area-type group: source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + area.parent = GroupFactory() + area.save() + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(group=area, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + + def test_source_iab(self): + """IAB-stream RFC: source is 'IAB'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="iab", group=GroupFactory(acronym="iab")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IAB") + + def test_source_ise(self): + """ISE-stream RFC: source is 'INDEPENDENT'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="ise", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "INDEPENDENT") + + def test_source_irtf_rg(self): + """IRTF-stream RG RFC: source is the group's full name.""" + rfc = PublishedRfcDocEventFactory(doc=RgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], rfc.group.name) + + def test_source_irtf_no_rg(self): + """IRTF-stream RFC with no specific RG (group acronym 'none'): source is 'IRTF'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="irtf", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IRTF") + + def test_pub_levels_passed_in(self): + """When pub_levels is passed in, get_publication_std_levels() is not called.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_empty_errata() + + ps_level = StdLevelName.objects.get(slug="ps") + pub_levels = {rfc.rfc_number: ps_level} + + with mock.patch( + "ietf.doc.utils_rfc_json.get_publication_std_levels" + ) as mock_get: + generate_rfc_json(rfc.rfc_number, pub_levels=pub_levels) + mock_get.assert_not_called() + + data = _read_json(rfc.rfc_number) + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + + def test_pub_levels_fetch_failure_returns_without_writing(self): + """If get_publication_std_levels() raises, function logs and returns without writing a blob.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + _put_empty_errata() + + with mock.patch( + "ietf.doc.utils_rfc_json.get_publication_std_levels", + side_effect=FileNotFoundError("not found"), + ): + generate_rfc_json(rfc.rfc_number) # must not raise + + from ietf.blobdb.models import Blob + + self.assertFalse( + Blob.objects.filter( + bucket="rfc", name=f"json/rfc{rfc.rfc_number}.json" + ).exists() + ) + + def test_pub_status_fallback_to_status_for_recent_rfc(self): + """RFC missing from pub_levels but published within 2 days: pub_status falls back to current std_level.""" + now = timezone.now() + rfc = PublishedRfcDocEventFactory( + time=now - datetime.timedelta(hours=1), + doc=WgRfcFactory(std_level_id="ps"), + ).doc + _put_empty_errata() + + with mock.patch("ietf.doc.utils_rfc_json.timezone") as mock_tz: + mock_tz.now.return_value = now + generate_rfc_json(rfc.rfc_number, pub_levels={}) + + data = _read_json(rfc.rfc_number) + self.assertEqual(data["pub_status"], "PROPOSED STANDARD") + + def test_pub_status_unknown_for_old_rfc_missing_from_levels(self): + """RFC missing from pub_levels and published more than 2 days ago: pub_status is UNKNOWN.""" + rfc = PublishedRfcDocEventFactory( + time="2020-01-01T00:00:00Z", + doc=WgRfcFactory(std_level_id="ps"), + ).doc + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number, pub_levels={}) + + data = _read_json(rfc.rfc_number) + self.assertEqual(data["pub_status"], "UNKNOWN") + + def _assert_no_blob_written(self, rfc): + from ietf.blobdb.models import Blob + + self.assertFalse( + Blob.objects.filter( + bucket="rfc", name=f"json/rfc{rfc.rfc_number}.json" + ).exists() + ) + + def test_source_malformed_no_stream(self): + """RFC with stream=None triggers assertion (logged to admins in production) and no blob is written.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + rfc.stream = None + rfc.save() + ps_level = StdLevelName.objects.get(slug="ps") + + with self.assertRaises(AssertionError): + generate_rfc_json(rfc.rfc_number, pub_levels={rfc.rfc_number: ps_level}) + + self._assert_no_blob_written(rfc) + + def test_source_malformed_no_group(self): + """RFC with group=None triggers assertion (logged to admins in production) and no blob is written.""" + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + rfc.group = None + rfc.save() + ps_level = StdLevelName.objects.get(slug="ps") + + with self.assertRaises(AssertionError): + generate_rfc_json(rfc.rfc_number, pub_levels={rfc.rfc_number: ps_level}) + + self._assert_no_blob_written(rfc) + + def test_source_ietf_malformed_no_area(self): + """IETF-stream RFC whose group has no parent triggers assertion (logged to admins in production) and no blob is written.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="ietf", group=GroupFactory()), + ).doc + rfc.group.parent = None + rfc.group.save() + ps_level = StdLevelName.objects.get(slug="ps") + + with self.assertRaises(AssertionError): + generate_rfc_json(rfc.rfc_number, pub_levels={rfc.rfc_number: ps_level}) + + self._assert_no_blob_written(rfc) diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 8e9c0569e22..0c13503b787 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -99,6 +99,8 @@ url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main), url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex), + url(r'^rfc(?P[0-9]+)/notprepped/$' , views_doc.rfcxml_notprepped), + url(r'^rfc(?P[0-9]+)/notprepped-wrapper/$', views_doc.rfcxml_notprepped_wrapper), url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state), url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref), url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml), @@ -125,6 +127,7 @@ url(r'^%(name)s/edit/info/$' % settings.URL_REGEXPS, views_draft.edit_info), url(r'^%(name)s/edit/requestresurrect/$' % settings.URL_REGEXPS, views_draft.request_resurrect), url(r'^%(name)s/edit/submit-to-iesg/$' % settings.URL_REGEXPS, views_draft.to_iesg), + url(r'^%(name)s/edit/issue-wg-lc/$' % settings.URL_REGEXPS, views_draft.issue_wg_lc), url(r'^%(name)s/edit/resurrect/$' % settings.URL_REGEXPS, views_draft.resurrect), url(r'^%(name)s/edit/addcomment/$' % settings.URL_REGEXPS, views_doc.add_comment), @@ -143,9 +146,13 @@ url(r'^%(name)s/edit/shepherdemail/$' % settings.URL_REGEXPS, views_draft.change_shepherd_email), url(r'^%(name)s/edit/shepherdwriteup/$' % settings.URL_REGEXPS, views_draft.edit_shepherd_writeup), url(r'^%(name)s/edit/requestpublication/$' % settings.URL_REGEXPS, views_draft.request_publication), + url(r'^%(name)s/edit/ask-about-ietf-adoption/$' % settings.URL_REGEXPS, views_draft.ask_about_ietf_adoption_call), url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft), + url(r'^%(name)s/edit/issue-wg-call-for-adoption/%(acronym)s/$' % settings.URL_REGEXPS, views_draft.issue_wg_call_for_adoption), + url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft), url(r'^%(name)s/edit/state/(?Pdraft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state), + url(r'^%(name)s/edit/wg-action-helpers/$' % settings.URL_REGEXPS, views_draft.offer_wg_action_helpers), url(r'^%(name)s/edit/state/statement/$' % settings.URL_REGEXPS, views_statement.change_statement_state), url(r'^%(name)s/edit/clearballot/(?P[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot), diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 115b28b09b9..5f8f587c59f 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -4,6 +4,7 @@ import datetime import io +import json import math import os import re @@ -13,7 +14,7 @@ from dataclasses import dataclass from hashlib import sha384 from pathlib import Path -from typing import Iterator, Optional, Union +from typing import Iterator, Optional, Union, Iterable from zoneinfo import ZoneInfo from django.conf import settings @@ -33,14 +34,25 @@ from ietf.community.models import CommunityList from ietf.community.utils import docs_tracked_by_community_list -from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor +from ietf.doc.models import ( + DocHistory, + DocHistoryAuthor, + Document, + DocumentAuthor, + EditedRfcAuthorsDocEvent, + RfcAuthor, + State, + StoredObject, +) from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent -from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent +from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent +from ietf.doc.storage_utils import force_replication from ietf.name.models import DocReminderTypeName, DocRelationshipName from ietf.group.models import Role, Group, GroupFeatures from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor from ietf.person.models import Email, Person +from ietf.person.utils import get_active_balloters from ietf.review.models import ReviewWish from ietf.utils import draft, log from ietf.utils.mail import parseaddr, send_mail @@ -533,7 +545,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.clear() if tags.removed("need-rev"): # Removed the 'need-rev' tag - drop authors from the action holders list - DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete() + DocumentActionHolder.objects.filter(document=doc, person__in=doc.author_persons()).delete() elif tags.added("need-rev"): # Remove the AD if we're asking for a new revision DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete() @@ -548,7 +560,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.add(doc.ad) # Authors get the action if a revision is needed if tags.added("need-rev"): - for auth in doc.authors(): + for auth in doc.author_persons(): doc.action_holders.add(auth) # Now create an event if we changed the set @@ -560,6 +572,40 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, ) +def _change_field_and_describe( + author: DocumentAuthor | RfcAuthor, + field: str, + newval, + field_display_name: str | None = None, +): + # make the change + oldval = getattr(author, field) + setattr(author, field, newval) + + was_empty = oldval is None or len(str(oldval)) == 0 + now_empty = newval is None or len(str(newval)) == 0 + + # describe the change + if oldval == newval: + return None + else: + if field_display_name is None: + field_display_name = field + + if was_empty and not now_empty: + return 'set {field} to "{new}"'.format( + field=field_display_name, new=newval + ) + elif now_empty and not was_empty: + return 'cleared {field} (was "{old}")'.format( + field=field_display_name, old=oldval + ) + else: + return 'changed {field} from "{old}" to "{new}"'.format( + field=field_display_name, old=oldval, new=newval + ) + + def update_documentauthors(doc, new_docauthors, by=None, basis=None): """Update the list of authors for a document @@ -572,27 +618,6 @@ def update_documentauthors(doc, new_docauthors, by=None, basis=None): used. These objects will not be saved, their attributes will be used to create new DocumentAuthor instances. (The document and order fields will be ignored.) """ - def _change_field_and_describe(auth, field, newval): - # make the change - oldval = getattr(auth, field) - setattr(auth, field, newval) - - was_empty = oldval is None or len(str(oldval)) == 0 - now_empty = newval is None or len(str(newval)) == 0 - - # describe the change - if oldval == newval: - return None - else: - if was_empty and not now_empty: - return 'set {field} to "{new}"'.format(field=field, new=newval) - elif now_empty and not was_empty: - return 'cleared {field} (was "{old}")'.format(field=field, old=oldval) - else: - return 'changed {field} from "{old}" to "{new}"'.format( - field=field, old=oldval, new=newval - ) - persons = [] changes = [] # list of change descriptions @@ -636,6 +661,123 @@ def _change_field_and_describe(auth, field, newval): ) for change in changes ] + +def update_rfcauthors( + rfc: Document, new_rfcauthors: Iterable[RfcAuthor], by: Person | None = None +) -> Iterable[EditedRfcAuthorsDocEvent]: + def _find_matching_author( + author_to_match: RfcAuthor, existing_authors: Iterable[RfcAuthor] + ) -> RfcAuthor | None: + """Helper to find a matching existing author""" + if author_to_match.person_id is not None: + for candidate in existing_authors: + if candidate.person_id == author_to_match.person_id: + return candidate + return None # no match + # author does not have a person, match on titlepage name + for candidate in existing_authors: + if candidate.titlepage_name == author_to_match.titlepage_name: + return candidate + return None # no match + + def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor: + """Helper to create an equivalent RfcAuthor from a DocumentAuthor""" + return RfcAuthor( + document_id=docauthor.document_id, + titlepage_name=docauthor.person.plain_name(), # closest thing we have + is_editor=False, + person_id=docauthor.person_id, + affiliation=docauthor.affiliation, + country=docauthor.country, + order=docauthor.order, + ) + + # Is this the first time this document is getting an RfcAuthor? If so, the + # updates will need to account for the model change. + converting_from_docauthors = not rfc.rfcauthor_set.exists() + + if converting_from_docauthors: + original_authors = [ + _rfcauthor_from_documentauthor(da) for da in rfc.documentauthor_set.all() + ] + else: + original_authors = list(rfc.rfcauthor_set.all()) + + authors_to_commit = [] + changes = [] + for order, new_author in enumerate(new_rfcauthors): + matching_author = _find_matching_author(new_author, original_authors) + if matching_author is not None: + # Update existing matching author using new_author data + authors_to_commit.append(matching_author) + original_authors.remove(matching_author) # avoid reuse + # Describe changes to this author + author_changes = [] + # Update fields other than order + for field in ["titlepage_name", "is_editor", "affiliation", "country"]: + author_changes.append( + _change_field_and_describe( + matching_author, + field, + getattr(new_author, field), + # List titlepage_name as "name" in logs + "name" if field == "titlepage_name" else field, + ) + ) + # Update order + author_changes.append( + _change_field_and_describe(matching_author, "order", order + 1) + ) + matching_author.save() + author_change_summary = ", ".join( + [ch for ch in author_changes if ch is not None] + ) + if len(author_change_summary) > 0: + changes.append( + 'Changed author "{name}": {summary}'.format( + name=matching_author.titlepage_name, + summary=author_change_summary, + ) + ) + else: + # No author matched, so update the new_author and use that + new_author.document = rfc + new_author.order = order + 1 + new_author.save() + if new_author.person_id is not None: + person_desc = f"Person {new_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Added "{new_author.titlepage_name}" ({person_desc}) as author' + ) + # Any authors left in original_authors are no longer in the list, so remove them + for removed_author in original_authors: + # Skip actual removal of old authors if we are converting from the + # DocumentAuthor models - the original_authors were just stand-ins anyway. + if not converting_from_docauthors: + removed_author.delete() + if removed_author.person_id is not None: + person_desc = f"Person {removed_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Removed "{removed_author.titlepage_name}" ({person_desc}) as author' + ) + # Create DocEvents, but leave it up to caller to save + if by is None: + by = Person.objects.get(name="(System)") + return [ + EditedRfcAuthorsDocEvent( + type="edited_authors", + by=by, + doc=rfc, + desc=change, + ) + for change in changes + ] + + def update_reminder(doc, reminder_type_slug, event, due_date): reminder_type = DocReminderTypeName.objects.get(slug=reminder_type_slug) @@ -687,6 +829,22 @@ def nice_consensus(consensus): } return mapping[consensus] +def last_ballot_doc_revision(doc, person): + """ Return the document revision for the most recent ballot position + by the provided user. """ + ballot = doc.active_ballot() + if ballot is None or person is None: + return None + balloters = get_active_balloters(ballot.ballot_type) + if person not in balloters: + return None + position_queryset = BallotPositionDocEvent.objects.filter(type="changed_ballot_position", balloter=person, ballot=ballot).order_by("-time") + if not position_queryset.exists(): + return None + ballot_time = position_queryset.first().time + doc_rev = NewRevisionDocEvent.objects.filter(doc=doc, time__lte=ballot_time).order_by('-time').first().rev + return doc_rev + def has_same_ballot(doc, date1, date2=None): """ Test if the most recent ballot created before the end of date1 is the same as the most recent ballot created before the @@ -799,50 +957,93 @@ def rebuild_reference_relations(doc, filenames): filenames should be a dict mapping file ext (i.e., type) to the full path of each file. """ - if doc.type.slug != 'draft': + if doc.type.slug not in ["draft", "rfc"]: + log.log(f"rebuild_reference_relations called for non draft/rfc doc {doc.name}") return None - # try XML first - if 'xml' in filenames: - refs = XMLDraft(filenames['xml']).get_refs() - elif 'txt' in filenames: - filename = filenames['txt'] - try: - refs = draft.PlaintextDraft.from_file(filename).get_refs() - except IOError as e: - return { 'errors': ["%s :%s" % (e.strerror, filename)] } - else: - return {'errors': ['No Internet-Draft text available for rebuilding reference relations. Need XML or plaintext.']} - doc.relateddocument_set.filter(relationship__slug__in=['refnorm','refinfo','refold','refunk']).delete() + if "xml" not in filenames and "txt" not in filenames: + log.log(f"rebuild_reference_relations error: no file available for {doc.name}") + return { + "errors": [ + "No file available for rebuilding reference relations. Need XML or plaintext." + ] + } + else: + try: + # try XML first + if "xml" in filenames: + refs = XMLDraft(filenames["xml"]).get_refs() + elif "txt" in filenames: + filename = filenames["txt"] + refs = draft.PlaintextDraft.from_file(filename).get_refs() + except (IOError, UnicodeDecodeError) as e: + log.log(f"rebuild_reference_relations error: On {doc.name}: {e}") + return {"errors": [f"{e}: {filename}"]} + + before = set(doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).values_list("relationship__slug","target__name")) warnings = [] errors = [] unfound = set() - for ( ref, refType ) in refs.items(): - refdoc = Document.objects.filter(name=ref) - if not refdoc and re.match(r"^draft-.*-\d{2}$", ref): - refdoc = Document.objects.filter(name=ref[:-3]) + intended = set() + names = [ref for ref in refs] + names.extend([ref[:-3] for ref in refs if re.match(r"^draft-.*-\d{2}$", ref)]) + queryset = Document.objects.filter(name__in=names) + for ref, refType in refs.items(): + refdoc = queryset.filter(name=ref) + if not refdoc.exists() and re.match(r"^draft-.*-\d{2}$", ref): + refdoc = queryset.filter(name=ref[:-3]) count = refdoc.count() if count == 0: - unfound.add( "%s" % ref ) + unfound.add("%s" % ref) continue elif count > 1: - errors.append("Too many Document objects found for %s"%ref) + log.unreachable("2026-3-16") # This branch is holdover from DocAlias + errors.append("Too many Document objects found for %s" % ref) else: # Don't add references to ourself if doc != refdoc[0]: - RelatedDocument.objects.get_or_create( source=doc, target=refdoc[ 0 ], relationship=DocRelationshipName.objects.get( slug='ref%s' % refType ) ) + intended.add((f"ref{refType}", refdoc[0].name)) + if unfound: - warnings.append('There were %d references with no matching Document'%len(unfound)) + warnings.append( + "There were %d references with no matching Document" % len(unfound) + ) + + if intended != before: + for slug, name in before-intended: + doc.relateddocument_set.filter(target__name=name, relationship_id=slug).delete() + for slug, name in intended-before: + doc.relateddocument_set.create( + target=queryset.get(name=name), + relationship_id=slug + ) + after = set(doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).values_list("relationship__slug","target__name")) + if after != intended: + errors.append("Attempted changed didn't achieve intended results") + changed_references = True + else: + changed_references = False ret = {} if errors: - ret['errors']=errors + ret["errors"] = errors if warnings: - ret['warnings']=warnings + ret["warnings"] = warnings if unfound: - ret['unfound']=list(unfound) + ret["unfound"] = list(unfound) + + logmsg = f"rebuild_reference_relations for {doc.name}: " + logmsg += "changed references" if changed_references else "references unchanged" + if ret: + logmsg += f" {json.dumps(ret)}" + + log.log(logmsg) return ret @@ -1073,9 +1274,6 @@ def build_file_urls(doc: Union[Document, DocHistory]): label = "plain text" if t == "txt" else t file_urls.append((label, base + doc.name + "." + t)) - if "pdf" not in found_types and "txt" in found_types: - file_urls.append(("pdf", base + "pdfrfc/" + doc.name + ".txt.pdf")) - if "txt" in found_types: file_urls.append(("htmlized", urlreverse('ietf.doc.views_doc.document_html', kwargs=dict(name=doc.name)))) if doc.tags.filter(slug="verified-errata").exists(): @@ -1515,3 +1713,23 @@ def update_or_create_draft_bibxml_file(doc, rev): def ensure_draft_bibxml_path_exists(): (Path(settings.BIBXML_BASE_PATH) / "bibxml-ids").mkdir(exist_ok=True) + + +def replicate_stored_objects_for_document(doc: Document) -> int: + """Sync all StoredObjects associated with doc to the replica blob store + + Returns count of StoredObjects queued for replication (which may or may not + be replicated, depending on whether replication is enabled / the storages are + actually BlobdbStorage instances, etc). + """ + # n.b., StoredObjects have a nullable doc_rev field, but Documents do not. + # Until / unless we straighten that out, treat "" and None equivalently when + # matching rev. + qs_matching_rev = StoredObject.objects.filter(doc_rev=doc.rev) + if doc.rev == "": + qs_matching_rev |= StoredObject.objects.filter(doc_rev__isnull=True) + count = 0 + for stored_object in qs_matching_rev.filter(doc_name=doc.name): + force_replication(kind=stored_object.store, name=stored_object.name) + count += 1 + return count diff --git a/ietf/doc/utils_bofreq.py b/ietf/doc/utils_bofreq.py index aec8f60ad67..d01b039b8e8 100644 --- a/ietf/doc/utils_bofreq.py +++ b/ietf/doc/utils_bofreq.py @@ -1,12 +1,149 @@ -# Copyright The IETF Trust 2021 All Rights Reserved +# Copyright The IETF Trust 2021-2026 All Rights Reserved +import datetime +from pathlib import Path -from ietf.doc.models import BofreqEditorDocEvent, BofreqResponsibleDocEvent +from django.conf import settings + +from ietf.doc.models import ( + BofreqEditorDocEvent, + BofreqResponsibleDocEvent, + DocEvent, + DocHistory, + Document, +) from ietf.person.models import Person +from ietf.utils import log + def bofreq_editors(bofreq): e = bofreq.latest_event(BofreqEditorDocEvent) return e.editors.all() if e else Person.objects.none() + def bofreq_responsible(bofreq): e = bofreq.latest_event(BofreqResponsibleDocEvent) - return e.responsible.all() if e else Person.objects.none() \ No newline at end of file + return e.responsible.all() if e else Person.objects.none() + + +def fixup_bofreq_timestamps(): # pragma: nocover + """Fixes bofreq event / document timestamps + + Timestamp errors resulted from the bug fixed by + https://github.com/ietf-tools/datatracker/pull/10333 + + Does not fix up -00 revs because the timestamps on these were not affected by + the bug. Replacing their timestamps creates a confusing event history because the + filesystem timestamp is usually a fraction of a second later than other events + created upon the initial rev creation. This causes the "New revision available" + event to appear _after_ these events in the history. Better to leave them as is. + """ + FIX_DEPLOYMENT_TIME = "2026-02-03T01:16:00+00:00" # 12.58.0 -> production + + def _get_doc_time(doc_name: str, rev: str): + path = Path(settings.BOFREQ_PATH) / f"{doc_name}-{rev}.md" + return datetime.datetime.fromtimestamp(path.stat().st_mtime, datetime.UTC) + + # Find affected DocEvents and DocHistories + new_bofreq_events = ( + DocEvent.objects.filter( + doc__type="bofreq", type="new_revision", time__lt=FIX_DEPLOYMENT_TIME + ) + .exclude(rev="00") # bug did not affect rev 00 events + .order_by("doc__name", "rev") + ) + log.log( + f"fixup_bofreq_timestamps: found {new_bofreq_events.count()} " + f"new_revision events before {FIX_DEPLOYMENT_TIME}" + ) + document_fixups = {} + for e in new_bofreq_events: + name = e.doc.name + rev = e.rev + filesystem_time = _get_doc_time(name, rev) + assert e.time < filesystem_time, ( + f"Rev {rev} event timestamp for {name} unexpectedly later than the " + "filesystem timestamp!" + ) + try: + dochistory = DocHistory.objects.filter( + name=name, time__lt=filesystem_time + ).get(rev=rev) + except DocHistory.MultipleObjectsReturned as err: + raise RuntimeError( + f"Multiple DocHistories for {name} rev {rev} exist earlier than the " + "filesystem timestamp!" + ) from err + except DocHistory.DoesNotExist as err: + if rev == "00": + # Unreachable because we don't adjust -00 revs, but could be needed + # if we did, in theory. In practice it's still not reached, but + # keeping the case for completeness. + dochistory = None + else: + raise RuntimeError( + f"No DocHistory for {name} rev {rev} exists earlier than the " + f"filesystem timestamp!" + ) from err + + if name not in document_fixups: + document_fixups[name] = [] + document_fixups[name].append( + { + "event": e, + "dochistory": dochistory, + "filesystem_time": filesystem_time, + } + ) + + # Now do the actual fixup + system_person = Person.objects.get(name="(System)") + for doc_name, fixups in document_fixups.items(): + bofreq = Document.objects.get(type="bofreq", name=doc_name) + log_msg_parts = [] + adjusted_revs = [] + for fixup in fixups: + event_to_fix = fixup["event"] + dh_to_fix = fixup["dochistory"] + new_time = fixup["filesystem_time"] + adjusted_revs.append(event_to_fix.rev) + + # Fix up the event + event_to_fix.time = new_time + event_to_fix.save() + log_msg_parts.append(f"rev {event_to_fix.rev} DocEvent") + + # Fix up the DocHistory + if dh_to_fix is not None: + dh_to_fix.time = new_time + dh_to_fix.save() + log_msg_parts.append(f"rev {dh_to_fix.rev} DocHistory") + + if event_to_fix.rev == bofreq.rev and bofreq.time < new_time: + # Update the Document without calling save(). Only update if + # the time has not changed so we don't inadvertently overwrite + # a concurrent update. + Document.objects.filter(pk=bofreq.pk, time=bofreq.time).update( + time=new_time + ) + bofreq.refresh_from_db() + if bofreq.rev == event_to_fix.rev: + log_msg_parts.append(f"rev {bofreq.rev} Document") + else: + log.log( + "fixup_bofreq_timestamps: WARNING: bofreq Document rev " + f"changed for {bofreq.name}" + ) + log.log(f"fixup_bofreq_timestamps: {bofreq.name}: " + ", ".join(log_msg_parts)) + + # Fix up the Document, if necessary, and add a record of the adjustment + DocEvent.objects.create( + type="added_comment", + by=system_person, + doc=bofreq, + rev=bofreq.rev, + desc=( + "Corrected inaccurate document and new revision event timestamps for " + + ("version " if len(adjusted_revs) == 1 else "versions ") + + ", ".join(adjusted_revs) + ), + ) diff --git a/ietf/doc/utils_errata.py b/ietf/doc/utils_errata.py new file mode 100644 index 00000000000..539262151ff --- /dev/null +++ b/ietf/doc/utils_errata.py @@ -0,0 +1,35 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def signal_update_rfc_metadata(rfc_number_list=()): + key = getattr(settings, "ERRATA_METADATA_NOTIFICATION_API_KEY", None) + if key is not None: + headers = {"X-Api-Key": settings.ERRATA_METADATA_NOTIFICATION_API_KEY} + post_dict = { + "rfc_number_list": list(rfc_number_list), + } + try: + response = requests.post( + settings.ERRATA_METADATA_NOTIFICATION_URL, + headers=headers, + json=post_dict, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log( + f"POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {e}" + ) + # raise RuntimeError(f'POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL}') from e + return + if response.status_code != 200: + log( + f"POST request failed for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {response.status_code} {response.text}" + ) + else: + log("No API key configured for errata metadata notification, skipping") diff --git a/ietf/doc/utils_r2.py b/ietf/doc/utils_r2.py new file mode 100644 index 00000000000..53fb978303d --- /dev/null +++ b/ietf/doc/utils_r2.py @@ -0,0 +1,17 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.core.files.storage import storages + +from ietf.doc.models import StoredObject + + +def rfcs_are_in_r2(rfc_number_list=()): + r2_rfc_bucket = storages["r2-rfc"] + for rfc_number in rfc_number_list: + stored_objects = StoredObject.objects.filter( + store="rfc", doc_name=f"rfc{rfc_number}" + ) + for stored_object in stored_objects: + if not r2_rfc_bucket.exists(stored_object.name): + return False + return True diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py new file mode 100644 index 00000000000..5c5879d688d --- /dev/null +++ b/ietf/doc/utils_red.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def trigger_red_precomputer(rfc_number_list=()): + url = getattr(settings, "TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) + if url is not None: + payload = { + "rfcs": ",".join([str(n) for n in rfc_number_list]), + } + try: + log(f"Triggering red precompute multiple for RFCs {rfc_number_list}") + response = requests.post( + url=url, + json=payload, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log(f"POST request timed out for {url} : {e}") + return + if response.status_code // 100 != 2: # 2xx status codes are ok + log( + f"POST request failed for {url} : status_code={response.status_code}" + ) + else: + log("No URL configured for triggering red precompute multiple, skipping") diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py new file mode 100644 index 00000000000..1f134556869 --- /dev/null +++ b/ietf/doc/utils_rfc_json.py @@ -0,0 +1,231 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import json +from pathlib import Path + +from django.conf import settings +from django.utils import timezone + +from ietf.doc.models import Document, RelatedDocument +from ietf.name.models import StdLevelName +from ietf.doc.storage_utils import exists_in_storage, store_bytes +from ietf.sync.errata import errata_map_from_json, get_errata_data +from ietf.sync.rfcindex import get_april1_rfc_numbers, get_publication_std_levels +from ietf.utils.log import assertion, log + + +_FORMAT_CHECKS = [ + ("xml", "XML"), + ("txt", "TEXT"), + ("html", "HTML"), + ("pdf", "PDF"), +] + + +def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: + """Generate and store the JSON metadata file for a published RFC. + + Reads RFC metadata from the DB and errata data from the red bucket, combines + them, and writes json/rfc{N}.json to the "rfc" blob bucket (overwriting any + existing file). + + pub_levels, if provided, should be the defaultdict returned by + get_publication_std_levels(). Pass it when generating JSON for multiple RFCs + to avoid a redundant blob read per call. + """ + try: + rfc = ( + Document.objects.select_related("std_level", "stream", "group__parent") + .prefetch_related("rfcauthor_set") + .get(type_id="rfc", rfc_number=rfc_number) + ) + except Document.DoesNotExist: + log(f"generate_rfc_json: no RFC found for rfc_number={rfc_number}") + return + + if pub_levels is None: + try: + pub_levels = get_publication_std_levels() + except Exception as e: + log(f"generate_rfc_json: failed to get publication std levels: {e}") + return + + doc_id = f"RFC{rfc_number}" + + # draft name + draft_doc = rfc.came_from_draft() + draft = f"{draft_doc.name}-{draft_doc.rev}" if draft_doc else None + + # authors: ordered list of display strings + authors = [] + for author in rfc.rfcauthor_set.order_by("order"): + name = author.titlepage_name + if author.is_editor: + name = f"{name}, Ed." + authors.append(name) + + # format: check which file blobs are present + formats = [ + label + for ext, label in _FORMAT_CHECKS + if exists_in_storage(kind="rfc", name=f"{ext}/rfc{rfc_number}.{ext}") + ] + + # page_count + page_count = str(rfc.pages) if rfc.pages is not None else "" + + # status: current std_level + status = rfc.std_level.name.upper() if rfc.std_level else "" + + # pub_status from publication-std-levels.json in the red bucket + # but guard against recent publication not having updated the bucket yet + pub_event = rfc.latest_event(type="published_rfc") + if rfc_number in pub_levels: + pub_status = pub_levels[rfc_number].name.upper() + else: + if ( + pub_event is not None + and timezone.now() - pub_event.time < datetime.timedelta(days=2) + ): + pub_status = status + else: + log(f"Assuming an unknown publication status for rfc{rfc_number}") + pub_status = StdLevelName.objects.get(slug="unkn").name.upper() + + # source: adapted from errata system's display_source() logic + if rfc.stream is None or rfc.group is None: + # Basic expectations (should be constraints) on RFC Document objects + # have been violated. + assertion("rfc.stream is not None and rfc.group is not None") + log( + f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" + ) + return + stream_slug = rfc.stream.slug + group_acronym = rfc.group.acronym + + if stream_slug == "ietf": + if rfc.group.parent is None: + assertion("rfc.group.parent is not None") + log( + f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" + ) + return + + if stream_slug == "ise": + source = "INDEPENDENT" + elif stream_slug == "iab": + source = "IAB" + elif stream_slug == "ietf" and ( + group_acronym == "none" or rfc.group.type_id == "area" + ): + source = "IETF - NON WORKING GROUP" + elif stream_slug == "irtf": + if group_acronym == "none": + source = "IRTF" + else: + source = rfc.group.name + elif group_acronym not in ("none", "") and stream_slug in ["ietf", "editorial"]: + source = rfc.group.name + elif stream_slug: + source = "Legacy" if stream_slug == "legacy" else stream_slug.upper() + else: + source = "" + + # pub_date: month/year of publication, with April 1st special-casing + pub_date = None + if pub_event: + dt = pub_event.time + try: + april_first_numbers = get_april1_rfc_numbers() + except Exception: + april_first_numbers = [] + if dt.month == 4 and rfc_number in april_first_numbers: + pub_date = dt.strftime("1 %B %Y") + else: + pub_date = dt.strftime("%B %Y") + + # relationship lists — sorted by RFC number + def _rfc_list(qs, attr): + numbers = [ + getattr(rd, attr).rfc_number + for rd in qs + if getattr(rd, attr).rfc_number is not None + ] + return [f"RFC{n}" for n in sorted(numbers)] + + obsoletes = _rfc_list( + RelatedDocument.objects.filter( + source=rfc, relationship_id="obs" + ).select_related("target"), + "target", + ) + obsoleted_by = _rfc_list( + RelatedDocument.objects.filter( + target=rfc, relationship_id="obs" + ).select_related("source"), + "source", + ) + updates = _rfc_list( + RelatedDocument.objects.filter( + source=rfc, relationship_id="updates" + ).select_related("target"), + "target", + ) + updated_by = _rfc_list( + RelatedDocument.objects.filter( + target=rfc, relationship_id="updates" + ).select_related("source"), + "source", + ) + + # errata_url: non-None if any errata entry exists for this RFC (any status) + try: + errata_data = get_errata_data() + errata_map = errata_map_from_json(errata_data) + errata_url = ( + settings.RFC_EDITOR_ERRATA_BASE_URL + f"rfc{rfc_number}" + if rfc_number in errata_map + else None + ) + except Exception: + log(f"generate_rfc_json: could not load errata data for RFC {rfc_number}") + errata_url = None + + data = { + "draft": draft, + "doc_id": doc_id, + "title": rfc.title, + "authors": authors, + "format": formats, + "page_count": page_count, + "pub_status": pub_status, + "status": status, + "source": source, + "abstract": rfc.abstract, + "pub_date": pub_date, + "keywords": [kw for kw in rfc.keywords if kw], + "obsoletes": obsoletes, + "obsoleted_by": obsoleted_by, + "updates": updates, + "updated_by": updated_by, + "see_also": [], + "doi": f"10.17487/{doc_id}", + "errata_url": errata_url, + } + + content = json.dumps(data, indent=2).encode("utf-8") + store_bytes( + kind="rfc", + name=f"json/rfc{rfc_number}.json", + content=content, + allow_overwrite=True, + doc_name=f"rfc{rfc_number}", + doc_rev=None, + mtime=timezone.now(), + ) + fs_path = Path(settings.RFC_PATH) / f"rfc{rfc_number}.json" + if settings.SERVER_MODE != "production" and not fs_path.parent.exists(): + fs_path.parent.mkdir() + fs_path.write_bytes(content) diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 03cf01a4a16..ec593934131 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -425,7 +425,10 @@ def build_position_email_from_dict(pos_dict): pos=pos_name, blocking_name=blocking_name, settings=settings)) - frm = balloter.role_email("ad").formatted_email() + if doc.stream_id == "ietf": + frm = balloter.role_email("ad").formatted_email() + else: + frm = balloter.formatted_email() if doc.stream_id == "irtf": addrs = gather_address_lists('irsg_ballot_saved',doc=doc) @@ -939,16 +942,6 @@ def approve_ballot(request, name): if ballot_writeup_event.pk == None: ballot_writeup_event.save() - if new_state.slug == "ann" and new_state.slug != prev_state.slug: - # start by notifying the RFC Editor - import ietf.sync.rfceditor - response, error = ietf.sync.rfceditor.post_approved_draft(settings.RFC_EDITOR_SYNC_NOTIFICATION_URL, doc.name) - if error: - return render(request, 'doc/draft/rfceditor_post_approved_draft_failed.html', - dict(name=doc.name, - response=response, - error=error)) - doc.set_state(new_state) doc.tags.remove(*prev_tags) diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 71cbe304913..94e3960dfa4 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -91,7 +91,6 @@ def submit(request, name): by=request.user.person, rev=bofreq.rev, desc='New revision available', - time=bofreq.time, ) bofreq.save_with_history([e]) bofreq_submission = form.cleaned_data['bofreq_submission'] diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 4a20db3c890..af056f6a96b 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2024, All Rights Reserved +# Copyright The IETF Trust 2009-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). @@ -43,9 +43,10 @@ from celery.result import AsyncResult from django.core.cache import caches +from django.core.files.base import ContentFile from django.core.exceptions import PermissionDenied from django.db.models import Max -from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse +from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.template.loader import render_to_string from django.urls import reverse as urlreverse @@ -57,7 +58,7 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType, - ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, + ConsensusDocEvent, NewRevisionDocEvent, StoredObject, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor, RelatedDocument, RelatedDocHistory) from ietf.doc.tasks import investigate_fragment_task @@ -79,19 +80,21 @@ from ietf.doc.views_ballot import parse_ballot_edit_return_point from ietf.doc.forms import InvestigateForm, TelechatForm, NotifyForm, ActionHoldersForm, DocAuthorForm, DocAuthorChangeBasisForm from ietf.doc.mails import email_comment, email_remind_action_holders +from ietf.doc.utils import last_ballot_doc_revision from ietf.mailtrigger.utils import gather_relevant_expansions from ietf.meeting.models import Session, SessionPresentation from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, sort_sessions, add_event_info_to_session_qs from ietf.review.models import ReviewAssignment from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc +from ietf.doc.storage_utils import retrieve_bytes from ietf.utils import markup_txt, log, markdown from ietf.utils.draft import get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split from ietf.utils.timezone import date_today - +from ietf.utils.unicodenormalize import normalize_for_sorting def render_document_top(request, doc, tab, name): tabs = [] @@ -255,7 +258,7 @@ def document_main(request, name, rev=None, document_html=False): interesting_relations_that, interesting_relations_that_doc = interesting_doc_relations(doc) can_edit = has_role(request.user, ("Area Director", "Secretariat")) - can_edit_authors = has_role(request.user, ("Secretariat")) + can_edit_authors = has_role(request.user, ("Secretariat")) and not doc.rfcauthor_set.exists() stream_slugs = StreamName.objects.values_list("slug", flat=True) # For some reason, AnonymousUser has __iter__, but is not iterable, @@ -514,13 +517,17 @@ def document_main(request, name, rev=None, document_html=False): # remaining actions actions = [] - if can_adopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: + if can_adopt_draft(request.user, doc) and doc.get_state_slug() not in ["rfc"] and not snapshot: + target = urlreverse("ietf.doc.views_draft.adopt_draft", kwargs=dict(name=doc.name)) if doc.group and doc.group.acronym != 'none': # individual submission # already adopted in one group button_text = "Switch adoption" else: button_text = "Manage adoption" - actions.append((button_text, urlreverse('ietf.doc.views_draft.adopt_draft', kwargs=dict(name=doc.name)))) + # can_adopt_draft currently returns False for Area Directors + if has_role(request.user, ["Secretariat", "WG Chair"]): + target = urlreverse("ietf.doc.views_draft.ask_about_ietf_adoption_call", kwargs=dict(name=doc.name)) + actions.append((button_text, target)) if can_unadopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: if doc.get_state_slug('draft-iesg') == 'idexists': @@ -1225,6 +1232,10 @@ def document_history(request, name): request.user, ("Area Director", "Secretariat", "IRTF Chair") ) + # if the current user has balloted on this document, give them a revision hint + ballot_doc_rev = None + if request.user.is_authenticated: + ballot_doc_rev = last_ballot_doc_revision(doc, request.user.person) return render( request, @@ -1235,6 +1246,7 @@ def document_history(request, name): "diff_revisions": diff_revisions, "events": events, "can_add_comment": can_add_comment, + "ballot_doc_rev": ballot_doc_rev, }, ) @@ -1275,9 +1287,7 @@ def document_bibtex(request, name, rev=None): break elif doc.type_id == "rfc": - # This needs to be replaced with a lookup, as the mapping may change - # over time. - doi = f"10.17487/RFC{doc.rfc_number:04d}" + doi = doc.doi if doc.is_dochistory(): latest_event = doc.latest_event(type='new_revision', rev=rev) @@ -1500,7 +1510,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True): position_groups = [] for n in BallotPositionName.objects.filter(slug__in=[p.pos_id for p in positions]).order_by('order'): g = (n, [p for p in positions if p.pos_id == n.slug]) - g[1].sort(key=lambda p: (p.is_old_pos, p.balloter.plain_name())) + g[1].sort(key=lambda p: (p.is_old_pos, normalize_for_sorting(p.balloter.plain_name()))) if n.blocking: position_groups.insert(0, g) else: @@ -1643,11 +1653,18 @@ def extract_name(s): data["state"] = extract_name(doc.get_state()) data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) + author_qs = ( + doc.rfcauthor_set + if doc.type_id == "rfc" and doc.rfcauthor_set.exists() + else doc.documentauthor_set + ).select_related("person").prefetch_related("person__email_set").order_by("order") data["authors"] = [ - dict(name=author.person.name, - email=author.email.address if author.email else None, - affiliation=author.affiliation) - for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + { + "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name, + "email": author.email.address if author.email else None, + "affiliation": author.affiliation, + } + for author in author_qs ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None @@ -1825,12 +1842,15 @@ def add_fields(self, form, index): if fh in form.fields: form.fields[fh].widget = forms.HiddenInput() + doc = get_object_or_404(Document, name=name) + if doc.rfcauthor_set.exists(): + return HttpResponseForbidden("Contact the RFC Editor to change RFC Author information") + AuthorFormSet = forms.formset_factory(DocAuthorForm, formset=_AuthorsBaseFormSet, can_delete=True, can_order=True, extra=0) - doc = get_object_or_404(Document, name=name) if request.method == 'POST': change_basis_form = DocAuthorChangeBasisForm(request.POST) @@ -1931,9 +1951,9 @@ def edit_action_holders(request, name): role_ids = dict() # maps role slug to list of Person IDs (assumed numeric in the JavaScript) extra_prefetch = [] # list of Person objects to prefetch for select2 field - if len(doc.authors()) > 0: + authors = doc.author_persons() + if len(authors) > 0: doc_role_labels.append(dict(slug='authors', label='Authors')) - authors = doc.authors() role_ids['authors'] = [p.pk for p in authors] extra_prefetch += authors @@ -2341,3 +2361,29 @@ def investigate(request): "results": results, }, ) + +def rfcxml_notprepped(request, number): + number = int(number) + if number < settings.FIRST_V3_RFC: + raise Http404 + rfc = Document.objects.filter(type="rfc", rfc_number=number).first() + if rfc is None: + raise Http404 + name = f"notprepped/rfc{number}.notprepped.xml" + if not StoredObject.objects.filter(name=name).exists(): + raise Http404 + try: + bytes = retrieve_bytes("rfc", name) + except FileNotFoundError: + raise Http404 + return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True) + + +def rfcxml_notprepped_wrapper(request, number): + number = int(number) + if number < settings.FIRST_V3_RFC: + raise Http404 + rfc = Document.objects.filter(type="rfc", rfc_number=number).first() + if rfc is None: + raise Http404 + return render(request, "doc/notprepped_wrapper.html", context={"rfc": rfc}) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 16d04ee66aa..a64d0a53fe9 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -28,12 +28,12 @@ IanaExpertDocEvent, IESG_SUBSTATE_TAGS) from ietf.doc.mails import ( email_pulled_from_rfc_queue, email_resurrect_requested, email_resurrection_completed, email_state_changed, email_stream_changed, - email_wg_call_for_adoption_issued, email_wg_last_call_issued, email_stream_state_changed, email_stream_tags_changed, extra_automation_headers, generate_publication_request, email_adopted, email_intended_status_changed, email_iesg_processing_document, email_ad_approved_doc, email_iana_expert_review_state_changed ) from ietf.doc.storage_utils import retrieve_bytes, store_bytes +from ietf.doc.templatetags.ietf_filters import is_doc_ietf_adoptable from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft, get_tags_for_stream_id, nice_consensus, update_action_holders, update_reminder, update_telechat, make_notify_changed_event, get_initial_notify, @@ -51,12 +51,12 @@ from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName from ietf.person.fields import SearchableEmailField from ietf.person.models import Person, Email -from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of +from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of, send_mail_text from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils import log -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField, MultiEmailField from ietf.utils.response import permission_denied -from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO +from ietf.utils.timezone import date_today, datetime_from_date, datetime_today, DEADLINE_TZINFO class ChangeStateForm(forms.Form): @@ -1276,15 +1276,6 @@ class PublicationForm(forms.Form): if form.is_valid(): events = [] - # start by notifying the RFC Editor - import ietf.sync.rfceditor - response, error = ietf.sync.rfceditor.post_approved_draft(settings.RFC_EDITOR_SYNC_NOTIFICATION_URL, doc.name) - if error: - return render(request, 'doc/draft/rfceditor_post_approved_draft_failed.html', - dict(name=doc.name, - response=response, - error=error)) - m.subject = form.cleaned_data["subject"] m.body = form.cleaned_data["body"] m.save() @@ -1564,7 +1555,7 @@ def adopt_draft(request, name): events.append(e) due_date = None - if form.cleaned_data["weeks"] != None: + if form.cleaned_data["weeks"] is not None: due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"]) update_reminder(doc, "stream-s", e, due_date) @@ -1573,11 +1564,6 @@ def adopt_draft(request, name): # setting states that are _not_ the adopted state. email_adopted(request, doc, prev_state, new_state, by, comment) - # Currently only the IETF stream uses the c-adopt state - guard against other - # streams starting to use it asthe IPR rules for those streams will be different. - if doc.stream_id == "ietf" and new_state.slug == "c-adopt": - email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) - # comment if comment: e = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=by) @@ -1689,11 +1675,14 @@ def __init__(self, *args, **kwargs): f.queryset = f.queryset.exclude(pk__in=unused_states) f.label = state_type.label if self.stream.slug == 'ietf': + help_text_items = [] if self.can_set_sub_pub: - f.help_text = "Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication." + help_text_items.append("Only select 'Submitted to IESG for Publication' to correct errors. This is not how to submit a document to the IESG.") else: f.queryset = f.queryset.exclude(slug='sub-pub') - f.help_text = "You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication." + help_text_items.append("You may not set the 'Submitted to IESG for Publication' using this form - Use the button above or the document's main page to request publication.") + help_text_items.append("Only use this form in unusual circumstances when issuing call for adoption or working group last call.") + f.help_text = " ".join(help_text_items) f = self.fields['tags'] f.queryset = f.queryset.filter(slug__in=get_tags_for_stream_id(doc.stream_id)) @@ -1704,7 +1693,7 @@ def __init__(self, *args, **kwargs): def clean_new_state(self): new_state = self.cleaned_data.get('new_state') if new_state.slug=='sub-pub' and not self.can_set_sub_pub: - raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) + raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for Publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) return new_state @@ -1730,6 +1719,19 @@ def next_states_for_stream_state(doc, state_type, current_state): return next_states +@login_required +def offer_wg_action_helpers(request, name): + doc = get_object_or_404(Document, type="draft", name=name) + if doc.stream is None or doc.stream_id != "ietf" or doc.became_rfc() is not None: + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + return render(request, "doc/draft/wg_action_helpers.html", + {"doc": doc, + }) + @login_required def change_stream_state(request, name, state_type): doc = get_object_or_404(Document, type="draft", name=name) @@ -1744,10 +1746,17 @@ def change_stream_state(request, name, state_type): prev_state = doc.get_state(state_type.slug) next_states = next_states_for_stream_state(doc, state_type, prev_state) + # These tell the form to allow directly setting the state to fix up errors. can_set_sub_pub = has_role(request.user,('Secretariat','Area Director')) or (prev_state and prev_state.slug=='sub-pub') if request.method == 'POST': - form = ChangeStreamStateForm(request.POST, doc=doc, state_type=state_type,can_set_sub_pub=can_set_sub_pub,stream=doc.stream) + form = ChangeStreamStateForm( + request.POST, + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) if form.is_valid(): by = request.user.person events = [] @@ -1768,14 +1777,7 @@ def change_stream_state(request, name, state_type): update_reminder(doc, "stream-s", e, due_date) email_stream_state_changed(request, doc, prev_state, new_state, by, comment) - - if doc.stream_id == "ietf": - if new_state.slug == "c-adopt": - email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) - if new_state.slug == "wg-lc": - email_wg_last_call_issued(request, doc, wglc_duration_weeks=form.cleaned_data["weeks"]) - # tags existing_tags = set(doc.tags.all()) new_tags = set(form.cleaned_data["tags"]) @@ -1811,8 +1813,15 @@ def change_stream_state(request, name, state_type): else: form.add_error(None, "No change in state or tags found, and no comment provided -- nothing to do.") else: - form = ChangeStreamStateForm(initial=dict(new_state=prev_state.pk if prev_state else None, tags= doc.tags.all()), - doc=doc, state_type=state_type, can_set_sub_pub = can_set_sub_pub,stream = doc.stream) + form = ChangeStreamStateForm( + initial=dict( + new_state=prev_state.pk if prev_state else None, tags=doc.tags.all() + ), + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) milestones = doc.groupmilestone_set.all() @@ -1857,3 +1866,326 @@ def set_intended_status_level(request, doc, new_level, old_level, comment): msg = "\n".join(e.desc for e in events) email_intended_status_changed(request, doc, msg) + +class IssueWorkingGroupLastCallForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Last Call closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Last Call message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Last Call message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + + +@login_required +def issue_wg_lc(request, name): + doc = get_object_or_404(Document, name=name) + + if doc.stream_id != "ietf": + raise Http404 + if doc.type_id != "draft" or doc.group.type_id != "wg": + raise Http404 + if doc.get_state_slug("draft-stream-ietf") == "wg-lc": + raise Http404 + if doc.get_state_slug("draft") == "rfc": + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + if request.method == "POST": + form = IssueWorkingGroupLastCallForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + prev_state = doc.get_state("draft-stream-ietf") + events = [] + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + e = add_state_change_event(doc, by, prev_state, wglc_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, wglc_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_last_call_issued.txt", + dict( + doc=doc, + end_date=end_date, + wg_list=doc.group.list_email, + url=f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt", + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) + + form = IssueWorkingGroupLastCallForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_last_call.html", + dict( + doc=doc, + form=form, + ), + ) + +class IssueCallForAdoptionForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Call for Adoption closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Call for Adoption message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Call for Adoption message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + +@login_required +def issue_wg_call_for_adoption(request, name, acronym): + doc = get_object_or_404(Document, name=name) + group = Group.objects.filter(acronym=acronym, type_id="wg").first() + reject = False + if group is None or doc.type_id != "draft" or not is_doc_ietf_adoptable(doc): + reject = True + if doc.stream is None: + if not can_adopt_draft(request.user, doc): + reject = True + elif doc.stream_id != "ietf": + reject = True + else: # doc.stream_id == "ietf" + if not is_authorized_in_doc_stream(request.user, doc): + reject = True + if reject: + raise permission_denied(request, f"You can't issue a {acronym} wg call for adoption for this document.") + + if request.method == "POST": + form = IssueCallForAdoptionForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + + events = [] + if doc.stream_id != "ietf": + stream = StreamName.objects.get(slug="ietf") + doc.stream = stream + e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed stream to {stream.name}" # Propogates embedding html in DocEvent.desc for consistency + e.save() + events.append(e) + if doc.group != group: + doc.group = group + e = DocEvent(type="changed_group", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed group to {group.name} ({group.acronym.upper()})" # Even if it makes the cats cry + e.save() + events.append(e) + prev_state = doc.get_state("draft-stream-ietf") + c_adopt_state = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + doc.set_state(c_adopt_state) + e = add_state_change_event(doc, by, prev_state, c_adopt_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, c_adopt_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_call_for_adoption_issued.txt", + dict( + doc=doc, + group=group, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) + if doc.group.acronym == "none": + to.insert(0, f"{group.acronym}-chairs@ietf.org") + to.insert(0, group.list_email) + form = IssueCallForAdoptionForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_call_for_adoption.html", + dict( + doc=doc, + form=form, + ), + ) + +class GroupModelChoiceField(forms.ModelChoiceField): + def label_from_instance(self, obj): + return f"{obj.acronym} - {obj.name}" + + +class WgForm(forms.Form): + group = GroupModelChoiceField( + queryset=Group.objects.filter(type_id="wg", state="active") + .order_by("acronym") + .distinct(), + required=True, + empty_label="Select IETF Working Group", + ) + + def __init__(self, *args, **kwargs): + user = kwargs.pop("user") + super(WgForm, self).__init__(*args, **kwargs) + if not has_role(user, ["Secretariat", "Area Director"]): + self.fields["group"].queryset = self.fields["group"].queryset.filter( + role__name_id="chair", role__person=user.person + ) + + +@role_required("Secretariat", "WG Chair") +def ask_about_ietf_adoption_call(request, name): + doc = get_object_or_404(Document, name=name) + if doc.stream is not None or doc.group.acronym != "none": + raise Http404 + if request.method == "POST": + form = WgForm(request.POST, user=request.user) + if form.is_valid(): + group = form.cleaned_data["group"] + return redirect(issue_wg_call_for_adoption, name=doc.name, acronym=group.acronym) + else: + form = WgForm(initial={"group": None}, user=request.user) + return render( + request, + "doc/draft/ask_about_ietf_adoption.html", + dict( + doc=doc, + form=form, + ), + ) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 2144c23e06d..4232d77f6c4 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -74,7 +74,7 @@ from ietf.utils.log import log from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD from ietf.ietfauth.utils import has_role - +from ietf.utils.unicodenormalize import normalize_for_sorting class SearchForm(forms.Form): name = forms.CharField(required=False) @@ -219,7 +219,7 @@ def retrieve_search_results(form, all_types=False): queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")]) combined_query = reduce(operator.or_, queries) - docs = docs.filter(combined_query).distinct() + docs = docs.filter(combined_query) # rfc/active/old check buttons allowed_draft_states = [] @@ -229,20 +229,23 @@ def retrieve_search_results(form, all_types=False): allowed_draft_states.extend(['repl', 'expired', 'auth-rm', 'ietf-rm']) docs = docs.filter(Q(states__slug__in=allowed_draft_states) | - ~Q(type__slug='draft')).distinct() + ~Q(type__slug='draft')) # radio choices by = query["by"] if by == "author": docs = docs.filter( Q(documentauthor__person__alias__name__icontains=query["author"]) | - Q(documentauthor__person__email__address__icontains=query["author"]) + Q(documentauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__person__alias__name__icontains=query["author"]) | + Q(rfcauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__titlepage_name__icontains=query["author"]) ) elif by == "group": docs = docs.filter(group__acronym__iexact=query["group"]) elif by == "area": docs = docs.filter(Q(group__type="wg", group__parent=query["area"]) | - Q(group=query["area"])).distinct() + Q(group=query["area"])) elif by == "ad": docs = docs.filter(ad=query["ad"]) elif by == "state": @@ -255,6 +258,8 @@ def retrieve_search_results(form, all_types=False): elif by == "stream": docs = docs.filter(stream=query["stream"]) + docs=docs.distinct() + return docs @@ -480,6 +485,7 @@ def _state_to_doc_type(state): ).distinct(): if p in get_active_ads(): ads.append(p) + ads.sort(key=lambda p: normalize_for_sorting(p.plain_name())) bucket_template = { dt: {state: [[] for _ in range(days)] for state in STATE_SLUGS[dt].values()} diff --git a/ietf/group/admin.py b/ietf/group/admin.py index fedec49d850..685c10aeea1 100644 --- a/ietf/group/admin.py +++ b/ietf/group/admin.py @@ -26,14 +26,15 @@ MilestoneGroupEvent, GroupExtResource, Appeal, AppealArtifact ) from ietf.name.models import GroupTypeName -from ietf.utils.validators import validate_external_resource_value +from ietf.utils.admin import SaferTabularInline from ietf.utils.response import permission_denied +from ietf.utils.validators import validate_external_resource_value -class RoleInline(admin.TabularInline): +class RoleInline(SaferTabularInline): model = Role raw_id_fields = ["person", "email"] -class GroupURLInline(admin.TabularInline): +class GroupURLInline(SaferTabularInline): model = GroupURL class GroupForm(forms.ModelForm): diff --git a/ietf/group/models.py b/ietf/group/models.py index 2d5e7c4e6f9..a7e3c6616e7 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -111,6 +111,9 @@ def active_wgs(self): def closed_wgs(self): return self.wgs().exclude(state__in=Group.ACTIVE_STATE_IDS) + def areas(self): + return self.get_queryset().filter(type="area") + def with_meetings(self): return self.get_queryset().filter(type__features__has_meetings=True) diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py new file mode 100644 index 00000000000..e789ba46bf7 --- /dev/null +++ b/ietf/group/serializers.py @@ -0,0 +1,50 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.person.models import Email +from .models import Group, Role + + +class GroupSerializer(serializers.ModelSerializer): + class Meta: + model = Group + fields = ["acronym", "name", "type", "list_email"] + + +class AreaDirectorSerializer(serializers.Serializer): + """Serialize an area director + + Works with Email or Role + """ + + name = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_name(self, instance: Email | Role): + person = getattr(instance, 'person', None) + return person.plain_name() if person else None + + @extend_schema_field(serializers.EmailField) + def get_email(self, instance: Email | Role): + if isinstance(instance, Role): + return instance.email.email_address() + return instance.email_address() + + +class AreaSerializer(serializers.ModelSerializer): + ads = serializers.SerializerMethodField() + + class Meta: + model = Group + fields = ["acronym", "name", "ads"] + + @extend_schema_field(AreaDirectorSerializer(many=True)) + def get_ads(self, area: Group): + return AreaDirectorSerializer( + area.ads if area.is_active else Role.objects.none(), + many=True, + ).data diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 34f8500854b..97ec7ebdb15 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, WgRfcFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -396,6 +396,7 @@ def test_group_documents(self): draft7 = WgDraftFactory(group=group) draft7.set_state(State.objects.get(type='draft', slug='expired')) draft7.set_state(State.objects.get(type='draft-stream-%s' % draft7.stream_id, slug='dead')) # Expired WG draft, marked as dead + rfc = WgRfcFactory(group=group) clist = CommunityList.objects.get(group=group) related_docs_rule = clist.searchrule_set.get(rule_type='name_contains') @@ -426,6 +427,12 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # RFC rows must use the RFC number as the sort key so that numeric sort + # is not disrupted by the page-count text that precedes the name in the cell. + # Draft rows must use the document name. + self.assertTrue(q(f'td.doc[data-sort-number="{rfc.rfc_number}"]')) + self.assertTrue(q(f'td.doc[data-sort-number="{draft.name}"]')) + # Let's also check the IRTF stream rg = GroupFactory(type_id='rg') setup_default_community_list_for_group(rg) @@ -543,6 +550,25 @@ def verify_can_edit_group(url, group, username): for username in list(set(interesting_users)-set(can_edit[group.type_id])): verify_cannot_edit_group(url, group, username) + def test_group_about_team_parent(self): + """Team about page should show parent when parent is not an area""" + GroupFactory(type_id='team', parent=GroupFactory(type_id='area', acronym='gen')) + GroupFactory(type_id='team', parent=GroupFactory(type_id='ietf', acronym='iab')) + GroupFactory(type_id='team', parent=None) + + for team in Group.objects.filter(type='team').select_related('parent'): + url = urlreverse('ietf.group.views.group_about', kwargs=dict(acronym=team.acronym)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + if team.parent and team.parent.type_id != 'area': + self.assertContains(r, 'Parent') + self.assertContains(r, team.parent.acronym) + elif team.parent and team.parent.type_id == 'area': + self.assertContains(r, team.parent.name) + self.assertNotContains(r, '>Parent<') + else: + self.assertNotContains(r, '>Parent<') + def test_group_about_personnel(self): """Correct personnel should appear on the group About page""" group = GroupFactory() diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index 89c755bb264..bb9b79a416a 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -888,10 +888,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -907,10 +907,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertNotContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') - self.assertNotContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') + self.assertNotContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertNotContains(r, escape(assignment2.reviewer.person.name)) @@ -926,10 +926,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertNotContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -940,9 +940,9 @@ def test_requests_history_filter_page(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') def test_requests_history_invalid_filter_parameters(self): # First assignment as assigned diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py new file mode 100644 index 00000000000..b584a17ae21 --- /dev/null +++ b/ietf/group/tests_serializers.py @@ -0,0 +1,96 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.person.factories import EmailFactory +from ietf.utils.test_utils import TestCase + + +class GroupSerializerTests(TestCase): + def test_serializes(self): + wg = GroupFactory() + serialized = GroupSerializer(wg).data + self.assertEqual( + serialized, + { + "acronym": wg.acronym, + "name": wg.name, + "type": "wg", + "list_email": wg.list_email, + }, + ) + + +class AreaDirectorSerializerTests(TestCase): + def test_serializes_role(self): + """Should serialize a Role correctly""" + role = RoleFactory(group__type_id="area", name_id="ad") + serialized = AreaDirectorSerializer(role).data + self.assertEqual( + serialized, + {"email": role.email.email_address(), "name": role.person.plain_name()}, + ) + + def test_serializes_email(self): + """Should serialize an Email correctly""" + email = EmailFactory() + serialized = AreaDirectorSerializer(email).data + self.assertEqual( + serialized, + { + "email": email.email_address(), + "name": email.person.plain_name() if email.person else None, + }, + ) + + +class AreaSerializerTests(TestCase): + def test_serializes_active_area(self): + """Should serialize an active area correctly""" + area = GroupFactory(type_id="area", state_id="active") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + ad_roles = RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual(serialized["acronym"], area.acronym) + self.assertEqual(serialized["name"], area.name) + self.assertCountEqual( + serialized["ads"], + [ + {"email": ad.email.email_address(), "name": ad.person.plain_name()} + for ad in ad_roles + ], + ) + + def test_serializes_inactive_area(self): + """Should serialize an inactive area correctly""" + area = GroupFactory(type_id="area", state_id="conclude") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 29cfff2c2dd..b1675de659e 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -237,9 +237,11 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Review requests", urlreverse(ietf.group.views.review_requests, kwargs=kwargs))) entries.append(("Reviewers", urlreverse(ietf.group.views.reviewer_overview, kwargs=kwargs))) entries.append(("Reviews History", urlreverse(ietf.group.views.review_requests_history, kwargs=kwargs))) - if group.features.has_meetings: entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) + if group.acronym in ["iesg"]: + entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) + entries.append(("Decisions", urlreverse("ietf.iesg.views.review_decisions"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) @@ -250,7 +252,6 @@ def construct_group_menu_context(request, group, selected, group_type, others): if is_valid_url(group.list_archive): entries.append((mark_safe("List archive »"), group.list_archive)) - # actions actions = [] diff --git a/ietf/group/views.py b/ietf/group/views.py index efe3eca15d5..8561a5059fc 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -245,10 +245,19 @@ def active_review_dirs(request): return render(request, 'group/active_review_dirs.html', {'dirs' : dirs }) def active_teams(request): - teams = Group.objects.filter(type="team", state="active").order_by("name") + parent_type_order = {"area": 1, "adm": 3, None: 4} + + def team_sort_key(group): + type_id = group.parent.type_id if group.parent else None + return (parent_type_order.get(type_id, 2), group.parent.name if group.parent else "", group.name) + + teams = sorted( + Group.objects.filter(type="team", state="active").select_related("parent"), + key=team_sort_key, + ) for group in teams: group.chairs = sorted(roles(group, "chair"), key=extract_last_name) - return render(request, 'group/active_teams.html', {'teams' : teams }) + return render(request, 'group/active_teams.html', {'teams': teams}) def active_iab(request): iabgroups = Group.objects.filter(type__in=("program","iabasg","iabworkshop"), state="active").order_by("-type_id","name") diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py index 587713089f3..ace4c9ec402 100644 --- a/ietf/iesg/agenda.py +++ b/ietf/iesg/agenda.py @@ -133,7 +133,7 @@ def agenda_sections(): ('4.2', {'title':"WG rechartering"}), ('4.2.1', {'title':"Under evaluation for IETF review", 'docs':[]}), ('4.2.2', {'title':"Proposed for approval", 'docs':[]}), - ('5', {'title':"IAB news we can use"}), + ('5', {'title':"IESG Liaison News"}), ('6', {'title':"Management issues"}), ('7', {'title':"Any Other Business (WG News, New Proposals, etc.)"}), ]) diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index f3778d1ded1..e5fbe5da7bd 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -281,12 +281,13 @@ def test_working_groups(self): wg_summary, ) = get_wg_dashboard_info() + # checks for the expected result with area sorted by name self.assertEqual( area_summary, [ { - "area": "red", - "groups_in_area": 1, + "area": "blue", + "groups_in_area": 3, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -295,8 +296,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "orange", - "groups_in_area": 4, + "area": "green", + "groups_in_area": 3, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -305,8 +306,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "yellow", - "groups_in_area": 2, + "area": "orange", + "groups_in_area": 4, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -315,8 +316,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "green", - "groups_in_area": 3, + "area": "red", + "groups_in_area": 1, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -325,8 +326,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "blue", - "groups_in_area": 3, + "area": "violet", + "groups_in_area": 4, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -335,8 +336,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "violet", - "groups_in_area": 4, + "area": "yellow", + "groups_in_area": 2, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -1192,34 +1193,14 @@ def test_working_groups(self): area_summary, [ { - "area": "red", - "groups_in_area": 1, - "groups_with_docs": 1, - "doc_count": 1, - "page_count": 7, - "group_percent": 6.25, - "doc_percent": 3.571428571428571, - "page_percent": 3.5897435897435894, - }, - { - "area": "orange", - "groups_in_area": 4, + "area": "blue", + "groups_in_area": 3, "groups_with_docs": 3, - "doc_count": 4, - "page_count": 29, + "doc_count": 6, + "page_count": 40, "group_percent": 18.75, - "doc_percent": 14.285714285714285, - "page_percent": 14.871794871794872, - }, - { - "area": "yellow", - "groups_in_area": 2, - "groups_with_docs": 2, - "doc_count": 2, - "page_count": 17, - "group_percent": 12.5, - "doc_percent": 7.142857142857142, - "page_percent": 8.717948717948717, + "doc_percent": 21.428571428571427, + "page_percent": 20.51282051282051, }, { "area": "green", @@ -1232,14 +1213,24 @@ def test_working_groups(self): "page_percent": 11.282051282051283, }, { - "area": "blue", - "groups_in_area": 3, + "area": "orange", + "groups_in_area": 4, "groups_with_docs": 3, - "doc_count": 6, - "page_count": 40, + "doc_count": 4, + "page_count": 29, "group_percent": 18.75, - "doc_percent": 21.428571428571427, - "page_percent": 20.51282051282051, + "doc_percent": 14.285714285714285, + "page_percent": 14.871794871794872, + }, + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.25, + "doc_percent": 3.571428571428571, + "page_percent": 3.5897435897435894, }, { "area": "violet", @@ -1251,6 +1242,16 @@ def test_working_groups(self): "doc_percent": 35.714285714285715, "page_percent": 41.02564102564102, }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 2, + "doc_count": 2, + "page_count": 17, + "group_percent": 12.5, + "doc_percent": 7.142857142857142, + "page_percent": 8.717948717948717, + }, ], ) self.assertEqual( diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py index 9051cf92b28..1d24ecac8e1 100644 --- a/ietf/iesg/utils.py +++ b/ietf/iesg/utils.py @@ -12,7 +12,7 @@ from ietf.group.models import Group from ietf.iesg.agenda import get_doc_section from ietf.person.utils import get_active_ads - +from ietf.utils.unicodenormalize import normalize_for_sorting TelechatPageCount = namedtuple( "TelechatPageCount", @@ -192,6 +192,7 @@ def get_wg_dashboard_info(): else 0, } ) + area_summary.sort(key=lambda r: r["area"]) area_totals = { "group_count": groups_total, "doc_count": docs_total, @@ -238,7 +239,7 @@ def get_wg_dashboard_info(): else 0, } ) - noad_summary.sort(key=lambda r: (r["ad"], r["area"])) + noad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) ad_summary = [] ad_totals = { @@ -278,7 +279,7 @@ def get_wg_dashboard_info(): else 0, } ) - ad_summary.sort(key=lambda r: (r["ad"], r["area"])) + ad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) rfc_counter = Counter( Document.objects.filter(type="rfc").values_list("group__acronym", flat=True) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index 014b2904259..af04ad06fd5 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). @@ -59,6 +59,7 @@ from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES from ietf.doc.utils import update_telechat, augment_events_with_revision from ietf.group.models import GroupMilestone, Role +from ietf.group.utils import construct_group_menu_context, get_group_or_404 from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count @@ -68,6 +69,7 @@ from ietf.meeting.utils import get_activity_stats from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date from ietf.utils.timezone import date_today, datetime_from_date +from ietf.utils.unicodenormalize import normalize_for_sorting def review_decisions(request, year=None): events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved")) @@ -88,13 +90,15 @@ def review_decisions(request, year=None): #doc_levels = ["exp", "inf"] timeframe = "%s" % year if year else "the past 6 months" + group_type = None + group = get_group_or_404("iesg", group_type) return render(request, 'iesg/review_decisions.html', - dict(events=events, - years=years, - year=year, - timeframe=timeframe), - ) + construct_group_menu_context(request, group, "decisions", group_type, { + "events": events, + "years": years, + "year": year, + "timeframe": timeframe})) def agenda_json(request, date=None): data = agenda_data(date) @@ -547,7 +551,7 @@ def milestones_needing_review(request): ) return render(request, 'iesg/milestones_needing_review.html', - dict(ads=sorted(ad_list, key=lambda ad: ad.plain_name()),)) + dict(ads=sorted(ad_list, key=lambda ad: normalize_for_sorting(ad.plain_name())),)) def photos(request): roles = sorted(Role.objects.filter(group__type='area', group__state='active', name_id='ad'),key=lambda x: "" if x.group.acronym=="gen" else x.group.acronym) @@ -631,8 +635,17 @@ def working_groups(request): area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() - return render( - request, - "iesg/working_groups.html", - dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), - ) + group_type = None + group = get_group_or_404("iesg", group_type) + + return render(request, 'iesg/working_groups.html', + construct_group_menu_context(request, group, "working groups", group_type, { + "area_summary": area_summary, + "area_totals": area_totals, + "ad_summary": ad_summary, + "noad_summary": noad_summary, + "ad_totals": ad_totals, + "noad_totals": noad_totals, + "totals": totals, + "wg_summary": wg_summary, + })) diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index e2893a90f79..30d51cddd09 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs): | Q(name="atlarge", group__acronym="irsg") ), "RSAB Member": Q(name="member", group__acronym="rsab"), + "LLC Staff": Q(name="member", group__acronym="llc-staff"), "Robot": Q(name="robot", group__acronym="secretariat"), } @@ -211,9 +212,9 @@ def role_required(*role_names): # specific permissions + def is_authorized_in_doc_stream(user, doc): - """Return whether user is authorized to perform stream duties on - document.""" + """Is user authorized to perform stream duties on doc?""" if has_role(user, ["Secretariat"]): return True @@ -287,7 +288,7 @@ def is_individual_draft_author(user, doc): if not hasattr(user, 'person'): return False - if user.person in doc.authors(): + if user.person in doc.author_persons(): return True return False diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py index 1a8a908dcdf..d6a320203b5 100644 --- a/ietf/ipr/admin.py +++ b/ietf/ipr/admin.py @@ -17,6 +17,7 @@ NonDocSpecificIprDisclosure, LegacyMigrationIprEvent, ) +from ietf.utils.admin import SaferTabularInline # ------------------------------------------------------ # ModelAdmins @@ -29,13 +30,13 @@ class Meta: 'sections':forms.TextInput, } -class IprDocRelInline(admin.TabularInline): +class IprDocRelInline(SaferTabularInline): model = IprDocRel form = IprDocRelAdminForm raw_id_fields = ['document'] extra = 1 -class RelatedIprInline(admin.TabularInline): +class RelatedIprInline(SaferTabularInline): model = RelatedIpr raw_id_fields = ['target'] fk_name = 'source' diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 665c99dc43b..0a43ff2c279 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -81,7 +81,8 @@ def get_document_emails(ipr): addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) + # Get a list of author names for the salutation in the body of the email + author_names = ', '.join(doc.author_names()) context = dict( settings=settings, diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py index 21515ed1a30..d873cce5362 100644 --- a/ietf/liaisons/admin.py +++ b/ietf/liaisons/admin.py @@ -7,15 +7,16 @@ from ietf.liaisons.models import ( LiaisonStatement, LiaisonStatementEvent, RelatedLiaisonStatement, LiaisonStatementAttachment ) +from ietf.utils.admin import SaferTabularInline -class RelatedLiaisonStatementInline(admin.TabularInline): +class RelatedLiaisonStatementInline(SaferTabularInline): model = RelatedLiaisonStatement fk_name = 'source' raw_id_fields = ['target'] extra = 1 -class LiaisonStatementAttachmentInline(admin.TabularInline): +class LiaisonStatementAttachmentInline(SaferTabularInline): model = LiaisonStatementAttachment raw_id_fields = ['document'] extra = 1 diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 1747e55571e..6ceda5ad38d 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -23,7 +23,7 @@ from ietf.liaisons.fields import SearchableLiaisonStatementsField from ietf.liaisons.models import (LiaisonStatement, LiaisonStatementEvent, LiaisonStatementAttachment, LiaisonStatementPurposeName) -from ietf.liaisons.utils import get_person_for_user, is_authorized_individual, OUTGOING_LIAISON_ROLES, \ +from ietf.liaisons.utils import get_person_for_user, OUTGOING_LIAISON_ROLES, \ INCOMING_LIAISON_ROLES from ietf.liaisons.widgets import ButtonWidget, ShowAttachmentsWidget from ietf.name.models import DocRelationshipName @@ -466,25 +466,11 @@ def set_to_fields(self): assert NotImplemented class IncomingLiaisonForm(LiaisonModelForm): - def clean(self): - if 'send' in list(self.data.keys()) and self.get_post_only(): - raise forms.ValidationError('As an IETF Liaison Manager you can not send incoming liaison statements, you only can post them') - return super(IncomingLiaisonForm, self).clean() def is_approved(self): '''Incoming Liaison Statements do not required approval''' return True - def get_post_only(self): - from_groups = self.cleaned_data.get("from_groups") - if ( - has_role(self.user, "Secretariat") - or has_role(self.user, "Liaison Coordinator") - or is_authorized_individual(self.user, from_groups) - ): - return False - return True - def set_from_fields(self): """Configure from "From" fields based on user roles""" qs = external_groups_for_person(self.person) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 5478f6c3028..e29045443f2 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -110,6 +110,17 @@ def test_help_pages(self): self.assertEqual(self.client.get('/liaison/help/from_ietf/').status_code, 200) self.assertEqual(self.client.get('/liaison/help/to_ietf/').status_code, 200) + def test_list_other_sdo(self): + GroupFactory(type_id="sdo", state_id="conclude", acronym="third") + GroupFactory(type_id="sdo", state_id="active", acronym="second") + GroupFactory(type_id="sdo", state_id="active", acronym="first") + url = urlreverse("ietf.liaisons.views.list_other_sdo") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + q = PyQuery(r.content) + self.assertEqual(len(q("h1")), 2) + first_td_elements_text = [e.text for e in q("tr").find("td:first-child a")] + self.assertEqual(first_td_elements_text, ["first", "second", "third"]) class UnitTests(TestCase): def test_get_contacts_for_liaison_messages_for_group_primary(self): @@ -203,7 +214,6 @@ def test_ajax(self): self.assertEqual(r.status_code, 200) data = r.json() self.assertEqual(data["error"], False) - self.assertEqual(data["post_only"], False) self.assertTrue('cc' in data) self.assertTrue('needs_approval' in data) self.assertTrue('to_contacts' in data) @@ -871,38 +881,6 @@ def test_add_outgoing_liaison(self): self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('aread@' in outbox[-1]['To']) self.assertTrue(submitter.email_address(), outbox[-1]['Cc']) - - - def test_add_outgoing_liaison_unapproved_post_only(self): - RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') - mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group - RoleFactory(name_id='ad',group=mars) - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'}) - login_testing_unauthorized(self, "secretary", url) - - # add new - mailbox_before = len(outbox) - from_group = Group.objects.get(acronym="mars") - to_group = Group.objects.filter(type="sdo")[0] - submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.UTC) - r = self.client.post(url, - dict(from_groups=str(from_group.pk), - from_contact=submitter.email_address(), - to_groups=str(to_group.pk), - to_contacts='to_contacts@example.com', - approved="", - purpose="info", - title="title", - submitted_date=today.strftime("%Y-%m-%d"), - body="body", - post_only="1", - )) - self.assertEqual(r.status_code, 302) - l = LiaisonStatement.objects.all().order_by("-id")[0] - self.assertEqual(l.state.slug,'pending') - self.assertEqual(len(outbox), mailbox_before + 1) def test_liaison_add_attachment(self): liaison = LiaisonStatementFactory(deadline=date_today(DEADLINE_TZINFO)+datetime.timedelta(days=1)) @@ -1152,17 +1130,6 @@ def test_redirect_for_approval(self): # ------------------------------------------------- # Form validations # ------------------------------------------------- - def test_post_and_send_fail(self): - RoleFactory(name_id='liaiman',person__user__username='ulm-liaiman',group__type_id='sdo',group__acronym='ulm') - GroupFactory(type_id='wg',acronym='mars') - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'incoming'}) - login_testing_unauthorized(self, "ulm-liaiman", url) - - r = self.client.post(url,get_liaison_post_data(),follow=True) - - self.assertEqual(r.status_code, 200) - self.assertContains(r, 'As an IETF Liaison Manager you can not send incoming liaison statements') def test_deadline_field(self): '''Required for action, comment, not info, response''' diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py index 0fbd29425e4..498df3b965c 100644 --- a/ietf/liaisons/urls.py +++ b/ietf/liaisons/urls.py @@ -37,4 +37,5 @@ url(r'^add/$', views.redirect_add), url(r'^for_approval/$', views.redirect_for_approval), url(r'^for_approval/(?P\d+)/$', views.redirect_for_approval), + url(r"^list_other_sdo/$", views.list_other_sdo), ] diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index f54a0233574..59c6ea69fcd 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -118,23 +118,6 @@ def normalize_sort(request): return sort, order_by -def post_only(group,person): - '''Returns true if the user is restricted to post_only (vs. post_and_send) for this - group. This is for incoming liaison statements. - - Secretariat have full access. - - Authorized Individuals have full access for the group they are associated with - - Liaison Managers can post only - ''' - if group.type_id == "sdo" and ( - not ( - has_role(person.user, "Secretariat") - or has_role(person.user, "Liaison Coordinator") - or group.role_set.filter(name="auth", person=person) - ) - ): - return True - else: - return False # ------------------------------------------------- # Ajax Functions @@ -159,15 +142,13 @@ def ajax_get_liaison_info(request): cc = [] does_need_approval = [] - can_post_only = [] to_contacts = [] response_contacts = [] - result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'post_only': False, 'full_list': []} + result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'full_list': []} for group in from_groups: cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) does_need_approval.append(needs_approval(group,person)) - can_post_only.append(post_only(group,person)) response_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) for group in to_groups: @@ -183,12 +164,15 @@ def ajax_get_liaison_info(request): else: does_need_approval = True - result.update({'error': False, - 'cc': list(set(cc)), - 'response_contacts':list(set(response_contacts)), - 'to_contacts': list(set(to_contacts)), - 'needs_approval': does_need_approval, - 'post_only': any(can_post_only)}) + result.update( + { + "error": False, + "cc": list(set(cc)), + "response_contacts": list(set(response_contacts)), + "to_contacts": list(set(to_contacts)), + "needs_approval": does_need_approval, + } + ) json_result = json.dumps(result) return HttpResponse(json_result, content_type='application/json') @@ -525,3 +509,17 @@ def liaison_resend(request, object_id): messages.success(request,'Liaison Statement resent') return redirect('ietf.liaisons.views.liaison_list') + +@role_required("Secretariat", "IAB", "Liaison Coordinator", "Liaison Manager") +def list_other_sdo(request): + def _sdo_order_key(obj:Group)-> tuple[str,str]: + state_order = { + "active" : "a", + "conclude": "b", + } + return (state_order.get(obj.state.slug,f"c{obj.state.slug}"), obj.acronym) + + sdos = sorted(list(Group.objects.filter(type="sdo")),key = _sdo_order_key) + for sdo in sdos: + sdo.liaison_managers =[r.person for r in sdo.role_set.filter(name="liaiman")] + return render(request,"liaisons/list_other_sdo.html",dict(sdos=sdos)) diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index 74368e83f2b..48db8af0a31 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -26,7 +26,9 @@ def render(self, name, value, **kwargs): html += '%s' % conditional_escape(i) required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label) html += '%s' % conditional_escape(required_str) - html += '' % conditional_escape(self.label) + html += ''.format( + f"id_{name}", conditional_escape(self.label) + ) return mark_safe(html) diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index d886a9a4b68..03abf5c0299 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -10,6 +10,7 @@ SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint, ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket, AttendanceTypeName) +from ietf.utils.admin import SaferTabularInline class UrlResourceAdmin(admin.ModelAdmin): @@ -18,7 +19,7 @@ class UrlResourceAdmin(admin.ModelAdmin): raw_id_fields = ['room', ] admin.site.register(UrlResource, UrlResourceAdmin) -class UrlResourceInline(admin.TabularInline): +class UrlResourceInline(SaferTabularInline): model = UrlResource class RoomAdmin(admin.ModelAdmin): @@ -28,7 +29,7 @@ class RoomAdmin(admin.ModelAdmin): admin.site.register(Room, RoomAdmin) -class RoomInline(admin.TabularInline): +class RoomInline(SaferTabularInline): model = Room class MeetingAdmin(admin.ModelAdmin): @@ -93,7 +94,7 @@ def name_lower(self, instance): admin.site.register(Constraint, ConstraintAdmin) -class SchedulingEventInline(admin.TabularInline): +class SchedulingEventInline(SaferTabularInline): model = SchedulingEvent raw_id_fields = ["by"] @@ -244,7 +245,7 @@ def queryset(self, request, queryset): return queryset.filter(tickets__attendance_type__slug=self.value()).distinct() return queryset -class RegistrationTicketInline(admin.TabularInline): +class RegistrationTicketInline(SaferTabularInline): model = RegistrationTicket class RegistrationAdmin(admin.ModelAdmin): diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 7d9e318aabc..8cf386abf29 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -236,16 +236,24 @@ def get_proceedings_materials(self): document__states__slug='active', document__states__type_id='procmaterials' ).order_by('type__order') - def get_attendance(self): - """Get the meeting attendance from the Registrations + def get_attendees(self, sessions=None): + """Get the meeting attendees from the Registrations - Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable - for this meeting. + Returns a pair of sets containing Person objects for persons attending + the meeting either onsite or online (remote). Returns None if the record + is unavailable for this meeting. + + sessions: optional queryset of Sessions to restrict which Attended records + count toward attendance. Defaults to all sessions at this meeting. + Note: restricting sessions does not affect the registration-based path + (checkedin / attended flags), only the Attended-record path. """ number = self.get_number() if number is None or number < 110: return None - Attendance = namedtuple('Attendance', 'onsite remote') + + if sessions is None: + sessions = self.session_set.all() # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older @@ -254,11 +262,8 @@ def get_attendance(self): # Looking up by registration and attendance records separately and joining in # python is far faster than combining the Q objects in the query (~100x). # Further optimization may be possible, but the queries are tricky... - attended_per_meeting_registration = ( - Q(registration__meeting=self) & ( - Q(registration__attended=True) | - Q(registration__checkedin=True) - ) + attended_per_meeting_registration = Q(registration__meeting=self) & ( + Q(registration__attended=True) | Q(registration__checkedin=True) ) attendees_by_reg = set( Person.objects.filter(attended_per_meeting_registration).values_list( @@ -266,29 +271,81 @@ def get_attendance(self): ) ) - attended_per_meeting_attended = ( - Q(attended__session__meeting=self) - # Note that we are not filtering to plenary, wg, or rg sessions - # as we do for nomcom eligibility - if picking up a badge (see above) - # is good enough, just attending e.g. a training session is also good enough - ) attendees_by_att = set( - Person.objects.filter(attended_per_meeting_attended).values_list( + Person.objects.filter(attended__session__in=sessions).values_list( "pk", flat=True ) ) - - attendees = Person.objects.filter( - pk__in=attendees_by_att | attendees_by_reg + + attendees = Person.objects.filter(pk__in=attendees_by_att | attendees_by_reg) + onsite = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="onsite", + ) + ) + remote = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="remote", + ) ) - onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) - remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) - return Attendance( - onsite=len(onsite), - remote=len(remote) + return (onsite, remote) + + def get_attendance(self): + """Get the meeting attendance from the Registrations + + Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable + for this meeting. + """ + attendees = self.get_attendees() + if attendees is None: + return None + Attendance = namedtuple("Attendance", "onsite remote") + onsite, remote = attendees + return Attendance(onsite=len(onsite), remote=len(remote)) + + def nomcom_eligible_participants(self): + """Return (onsite_pks, remote_pks) as frozensets for nomcom eligibility. + + Onsite: has onsite ticket AND (checked in OR attended a qualifying session). + Remote: has remote ticket AND attended a qualifying plenary/wg/rg session. + The registration attended flag alone is not sufficient for remote. + """ + sessions = self.session_set.filter( + Q(type="plenary") | Q(group__type__in=["wg", "rg"]) + ) + attendees = self.get_attendees(sessions=sessions) + if attendees is None: + onsite_pks = frozenset( + self.registration_set.onsite() + .filter(checkedin=True) + .values_list("person", flat=True) + .distinct() + ) + remote_pks = ( + frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() + ) + - onsite_pks + ) + return (onsite_pks, remote_pks) + onsite, remote = attendees + onsite_pks = frozenset(p.pk for p in onsite) + # Remote requires actual qualifying session attendance; the registration + # attended flag alone (used as a proxy before Attended records existed) + # is not sufficient. + session_attended_pks = frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() ) + remote_pks = frozenset(p.pk for p in remote) & session_attended_pks + return (onsite_pks, remote_pks) @property def proceedings_format_version(self): diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index 88562a88feb..490b75f9253 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -21,7 +21,13 @@ Attended, Registration, RegistrationTicket) -from ietf.name.resources import MeetingTypeNameResource +from ietf.name.resources import ( + AttendanceTypeNameResource, + MeetingTypeNameResource, + RegistrationTicketTypeNameResource, +) + + class MeetingResource(ModelResource): type = ToOneField(MeetingTypeNameResource, 'type') schedule = ToOneField('ietf.meeting.resources.ScheduleResource', 'schedule', null=True) @@ -437,11 +443,16 @@ class Meta: } api.meeting.register(AttendedResource()) -from ietf.meeting.resources import MeetingResource from ietf.person.resources import PersonResource class RegistrationResource(ModelResource): meeting = ToOneField(MeetingResource, 'meeting') person = ToOneField(PersonResource, 'person', null=True) + tickets = ToManyField( + 'ietf.meeting.resources.RegistrationTicketResource', + 'tickets', + full=True, + ) + class Meta: queryset = Registration.objects.all() serializer = api.Serializer() @@ -456,13 +467,17 @@ class Meta: "country_code": ALL, "email": ALL, "attended": ALL, + "checkedin": ALL, "meeting": ALL_WITH_RELATIONS, "person": ALL_WITH_RELATIONS, + "tickets": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationResource()) class RegistrationTicketResource(ModelResource): registration = ToOneField(RegistrationResource, 'registration') + attendance_type = ToOneField(AttendanceTypeNameResource, 'attendance_type') + ticket_type = ToOneField(RegistrationTicketTypeNameResource, 'ticket_type') class Meta: queryset = RegistrationTicket.objects.all() serializer = api.Serializer() @@ -471,8 +486,8 @@ class Meta: ordering = ['id', ] filtering = { "id": ALL, - "ticket_type": ALL, - "attendance_type": ALL, + "ticket_type": ALL_WITH_RELATIONS, + "attendance_type": ALL_WITH_RELATIONS, "registration": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationTicketResource()) diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index c361325f9ab..a73763560bf 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -1,11 +1,14 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime -from celery import shared_task -# from django.db.models import QuerySet +from itertools import batched + +from celery import shared_task, chain +from django.db.models import IntegerField +from django.db.models.functions import Cast from django.utils import timezone from ietf.utils import log @@ -19,9 +22,56 @@ from .utils import fetch_attendance_from_meetings +@shared_task +def agenda_data_refresh_task(num=None): + """Refresh agenda data for one plenary meeting + + If `num` is `None`, refreshes data for the current meeting. + """ + log.log( + f"Refreshing agenda data for {f"IETF-{num}" if num else "current IETF meeting"}" + ) + try: + generate_agenda_data(num, force_refresh=True) + except Exception as err: + # Log and swallow exceptions so failure on one meeting won't break a chain of + # tasks. This is used by agenda_data_refresh_all_task(). + log.log(f"ERROR: Refreshing agenda data failed for num={num}: {err}") + + @shared_task def agenda_data_refresh(): - generate_agenda_data(force_refresh=True) + """Deprecated. Use agenda_data_refresh_task() instead. + + TODO remove this after switching the periodic task to the new name + """ + log.log("Deprecated agenda_data_refresh task called!") + agenda_data_refresh_task() + + +@shared_task +def agenda_data_refresh_all_task(*, batch_size=10): + """Refresh agenda data for all plenary meetings + + Executes as a chain of tasks, each computing up to `batch_size` meetings + in a single task. + """ + meeting_numbers = sorted( + Meeting.objects.annotate( + number_as_int=Cast("number", output_field=IntegerField()) + ) + .filter(type_id="ietf", number_as_int__gt=64) + .values_list("number_as_int", flat=True) + ) + # Batch using chained maps rather than celery.chunk so we only use one worker + # at a time. + batched_task_chain = chain( + *( + agenda_data_refresh_task.map(nums) + for nums in batched(meeting_numbers, batch_size) + ) + ) + batched_task_chain.delay() @shared_task @@ -55,7 +105,9 @@ def proceedings_content_refresh_task(*, all=False): @shared_task def fetch_meeting_attendance_task(): # fetch most recent two meetings - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2] + meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by( + "-date" + )[:2] try: stats = fetch_attendance_from_meetings(meetings) except RuntimeError as err: @@ -64,8 +116,11 @@ def fetch_meeting_attendance_task(): for meeting, meeting_stats in zip(meetings, stats): log.log( "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format( - meeting.number, meeting_stats['created'], meeting_stats['updated'], meeting_stats['deleted'], - meeting_stats['processed'] + meeting.number, + meeting_stats["created"], + meeting_stats["updated"], + meeting_stats["deleted"], + meeting_stats["processed"], ) ) @@ -73,7 +128,7 @@ def fetch_meeting_attendance_task(): def _select_meetings( meetings: list[str] | None = None, meetings_since: str | None = None, - meetings_until: str | None = None + meetings_until: str | None = None, ): # nyah """Select meetings by number or date range""" # IETF-1 = 1986-01-16 @@ -130,15 +185,15 @@ def _select_meetings( @shared_task def resolve_meeting_materials_task( *, # only allow kw arguments - meetings: list[str] | None=None, - meetings_since: str | None=None, - meetings_until: str | None=None + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, ): """Run materials resolver on meetings - + Can request a set of meetings by number by passing a list in the meetings arg, or by range by passing an iso-format timestamps in meetings_since / meetings_until. - To select all meetings, set meetings_since="zero" and omit other parameters. + To select all meetings, set meetings_since="zero" and omit other parameters. """ meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) for meeting in meetings_qs.order_by("date"): @@ -155,7 +210,9 @@ def resolve_meeting_materials_task( f"meeting {meeting.number}: {err}" ) else: - log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + log.log( + f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) @shared_task @@ -163,13 +220,13 @@ def store_meeting_materials_as_blobs_task( *, # only allow kw arguments meetings: list[str] | None = None, meetings_since: str | None = None, - meetings_until: str | None = None + meetings_until: str | None = None, ): """Push meeting materials into the blob store Can request a set of meetings by number by passing a list in the meetings arg, or by range by passing an iso-format timestamps in meetings_since / meetings_until. - To select all meetings, set meetings_since="zero" and omit other parameters. + To select all meetings, set meetings_since="zero" and omit other parameters. """ meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) for meeting in meetings_qs.order_by("date"): @@ -187,4 +244,5 @@ def store_meeting_materials_as_blobs_task( ) else: log.log( - f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 262b47652cf..3269342924e 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1260,7 +1260,7 @@ def _assert_ietf_tz_correct(meetings, tz): # to inherit Django's settings.TIME_ZONE but I don't know whether that's guaranteed to be consistent. # To avoid test fragility, ask Moment what it considers local and expect that. local_tz = self.driver.execute_script('return moment.tz.guess();') - local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value=%s]' % local_tz) + local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) local_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) # Should start off in local time zone diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 869d9ec814e..0b9ce3f607f 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2021-2024, All Rights Reserved # -*- coding: utf-8 -*- """Tests of models in the Meeting application""" + import datetime from unittest.mock import patch @@ -10,7 +11,12 @@ import ietf.meeting.models from ietf.group.factories import GroupFactory, GroupHistoryFactory -from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory +from ietf.meeting.factories import ( + MeetingFactory, + SessionFactory, + AttendedFactory, + SessionPresentationFactory, +) from ietf.meeting.factories import RegistrationFactory from ietf.meeting.models import Session from ietf.utils.test_utils import TestCase @@ -18,105 +24,200 @@ class MeetingTests(TestCase): - def test_get_attendance_pre110(self): - """Pre-110 meetings do not calculate attendance""" - meeting = MeetingFactory(type_id='ietf', number='109') - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}) - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}) - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}) - self.assertIsNone(meeting.get_attendance()) - - def test_get_attendance_110(self): - """Look at attendance as captured at 110""" - meeting = MeetingFactory(type_id='ietf', number='110') - - # start with attendees that should be ignored - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 0) - - # add online attendees with at least one who registered but did not attend - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 0) - - # and the same for onsite attendees - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 5) - - # and once more after removing all the online attendees + def test_get_attendees_pre110(self): + """Pre-110 meetings return None""" + meeting = MeetingFactory(type_id="ietf", number="109") + RegistrationFactory.create_batch( + 3, meeting=meeting, with_ticket={"attendance_type_id": "unknown"} + ) + RegistrationFactory.create_batch( + 4, meeting=meeting, with_ticket={"attendance_type_id": "remote"} + ) + RegistrationFactory.create_batch( + 5, meeting=meeting, with_ticket={"attendance_type_id": "onsite"} + ) + self.assertIsNone(meeting.get_attendees()) + + def test_get_attendees_110(self): + """Registration-based attendance (attended flag) used at IETF 110""" + meeting = MeetingFactory(type_id="ietf", number="110") + + # Unknown-ticket attendees are excluded regardless of attended flag + RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual(len(onsite), 0) + self.assertEqual(len(remote), 0) + + # Remote attendees: only those with attended=True are included + remote_regs = RegistrationFactory.create_batch( + 4, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, set()) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Onsite attendees: only those with attended=True are included + onsite_regs = RegistrationFactory.create_batch( + 5, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Deleting remote registrations empties the remote set meeting.registration_set.remote().delete() - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 5) - - def test_get_attendance_113(self): - """Simulate IETF 113 attendance gathering data""" - meeting = MeetingFactory(type_id='ietf', number='113') - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True, checkedin=False) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=True) - p1 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p1) - p2 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p2) - attendance = meeting.get_attendance() - self.assertEqual(attendance.onsite, 3) - self.assertEqual(attendance.remote, 1) - - def test_get_attendance_keeps_meetings_distinct(self): + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual(len(remote), 0) + + def test_get_attendees_113(self): + """Simulate IETF 113: mix of attended flag, checkedin flag, and Attended records""" + meeting = MeetingFactory(type_id="ietf", number="113") + p_attended = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + checkedin=False, + ).person + p_checkedin = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=True, + ).person + p_onsite_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_onsite_session) + p_remote_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_remote_session) + onsite, remote = meeting.get_attendees() + self.assertEqual( + {p.pk for p in onsite}, {p_attended.pk, p_checkedin.pk, p_onsite_session.pk} + ) + self.assertEqual({p.pk for p in remote}, {p_remote_session.pk}) + + def test_get_attendees_keeps_meetings_distinct(self): """No cross-talk between attendance for different meetings""" - # numbers are arbitrary here - first_mtg = MeetingFactory(type_id='ietf', number='114') - second_mtg = MeetingFactory(type_id='ietf', number='115') + first_mtg = MeetingFactory(type_id="ietf", number="114") + second_mtg = MeetingFactory(type_id="ietf", number="115") - # Create a person who attended a remote session for first_mtg and onsite for second_mtg without - # checking in for either. - p = RegistrationFactory(meeting=second_mtg, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person + # Person with remote ticket at first_mtg and onsite ticket at second_mtg, attended both via Attended records + p = RegistrationFactory( + meeting=second_mtg, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + RegistrationFactory( + meeting=first_mtg, + person=p, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ) AttendedFactory(session__meeting=first_mtg, person=p) - RegistrationFactory(meeting=first_mtg, person=p, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False) AttendedFactory(session__meeting=second_mtg, person=p) - att = first_mtg.get_attendance() - self.assertEqual(att.onsite, 0) - self.assertEqual(att.remote, 1) + first_onsite, first_remote = first_mtg.get_attendees() + self.assertEqual({q.pk for q in first_onsite}, set()) + self.assertEqual({q.pk for q in first_remote}, {p.pk}) + + second_onsite, second_remote = second_mtg.get_attendees() + self.assertEqual({q.pk for q in second_onsite}, {p.pk}) + self.assertEqual({q.pk for q in second_remote}, set()) + + def test_get_attendance(self): + """get_attendance delegates to get_attendees and returns counts as a NamedTuple""" + meeting = MeetingFactory(type_id="ietf", number="120") + onsite_regs = RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + remote_regs = RegistrationFactory.create_batch( + 2, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + att = meeting.get_attendance() + self.assertIsNotNone(att) + self.assertEqual(att.onsite, len(onsite_regs)) + self.assertEqual(att.remote, len(remote_regs)) - att = second_mtg.get_attendance() - self.assertEqual(att.onsite, 1) - self.assertEqual(att.remote, 0) + old_meeting = MeetingFactory(type_id="ietf", number="109") + self.assertIsNone(old_meeting.get_attendance()) def test_vtimezone(self): # normal time zone that should have a zoneinfo file - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNotNone(vtz) self.assertGreater(len(vtz), 0) # time zone that does not have a zoneinfo file should return None - meeting = MeetingFactory(type_id='ietf', time_zone='Fake/Time_Zone', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="Fake/Time_Zone", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNone(vtz) # ioerror trying to read zoneinfo should return None - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) - with patch('ietf.meeting.models.io.open', side_effect=IOError): + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) + with patch("ietf.meeting.models.io.open", side_effect=IOError): vtz = meeting.vtimezone() self.assertIsNone(vtz) def test_group_at_the_time(self): - m = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=10)) + m = MeetingFactory( + type_id="ietf", date=date_today() - datetime.timedelta(days=10) + ) cached_groups = GroupFactory.create_batch(2) m.cached_groups_at_the_time = {g.pk: g for g in cached_groups} # fake the cache - uncached_group_hist = GroupHistoryFactory(time=datetime_today() - datetime.timedelta(days=30)) - self.assertEqual(m.group_at_the_time(uncached_group_hist.group), uncached_group_hist) + uncached_group_hist = GroupHistoryFactory( + time=datetime_today() - datetime.timedelta(days=30) + ) + self.assertEqual( + m.group_at_the_time(uncached_group_hist.group), uncached_group_hist + ) self.assertIn(uncached_group_hist.group.pk, m.cached_groups_at_the_time) @@ -127,27 +228,36 @@ def test_chat_archive_url(self): meeting__number=120, # needs to use proceedings_format_version > 1 ) with override_settings(): - if hasattr(settings, 'CHAT_ARCHIVE_URL_PATTERN'): + if hasattr(settings, "CHAT_ARCHIVE_URL_PATTERN"): del settings.CHAT_ARCHIVE_URL_PATTERN self.assertEqual(session.chat_archive_url(), session.chat_room_url()) - settings.CHAT_ARCHIVE_URL_PATTERN = 'http://chat.example.com' - self.assertEqual(session.chat_archive_url(), 'http://chat.example.com') - chatlog = SessionPresentationFactory(session=session, document__type_id='chatlog').document + settings.CHAT_ARCHIVE_URL_PATTERN = "http://chat.example.com" + self.assertEqual(session.chat_archive_url(), "http://chat.example.com") + chatlog = SessionPresentationFactory( + session=session, document__type_id="chatlog" + ).document self.assertEqual(session.chat_archive_url(), chatlog.get_href()) # datatracker 8.8.0 rolled out on 2022-07-15. Before that, chat logs were jabber logs hosted at www.ietf.org. - session_with_jabber = SessionFactory(group__acronym='fakeacronym', meeting__date=datetime.date(2022,7,14)) - self.assertEqual(session_with_jabber.chat_archive_url(), 'https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D') - chatlog = SessionPresentationFactory(session=session_with_jabber, document__type_id='chatlog').document + session_with_jabber = SessionFactory( + group__acronym="fakeacronym", meeting__date=datetime.date(2022, 7, 14) + ) + self.assertEqual( + session_with_jabber.chat_archive_url(), + "https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D", + ) + chatlog = SessionPresentationFactory( + session=session_with_jabber, document__type_id="chatlog" + ).document self.assertEqual(session_with_jabber.chat_archive_url(), chatlog.get_href()) def test_chat_room_name(self): - session = SessionFactory(group__acronym='xyzzy') - self.assertEqual(session.chat_room_name(), 'xyzzy') - session.type_id = 'plenary' - self.assertEqual(session.chat_room_name(), 'plenary') - session.chat_room = 'fnord' - self.assertEqual(session.chat_room_name(), 'fnord') + session = SessionFactory(group__acronym="xyzzy") + self.assertEqual(session.chat_room_name(), "xyzzy") + session.type_id = "plenary" + self.assertEqual(session.chat_room_name(), "plenary") + session.chat_room = "fnord" + self.assertEqual(session.chat_room_name(), "fnord") def test_alpha_str(self): self.assertEqual(Session._alpha_str(0), "a") @@ -157,7 +267,11 @@ def test_alpha_str(self): self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz") self.assertEqual(Session._alpha_str(27 * 26), "aaa") - @patch.object(ietf.meeting.models.Session, "_session_recording_url_label", return_value="LABEL") + @patch.object( + ietf.meeting.models.Session, + "_session_recording_url_label", + return_value="LABEL", + ) def test_session_recording_url(self, mock): for session_type in ["ietf", "interim"]: session = SessionFactory(meeting__type_id=session_type) @@ -165,20 +279,29 @@ def test_session_recording_url(self, mock): if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): del settings.MEETECHO_SESSION_RECORDING_URL self.assertIsNone(session.session_recording_url()) - + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" - self.assertEqual(session.session_recording_url(), "http://player.example.com") - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" - self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL") + self.assertEqual( + session.session_recording_url(), "http://player.example.com" + ) + + settings.MEETECHO_SESSION_RECORDING_URL = ( + "http://player.example.com?{session_label}" + ) + self.assertEqual( + session.session_recording_url(), "http://player.example.com?LABEL" + ) - session.meetecho_recording_name="actualname" + session.meetecho_recording_name = "actualname" session.save() - self.assertEqual(session.session_recording_url(), "http://player.example.com?actualname") + self.assertEqual( + session.session_recording_url(), + "http://player.example.com?actualname", + ) def test_session_recording_url_label_ietf(self): session = SessionFactory( - meeting__type_id='ietf', + meeting__type_id="ietf", meeting__date=date_today(), meeting__number="123", group__acronym="acro", @@ -186,15 +309,17 @@ def test_session_recording_url_label_ietf(self): session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF123-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) def test_session_recording_url_label_interim(self): session = SessionFactory( - meeting__type_id='interim', + meeting__type_id="interim", meeting__date=date_today(), group__acronym="acro", ) session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) diff --git a/ietf/meeting/tests_session_requests.py b/ietf/meeting/tests_session_requests.py index 0cb092d2f87..42dbee5f230 100644 --- a/ietf/meeting/tests_session_requests.py +++ b/ietf/meeting/tests_session_requests.py @@ -236,7 +236,7 @@ def test_edit(self): self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database - sessions = Session.objects.filter(meeting=meeting, group=mars) + sessions = Session.objects.filter(meeting=meeting, group=mars).order_by("id") self.assertEqual(len(sessions), 2) session = sessions[0] self.assertFalse(session.constraints().filter(name='time_relation')) diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index a5da00ecbf5..2c5120a39d5 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -5,23 +5,63 @@ from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today from .factories import MeetingFactory -from .tasks import proceedings_content_refresh_task, agenda_data_refresh +from .tasks import ( + proceedings_content_refresh_task, + agenda_data_refresh_task, + agenda_data_refresh_all_task, +) from .tasks import fetch_meeting_attendance_task class TaskTests(TestCase): @patch("ietf.meeting.tasks.generate_agenda_data") - def test_agenda_data_refresh(self, mock_generate): - agenda_data_refresh() + def test_agenda_data_refresh_task(self, mock_generate): + agenda_data_refresh_task() self.assertTrue(mock_generate.called) - self.assertEqual(mock_generate.call_args, call(force_refresh=True)) + self.assertEqual(mock_generate.call_args, call(None, force_refresh=True)) + + mock_generate.reset_mock() + mock_generate.side_effect = RuntimeError + try: + agenda_data_refresh_task() + except Exception as err: + self.fail( + f"agenda_data_refresh_task should not raise exceptions (got {repr(err)})" + ) + + @patch("ietf.meeting.tasks.agenda_data_refresh_task") + @patch("ietf.meeting.tasks.chain") + def test_agenda_data_refresh_all_task(self, mock_chain, mock_agenda_data_refresh): + # Patch the agenda_data_refresh_task task with a mock whose `.map` attribute + # converts its argument, which is expected to be an iterator, to a list + # and returns it. We'll use this to check that the expected task chain + # was set up, but we don't actually run any celery tasks. + mock_agenda_data_refresh.map.side_effect = lambda x: list(x) + + meetings = MeetingFactory.create_batch(5, type_id="ietf") + numbers = sorted(int(m.number) for m in meetings) + agenda_data_refresh_all_task(batch_size=2) + self.assertTrue(mock_chain.called) + # The lists in the call() below are the output of the lambda we patched in + # via mock_agenda_data_refresh.map.side_effect above. I.e., this tests that + # map() was called with the correct batched data. + self.assertEqual( + mock_chain.call_args, + call( + [numbers[0], numbers[1]], + [numbers[2], numbers[3]], + [numbers[4]], + ), + ) + self.assertEqual(mock_agenda_data_refresh.call_count, 0) + self.assertEqual(mock_agenda_data_refresh.map.call_count, 3) @patch("ietf.meeting.tasks.generate_proceedings_content") def test_proceedings_content_refresh_task(self, mock_generate): # Generate a couple of meetings meeting120 = MeetingFactory(type_id="ietf", number="120") # 24 * 5 meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 - + # Times to be returned now_utc = datetime.datetime.now(tz=datetime.UTC) hour_00_utc = now_utc.replace(hour=0) @@ -34,19 +74,19 @@ def test_proceedings_content_refresh_task(self, mock_generate): self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True)) mock_generate.reset_mock() - + # hour 01 - should call no meetings with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 0) - + # hour 07 - should call meeting with number % 24 == 0 with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True)) mock_generate.reset_mock() - + # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task(all=True) @@ -61,10 +101,10 @@ def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), ] data = { - 'created': 1, - 'updated': 2, - 'deleted': 0, - 'processed': 3, + "created": 1, + "updated": 2, + "deleted": 0, + "processed": 3, } mock_fetch_attendance.return_value = [data, data] diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index b1bbc629071..eea08be8c77 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -33,6 +33,7 @@ from django.http import QueryDict, FileResponse from django.template import Context, Template from django.utils import timezone +from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.text import slugify @@ -49,8 +50,12 @@ from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data -from ietf.meeting.utils import condition_slide_order, generate_proceedings_content -from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting +from ietf.meeting.utils import ( + condition_slide_order, + generate_proceedings_content, + diff_meeting_schedules, +) +from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data @@ -4750,7 +4755,7 @@ def _approval_url(slidesub): 0, "second session proposed slides should be linked for approval", ) - + class EditScheduleListTests(TestCase): def setUp(self): @@ -4765,73 +4770,151 @@ def test_list_schedules(self): self.assertTrue(r.status_code, 200) def test_diff_schedules(self): - meeting = make_meeting_test_data() - - url = urlreverse('ietf.meeting.views.diff_schedules',kwargs={'num':meeting.number}) - login_testing_unauthorized(self,"secretary", url) - r = self.client.get(url) - self.assertTrue(r.status_code, 200) - - from_schedule = Schedule.objects.get(meeting=meeting, name="test-unofficial-schedule") - - session1 = Session.objects.filter(meeting=meeting, group__acronym='mars').first() - session2 = Session.objects.filter(meeting=meeting, group__acronym='ames').first() - session3 = SessionFactory(meeting=meeting, group=Group.objects.get(acronym='mars'), - attendees=10, requested_duration=datetime.timedelta(minutes=70), - add_to_schedule=False) - SchedulingEvent.objects.create(session=session3, status_id='schedw', by=Person.objects.first()) - - slot2 = TimeSlot.objects.filter(meeting=meeting, type='regular').order_by('-time').first() - slot3 = TimeSlot.objects.create( - meeting=meeting, type_id='regular', location=slot2.location, - duration=datetime.timedelta(minutes=60), - time=slot2.time + datetime.timedelta(minutes=60), + # Create meeting and some time slots + meeting = MeetingFactory(type_id="ietf", populate_schedule=False) + rooms = RoomFactory.create_batch(2, meeting=meeting) + # first index is room, second is time + timeslots = [ + [ + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(9, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(10, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(11, 0, tzinfo=datetime.UTC) + ) + ), + ] + for room in rooms + ] + sessions = SessionFactory.create_batch( + 5, meeting=meeting, add_to_schedule=False ) - # copy - new_url = urlreverse("ietf.meeting.views.new_meeting_schedule", kwargs=dict(num=meeting.number, owner=from_schedule.owner_email(), name=from_schedule.name)) - r = self.client.post(new_url, { - 'name': "newtest", - 'public': "on", - }) - self.assertNoFormPostErrors(r) + from_schedule = ScheduleFactory(meeting=meeting) + to_schedule = ScheduleFactory(meeting=meeting) - to_schedule = Schedule.objects.get(meeting=meeting, name='newtest') + # sessions[0]: not scheduled in from_schedule, scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[0], + timeslot=timeslots[0][0], + ) + # sessions[1]: scheduled in from_schedule, not scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[1], + timeslot=timeslots[0][0], + ) + # sessions[2]: moves rooms, not time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[2], + timeslot=timeslots[0][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[2], + timeslot=timeslots[1][1], + ) + # sessions[3]: moves time, not room + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[3], + timeslot=timeslots[1][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[3], + timeslot=timeslots[1][2], + ) + # sessions[4]: moves room and time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[4], + timeslot=timeslots[1][0], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[4], + timeslot=timeslots[0][2], + ) - # make some changes + # Check the raw diffs + raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) + self.assertCountEqual( + raw_diffs, + [ + { + "change": "schedule", + "session": sessions[0].pk, + "to": timeslots[0][0].pk, + }, + { + "change": "unschedule", + "session": sessions[1].pk, + "from": timeslots[0][0].pk, + }, + { + "change": "move", + "session": sessions[2].pk, + "from": timeslots[0][1].pk, + "to": timeslots[1][1].pk, + }, + { + "change": "move", + "session": sessions[3].pk, + "from": timeslots[1][1].pk, + "to": timeslots[1][2].pk, + }, + { + "change": "move", + "session": sessions[4].pk, + "from": timeslots[1][0].pk, + "to": timeslots[0][2].pk, + }, + ] + ) - edit_url = urlreverse("ietf.meeting.views.edit_meeting_schedule", kwargs=dict(num=meeting.number, owner=to_schedule.owner_email(), name=to_schedule.name)) + # Check the view + url = urlreverse("ietf.meeting.views.diff_schedules", + kwargs={"num": meeting.number}) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(r.status_code, 200) - # schedule session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot3.pk, - 'session': session3.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # unschedule session - r = self.client.post(edit_url, { - 'action': 'unassign', - 'session': session1.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # move session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot2.pk, - 'session': session2.pk, + # with show room changes disabled - does not show sessions[2] because it did + # not change time + r = self.client.get(url, { + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, }) - self.assertEqual(json.loads(r.content)['success'], True) + self.assertTrue(r.status_code, 200) + q = PyQuery(r.content) + self.assertEqual(len(q(".schedule-diffs tr")), 4 + 1) - # now get differences + # with show room changes enabled - shows all changes r = self.client.get(url, { - 'from_schedule': from_schedule.name, - 'to_schedule': to_schedule.name, + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, + "show_room_changes": "on", }) self.assertTrue(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q(".schedule-diffs tr")), 3+1) + self.assertEqual(len(q(".schedule-diffs tr")), 5 + 1) def test_delete_schedule(self): url = urlreverse('ietf.meeting.views.delete_schedule', @@ -5220,7 +5303,9 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertContains(r, 'Remote instructions: https://someurl.example.com') + # Unfold long lines that might have been folded by iCal + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertIn('Remote instructions: https://someurl.example.com', content_unfolded) Session.objects.filter(meeting__type_id='interim').update(remote_instructions='') r = self.client.get(url) @@ -5228,7 +5313,8 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertNotContains(r, 'Remote instructions:') + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertNotIn('Remote instructions:', content_unfolded) updated = meeting.updated() self.assertIsNotNone(updated) @@ -7066,7 +7152,7 @@ def test_disapprove_proposed_slides(self): self.assertFalse(exists_in_storage("staging", submission.filename)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+rejected") + self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+declined") @override_settings(MEETECHO_API_CONFIG="fake settings") # enough to trigger API calls @patch("ietf.meeting.views.SlidesManager") @@ -7260,6 +7346,67 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): fd.close() self.assertIn('third version', contents) + @override_settings( + MEETECHO_API_CONFIG="fake settings" + ) # enough to trigger API calls + @patch("ietf.meeting.views.SlidesManager") + def test_notify_meetecho_of_all_slides(self, mock_slides_manager_cls): + for meeting_type in ["ietf", "interim"]: + # Reset for the sake of the second iteration + self.client.logout() + mock_slides_manager_cls.reset_mock() + + session = SessionFactory(meeting__type_id=meeting_type) + meeting = session.meeting + + # bad meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": 9999, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + r = self.client.post(url) + self.assertEqual(r.status_code, 404) + self.assertFalse(mock_slides_manager_cls.called) + self.client.logout() + + # good meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": meeting.number, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + self.assertFalse(mock_slides_manager_cls.called) + mock_slides_manager = mock_slides_manager_cls.return_value + mock_slides_manager.send_update.return_value = True + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertEqual( + str(messages[0]), f"Notified Meetecho about slides for {session}" + ) + + mock_slides_manager.send_update.reset_mock() + mock_slides_manager.send_update.return_value = False + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertIn( + "No sessions were eligible for Meetecho slides update.", str(messages[0]) + ) + @override_settings(IETF_NOTES_URL='https://notes.ietf.org/') class ImportNotesTests(TestCase): @@ -8861,6 +9008,8 @@ def test_proceedings_attendees(self): - assert onsite checkedin=True appears, not onsite checkedin=False - assert remote attended appears, not remote not attended - prefer onsite checkedin=True to remote attended when same person has both + - summary stats row shows correct counts + - chart data JSON is embedded with correct values """ m = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118") @@ -8882,6 +9031,17 @@ def test_proceedings_attendees(self): text = q('#id_attendees tbody tr').text().replace('\n', ' ') self.assertEqual(text, f"A Person {areg.affiliation} {areg.country_code} onsite C Person {creg.affiliation} {creg.country_code} remote") + # Summary stats row: Onsite / Remote / Total (matches registration.ietf.org) + self.assertContains(response, 'Onsite:') + self.assertContains(response, 'Remote:') + self.assertContains(response, 'Total:') + self.assertContains(response, '1') # onsite and remote + self.assertContains(response, '2') # total + + # Chart data embedded in page + chart_json = json.loads(q('#attendees-chart-data').text()) + self.assertEqual(chart_json['type'], [['Onsite', 1], ['Remote', 1]]) + def test_proceedings_overview(self): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. @@ -9280,20 +9440,32 @@ def test_get_next_sequence(self): sequence = get_next_sequence(group,meeting,'recording') self.assertEqual(sequence,1) - def test_participants_for_meeting(self): + def test_nomcom_eligible_participants_for_meeting(self): m = MeetingFactory.create(type_id='ietf') areg = RegistrationFactory(meeting=m, checkedin=True, with_ticket={'attendance_type_id': 'onsite'}) breg = RegistrationFactory(meeting=m, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) creg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) dreg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) AttendedFactory(session__meeting=m, session__type_id='plenary', person=creg.person) - checked_in, attended = participants_for_meeting(m) - self.assertIn(areg.person.pk, checked_in) - self.assertNotIn(breg.person.pk, checked_in) - self.assertNotIn(areg.person.pk, attended) - self.assertNotIn(breg.person.pk, attended) - self.assertIn(creg.person.pk, attended) - self.assertNotIn(dreg.person.pk, attended) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertIn(areg.person.pk, onsite_pks) + self.assertNotIn(breg.person.pk, onsite_pks) + self.assertNotIn(areg.person.pk, remote_pks) + self.assertNotIn(breg.person.pk, remote_pks) + self.assertIn(creg.person.pk, remote_pks) + self.assertNotIn(dreg.person.pk, remote_pks) + + def test_nomcom_eligible_participants_remote_requires_session(self): + """Remote registration.attended=True without a qualifying session does not count (>= 110 path)""" + m = MeetingFactory.create(type_id='ietf', number='119') + # Remote with attended=True but no Attended record: should not be eligible + no_session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + # Remote with attended=True AND a qualifying Attended record: should be eligible + session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=session_reg.person) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertNotIn(no_session_reg.person.pk, remote_pks) + self.assertIn(session_reg.person.pk, remote_pks) def test_session_attendance(self): meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number='118') @@ -9332,7 +9504,7 @@ def test_session_attendance(self): self.assertEqual(r.status_code, 200) self.assertContains(r, '3 attendees') for person in persons: - self.assertContains(r, person.plain_name()) + self.assertContains(r, escape(person.plain_name())) # Test for the "I was there" button. def _test_button(person, expected): @@ -9352,14 +9524,14 @@ def _test_button(person, expected): # attempt to POST anyway is ignored r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertNotContains(r, persons[3].plain_name()) + self.assertNotContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 3) # button is shown, and POST is accepted meeting.importantdate_set.update(name_id='revsub',date=date_today() + datetime.timedelta(days=20)) _test_button(persons[3], True) r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertContains(r, persons[3].plain_name()) + self.assertContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 4) # When the meeting is finalized, a bluesheet file is generated, diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index af36a6656c3..a038e1cfe6e 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -15,6 +15,7 @@ def get_redirect_url(self, *args, **kwargs): safe_for_all_meeting_types = [ url(r'^session/(?P[-a-z0-9]+)/?$', views.session_details), + url(r'^session/(?P[-a-z0-9]+)/send_slide_notifications$', views.notify_meetecho_of_all_slides), url(r'^session/(?P\d+)/drafts$', views.add_session_drafts), url(r'^session/(?P\d+)/recordings$', views.add_session_recordings), url(r'^session/(?P\d+)/attendance$', views.session_attendance), @@ -30,7 +31,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^session/(?P\d+)/doc/%(name)s/remove$' % settings.URL_REGEXPS, views.remove_sessionpresentation), url(r'^session/(?P\d+)\.ics$', views.agenda_ical), url(r'^sessions/(?P[-a-z0-9]+)\.ics$', views.agenda_ical), - url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides) + url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides), ] diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index bdf3d3d3d3a..ffd37fc363d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1025,9 +1025,18 @@ def resolve_materials_for_one_meeting(meeting: Meeting): ) def resolve_uploaded_material(meeting: Meeting, doc: Document): - resolved = [] + resolved: list[ResolvedMaterial] = [] + remove = ResolvedMaterial.objects.none() blob = resolve_one_material(doc, rev=None, ext=None) - if blob is not None: + if blob is None: + # Versionless file does not exist. Remove the versionless ResolvedMaterial + # if it existed. This is to avoid leaving behind a stale link to a replaced + # version. This comes up e.g. if a ProceedingsMaterial is changed from having + # an uploaded file to being an external URL. + remove = ResolvedMaterial.objects.filter( + name=doc.name, meeting_number=meeting.number + ) + else: resolved.append( ResolvedMaterial( name=doc.name, @@ -1047,12 +1056,15 @@ def resolve_uploaded_material(meeting: Meeting, doc: Document): blob=blob.name, ) ) + # Create the new record(s) ResolvedMaterial.objects.bulk_create( resolved, update_conflicts=True, unique_fields=["name", "meeting_number"], update_fields=["bucket", "blob"], ) + # and remove one if necessary (will be a none() queryset if not) + remove.delete() def store_blob_for_one_material_file(doc: Document, rev: str, filepath: Path): @@ -1350,17 +1362,6 @@ def post_process(doc): doc.save_with_history([e]) -def participants_for_meeting(meeting): - """ Return a tuple (checked_in, attended) - checked_in = queryset of onsite, checkedin participants values_list('person') - attended = queryset of remote participants who attended a session values_list('person') - """ - checked_in = meeting.registration_set.onsite().filter(checkedin=True).values_list('person', flat=True).distinct() - sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg'])) - attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct() - return (checked_in, attended) - - def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 69635d62194..913c8bd3021 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -40,7 +40,7 @@ from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.core.validators import URLValidator -from django.urls import reverse,reverse_lazy +from django.urls import reverse, reverse_lazy, NoReverseMatch from django.db.models import F, Max, Q from django.forms.models import modelform_factory, inlineformset_factory from django.template import TemplateDoesNotExist @@ -107,9 +107,9 @@ from ietf.meeting.utils import preprocess_meeting_important_dates from ietf.meeting.utils import new_doc_for_session, write_doc_for_session from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording -from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet +from ietf.meeting.utils import generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message -from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName +from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown from ietf.utils.decorators import require_api_key from ietf.utils.hedgedoc import Note, NoteError @@ -1675,6 +1675,11 @@ def list_schedules(request, num): class DiffSchedulesForm(forms.Form): from_schedule = forms.ChoiceField() to_schedule = forms.ChoiceField() + show_room_changes = forms.BooleanField( + initial=False, + required=False, + help_text="Include changes to room without a date or time change", + ) def __init__(self, meeting, user, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1707,6 +1712,14 @@ def diff_schedules(request, num): raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) diffs = prefetch_schedule_diff_objects(raw_diffs) + if not form.cleaned_data["show_room_changes"]: + # filter out room-only changes + diffs = [ + d + for d in diffs + if (d["change"] != "move") or (d["from"].time != d["to"].time) + ] + for d in diffs: s = d['session'] s.session_label = s.short_name @@ -1846,18 +1859,22 @@ def generate_agenda_data(num=None, force_refresh=False): :num: meeting number :force_refresh: True to force a refresh of the cache """ - cache = caches["default"] - cache_timeout = 6 * 60 - meeting = get_ietf_meeting(num) if meeting is None: raise Http404("No such full IETF meeting") elif int(meeting.number) <= 64: - return Http404("Pre-IETF 64 meetings are not available through this API") - else: - pass + raise Http404("Pre-IETF 64 meetings are not available through this API") + is_current_meeting = meeting.number == get_current_ietf_meeting_num() + + cache = caches["agenda"] + cache_timeout = ( + settings.AGENDA_CACHE_TIMEOUT_CURRENT_MEETING + if is_current_meeting + else settings.AGENDA_CACHE_TIMEOUT_DEFAULT + ) + cache_format = "1" # bump this on backward-incompatible data format changes - cache_key = f"generate_agenda_data_{meeting.number}" + cache_key = f"generate_agenda_data:{meeting.number}:v{cache_format}" if not force_refresh: cached_value = cache.get(cache_key) if cached_value is not None: @@ -1877,8 +1894,6 @@ def generate_agenda_data(num=None, force_refresh=False): filter_organizer = AgendaFilterOrganizer(assignments=filtered_assignments) - is_current_meeting = (num is None) or (num == get_current_ietf_meeting_num()) - # Get Floor Plans floors = FloorPlan.objects.filter(meeting=meeting).order_by('order') @@ -1953,21 +1968,32 @@ def api_get_session_materials(request, session_id=None): ) -def agenda_extract_schedule (item): +def agenda_extract_schedule(item): + if item.session.current_status == "resched": + resched_to = item.session.tombstone_for.official_timeslotassignment() + else: + resched_to = None return { "id": item.id, + "slug": item.slug(), "sessionId": item.session.id, - "room": item.room_name if item.timeslot.show_location else None, + "room": (item.timeslot.get_location() or None) if item.timeslot else None, "location": { "short": item.timeslot.location.floorplan.short, "name": item.timeslot.location.floorplan.name, } if (item.timeslot.show_location and item.timeslot.location and item.timeslot.location.floorplan) else {}, "acronym": item.acronym, - "duration": item.timeslot.duration.seconds, + "duration": item.timeslot.duration.total_seconds(), "name": item.session.name, + "slotId": item.timeslot.id, "slotName": item.timeslot.name, + "slotModified": item.timeslot.modified.isoformat(), "startDateTime": item.timeslot.time.isoformat(), "status": item.session.current_status, + "rescheduledTo": { + "startDateTime": resched_to.timeslot.time.isoformat(), + "duration": resched_to.timeslot.duration.total_seconds(), + } if resched_to is not None else {}, "type": item.session.type.slug, "purpose": item.session.purpose.slug, "isBoF": item.session.group_at_the_time().state_id == "bof", @@ -1985,7 +2011,7 @@ def agenda_extract_schedule (item): "showAgenda": True if (item.session.agenda() is not None or item.session.remote_instructions) else False }, "agenda": { - "url": item.session.agenda().get_href() + "url": item.session.agenda().get_versionless_href() } if item.session.agenda() is not None else { "url": None }, @@ -2277,10 +2303,131 @@ def ical_session_status(assignment): else: return "CONFIRMED" + +def render_icalendar_precomp(agenda_data): + ical_content = generate_agenda_ical_precomp(agenda_data) + return HttpResponse(ical_content, content_type="text/calendar") + + def render_icalendar(schedule, assignments): ical_content = generate_agenda_ical(schedule, assignments) return HttpResponse(ical_content, content_type="text/calendar") + +def generate_agenda_ical_precomp(agenda_data): + """Generate iCalendar from precomputed data using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + meeting_data = agenda_data["meeting"] + for item in agenda_data["schedule"]: + event = Event() + + uid = f"ietf-{meeting_data["number"]}-{item["slotId"]}-{item["acronym"]}" + event.add("uid", uid) + + # add custom field with meeting's local TZ + event.add("x-meeting-tz", meeting_data["timezone"]) + + if item["name"]: + summary = item["name"] + else: + summary = f"{item["groupAcronym"]} - {item["groupName"]}" + + if item["note"]: + summary += f" ({item["note"]})" + + event.add("summary", summary) + + if item["room"]: + event.add("location", item["room"]) # room name + + if item["status"] == "canceled": + status = "CANCELLED" + elif item["status"] == "resched": + resched_to = item["rescheduledTo"] + if resched_to is None: + status = "RESCHEDULED" + else: + resched_start = datetime.datetime.fromisoformat( + resched_to["startDateTime"] + ) + dur = datetime.timedelta(seconds=resched_to["duration"]) + resched_end = resched_start + dur + formatted_start = resched_start.strftime("%A %H:%M").upper() + formatted_end = resched_end.strftime("%H:%M") + status = f"RESCHEDULED TO {formatted_start}-{formatted_end}" + else: + status = "CONFIRMED" + event.add("status", status) + + event.add("class", "PUBLIC") + + start_time = datetime.datetime.fromisoformat(item["startDateTime"]) + duration = datetime.timedelta(seconds=item["duration"]) + event.add("dtstart", start_time) + event.add("dtend", start_time + duration) + + # DTSTAMP: when the event was created or last modified (in UTC) + # n.b. timeslot.modified may not be an accurate measure of this + event.add("dtstamp", datetime.datetime.fromisoformat(item["slotModified"])) + + description_parts = [item["slotName"]] + + if item["note"]: + description_parts.append(f"Note: {item["note"]}") + + links = item["links"] + if links["onsiteTool"]: + description_parts.append(f"Onsite tool: {links["onsiteTool"]}") + + if links["videoStream"]: + description_parts.append(f"Meetecho: {links["videoStream"]}") + + if links["webex"]: + description_parts.append(f"Webex: {links["webex"]}") + + if item["remoteInstructions"]: + description_parts.append( + f"Remote instructions: {item["remoteInstructions"]}" + ) + + try: + materials_url = absurl( + "ietf.meeting.views.session_details", + num=meeting_data["number"], + acronym=item["acronym"], + ) + except NoReverseMatch: + pass + else: + description_parts.append(f"Session materials: {materials_url}") + event.add("url", materials_url) + + if meeting_data["number"].isdigit(): + try: + agenda_url = absurl("agenda", num=meeting_data["number"]) + except NoReverseMatch: + pass + else: + description_parts.append(f"See in schedule: {agenda_url}#row-{item["slug"]}") + + if item["agenda"] and item["agenda"]["url"]: + description_parts.append(f"Agenda {item["agenda"]["url"]}") + + # Join all description parts with 2 newlines + description = "\n\n".join(description_parts) + event.add("description", description) + + # Add event to calendar + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + + def generate_agenda_ical(schedule, assignments): """Generate iCalendar using the icalendar library""" @@ -2415,10 +2562,66 @@ def parse_agenda_filter_params(querydict): def should_include_assignment(filter_params, assignment): """Decide whether to include an assignment""" - shown = len(set(filter_params['show']).intersection(assignment.filter_keywords)) > 0 - hidden = len(set(filter_params['hide']).intersection(assignment.filter_keywords)) > 0 + if hasattr(assignment, "filter_keywords"): + kw = assignment.filter_keywords + elif isinstance(assignment, dict): + kw = assignment.get("filterKeywords", []) + else: + raise ValueError("Unsupported assignment instance") + shown = len(set(filter_params['show']).intersection(kw)) > 0 + hidden = len(set(filter_params['hide']).intersection(kw)) > 0 return shown and not hidden + +def agenda_ical_ietf(meeting, filt_params, acronym=None, session_id=None): + agenda_data = generate_agenda_data(meeting.number, force_refresh=False) + if acronym: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["groupAcronym"] == acronym + ] + elif session_id: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["sessionId"] == session_id + ] + if filt_params is not None: + # Apply the filter + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if should_include_assignment(filt_params, item) + ] + return render_icalendar_precomp(agenda_data) + + +def agenda_ical_interim(meeting, filt_params, acronym=None, session_id=None): + schedule = get_schedule(meeting) + + if schedule is None and acronym is None and session_id is None: + raise Http404 + + assignments = SchedTimeSessAssignment.objects.filter( + schedule__in=[schedule, schedule.base], + session__on_agenda=True, + ) + assignments = preprocess_assignments_for_agenda(assignments, meeting) + AgendaKeywordTagger(assignments=assignments).apply() + + if filt_params is not None: + # Apply the filter + assignments = [a for a in assignments if should_include_assignment(filt_params, a)] + + if acronym: + assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] + elif session_id: + assignments = [ a for a in assignments if a.session_id == int(session_id) ] + + return render_icalendar(schedule, assignments) + + def agenda_ical(request, num=None, acronym=None, session_id=None): """Agenda ical view @@ -2446,33 +2649,20 @@ def agenda_ical(request, num=None, acronym=None, session_id=None): raise Http404 else: meeting = get_meeting(num, type_in=None) # get requested meeting, whatever its type - schedule = get_schedule(meeting) - if schedule is None and acronym is None and session_id is None: - raise Http404 - - assignments = SchedTimeSessAssignment.objects.filter( - schedule__in=[schedule, schedule.base], - session__on_agenda=True, - ) - assignments = preprocess_assignments_for_agenda(assignments, meeting) - AgendaKeywordTagger(assignments=assignments).apply() + if isinstance(session_id, str) and session_id.isdigit(): + session_id = int(session_id) try: filt_params = parse_agenda_filter_params(request.GET) except ValueError as e: return HttpResponseBadRequest(str(e)) - if filt_params is not None: - # Apply the filter - assignments = [a for a in assignments if should_include_assignment(filt_params, a)] - - if acronym: - assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] - elif session_id: - assignments = [ a for a in assignments if a.session_id == int(session_id) ] + if meeting.type_id == "ietf": + return agenda_ical_ietf(meeting, filt_params, acronym, session_id) + else: + return agenda_ical_interim(meeting, filt_params, acronym, session_id) - return render_icalendar(schedule, assignments) @cache_page(15 * 60) def agenda_json(request, num=None): @@ -4417,21 +4607,137 @@ def upcoming_ical(request): else: ietfs = [] - meeting_vtz = {meeting.vtimezone() for meeting in meetings} - meeting_vtz.discard(None) - - # icalendar response file should have '\r\n' line endings per RFC5545 - response = render_to_string('meeting/upcoming.ics', { - 'vtimezones': ''.join(sorted(meeting_vtz)), - 'assignments': assignments, - 'ietfs': ietfs, - }, request=request) - response = parse_ical_line_endings(response) + response = render_upcoming_ical(assignments, ietfs, request) response = HttpResponse(response, content_type='text/calendar') response['Content-Disposition'] = 'attachment; filename="upcoming.ics"' return response +def render_important_dates_ical(meetings, request): + """Generate important dates using the icalendar library""" + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical importantdates//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + for meeting in meetings: + for important_date in meeting.important_dates: + event = Event() + event.add("uid", f"ietf-{meeting.number}-{important_date.name_id}-" + f"{important_date.date.isoformat()}") + event.add("summary", f"IETF {meeting.number}: {important_date.name.name}") + event.add("class", "PUBLIC") + + if not important_date.midnight_cutoff: + event.add("dtstart", important_date.date) + else: + event.add("dtstart", datetime.datetime.combine( + important_date.date, + datetime.time(23, 59, 0, tzinfo=pytz.UTC)) + ) + + event.add("transp", "TRANSPARENT") + event.add("dtstamp", meeting.cached_updated) + description_lines = [important_date.name.desc] + if important_date.name.slug in ('openreg', 'earlybird'): + description_lines.append( + "Register here: https://www.ietf.org/how/meetings/register/") + if important_date.name.slug == 'opensched': + description_lines.append("To request a Working Group session, use the " + "IETF Meeting Session Request Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views_session_request.list_view')}") + description_lines.append("If you are working on a BOF request, it is " + "highly recommended to tell the IESG") + description_lines.append("now by sending an email to iesg@ietf.org " + "to get advance help with the request.") + if important_date.name.slug == 'cutoffwgreq': + description_lines.append("To request a Working Group session, use the " + "IETF Meeting Session Request Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views_session_request.list_view')}") + if important_date.name.slug == 'cutoffbofreq': + description_lines.append("To request a BOF, please see instructions on " + "Requesting a BOF:") + description_lines.append("https://www.ietf.org/how/bofs/bof-procedures/") + if important_date.name.slug == 'idcutoff': + description_lines.append("Upload using the I-D Submission Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.submit.views.upload_submission')}") + if important_date.name.slug in ( + 'draftwgagenda', + 'revwgagenda', + 'procsub', + 'revslug' + ): + description_lines.append("Upload using the Meeting Materials " + "Management Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views.materials', + kwargs={'num': meeting.number})}") + + event.add("description", "\n".join(description_lines)) + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + +def render_upcoming_ical(assignments, meetings, request): + """Generate upcoming using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical upcoming//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + for item in assignments: + event = Event() + + event.add("uid", f"ietf-{item.session.meeting.number}-{item.timeslot.pk}") + event.add("summary", f"{item.session.group.acronym.lower()} - {item.session.name if item.session.name else item.session.group.name}") + + if item.schedule.meeting.city: + event.add("location", f"{item.schedule.meeting.city},{item.schedule.meeting.country}") + + event.add("status", item.session.ical_status) + event.add("class", "PUBLIC") + + event.add("dtstart", item.timeslot.utc_start_time()) + event.add("dtend", item.timeslot.utc_end_time()) + event.add("dtstamp", item.timeslot.modified) + if item.session.agenda(): + event.add("url", item.session.agenda().get_href()) + + description_lines = [] + if item.timeslot.name: + description_lines.append(f"{item.timeslot.name}") + if item.session.agenda_note: + description_lines.append(f"Note: {item.session.agenda_note}") + + for material in item.session.materials.all(): + title_part = f" ({material.title})" if material.type.name != "Agenda" else "" + description_lines.append(f"{material.type}{title_part}: {material.get_href()}") + + if item.session.remote_instructions: + description_lines.append(f"Remote instructions: {item.session.remote_instructions}") + + event.add("description", "\n".join(description_lines)) + cal.add_component(event) + + for meeting in meetings: + event = Event() + event.add("uid", f"ietf-{meeting.number}") + event.add("summary", f"IETF {meeting.number}") + if meeting.city: + event.add("location", f"{meeting.city},{meeting.country}") + event.add("class", "PUBLIC") + event.add("dtstart", meeting.date) + event.add("dtend", meeting.end_date() + datetime.timedelta(days=1)) + event.add("dtstamp", meeting.cached_updated) + event.add("url", f"{request.scheme}://{request.get_host()}{reverse('agenda', kwargs={'num': meeting.number})}") + + cal.add_component(event) + + return cal.to_ical().decode("utf-8") def upcoming_json(request): '''Return Upcoming meetings in json format''' @@ -4506,15 +4812,45 @@ def proceedings_attendees(request, num=None): template = None registrations = None + stats = None + chart_data = None + if int(meeting.number) >= 118: - checked_in, attended = participants_for_meeting(meeting) - regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) + onsite, remote = meeting.get_attendees() + onsite_count = len(onsite) + remote_count = len(remote) + onsite_pks = frozenset(p.pk for p in onsite) + remote_pks = frozenset(p.pk for p in remote) + + regs = [ + reg + for reg in Registration.objects.onsite() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in onsite_pks + ] + [ + reg + for reg in Registration.objects.remote() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in remote_pks + ] + registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) - for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person'): - if reg.person.pk in attended and reg.person.pk not in checked_in: - regs.append(reg) + country_codes = [r.country_code for r in registrations if r.country_code] + stats = { + 'total': onsite_count + remote_count, + 'onsite': onsite_count, + 'remote': remote_count, + } - registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) + code_to_name = dict(CountryName.objects.values_list('slug', 'name')) + country_counts = Counter(code_to_name.get(c, c) for c in country_codes).most_common() + + chart_data = { + 'type': [['Onsite', onsite_count], ['Remote', remote_count]], + 'countries': country_counts, + } else: overview_template = "/meeting/proceedings/%s/attendees.html" % meeting.number try: @@ -4526,6 +4862,8 @@ def proceedings_attendees(request, num=None): 'meeting': meeting, 'registrations': registrations, 'template': template, + 'stats': stats, + 'chart_data': chart_data, }) def proceedings_overview(request, num=None): @@ -5050,11 +5388,8 @@ def important_dates(request, num=None, output_format=None): if output_format == 'ics': preprocess_meeting_important_dates(meetings) - ics = render_to_string('meeting/important_dates.ics', { - 'meetings': meetings, - }, request=request) - # icalendar response file should have '\r\n' line endings per RFC5545 - response = HttpResponse(parse_ical_line_endings(ics), content_type='text/calendar') + response = HttpResponse(render_important_dates_ical(meetings, request), + content_type='text/calendar') response['Content-Disposition'] = 'attachment; filename="important-dates.ics"' return response @@ -5291,7 +5626,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): if request.POST.get('approve'): # Ensure that we have a file to approve. The system gets cranky otherwise. if submission.filename is None or submission.filename == '' or not os.path.isfile(submission.staged_filepath()): - return HttpResponseNotFound("The slides you attempted to approve could not be found. Please disapprove and delete them instead.") + return HttpResponseNotFound("The slides you attempted to approve could not be found. Please decline and delete them instead.") title = form.cleaned_data['title'] if existing_doc: doc = Document.objects.get(name=name) @@ -5407,6 +5742,52 @@ def approve_proposed_slides(request, slidesubmission_id, num): }) +@role_required("Secretariat") +def notify_meetecho_of_all_slides(request, num, acronym): + """Notify meetecho of state of all slides for the group + + Respects the usual notification window around each session. Meetecho will ignore + notices outside that window anyway, so no sense sending them. + """ + meeting = get_meeting(num=num, type_in=None) # raises 404 + if request.method != "POST": + return HttpResponseNotAllowed( + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=("POST",), + ) + scheduled_sessions = [ + session + for session in get_sessions(meeting.number, acronym) + if session.current_status == "sched" + ] + sm = SlidesManager(api_config=settings.MEETECHO_API_CONFIG) + updated = [] + for session in scheduled_sessions: + if sm.send_update(session): + updated.append(session) + if len(updated) > 0: + messages.success( + request, + f"Notified Meetecho about slides for {','.join(str(s) for s in updated)}", + ) + elif sm.slides_notify_time is not None: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + f"only sent within {sm.slides_notify_time} before or after the session.", + ) + else: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + "currently disabled.", + ) + return redirect( + "ietf.meeting.views.session_details", num=meeting.number, acronym=acronym + ) + + def import_session_minutes(request, session_id, num): """Import session minutes from the ietf.notes.org site @@ -5454,6 +5835,7 @@ def import_session_minutes(request, session_id, num): except SaveMaterialsError as err: form.add_error(None, str(err)) else: + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: diff --git a/ietf/meeting/views_proceedings.py b/ietf/meeting/views_proceedings.py index d1169bff2d6..639efa1da46 100644 --- a/ietf/meeting/views_proceedings.py +++ b/ietf/meeting/views_proceedings.py @@ -14,7 +14,7 @@ from ietf.meeting.models import Meeting, MeetingHost from ietf.meeting.helpers import get_meeting from ietf.name.models import ProceedingsMaterialTypeName -from ietf.meeting.utils import handle_upload_file +from ietf.meeting.utils import handle_upload_file, resolve_uploaded_material from ietf.utils.text import xslugify class UploadProceedingsMaterialForm(FileUploadForm): @@ -150,7 +150,7 @@ def save_proceedings_material_doc(meeting, material_type, title, request, file=N if events: doc.save_with_history(events) - + resolve_uploaded_material(meeting, doc) return doc diff --git a/ietf/message/admin.py b/ietf/message/admin.py index 250e1eb5963..6a876cdc70d 100644 --- a/ietf/message/admin.py +++ b/ietf/message/admin.py @@ -27,7 +27,8 @@ def queryset(self, request, queryset): class MessageAdmin(admin.ModelAdmin): - list_display = ["sent_status", "subject", "by", "time", "groups"] + list_display = ["sent_status", "display_subject", "by", "time", "groups"] + list_display_links = ["display_subject"] search_fields = ["subject", "body"] raw_id_fields = ["by", "related_groups", "related_docs"] list_filter = [ @@ -37,6 +38,10 @@ class MessageAdmin(admin.ModelAdmin): ordering = ["-time"] actions = ["retry_send"] + @admin.display(description="Subject", empty_value="(no subject)") + def display_subject(self, instance): + return instance.subject or None # None triggers the empty_value + def groups(self, instance): return ", ".join(g.acronym for g in instance.related_groups.all()) diff --git a/ietf/middleware.py b/ietf/middleware.py index a4b7a0d24c6..fa2e8efd0c5 100644 --- a/ietf/middleware.py +++ b/ietf/middleware.py @@ -8,6 +8,7 @@ from django.http import HttpResponsePermanentRedirect from ietf.utils.log import log, exc_parts from ietf.utils.mail import log_smtp_exception +from opentelemetry.propagate import inject import re import smtplib import unicodedata @@ -99,3 +100,12 @@ def add_header(request): return response return add_header + +def add_otel_traceparent_header(get_response): + """Middleware to add the OpenTelemetry traceparent id header to the response""" + def add_header(request): + response = get_response(request) + inject(response) + return response + + return add_header diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 4336e0569c7..b89d6d141cf 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -57,6 +57,7 @@ from ietf.stats.models import CountryAlias +from ietf.utils.admin import SaferTabularInline class NameAdmin(admin.ModelAdmin): @@ -86,7 +87,7 @@ class GroupTypeNameAdmin(NameAdmin): admin.site.register(GroupTypeName, GroupTypeNameAdmin) -class CountryAliasInline(admin.TabularInline): +class CountryAliasInline(SaferTabularInline): model = CountryAlias extra = 1 diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 64e26e503ad..798fa9178e1 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -666,7 +666,7 @@ }, { "fields": { - "desc": "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", + "desc": "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", "name": "Adopted by a WG", "next_states": [ 38 @@ -694,7 +694,7 @@ }, { "fields": { - "desc": "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", + "desc": "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", "name": "WG Document", "next_states": [ 39, @@ -759,7 +759,7 @@ }, { "fields": { - "desc": "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", + "desc": "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", "name": "Waiting for WG Chair Go-Ahead", "next_states": [ 41, @@ -790,7 +790,7 @@ }, { "fields": { - "desc": "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + "desc": "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", "name": "Submitted to IESG for Publication", "next_states": [ 38 @@ -2656,6 +2656,32 @@ "model": "doc.state", "pk": 183 }, + { + "fields": { + "desc": "", + "name": "In Progress", + "next_states": [], + "order": 0, + "slug": "in_progress", + "type": "draft-rfceditor", + "used": true + }, + "model": "doc.state", + "pk": 216 + }, + { + "fields": { + "desc": "", + "name": "Blocked", + "next_states": [], + "order": 0, + "slug": "blocked", + "type": "draft-rfceditor", + "used": true + }, + "model": "doc.state", + "pk": 217 + }, { "fields": { "label": "State" @@ -5816,6 +5842,20 @@ "model": "mailtrigger.mailtrigger", "pk": "review_completed_artart_telechat" }, + { + "fields": { + "cc": [ + "review_doc_all_parties", + "review_doc_group_mail_list" + ], + "desc": "Recipients when a bgpdir Early review is completed", + "to": [ + "review_team_mail_list" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "review_completed_bgpdir_early" + }, { "fields": { "cc": [ @@ -6124,6 +6164,20 @@ "model": "mailtrigger.mailtrigger", "pk": "review_completed_opsdir_telechat" }, + { + "fields": { + "cc": [ + "review_doc_all_parties", + "review_doc_group_mail_list" + ], + "desc": "Recipients when a perfmetrdir Early review is completed", + "to": [ + "review_team_mail_list" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "review_completed_perfmetrdir_early" + }, { "fields": { "cc": [ @@ -14038,6 +14092,16 @@ "model": "name.reviewresultname", "pk": "almost-ready" }, + { + "fields": { + "desc": "", + "name": "Clarification Needed", + "order": 10, + "used": true + }, + "model": "name.reviewresultname", + "pk": "clarification-needed" + }, { "fields": { "desc": "", @@ -17757,5 +17821,13 @@ }, "model": "stats.countryalias", "pk": 303 + }, + { + "fields": { + "alias": "czechia", + "country": "CZ" + }, + "model": "stats.countryalias", + "pk": 304 } ] diff --git a/ietf/name/serializers.py b/ietf/name/serializers.py new file mode 100644 index 00000000000..a764f560512 --- /dev/null +++ b/ietf/name/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import StreamName + + +class StreamNameSerializer(serializers.ModelSerializer): + class Meta: + model = StreamName + fields = ["slug", "name", "desc"] diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index dcdb9ef836f..ac86d7cf4b4 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2012-2023, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2012-2025, All Rights Reserved import datetime @@ -27,8 +26,15 @@ from ietf.api.views import EmailIngestionError from ietf.dbtemplate.factories import DBTemplateFactory from ietf.dbtemplate.models import DBTemplate -from ietf.doc.factories import DocEventFactory, WgDocumentAuthorFactory, \ - NewRevisionDocEventFactory, DocumentAuthorFactory +from ietf.doc.factories import ( + DocEventFactory, + WgDocumentAuthorFactory, + NewRevisionDocEventFactory, + DocumentAuthorFactory, + RfcAuthorFactory, + WgDraftFactory, WgRfcFactory, +) +from ietf.doc.models import RfcAuthor from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory @@ -45,10 +51,20 @@ nomcom_kwargs_for_year, provide_private_key_to_test_client, \ key from ietf.nomcom.tasks import send_nomcom_reminders_task -from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \ - get_hash_nominee_position, is_eligible, list_eligible, \ - get_eligibility_date, suggest_affiliation, ingest_feedback_email, \ - decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder +from ietf.nomcom.utils import ( + get_nomcom_by_year, + make_nomineeposition, + get_hash_nominee_position, + is_eligible, + list_eligible, + get_eligibility_date, + suggest_affiliation, + ingest_feedback_email, + decorate_volunteers_with_qualifications, + send_reminders, + _is_time_to_send_reminder, + get_qualified_author_queryset, +) from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.utils.mail import outbox, empty_outbox, get_payload_text @@ -1175,8 +1191,8 @@ def setUp(self): today = datetime_today() t_minus_3 = today - datetime.timedelta(days=3) t_minus_4 = today - datetime.timedelta(days=4) - e1 = EmailFactory(address="nominee1@example.org", person=PersonFactory(name="Nominee 1"), origin='test') - e2 = EmailFactory(address="nominee2@example.org", person=PersonFactory(name="Nominee 2"), origin='test') + e1 = EmailFactory(address="nominee1@example.org", person__name="Nominee 1", origin='test', primary=True) + e2 = EmailFactory(address="nominee2@example.org", person__name="Nominee 2", origin='test', primary=True) n = make_nomineeposition(self.nomcom,e1.person,gen,None) np = n.nomineeposition_set.get(position=gen) np.time = t_minus_3 @@ -1625,6 +1641,20 @@ def test_feedback_topic_badges(self): q = PyQuery(response.content) self.assertEqual( len(q('.text-bg-success')), 0 ) + def test_feedback_index_sort_keys(self): + url = reverse('ietf.nomcom.views.view_feedback', kwargs={'year': self.nc.year()}) + login_testing_unauthorized(self, self.member.user.username, url) + provide_private_key_to_test_client(self) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + # Feedback count cells must carry a numeric data-sort-number so that + # a "New" badge appearing before the count doesn't corrupt the sort key. + sort_cells = q('td[data-sort-number]') + self.assertTrue(len(sort_cells) > 0) + for cell in sort_cells.items(): + self.assertRegex(cell.attr('data-sort-number'), r'^\d+$') + class NewActiveNomComTests(TestCase): def setUp(self): @@ -2440,6 +2470,72 @@ def test_get_eligibility_date(self): NomComFactory(group__acronym=f'nomcom{this_year}', first_call_for_volunteers=datetime.date(this_year,5,6)) self.assertEqual(get_eligibility_date(),datetime.date(this_year,5,6)) + def test_get_qualified_author_queryset(self): + """get_qualified_author_queryset implements the eligiblity rules correctly + + This is not an exhaustive test of corner cases. Overlaps considerably with + rfc8989EligibilityTests.test_elig_by_author(). + """ + people = PersonFactory.create_batch(2) + extra_person = PersonFactory() + base_qs = Person.objects.filter(pk__in=[person.pk for person in people]) + now = datetime.datetime.now(tz=datetime.UTC) + one_year = datetime.timedelta(days=365) + + # Authors with no qualifying drafts + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Authors with one qualifying draft + approved_draft = WgDraftFactory(authors=people, states=[("draft", "active")]) + DocEventFactory( + type="iesg_approved", + doc=approved_draft, + time=now - 4 * one_year, + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Create a draft that was published into an RFC. Give it an extra author who + # should not be eligible. + published_draft = WgDraftFactory(authors=people, states=[("draft", "rfc")]) + DocEventFactory( + type="iesg_approved", + doc=published_draft, + time=now - 5.5 * one_year, # < 6 years ago + ) + rfc = WgRfcFactory( + authors=people + [extra_person], + group=published_draft.group, + ) + DocEventFactory( + type="published_rfc", + doc=rfc, + time=now - 0.5 * one_year, # < 1 year ago + ) + # Period 6 years ago to 1 year ago - authors are eligible due to the + # iesg-approved draft in this window + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now - one_year), + people, + ) + + # Period 5 years ago to now - authors are eligible due to the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + people, + ) + + # Use the extra_person to check that a single doc can't count both as an + # RFC _and_ an approved draft. Use an eligibility interval that includes both + # the approval and the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now), + people, # does not include extra_person! + ) + class rfc8713EligibilityTests(TestCase): @@ -2724,33 +2820,41 @@ def test_elig_by_author(self): ineligible = set() p = PersonFactory() - ineligible.add(p) - + ineligible.add(p) # no RFCs or iesg-approved drafts p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # only one RFC p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=first_date) - eligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=first_date) + eligible.add(p) # one RFC and one iesg-approved draft p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=middle_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=day_before_first_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=day_before_first_date) + ineligible.add(p) # RFC is out of the eligibility window p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=day_after_last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # iesg approval is outside the eligibility window for person in eligible: self.assertTrue(is_eligible(person,nomcom)) @@ -2759,7 +2863,9 @@ def test_elig_by_author(self): self.assertFalse(is_eligible(person,nomcom)) self.assertEqual(set(list_eligible(nomcom=nomcom)),set(eligible)) - Person.objects.filter(pk__in=[p.pk for p in eligible.union(ineligible)]).delete() + people_pks_to_delete = [p.pk for p in eligible.union(ineligible)] + RfcAuthor.objects.filter(person__pk__in=people_pks_to_delete).delete() + Person.objects.filter(pk__in=people_pks_to_delete).delete() class rfc9389EligibilityTests(TestCase): @@ -2878,15 +2984,38 @@ def test_volunteer(self): def test_suggest_affiliation(self): person = PersonFactory() - self.assertEqual(suggest_affiliation(person), '') - da = DocumentAuthorFactory(person=person,affiliation='auth_affil') + self.assertEqual(suggest_affiliation(person), "") + rfc_da = DocumentAuthorFactory( + person=person, + document__type_id="rfc", + affiliation="", + ) + rfc = rfc_da.document + DocEventFactory(doc=rfc, type="published_rfc") + self.assertEqual(suggest_affiliation(person), "") + + rfc_da.affiliation = "rfc_da_affil" + rfc_da.save() + self.assertEqual(suggest_affiliation(person), "rfc_da_affil") + + rfc_ra = RfcAuthorFactory(person=person, document=rfc, affiliation="") + self.assertEqual(suggest_affiliation(person), "") + + rfc_ra.affiliation = "rfc_ra_affil" + rfc_ra.save() + self.assertEqual(suggest_affiliation(person), "rfc_ra_affil") + + da = DocumentAuthorFactory(person=person, affiliation="auth_affil") NewRevisionDocEventFactory(doc=da.document) - self.assertEqual(suggest_affiliation(person), 'auth_affil') + self.assertEqual(suggest_affiliation(person), "auth_affil") + nc = NomComFactory() - nc.volunteer_set.create(person=person,affiliation='volunteer_affil') - self.assertEqual(suggest_affiliation(person), 'volunteer_affil') - RegistrationFactory(person=person, affiliation='meeting_affil') - self.assertEqual(suggest_affiliation(person), 'meeting_affil') + nc.volunteer_set.create(person=person, affiliation="volunteer_affil") + self.assertEqual(suggest_affiliation(person), "volunteer_affil") + + RegistrationFactory(person=person, affiliation="meeting_affil") + self.assertEqual(suggest_affiliation(person), "meeting_affil") + class VolunteerDecoratorUnitTests(TestCase): def test_decorate_volunteers_with_qualifications(self): @@ -2922,10 +3051,10 @@ def test_decorate_volunteers_with_qualifications(self): author_person = PersonFactory() for i in range(2): - da = WgDocumentAuthorFactory(person=author_person) + doc = WgRfcFactory(authors=[author_person]) DocEventFactory( type='published_rfc', - doc=da.document, + doc=doc, time=datetime.datetime( elig_date.year - 3, elig_date.month, diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index dd651c29413..46418714880 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -18,7 +18,7 @@ from email.utils import parseaddr from textwrap import dedent -from django.db.models import Q, Count +from django.db.models import Q, Count, F, QuerySet from django.conf import settings from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist @@ -27,12 +27,11 @@ from django.shortcuts import get_object_or_404 from ietf.dbtemplate.models import DBTemplate -from ietf.doc.models import DocEvent, NewRevisionDocEvent +from ietf.doc.models import DocEvent, NewRevisionDocEvent, Document from ietf.group.models import Group, Role from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists from ietf.meeting.models import Meeting -from ietf.meeting.utils import participants_for_meeting from ietf.utils.pipe import pipe from ietf.utils.mail import send_mail_text, send_mail, get_payload_text from ietf.utils.log import log @@ -576,6 +575,70 @@ def get_8989_eligibility_querysets(date, base_qs): def get_9389_eligibility_querysets(date, base_qs): return get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable=three_of_five_eligible_9389) + +def get_qualified_author_queryset( + base_qs: QuerySet[Person], + eligibility_period_start: datetime.datetime, + eligibility_period_end: datetime.datetime, +): + """Filter a Person queryset, keeping those qualified by RFC 8989's author path + + The author path is defined by "path 3" in section 4 of RFC 8989. It qualifies + a person who has been a front-page listed author or editor of at least two IETF- + stream RFCs within the last five years. An I-D in the RFC Editor queue that was + approved by the IESG is treated as an RFC, using the date of entry to the RFC + Editor queue as the date for qualification. + + This method does not strictly enforce "in the RFC Editor queue" for IESG-approved + drafts when computing eligibility. In the overwhelming majority of cases, an IESG- + approved draft immediately enters the queue and goes on to be published, so this + simplification makes the calculation much easier and virtually never affects + eligibility. + + Arguments eligibility_period_start and eligibility_period_end are datetimes that + mark the start and end of the eligibility period. These should be five years apart. + """ + # First, get the RFCs using publication date + qualifying_rfc_pub_events = DocEvent.objects.filter( + type='published_rfc', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_rfcs = Document.objects.filter( + type_id="rfc", + docevent__in=qualifying_rfc_pub_events + ).annotate( + rfcauthor_count=Count("rfcauthor") + ) + rfcs_with_rfcauthors = qualifying_rfcs.filter(rfcauthor_count__gt=0).distinct() + rfcs_without_rfcauthors = qualifying_rfcs.filter(rfcauthor_count=0).distinct() + + # Second, get the IESG-approved I-Ds excluding any we're already counting as rfcs + qualifying_approval_events = DocEvent.objects.filter( + type='iesg_approved', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_drafts = Document.objects.filter( + type_id="draft", + docevent__in=qualifying_approval_events, + ).exclude( + relateddocument__relationship_id="became_rfc", + relateddocument__target__in=qualifying_rfcs, + ).distinct() + + return base_qs.filter( + Q(documentauthor__document__in=qualifying_drafts) + | Q(rfcauthor__document__in=rfcs_with_rfcauthors) + | Q(documentauthor__document__in=rfcs_without_rfcauthors) + ).annotate( + document_author_count=Count('documentauthor'), + rfc_author_count=Count("rfcauthor") + ).annotate( + authorship_count=F("document_author_count") + F("rfc_author_count") + ).filter(authorship_count__gte=2) + + def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): if not base_qs: base_qs = Person.objects.all() @@ -608,14 +671,7 @@ def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): ) ).distinct() - rfc_pks = set(DocEvent.objects.filter(type='published_rfc', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk', flat=True)) - iesgappr_pks = set(DocEvent.objects.filter(type='iesg_approved', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk',flat=True)) - qualifying_pks = rfc_pks.union(iesgappr_pks.difference(rfc_pks)) - author_qs = base_qs.filter( - documentauthor__document__pk__in=qualifying_pks - ).annotate( - document_author_count = Count('documentauthor') - ).filter(document_author_count__gte=2) + author_qs = get_qualified_author_queryset(base_qs, five_years_ago, date_as_dt) return three_of_five_qs, officer_qs, author_qs def list_eligible_8989(date, base_qs=None): @@ -686,23 +742,47 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts = defaultdict(lambda: 0) for meeting in previous_five: - checked_in, attended = participants_for_meeting(meeting) - for id in set(checked_in) | set(attended): + onsite_pks, remote_pks = meeting.nomcom_eligible_participants() + for id in onsite_pks | remote_pks: counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) -def suggest_affiliation(person): +def suggest_affiliation(person) -> str: + """Heuristically suggest a current affiliation for a Person""" recent_meeting = person.registration_set.order_by('-meeting__date').first() - affiliation = recent_meeting.affiliation if recent_meeting else '' - if not affiliation: - recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() - if recent_volunteer: - affiliation = recent_volunteer.affiliation - if not affiliation: - recent_draft_revision = NewRevisionDocEvent.objects.filter(doc__type_id='draft',doc__documentauthor__person=person).order_by('-time').first() - if recent_draft_revision: - affiliation = recent_draft_revision.doc.documentauthor_set.filter(person=person).first().affiliation - return affiliation + if recent_meeting and recent_meeting.affiliation: + return recent_meeting.affiliation + + recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() + if recent_volunteer and recent_volunteer.affiliation: + return recent_volunteer.affiliation + + recent_draft_revision = NewRevisionDocEvent.objects.filter( + doc__type_id="draft", + doc__documentauthor__person=person, + ).order_by("-time").first() + if recent_draft_revision: + draft_author = recent_draft_revision.doc.documentauthor_set.filter( + person=person + ).first() + if draft_author and draft_author.affiliation: + return draft_author.affiliation + + recent_rfc_publication = DocEvent.objects.filter( + Q(doc__documentauthor__person=person) | Q(doc__rfcauthor__person=person), + doc__type_id="rfc", + type="published_rfc", + ).order_by("-time").first() + if recent_rfc_publication: + rfc = recent_rfc_publication.doc + if rfc.rfcauthor_set.exists(): + rfc_author = rfc.rfcauthor_set.filter(person=person).first() + else: + rfc_author = rfc.documentauthor_set.filter(person=person).first() + if rfc_author and rfc_author.affiliation: + return rfc_author.affiliation + return "" + def extract_volunteers(year): nomcom = get_nomcom_by_year(year) diff --git a/ietf/person/admin.py b/ietf/person/admin.py index cd8ca2abf16..f46edcf8aeb 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -7,6 +7,7 @@ from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, PersonApiKeyEvent, PersonExtResource from ietf.person.name import name_parts +from ietf.utils.admin import SaferStackedInline, SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -16,7 +17,7 @@ class EmailAdmin(simple_history.admin.SimpleHistoryAdmin): search_fields = ["address", "person__name", ] admin.site.register(Email, EmailAdmin) -class EmailInline(admin.TabularInline): +class EmailInline(SaferTabularInline): model = Email class AliasAdmin(admin.ModelAdmin): @@ -25,7 +26,7 @@ class AliasAdmin(admin.ModelAdmin): raw_id_fields = ["person"] admin.site.register(Alias, AliasAdmin) -class AliasInline(admin.StackedInline): +class AliasInline(SaferStackedInline): model = Alias class PersonAdmin(simple_history.admin.SimpleHistoryAdmin): diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 98756f26c8a..655f25994b3 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -95,7 +95,7 @@ def default_emails(obj, create, extracted, **kwargs): # pylint: disable=no-self- extracted = True if create and extracted: make_email = getattr(EmailFactory, 'create' if create else 'build') - make_email(person=obj, address=obj.user.email) + make_email(person=obj, address=obj.user.email, primary=True, **kwargs) @factory.post_generation def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument @@ -151,7 +151,7 @@ class Meta: django_get_or_create = ('address',) address = factory.Sequence(fake_email_address) - person = factory.SubFactory(PersonFactory) + person = factory.SubFactory(PersonFactory, default_emails=False) active = True primary = False diff --git a/ietf/person/forms.py b/ietf/person/forms.py index 81ee3625619..7eef8aa17b7 100644 --- a/ietf/person/forms.py +++ b/ietf/person/forms.py @@ -1,15 +1,26 @@ -# Copyright The IETF Trust 2018-2020, All Rights Reserved +# Copyright The IETF Trust 2018-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms + from ietf.person.models import Person +from ietf.utils.fields import MultiEmailField, NameAddrEmailField class MergeForm(forms.Form): source = forms.IntegerField(label='Source Person ID') target = forms.IntegerField(label='Target Person ID') + def __init__(self, *args, **kwargs): + self.readonly = False + if 'readonly' in kwargs: + self.readonly = kwargs.pop('readonly') + super().__init__(*args, **kwargs) + if self.readonly: + self.fields['source'].widget.attrs['readonly'] = True + self.fields['target'].widget.attrs['readonly'] = True + def clean_source(self): return self.get_person(self.cleaned_data['source']) @@ -21,3 +32,11 @@ def get_person(self, pk): return Person.objects.get(pk=pk) except Person.DoesNotExist: raise forms.ValidationError("ID does not exist") + + +class MergeRequestForm(forms.Form): + to = MultiEmailField() + frm = NameAddrEmailField() + reply_to = MultiEmailField() + subject = forms.CharField() + body = forms.CharField(widget=forms.Textarea) diff --git a/ietf/person/models.py b/ietf/person/models.py index 03cf0c87fb3..3ab89289a65 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -87,7 +87,7 @@ def short(self): else: prefix, first, middle, last, suffix = self.ascii_parts() return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") - def plain_name(self): + def plain_name(self) -> str: if not hasattr(self, '_cached_plain_name'): if self.plain: self._cached_plain_name = self.plain @@ -203,7 +203,10 @@ def has_drafts(self): def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(documentauthor__person=self, type='rfc')) + # When RfcAuthors are populated, this may over-return if an author is dropped + # from the author list between the final draft and the published RFC. Should + # ignore DocumentAuthors when an RfcAuthor exists for a draft. + rfcs = list(Document.objects.filter(type="rfc").filter(models.Q(documentauthor__person=self)|models.Q(rfcauthor__person=self)).distinct()) rfcs.sort(key=lambda d: d.name ) return rfcs @@ -266,11 +269,16 @@ def available_api_endpoints(self): def cdn_photo_url(self, size=80): if self.photo: if settings.SERVE_CDN_PHOTOS: + if settings.SERVER_MODE != "production": + original_media_dir = settings.MEDIA_URL + settings.MEDIA_URL = "https://www.ietf.org/lib/dt/media/" source_url = self.photo.url if source_url.startswith(settings.IETF_HOST_URL): source_url = source_url[len(settings.IETF_HOST_URL):] elif source_url.startswith('/'): source_url = source_url[1:] + if settings.SERVER_MODE != "production": + settings.MEDIA_URL = original_media_dir return f'{settings.IETF_HOST_URL}cdn-cgi/image/fit=scale-down,width={size},height={size}/{source_url}' else: datatracker_photo_path = urlreverse('ietf.person.views.photo', kwargs={'email_or_name': self.email()}) diff --git a/ietf/person/templatetags/person_filters.py b/ietf/person/templatetags/person_filters.py index 017b29c63ad..a7a6e8193a0 100644 --- a/ietf/person/templatetags/person_filters.py +++ b/ietf/person/templatetags/person_filters.py @@ -50,6 +50,7 @@ def person_link(person, **kwargs): title = kwargs.get("title", "") cls = kwargs.get("class", "") with_email = kwargs.get("with_email", True) + titlepage_name = kwargs.get("titlepage_name", None) if person is not None: plain_name = person.plain_name() name = ( @@ -61,6 +62,7 @@ def person_link(person, **kwargs): return { "name": name, "plain_name": plain_name, + "titlepage_name": titlepage_name, "email": email, "title": title, "class": cls, diff --git a/ietf/person/templatetags/tests.py b/ietf/person/templatetags/tests.py index 327cfad6ce6..7c35fd6b695 100644 --- a/ietf/person/templatetags/tests.py +++ b/ietf/person/templatetags/tests.py @@ -1,4 +1,6 @@ # Copyright The IETF Trust 2022, All Rights Reserved +from django.template.loader import render_to_string + from ietf.person.factories import PersonFactory from ietf.utils.test_utils import TestCase @@ -8,7 +10,6 @@ class PersonLinkTests(TestCase): # Tests of the person_link template tag. These assume it is implemented as an # inclusion tag. - # TODO test that the template actually renders the data in the dict def test_person_link(self): person = PersonFactory() self.assertEqual( @@ -16,6 +17,7 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': '', 'class': '', @@ -27,6 +29,7 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': '', 'class': '', @@ -38,6 +41,7 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': 'Random Title', 'class': '', @@ -50,12 +54,71 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': '', 'class': 'some-class', 'with_email': True, } ) + self.assertEqual( + person_link(person, titlepage_name='G. Surname'), + { + 'name': person.name, + 'plain_name': person.plain_name(), + 'titlepage_name': 'G. Surname', + 'email': person.email_address(), + 'title': '', + 'class': '', + 'with_email': True, + } + ) + + def test_person_link_renders(self): + """Verifies person/person_link.html renders context dict values correctly.""" + person = PersonFactory() + name = person.name + email = person.email_address() + base_context = { + 'name': name, + 'plain_name': person.plain_name(), + 'titlepage_name': None, + 'email': email, + 'title': '', + 'class': '', + 'with_email': True, + } + + # Default: name is used as link text with default title attribute + html = render_to_string('person/person_link.html', base_context) + self.assertIn(f'>{name}', html) + self.assertIn(f'Datatracker profile of {name}', html) + self.assertIn('bi-envelope', html) + + # titlepage_name overrides name as link text + html = render_to_string('person/person_link.html', {**base_context, 'titlepage_name': 'G. Surname'}) + self.assertIn('>G. Surname', html) + self.assertNotIn(f'>{name}', html) + + # with_email=False suppresses the envelope link + html = render_to_string('person/person_link.html', {**base_context, 'with_email': False}) + self.assertNotIn('bi-envelope', html) + + # Custom title appears in the anchor title attribute + html = render_to_string('person/person_link.html', {**base_context, 'title': 'Special Title'}) + self.assertIn('title="Special Title"', html) + + # Empty context (None person) renders (None) + self.assertInHTML( + '(None)', + render_to_string('person/person_link.html', {}), + ) + + # System email renders (System) + self.assertInHTML( + '(System)', + render_to_string('person/person_link.html', {'email': 'system@datatracker.ietf.org', 'name': ''}), + ) def test_invalid_person(self): """Generates correct context dict when input is invalid/missing""" diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 6326362fd8f..42c2c1547cb 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2022, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -10,7 +10,6 @@ from PIL import Image from pyquery import PyQuery - from django.core.exceptions import ValidationError from django.http import HttpRequest from django.test import override_settings @@ -23,6 +22,7 @@ from ietf.community.models import CommunityList from ietf.group.factories import RoleFactory from ietf.group.models import Group +from ietf.message.models import Message from ietf.nomcom.models import NomCom from ietf.nomcom.test_data import nomcom_test_data from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory @@ -78,7 +78,9 @@ def test_ajax_person_email_json(self): def test_default_email(self): person = PersonFactory() - primary = EmailFactory(person=person, primary=True, active=True) + primary = person.email_set.get() + self.assertEqual(primary.primary, True) + self.assertEqual(primary.active, True) EmailFactory(person=person, primary=False, active=True) EmailFactory(person=person, primary=False, active=False) self.assertTrue(primary.address in person.formatted_email()) @@ -208,13 +210,13 @@ def test_merge(self): def test_merge_with_params(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + "?source={}&target={}".format(p1.pk, p2.pk) + url = urlreverse("ietf.person.views.merge_submit") + "?source={}&target={}".format(p1.pk, p2.pk) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'retaining login', status_code=200) def test_merge_with_params_bad_id(self): - url = urlreverse("ietf.person.views.merge") + "?source=1000&target=2000" + url = urlreverse("ietf.person.views.merge_submit") + "?source=1000&target=2000" login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'ID does not exist', status_code=200) @@ -222,7 +224,7 @@ def test_merge_with_params_bad_id(self): def test_merge_post(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + url = urlreverse("ietf.person.views.merge_submit") expected_url = urlreverse("ietf.secr.rolodex.views.view", kwargs={'id': p2.pk}) login_testing_unauthorized(self, "secretary", url) data = {'source': p1.pk, 'target': p2.pk} @@ -358,7 +360,7 @@ def test_get_extra_primary(self): source = PersonFactory() target = PersonFactory() extra = get_extra_primary(source, target) - self.assertTrue(extra == list(source.email_set.filter(primary=True))) + self.assertEqual(set(extra), set(source.email_set.filter(primary=True))) def test_dedupe_aliases(self): person = PersonFactory() @@ -451,6 +453,30 @@ def test_dots(self): ncchair = RoleFactory(group__acronym='nomcom2020',group__type_id='nomcom',name_id='chair').person self.assertEqual(get_dots(ncchair),['nomcom']) + def test_send_merge_request(self): + empty_outbox() + message_count_before = Message.objects.count() + source = PersonFactory() + target = PersonFactory() + url = urlreverse('ietf.person.views.send_merge_request') + url = url + f'?source={source.pk}&target={target.pk}' + login_testing_unauthorized(self, 'secretary', url) + r = self.client.get(url) + initial = r.context['form'].initial + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + self.assertEqual(initial['to'], ', '.join([source.user.username, target.user.username])) + self.assertEqual(initial['subject'], subject) + self.assertEqual(initial['reply_to'], 'support@ietf.org') + self.assertEqual(r.status_code, 200) + r = self.client.post(url, data=initial) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + self.assertIn(source.user.username, outbox[0]['To']) + message_count_after = Message.objects.count() + message = Message.objects.last() + self.assertEqual(message_count_after, message_count_before + 1) + self.assertIn(source.user.username, message.to) + class TaskTests(TestCase): @mock.patch("ietf.person.tasks.log.log") diff --git a/ietf/person/urls.py b/ietf/person/urls.py index 867646fe395..f3eccd04b73 100644 --- a/ietf/person/urls.py +++ b/ietf/person/urls.py @@ -1,8 +1,12 @@ +# Copyright The IETF Trust 2009-2025, All Rights Reserved +# -*- coding: utf-8 -*- from ietf.person import views, ajax from ietf.utils.urls import url urlpatterns = [ url(r'^merge/?$', views.merge), + url(r'^merge/submit/?$', views.merge_submit), + url(r'^merge/send_request/?$', views.send_merge_request), url(r'^search/(?P(person|email))/$', views.ajax_select2_search), url(r'^(?P[0-9]+)/email.json$', ajax.person_email_json), url(r'^(?P[^/]+)$', views.profile), diff --git a/ietf/person/views.py b/ietf/person/views.py index a37b1643111..d0b5912431e 100644 --- a/ietf/person/views.py +++ b/ietf/person/views.py @@ -1,14 +1,16 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- from io import StringIO, BytesIO from PIL import Image +from django.conf import settings from django.contrib import messages from django.db.models import Q from django.http import HttpResponse, Http404 from django.shortcuts import render, redirect +from django.template.loader import render_to_string from django.utils import timezone import debug # pyflakes:ignore @@ -16,8 +18,9 @@ from ietf.ietfauth.utils import role_required from ietf.person.models import Email, Person from ietf.person.fields import select2_id_name_json -from ietf.person.forms import MergeForm +from ietf.person.forms import MergeForm, MergeRequestForm from ietf.person.utils import handle_users, merge_persons, lookup_persons +from ietf.utils.mail import send_mail_text def ajax_select2_search(request, model_name): @@ -98,16 +101,19 @@ def photo(request, email_or_name): @role_required("Secretariat") def merge(request): form = MergeForm() - method = 'get' + return render(request, 'person/merge.html', {'form': form}) + + +@role_required("Secretariat") +def merge_submit(request): change_details = '' warn_messages = [] source = None target = None if request.method == "GET": - form = MergeForm() if request.GET: - form = MergeForm(request.GET) + form = MergeForm(request.GET, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') target = form.cleaned_data.get('target') @@ -116,12 +122,9 @@ def merge(request): if source.user.last_login and target.user.last_login and source.user.last_login > target.user.last_login: warn_messages.append('WARNING: The most recently used login is being deleted!') change_details = handle_users(source, target, check_only=True) - method = 'post' - else: - method = 'get' if request.method == "POST": - form = MergeForm(request.POST) + form = MergeForm(request.POST, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') source_id = source.id @@ -136,11 +139,72 @@ def merge(request): messages.error(request, output) return redirect('ietf.secr.rolodex.views.view', id=target.pk) - return render(request, 'person/merge.html', { + return render(request, 'person/merge_submit.html', { 'form': form, - 'method': method, 'change_details': change_details, 'source': source, 'target': target, 'warn_messages': warn_messages, }) + + +@role_required("Secretariat") +def send_merge_request(request): + if request.method == 'GET': + merge_form = MergeForm(request.GET) + if merge_form.is_valid(): + source = merge_form.cleaned_data['source'] + target = merge_form.cleaned_data['target'] + to = [] + if source.email(): + to.append(source.email().address) + if target.email(): + to.append(target.email().address) + if source.user: + source_account = source.user.username + else: + source_account = source.email() + if target.user: + target_account = target.user.username + else: + target_account = target.email() + sender_name = request.user.person.name + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + context = { + 'source_account': source_account, + 'target_account': target_account, + 'sender_name': sender_name, + } + body = render_to_string('person/merge_request_email.txt', context) + initial = { + 'to': ', '.join(to), + 'frm': settings.DEFAULT_FROM_EMAIL, + 'reply_to': 'support@ietf.org', + 'subject': subject, + 'body': body, + 'by': request.user.person.pk, + } + form = MergeRequestForm(initial=initial) + else: + messages.error(request, "Error requesting merge email: " + merge_form.errors.as_text()) + return redirect("ietf.person.views.merge") + + if request.method == 'POST': + form = MergeRequestForm(request.POST) + if form.is_valid(): + extra = {"Reply-To": form.cleaned_data.get("reply_to")} + send_mail_text( + request, + form.cleaned_data.get("to"), + form.cleaned_data.get("frm"), + form.cleaned_data.get("subject"), + form.cleaned_data.get("body"), + extra=extra, + ) + + messages.success(request, "The merge confirmation email was sent.") + return redirect("ietf.person.views.merge") + + return render(request, "person/send_merge_request.html", { + "form": form, + }) diff --git a/ietf/review/resources.py b/ietf/review/resources.py index f79d6dfc221..546ab50a5ca 100644 --- a/ietf/review/resources.py +++ b/ietf/review/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-14 04:21 PDT -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index fa26d33a5c5..91ccde21879 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -256,7 +256,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): self.assertEqual(response.status_code,302) draft = Document.objects.get(name=draft.name) self.assertEqual(draft.get_state('draft-iesg').slug,'defer') - self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.filter(type='changed_action_holders').count(), 1) # Removing need-rev should remove authors @@ -273,7 +273,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): # Setting to approved should remove all action holders # noinspection DjangoOrm - draft.action_holders.add(*(draft.authors())) # add() with through model ok in Django 2.2+ + draft.action_holders.add(*(draft.author_persons())) # add() with through model ok in Django 2.2+ response = self.client.post(url,{ 'submit': 'update_state', 'state': State.objects.get(type_id='draft-iesg', slug='approved').pk, diff --git a/ietf/settings.py b/ietf/settings.py index eb5f9d2161e..7af9c7ae5be 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2025, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -13,6 +13,7 @@ import warnings from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +from django.http import UnreadablePostError # DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) @@ -22,6 +23,7 @@ warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider") warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie") warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 +warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)") # caused by django-filters when USE_DEPRECATED_PYTZ is true warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 @@ -225,159 +227,126 @@ BLOBSTORAGE_CONNECT_TIMEOUT = 10 # seconds; boto3 default is 60 BLOBSTORAGE_READ_TIMEOUT = 10 # seconds; boto3 default is 60 +# Caching for agenda data in seconds +AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days +AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) -FILE_UPLOAD_PERMISSIONS = 0o644 +FILE_UPLOAD_PERMISSIONS = 0o644 -# ------------------------------------------------------------------------ -# Django/Python Logging Framework Modifications +FIRST_V3_RFC = 8650 -# Filter out "Invalid HTTP_HOST" emails -# Based on http://www.tiwoc.de/blog/2013/03/django-prevent-email-notification-on-suspiciousoperation/ -from django.core.exceptions import SuspiciousOperation -def skip_suspicious_operations(record): - if record.exc_info: - exc_value = record.exc_info[1] - if isinstance(exc_value, SuspiciousOperation): - return False - return True -# Filter out UreadablePostError: -from django.http import UnreadablePostError +# +# Logging config +# + +# Callback to filter out UnreadablePostError: def skip_unreadable_post(record): if record.exc_info: - exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable + exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable if isinstance(exc_value, UnreadablePostError): return False return True -# Copied from DEFAULT_LOGGING as of Django 1.10.5 on 22 Feb 2017, and modified -# to incorporate html logging, invalid http_host filtering, and more. -# Changes from the default has comments. - -# The Python logging flow is as follows: -# (see https://docs.python.org/2.7/howto/logging.html#logging-flow) -# -# Init: get a Logger: logger = logging.getLogger(name) -# -# Logging call, e.g. logger.error(level, msg, *args, exc_info=(...), extra={...}) -# --> Logger (discard if level too low for this logger) -# (create log record from level, msg, args, exc_info, extra) -# --> Filters (discard if any filter attach to logger rejects record) -# --> Handlers (discard if level too low for handler) -# --> Filters (discard if any filter attached to handler rejects record) -# --> Formatter (format log record and emit) -# - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - # - 'loggers': { - 'django': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "celery": { + "handlers": ["console"], + "level": "INFO", }, - 'django.request': { - 'handlers': ['console'], - 'level': 'ERROR', + "datatracker": { + "handlers": ["console"], + "level": "INFO", }, - 'django.server': { - 'handlers': ['django.server'], - 'level': 'INFO', + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", }, - 'django.security': { - 'handlers': ['console', ], - 'level': 'INFO', + "django.request": {"level": "ERROR"}, # only log 5xx, ignore 4xx + "django.security": { + # SuspiciousOperation errors - log to console only + "handlers": ["console"], + "propagate": False, # no further handling please }, - 'oidc_provider': { - 'handlers': ['console', ], - 'level': 'DEBUG', + "django.server": { + # Only used by Django's runserver development server + "handlers": ["django.server"], + "level": "INFO", }, - 'datatracker': { - 'handlers': ['console'], - 'level': 'INFO', - }, - 'celery': { - 'handlers': ['console'], - 'level': 'INFO', + "oidc_provider": { + "handlers": ["console"], + "level": "DEBUG", }, }, - # - # No logger filters - # - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'debug_console': { - # Active only when DEBUG=True - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "debug_console": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'django.server': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'django.server', + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", }, - 'mail_admins': { - 'level': 'ERROR', - 'filters': [ - 'require_debug_false', - 'skip_suspicious_operations', # custom - 'skip_unreadable_posts', # custom + "mail_admins": { + "level": "ERROR", + "filters": [ + "require_debug_false", + "skip_unreadable_posts", ], - 'class': 'django.utils.log.AdminEmailHandler', - 'include_html': True, # non-default - } + "class": "django.utils.log.AdminEmailHandler", + "include_html": True, + }, }, - # # All these are used by handlers - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', - }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", }, - # custom filter, function defined above: - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", }, # custom filter, function defined above: - 'skip_unreadable_posts': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_unreadable_post, + "skip_unreadable_posts": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_unreadable_post, }, }, - # And finally the formatters - 'formatters': { - 'django.server': { - '()': 'django.utils.log.ServerFormatter', - 'format': '[%(server_time)s] %(message)s', + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", }, - 'plain': { - 'style': '{', - 'format': '{levelname}: {name}:{lineno}: {message}', + "plain": { + "style": "{", + "format": "{levelname}: {name}:{lineno}: {message}", }, - 'json' : { + "json": { "class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter", "style": "{", - "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}", - } + "format": ( + "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}" + "{process}{status_code}" + ), + }, }, } -# End logging -# ------------------------------------------------------------------------ - X_FRAME_OPTIONS = 'SAMEORIGIN' CSRF_TRUSTED_ORIGINS = [ @@ -411,6 +380,7 @@ def skip_unreadable_post(record): ], 'OPTIONS': { 'context_processors': [ + 'ietf.context_processors.traceparent_id', 'django.contrib.auth.context_processors.auth', 'django.template.context_processors.debug', # makes 'sql_queries' available in templates 'django.template.context_processors.i18n', @@ -443,12 +413,14 @@ def skip_unreadable_post(record): MIDDLEWARE = [ + "ietf.middleware.add_otel_traceparent_header", "django.middleware.csrf.CsrfViewMiddleware", "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it "django.middleware.common.CommonMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", + "ietf.middleware.is_authenticated_header_middleware", "django.middleware.http.ConditionalGetMiddleware", "simple_history.middleware.HistoryRequestMiddleware", # comment in this to get logging of SQL insert and update statements: @@ -459,7 +431,6 @@ def skip_unreadable_post(record): "django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.security.SecurityMiddleware", "ietf.middleware.unicode_nfkc_normalization_middleware", - "ietf.middleware.is_authenticated_header_middleware", ] ROOT_URLCONF = 'ietf.urls' @@ -498,6 +469,7 @@ def skip_unreadable_post(record): 'django_celery_results', 'corsheaders', 'django_markup', + 'django_filters', 'oidc_provider', 'drf_spectacular', 'drf_standardized_errors', @@ -805,6 +777,7 @@ def skip_unreadable_post(record): "polls", "procmaterials", "review", + "rfc", "slides", "staging", "statchg", @@ -829,6 +802,11 @@ def skip_unreadable_post(record): "slides", ] +# Other storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . @@ -916,16 +894,14 @@ def skip_unreadable_post(record): IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" IANA_SYNC_PROTOCOLS_URL = "https://www.iana.org/protocols/" -RFC_EDITOR_SYNC_PASSWORD="secret" -RFC_EDITOR_SYNC_NOTIFICATION_URL = "https://www.rfc-editor.org/parser/parser.php" RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org" -#RFC_EDITOR_GROUP_NOTIFICATION_URL = "https://www.rfc-editor.org/notification/group.php" -RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" -RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" +RFC_EDITOR_ERRATA_BASE_URL = "https://www.rfc-editor.org/errata/" RFC_EDITOR_INFO_BASE_URL = "https://www.rfc-editor.org/info/" +RFC_EDITOR_QUEUE_SITE_BASE_URL = "https://queue.rfc-editor.org" + # NomCom Tool settings ROLODEX_URL = "" @@ -1000,6 +976,10 @@ def skip_unreadable_post(record): ) RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES +# Paths in the red bucket +RFCINDEX_INPUT_PATH = "other/" +RFCINDEX_OUTPUT_PATH = "other/" + IDSUBMIT_MAX_DRAFT_SIZE = { 'txt': 2*1024*1024, # Max size of txt draft file in bytes 'xml': 3*1024*1024, # Max size of xml draft file in bytes @@ -1283,7 +1263,10 @@ def skip_unreadable_post(record): 'patch/change-oidc-provider-field-sizes-228.patch', 'patch/fix-oidc-access-token-post.patch', 'patch/fix-jwkest-jwt-logging.patch', - 'patch/django-cookie-delete-with-all-settings.patch', + # Patch includes old cookie-delete-with-all-settings and a backport of the fix + # to CVE-2026-35192 from Django 5.2. The patches conflict, so cannot be applied + # separately. + 'patch/django-cookie-delete-settings-and-CVE-2026-35192.patch', 'patch/tastypie-django22-fielderror-response.patch', ] if DEBUG: @@ -1293,7 +1276,7 @@ def skip_unreadable_post(record): except ImportError: pass -STATS_NAMES_LIMIT = 25 +STATS_TIMELINE_CACHE_TIMEOUT = 86400 UTILS_MEETING_CONFERENCE_DOMAINS = ['webex.com', 'zoom.us', 'jitsi.org', 'meetecho.com', 'gather.town', ] UTILS_TEST_RANDOM_STATE_FILE = '.factoryboy_random_state' @@ -1359,6 +1342,11 @@ def skip_unreadable_post(record): MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u" MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}" +# Errata system api configuration +# settings should provide +# ERRATA_METADATA_NOTIFICATION_URL +# ERRATA_METADATA_NOTIFICATION_API_KEY + # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. from ietf.settings_local import * # pyflakes:ignore pylint: disable=wildcard-import @@ -1393,6 +1381,16 @@ def skip_unreadable_post(record): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", @@ -1446,6 +1444,17 @@ def skip_unreadable_post(record): "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", @@ -1512,11 +1521,17 @@ def skip_unreadable_post(record): NOMCOM_APP_SECRET = b'\x9b\xdas1\xec\xd5\xa0SI~\xcb\xd4\xf5t\x99\xc4i\xd7\x9f\x0b\xa9\xe8\xfeY\x80$\x1e\x12tN:\x84' ALLOWED_HOSTS = ['*',] - + try: # see https://github.com/omarish/django-cprofile-middleware - import django_cprofile_middleware # pyflakes:ignore - MIDDLEWARE = MIDDLEWARE + ['django_cprofile_middleware.middleware.ProfilerMiddleware', ] + import django_cprofile_middleware # pyflakes:ignore + + MIDDLEWARE = MIDDLEWARE + [ + "django_cprofile_middleware.middleware.ProfilerMiddleware", + ] + DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = ( + False # Do not use this setting for a public site! + ) except ImportError: pass @@ -1529,3 +1544,5 @@ def skip_unreadable_post(record): YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com'] + +IETF_DOI_PREFIX = "10.17487" diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 6479069db02..e7ebc13eb28 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS +from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS, STORAGES import debug # pyflakes:ignore debug.debug = True @@ -114,3 +114,13 @@ def tempdir_with_cleanup(**kwargs): AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS except NameError: pass + +# Use InMemoryStorage for red bucket and r2-rfc storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} +STORAGES["r2-rfc"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "r2-rfc"}, +} diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index 40744a228df..edb978757a9 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -27,6 +27,9 @@ 'MAX_ENTRIES': 10000, }, }, + 'agenda': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, 'proceedings': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, diff --git a/ietf/static/css/datepicker.scss b/ietf/static/css/datepicker.scss index 88f9e835fdb..b193ccda3a8 100644 --- a/ietf/static/css/datepicker.scss +++ b/ietf/static/css/datepicker.scss @@ -4,3 +4,29 @@ $dp-cell-focus-background-color: $dropdown-link-hover-bg !default; @import "vanillajs-datepicker/sass/datepicker-bs5"; + +[data-bs-theme="dark"] .datepicker-picker { + .datepicker-header, + .datepicker-controls .btn, + .datepicker-main, + .datepicker-footer { + background-color: $gray-800; + } + + .datepicker-cell:hover { + background-color: $gray-700; + } + + .datepicker-cell.day.focused { + background-color: $gray-600; + } + + .datepicker-cell.day.selected.focused { + background-color: $blue; + } + + .datepicker-controls .btn:hover { + background-color:$gray-700; + color: $gray-400; + } +} diff --git a/ietf/static/css/document_html.scss b/ietf/static/css/document_html.scss index 47ef8d64b47..5a4282195f9 100644 --- a/ietf/static/css/document_html.scss +++ b/ietf/static/css/document_html.scss @@ -205,10 +205,6 @@ display: inline; } - .newpage { - margin-top: -1.25em; - } - } tbody.meta tr { diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss index df973863d53..b8c701eae1b 100644 --- a/ietf/static/css/ietf.scss +++ b/ietf/static/css/ietf.scss @@ -46,6 +46,11 @@ $bootstrap-icons-font-src: url("npm:bootstrap-icons/font/fonts/bootstrap-icons.w url("npm:bootstrap-icons/font/fonts/bootstrap-icons.woff") format("woff"); @import "bootstrap-icons/font/bootstrap-icons"; +// Disable contextual alternates (calt) +body { + font-feature-settings: "calt" off; +} + // Leave room for fixed-top navbar... body.navbar-offset { padding-top: 60px; @@ -1216,3 +1221,20 @@ iframe.status { .overflow-shadows--bottom-only { box-shadow: inset 0px -21px 18px -20px var(--bs-body-color); } + +#navbar-doc-search-wrapper { + position: relative; +} + +#navbar-doc-search-results { + max-height: 400px; + overflow-y: auto; + min-width: auto; + left: 0; + right: 0; + + .dropdown-item { + white-space: normal; + overflow-wrap: break-word; + } +} diff --git a/ietf/static/js/attendees-chart.js b/ietf/static/js/attendees-chart.js new file mode 100644 index 00000000000..fed3b1289cb --- /dev/null +++ b/ietf/static/js/attendees-chart.js @@ -0,0 +1,58 @@ +(function () { + var raw = document.getElementById('attendees-chart-data'); + if (!raw) return; + var chartData = JSON.parse(raw.textContent); + var chart = null; + var currentBreakdown = 'type'; + + // Override the global transparent background set by highcharts.js so the + // export menu and fullscreen view use the page background color. + var container = document.getElementById('attendees-pie-chart'); + var bodyBg = getComputedStyle(document.body).backgroundColor; + container.style.setProperty('--highcharts-background-color', bodyBg); + + function renderChart(breakdown) { + var seriesData = chartData[breakdown].map(function (item) { + return { name: item[0], y: item[1] }; + }); + if (chart) chart.destroy(); + chart = Highcharts.chart(container, { + chart: { type: 'pie', height: 400 }, + title: { text: null }, + tooltip: { pointFormat: '{point.name}: {point.y} ({point.percentage:.1f}%)' }, + plotOptions: { + pie: { + dataLabels: { + enabled: true, + format: '{point.name}
{point.y} ({point.percentage:.1f}%)', + }, + showInLegend: false, + } + }, + series: [{ name: 'Attendees', data: seriesData }], + }); + } + + var modal = document.getElementById('attendees-chart-modal'); + + // Render (or re-render) the chart each time the modal becomes fully visible, + // so Highcharts can measure the container dimensions correctly. + modal.addEventListener('shown.bs.modal', function () { + renderChart(currentBreakdown); + }); + + // Release the chart when the modal closes to avoid stale renders. + modal.addEventListener('hidden.bs.modal', function () { + if (chart) { + chart.destroy(); + chart = null; + } + }); + + document.querySelectorAll('[name="attendees-breakdown"]').forEach(function (radio) { + radio.addEventListener('change', function () { + currentBreakdown = this.value; + renderChart(currentBreakdown); + }); + }); +})(); diff --git a/ietf/static/js/document_html.js b/ietf/static/js/document_html.js index 6e8861739a6..3e609f3965d 100644 --- a/ietf/static/js/document_html.js +++ b/ietf/static/js/document_html.js @@ -117,4 +117,83 @@ document.addEventListener("DOMContentLoaded", function (event) { } }); } + + // Rewrite these CSS properties so that the values are available for restyling. + document.querySelectorAll("svg [style]").forEach(el => { + // Push these CSS properties into their own attributes + const SVG_PRESENTATION_ATTRS = new Set([ + 'alignment-baseline', 'baseline-shift', 'clip', 'clip-path', 'clip-rule', + 'color', 'color-interpolation', 'color-interpolation-filters', + 'color-rendering', 'cursor', 'direction', 'display', 'dominant-baseline', + 'fill', 'fill-opacity', 'fill-rule', 'filter', 'flood-color', + 'flood-opacity', 'font-family', 'font-size', 'font-size-adjust', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', + 'image-rendering', 'letter-spacing', 'lighting-color', 'marker-end', + 'marker-mid', 'marker-start', 'mask', 'opacity', 'overflow', 'paint-order', + 'pointer-events', 'shape-rendering', 'stop-color', 'stop-opacity', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', + 'text-anchor', 'text-decoration', 'text-rendering', 'unicode-bidi', + 'vector-effect', 'visibility', 'word-spacing', 'writing-mode', + ]); + + // Simple CSS splitter: respects quoted strings and parens so semicolons + // inside url(...) or "..." don't get treated as declaration boundaries. + function parseDeclarations(styleText) { + const decls = []; + let buf = ''; + let inStr = false; + let strChar = ''; + let escaped = false; + let depth = 0; + + for (const ch of styleText) { + if (inStr) { + if (escaped) { + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else if (ch === strChar) { + inStr = false; + } + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } else if (ch === ';' && depth === 0) { + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + buf = ''; + continue; + } + buf += ch; + } + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + return decls; + } + + const remainder = []; + for (const decl of parseDeclarations(el.getAttribute('style'))) { + const [prop, val] = decl.split(":", 2).map(v => v.trim()); + if (val && !/!important$/.test(val) && SVG_PRESENTATION_ATTRS.has(prop)) { + el.setAttribute(prop, val); + } else { + remainder.push(decl); + } + } + + if (remainder.length > 0) { + el.setAttribute('style', remainder.join('; ')); + } else { + el.removeAttribute('style'); + } + }); }); diff --git a/ietf/static/js/liaisons.js b/ietf/static/js/liaisons.js index e7a76660c96..6c46cb6dfcf 100644 --- a/ietf/static/js/liaisons.js +++ b/ietf/static/js/liaisons.js @@ -197,16 +197,6 @@ var liaisonForm = { } }, - checkPostOnly: function (post_only) { - if (post_only) { - $("button[name=send]") - .hide(); - } else { - $("button[name=send]") - .show(); - } - }, - updateInfo: function (first_time, sender) { // don't overwrite fields when editing existing liaison if (liaisonForm.is_edit_form) { @@ -239,7 +229,6 @@ var liaisonForm = { liaisonForm.toggleApproval(response.needs_approval); liaisonForm.response_contacts.val(response.response_contacts); } - liaisonForm.checkPostOnly(response.post_only); } } }); @@ -326,4 +315,4 @@ $(document) .each(liaisonForm.init); $('#liaison_search_form') .each(searchForm.init); - }); \ No newline at end of file + }); diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js index 756a75001a8..c03368cd728 100644 --- a/ietf/static/js/list.js +++ b/ietf/static/js/list.js @@ -16,7 +16,7 @@ function text_sort(a, b, options) { // sort by text content return prep(a, options).localeCompare(prep(b, options), "en", { sensitivity: "base", - ignorePunctuation: true, + ignorePunctuation: false, numeric: true }); } diff --git a/ietf/static/js/meeting_stats.js b/ietf/static/js/meeting_stats.js new file mode 100644 index 00000000000..cf43d08eb8a --- /dev/null +++ b/ietf/static/js/meeting_stats.js @@ -0,0 +1,58 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +import Chart from 'chart.js/auto' +import autocolors from 'chartjs-plugin-autocolors' + +document.addEventListener('DOMContentLoaded', () => { + Chart.register(autocolors) + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + new Chart(ctx, { + type: 'pie', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + plugins: { + autocolors: { + mode: 'data' // Required for Pie charts to color individual slices + }, + legend: { + position: 'bottom', + labels: { + padding: 20, + font: { size: 13 }, + color: '#475569', + generateLabels: function (chart) { + const dataset = chart.data.datasets[0] + return chart.data.labels.map((label, i) => ({ + text: `${label}: ${dataset.data[i]}`, + fillStyle: dataset.backgroundColor[i], + hidden: false, + index: i, + })) + } + } + }, + tooltip: { + callbacks: { + label: function (context) { + const label = context.label || '' + const value = context.raw + const total = context.dataset.data.reduce((a, b) => a + b, 0) + const percentage = ((value / total) * 100).toFixed(1) + + return `${label}: ${value} (${percentage}%)` + } + } + } + } + } + }) + } + + displayChart('totalRegistrationChart', totalChartData) + displayChart('inPersonRegistrationChart', inPersonChartData) +}) diff --git a/ietf/static/js/meeting_timeline.js b/ietf/static/js/meeting_timeline.js new file mode 100644 index 00000000000..713fb3ae707 --- /dev/null +++ b/ietf/static/js/meeting_timeline.js @@ -0,0 +1,92 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +import Chart from 'chart.js/auto' +import zoomPlugin from 'chartjs-plugin-zoom' + +document.addEventListener('DOMContentLoaded', () => { + Chart.register(zoomPlugin) // enable the zoom plugin + + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + const statsType = JSON.parse(document.getElementById('stats-type-data').textContent) + const stackedLines = statsType === 'total' + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + return new Chart(ctx, { + type: 'line', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + scales: { + y: { + stacked: stackedLines, + }, + x: { + title: { + display: true, + text: 'IETF Meeting Number', + }, + }, + }, + plugins: { + legend: { + position: 'bottom', + labels: { + usePointStyle: true, + padding: 15, + font: { size: 12 }, + }, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function (items) { + return `IETF Meeting ${items[0].label}` + }, + label: function (context) { + return ` ${context.dataset.label}: ${context.parsed.y} participants` + } + } + }, + zoom: { + zoom: { + wheel: { enabled: true }, // scroll to zoom + pinch: { enabled: true }, // pinch on mobile + drag: { // drag to select range + enabled: true, + modifierKey: 'alt' + }, + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) + } + + const totalChart = displayChart('totalRegistrationChart', totalChartData) + if (inPersonChartData !== null) { + inPersonChart = displayChart('inPersonRegistrationChart', inPersonChartData) + } + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + } + }) + document.getElementById('resetButton').addEventListener('click', () => { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + }) +}) diff --git a/ietf/static/js/navbar-doc-search.js b/ietf/static/js/navbar-doc-search.js new file mode 100644 index 00000000000..c36c0323103 --- /dev/null +++ b/ietf/static/js/navbar-doc-search.js @@ -0,0 +1,113 @@ +$(function () { + var $input = $('#navbar-doc-search'); + var $results = $('#navbar-doc-search-results'); + var ajaxUrl = $input.data('ajax-url'); + var debounceTimer = null; + var highlightedIndex = -1; + var keyboardHighlight = false; + var currentItems = []; + + function showDropdown() { + $results.addClass('show'); + } + + function hideDropdown() { + $results.removeClass('show'); + highlightedIndex = -1; + keyboardHighlight = false; + updateHighlight(); + } + + function updateHighlight() { + $results.find('.dropdown-item').removeClass('active'); + if (highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + $results.find('.dropdown-item').eq(highlightedIndex).addClass('active'); + } + } + + function doSearch(query) { + if (query.length < 2) { + hideDropdown(); + return; + } + $.ajax({ + url: ajaxUrl, + dataType: 'json', + data: { q: query }, + success: function (data) { + currentItems = data; + highlightedIndex = -1; + $results.empty(); + if (data.length === 0) { + $results.append('
  • No results found
    • '); + } else { + data.forEach(function (item) { + var $li = $('
  • '); + var $a = $('' + item.text + ''); + $li.append($a); + $results.append($li); + }); + } + showDropdown(); + } + }); + } + + $input.on('input', function () { + clearTimeout(debounceTimer); + var query = $(this).val().trim(); + debounceTimer = setTimeout(function () { + doSearch(query); + }, 250); + }); + + $input.on('keydown', function (e) { + if (e.key === 'ArrowDown') { + e.preventDefault(); + if (highlightedIndex < currentItems.length - 1) { + highlightedIndex++; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + if (highlightedIndex > 0) { + highlightedIndex--; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'Enter') { + e.preventDefault(); + if (keyboardHighlight && highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + window.location.href = currentItems[highlightedIndex].url; + } else { + var query = $(this).val().trim(); + if (query) { + window.location.href = '/doc/search/?name=' + encodeURIComponent(query) + '&rfcs=on&activedrafts=on&olddrafts=on'; + } + } + } else if (e.key === 'Escape') { + hideDropdown(); + $input.blur(); + } + }); + + // Hover highlights (visual only — Enter still submits the text) + $results.on('mouseenter', '.dropdown-item', function () { + highlightedIndex = $results.find('.dropdown-item').index(this); + keyboardHighlight = false; + updateHighlight(); + }); + + $results.on('mouseleave', '.dropdown-item', function () { + highlightedIndex = -1; + updateHighlight(); + }); + + // Click outside closes dropdown + $(document).on('click', function (e) { + if (!$(e.target).closest('#navbar-doc-search-wrapper').length) { + hideDropdown(); + } + }); +}); diff --git a/ietf/static/js/session_details.js b/ietf/static/js/session_details.js new file mode 100644 index 00000000000..03d1b2d3d91 --- /dev/null +++ b/ietf/static/js/session_details.js @@ -0,0 +1,53 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +// Relies on other scripts being loaded, see usage in session_details.html +document.addEventListener('DOMContentLoaded', () => { + // Init with best guess at local timezone. + ietf_timezone.set_tz_change_callback(timezone_changed) // cb is in upcoming.js + ietf_timezone.initialize('local') + + // Set up sortable elements if the user can manage materials + if (document.getElementById('can-manage-materials-flag')) { + const sortables = [] + const options = { + group: 'slides', + animation: 150, + handle: '.drag-handle', + onAdd: function (event) {onAdd(event)}, + onRemove: function (event) {onRemove(event)}, + onEnd: function (event) {onEnd(event)} + } + + function onAdd (event) { + const old_session = event.from.getAttribute('data-session') + const new_session = event.to.getAttribute('data-session') + $.post(event.to.getAttribute('data-add-to-session'), { + 'order': event.newIndex + 1, + 'name': event.item.getAttribute('name') + }) + $(event.item).find('td:eq(1)').find('a').each(function () { + $(this).attr('href', $(this).attr('href').replace(old_session, new_session)) + }) + } + + function onRemove (event) { + const old_session = event.from.getAttribute('data-session') + $.post(event.from.getAttribute('data-remove-from-session'), { + 'oldIndex': event.oldIndex + 1, + 'name': event.item.getAttribute('name') + }) + } + + function onEnd (event) { + if (event.to == event.from) { + $.post(event.from.getAttribute('data-reorder-in-session'), { + 'oldIndex': event.oldIndex + 1, + 'newIndex': event.newIndex + 1 + }) + } + } + + for (const elt of document.querySelectorAll('.slides tbody')) { + sortables.push(Sortable.create(elt, options)) + } + } +}) diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 59722c505ef..21e2c171acc 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2017-02-15 10:10 PST -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 48552c8fbac..57e3b76b39b 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,42 +1,118 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2016-2026, All Rights Reserved - -import calendar +import csv +import datetime +import io import json +from django.http import Http404 from pyquery import PyQuery import debug # pyflakes:ignore +from django.test import RequestFactory from django.urls import reverse as urlreverse +from django.utils import timezone +from ietf.meeting.models import Meeting from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views -from ietf.group.factories import RoleFactory -from ietf.person.factories import PersonFactory +from ietf.doc.factories import NewRevisionDocEventFactory +from ietf.group.factories import GroupFactory, RoleFactory +from ietf.person.factories import EmailFactory, PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory -from ietf.utils.timezone import date_today +from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory +from ietf.submit.factories import SubmissionFactory class StatisticsTests(TestCase): def test_stats_index(self): + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) def test_document_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.document_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + r = self.client.get(urlreverse(ietf.stats.views.document_stats)) + self.assertRedirects(r, urlreverse(ietf.stats.views.stats_index)) def test_meeting_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.meeting_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) + meeting124 = MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + meeting125 = MeetingFactory(type_id='ietf', number='125', date=timezone.now() + datetime.timedelta(days=120)) + RegistrationFactory.create_batch(15, meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=False) + RegistrationFactory.create_batch(14, meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(15, meeting=meeting125, affiliation='Test LLC', with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(25, meeting=meeting125, affiliation='Example, Ltd', with_ticket={'attendance_type_id': 'onsite'}, attended=False) + # Test the meeting specific statitistics per affiliation and per country + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Affiliation (31 in total)") + self.assertContains(r, "In Person Registrations by Affiliation (16 in total)") + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Country (31 in total)") + self.assertContains(r, "In Person Registrations by Country (16 in total)") + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + # Test the meetings timeline per country + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations by country") + # Test the meetings timeline per affiliation + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + self.assertContains(r, "This page provides a timeline of meeting registrations by affiliation") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + in_person_data = json.loads(pq.find("script#in-person-chart-data").text()) + self.assertTrue( + any( + ds["label"] == "Example" and ds["data"] == [0, 25] + for ds in in_person_data["datasets"] + ) + ) + # Test the global meetings timeline + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "total"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations.") + + def test_meeting_stats_for_bad_meeting(self): + self.assertFalse(Meeting.objects.filter(number=676767).exists()) + for stats_type in ["affiliation", "country"]: + r = self.client.get( + urlreverse( + "ietf.stats.views.meeting_stats", + kwargs={"meeting_number": 676767, "stats_type": stats_type}, + ) + ) + self.assertEqual(r.status_code, 404) + + # We don't have a URL for an interim, but make sure the view will 404 if + # somehow a non-interim gets selected... + interim_num = MeetingFactory(type_id="interim").number + request_factory = RequestFactory() + with self.assertRaises(Http404): + ietf.stats.views.meeting_stats( + request_factory.get(f"/stats/meeting/{interim_num}/{stats_type}"), + meeting_number=interim_num, + stats_type=stats_type, + ) - def test_known_country_list(self): # check redirect url = urlreverse(ietf.stats.views.known_countries_list) @@ -80,16 +156,19 @@ def test_review_stats(self): self.assertTrue(q('.review-stats td:contains("1")')) # check stacked chart - expected_date = date_today().replace(day=1) - expected_js_timestamp = calendar.timegm(expected_date.timetuple()) * 1000 url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "time" }) url += "?team={}".format(review_req.team.acronym) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [ - {"label": "in time", "color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}, - {"label": "late", "color": "#b42222", "data": [[expected_js_timestamp, 0]]} - ]) + stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(stacked_data), 2) + self.assertEqual(stacked_data[0]['label'], 'in time') + self.assertEqual(stacked_data[0]['color'], '#3d22b3') + self.assertEqual(stacked_data[0]['data'], [[stacked_data[0]['data'][0][0], 0]]) + self.assertEqual(stacked_data[1]['label'], 'late') + self.assertEqual(stacked_data[1]['color'], '#b42222') + self.assertEqual(stacked_data[1]['data'], [[stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) @@ -99,7 +178,11 @@ def test_review_stats(self): url += "&completion=not_completed" r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [{"color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}]) + non_stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(non_stacked_data), 1) + self.assertEqual(non_stacked_data[0]['color'], '#3d22b3') + self.assertEqual(non_stacked_data[0]['data'], [[non_stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) @@ -109,3 +192,139 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + +class AnnualReportInputsTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + self.non_member = PersonFactory() + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + + def test_access_unauthenticated(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + def test_access_non_member_forbidden(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + self.client.login( + username=self.non_member.user.username, + password=f"{self.non_member.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + + def test_access_member_allowed(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + + def test_default_year(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], datetime.date.today().year - 1) + + def test_year_param_redirects_to_year_url(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url, {"year": "2022"}) + self.assertRedirects( + r, + urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2022"}), + ) + + def test_summary_counts(self): + self._member_login() + year = 2021 + # author1 has a matching Person record; author2 does not + EmailFactory(address="author1@example.com") + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 6, 1), + submitter_email="submitter@example.com", + ) + sub.authors = [ + {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []}, + {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + # Same draft, second revision — should count once + extra = NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 15, tzinfo=datetime.timezone.utc), + doc=extra.doc, + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], year) + self.assertEqual(r.context["author_count"], 2) + self.assertEqual(r.context["author_person_count"], 1) + self.assertEqual(r.context["author_noperson_count"], 1) + self.assertEqual(r.context["submitter_count"], 1) + self.assertEqual(r.context["submitter_person_count"], 0) + self.assertEqual(r.context["submitter_noperson_count"], 1) + self.assertEqual(r.context["draft_count"], 3) + + def test_download_authors_csv(self): + self._member_login() + year = 2020 + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + ) + sub.authors = [ + {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "authors"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"authors-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvauthor@example.com", rows[0]) + + def test_download_submitters_csv(self): + self._member_login() + year = 2020 + SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + submitter_email="csvsubmitter@example.com", + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "submitters"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"submitters-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvsubmitter@example.com", rows[0]) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index d2993759d2a..3bb107813a6 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -11,7 +11,9 @@ url(r"^$", views.stats_index), url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), url(r"^knowncountries/$", views.known_countries_list), - url(r"^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), - url(r"^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), + url(r"^meeting/$", views.meetings_timeline), + url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), + url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), + url(r"^annual_report_inputs/(?:(?P\d{4})/)?$", views.annual_report_inputs), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 504d84e86d6..fe2fa82f554 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,19 +1,22 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- import calendar +import csv import datetime import itertools import json import dateutil.relativedelta from collections import defaultdict +from django.conf import settings from django.contrib.auth.decorators import login_required -from django.http import HttpResponseRedirect -from django.shortcuts import render +from django.core.cache import cache +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render, get_object_or_404 from django.urls import reverse as urlreverse - +from django.db.models import Count import debug # pyflakes:ignore @@ -25,15 +28,32 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName -from ietf.ietfauth.utils import has_role +from ietf.meeting.models import Registration, Meeting +from ietf.ietfauth.utils import has_role, role_required from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.meeting.helpers import get_current_ietf_meeting_num +# Color palette for lines +colors = [ + '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', + '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', + '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', + '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', +] def stats_index(request): - return render(request, "stats/index.html") + """Render the statistics index page with the current meeting number as it is required by the meeting menu item.""" + current_meeting = get_current_ietf_meeting_num() + return render(request, "stats/index.html", { + "current_meeting": current_meeting + }) def generate_query_string(query_dict, overrides): + """ + Returns: + A query string starting with '?' if there are parameters, empty string otherwise. + """ query_part = "" if query_dict or overrides: @@ -58,9 +78,20 @@ def generate_query_string(query_dict, overrides): return query_part def get_choice(request, get_parameter, possible_choices, multiple=False): - # the statistics are built with links to make navigation faster, - # so we don't really have a form in most cases, so just use this - # helper instead to select between the choices + """Extract a choice from the request GET parameters. + + Since statistics pages use links for navigation instead of forms, + this helper selects between possible choices from the URL parameters. + + Args: + request: The HTTP request object. + get_parameter: The name of the GET parameter. + possible_choices: List of tuples (value, label). + multiple: If True, return a list of found values; otherwise return the first found or None. + + Returns: + The selected value(s) or None. + """ values = request.GET.getlist(get_parameter) found = [t[0] for t in possible_choices if t[0] in values] @@ -73,75 +104,553 @@ def get_choice(request, get_parameter, possible_choices, multiple=False): return None def add_url_to_choices(choices, url_builder): + """Add URLs to a list of choices. + + Args: + choices: List of tuples (slug, label). + url_builder: Function that takes a slug and returns a URL. + + Returns: + List of tuples (slug, label, url). + """ return [ (slug, label, url_builder(slug)) for slug, label in choices] -def put_into_bin(value, bin_size): - if value is None: - return (0, '') +def document_stats(request, stats_type=None): + # timeline per year, or per specific year: streams, affiliation, rfc vs I-D + # could also be time between individual/WG I-D to rfc publication/IESG ballot + # DISCUSS resolution time + # Humm also split by authors (affiliation) / documents (the rest) probably + """Redirect to the stats index page. Deprecated view.""" + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - v = (value // bin_size) * bin_size - return (v, "{} - {}".format(v, v + bin_size - 1)) +def known_countries_list(request, stats_type=None, acronym=None): + """Render a list of known countries with their aliases.""" + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) -def prune_unknown_bin_with_known(bins): - # remove from the unknown bin all authors within the - # named/known bins - all_known = { n for b, names in bins.items() if b for n in names } - bins[""] = [name for name in bins[""] if name not in all_known] - if not bins[""]: - del bins[""] + return render(request, "stats/known_countries_list.html", { + "countries": countries, + }) -def count_bins(bins): - return len({ n for b, names in bins.items() if b for n in names }) +def canonicalize_affiliation(affiliation): + """Canonicalize an affiliation string by removing common suffixes and standardizing prefixes. + + Args: + affiliation: The affiliation string to canonicalize. + + Returns: + The canonicalized affiliation string, or None if input is None. + """ + if not affiliation or affiliation.lower() in ('n/a', 'none', 'unspecified'): + return None + for suffix in ('ab', 'ag', 'corp', 'corp.', 'corporation', 'gmbh', 'inc.', 'inc', 'international pte ltd', 'llc', 'ltd', 'ltd.', 'private limited', 'pty ltd', 'pvt ltd'): + if affiliation.lower().endswith(', ' + suffix): + affiliation = affiliation[:-(len(suffix)+2)] + elif affiliation.lower().endswith(' ' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + elif affiliation.lower().endswith(',' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + for prefix in ('akamai','apple', 'cisco', 'futurewei', 'google', 'hitachi', 'hpe', 'huawei', 'juniper', 'meta', 'nokia', 'ntt', 'siemens'): + if affiliation.lower().startswith(prefix + ' '): + affiliation = prefix + return affiliation.title() + +def get_affiliation_data_for_meetings(attendance_type=None): + """Get affiliation participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_affiliation_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 20 # could be a parameter, but would need to adjust cache handling + + # Get registration status details + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + registrations = registrations.values('affiliation', 'meeting__number') + + # Count per canonicalized affiliation + organization = dict() + meetings_set = set() + org_totals = defaultdict(int) + data_map = defaultdict(dict) # {org: {meeting: count}} + + for reg in registrations: + meeting = reg['meeting__number'] + meetings_set.add(meeting) + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 + data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_orgs = sorted( + org_totals.keys(), + key=lambda c: org_totals[c], + reverse=True + )[:top_n] + non_top_orgs = org_totals.keys() - top_orgs + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_orgs: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, org in enumerate(top_orgs): + color = colors[idx % len(colors)] + datasets.append({ + 'label': org, + 'data': [data_map[org].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) -def add_labeled_top_series_from_bins(chart_data, bins, limit): - """Take bins on the form (x, label): [name1, name2, ...], figure out - how many there are per label, take the overall top ones and put - them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" - aggregated_bins = defaultdict(set) - xs = set() - for (x, label), names in bins.items(): - xs.add(x) - aggregated_bins[label].update(names) + return sorted_meetings, datasets - xs = list(sorted(xs)) +def get_country_data_for_meetings(attendance_type=None): + """Get country participation data for meetings timeline chart. - sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) - top = [ label for label, names in list(sorted_bins)[:limit]] + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). - for label in top: - series_data = [] + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_country_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 10 # could be a parameter, but would need to adjust cache handling + # Get registration status counts, aggregated by country_code + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'country_code' # country code of the participant + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and country totals ── + meetings_set = set() + country_totals = defaultdict(int) + data_map = defaultdict(dict) # {country: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + country = row['country_code'] + count = row['participant_count'] + + meetings_set.add(meeting) + country_totals[country] += count + data_map[country][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_countries = sorted( + country_totals.keys(), + key=lambda c: country_totals[c], + reverse=True + )[:top_n] + + # -- Step 3.bis do the 'other' category -- + non_top_countries = country_totals.keys() - top_countries + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_countries: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, country in enumerate(top_countries): + color = colors[idx % len(colors)] + datasets.append({ + 'label': country, + 'data': [data_map[country].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) - for x in xs: - names = bins.get((x, label), set()) + return sorted_meetings, datasets + +def get_data_for_meetings(): + """Get total participation data by attendance type for meetings timeline chart. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = "stats:get_data_for_meetings" + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + # Get registration status counts, aggregated by ticket types + registrations = Registration.objects.filter(tickets__attendance_type__in=['onsite', 'remote']) + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'tickets__attendance_type' + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and tickets totals ── + meetings_set = set() + tickets_totals = defaultdict(int) + data_map = defaultdict(dict) # {ticket: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + ticket = row['tickets__attendance_type'] + count = row['participant_count'] + + meetings_set.add(meeting) + tickets_totals[ticket] += count + data_map[ticket][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + ticket_types = tickets_totals.keys() + + # ── Step 4: Build Chart.js datasets ── + # Color palette for lines + colors = [ '#FF6384', '#36A2EB'] + + datasets = [] + for idx, ticket_type in enumerate(ticket_types): + color = colors[idx % len(colors)] + datasets.append({ + 'label': ticket_type, + 'data': [data_map[ticket_type].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'backgroundColor': color + '99', # 60% opacity fill + 'fill': True, + 'tension': 0.0, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + return sorted_meetings, datasets + +def meetings_timeline(request, stats_type='country'): + """Render the meetings timeline page with participation statistics over time. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('country' or 'total'). + top_n: Number of top items to show (for country stats). + + Returns: + Rendered response for the meetings timeline template. + """ + if stats_type == 'total': + total_labels, total_data_sets = get_data_for_meetings() + in_person_labels = ([], []) + in_person_data_sets = ([], []) + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'affiliation': + total_labels, total_data_sets = get_affiliation_data_for_meetings() + in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'country': + total_labels, total_data_sets = get_country_data_for_meetings() + in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - series_data.append((x, len(names))) + total_chart_data = { + 'labels': total_labels, + 'datasets': total_data_sets, + } - chart_data.append({ - "data": series_data, - "name": label - }) + # On per country/affiliation have a separate graph for inperson + if stats_type == 'total': + in_person_chart_data = None + else: + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': in_person_data_sets, + } -def document_stats(request, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meetings_timeline, kwargs={'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meetings_timeline, kwargs={'stats_type': 'country'})), + ("total", "Total", urlreverse(meetings_timeline, kwargs={'stats_type': 'total'})), + ] + current_meeting = get_current_ietf_meeting_num() + if stats_type == 'total': + possible_stats_type = 'country' + else: + possible_stats_type = stats_type -def known_countries_list(request, stats_type=None, acronym=None): - countries = CountryName.objects.prefetch_related("countryalias_set") - for c in countries: - # the sorting is a bit of a hack - it puts the ISO code first - # since it was added in a migration - c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + possible_meeting_numbers = [(int(current_meeting)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)-1, 'stats_type': possible_stats_type})), + (int(current_meeting), urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting), 'stats_type': possible_stats_type})), + (int(current_meeting)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)+1, 'stats_type': possible_stats_type}))] - return render(request, "stats/known_countries_list.html", { - "countries": countries, + return render(request, "stats/meetings_timeline.html", { + "top_n": top_n, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "total_chart_data": total_chart_data, + "in_person_chart_data": in_person_chart_data, }) -def meeting_stats(request, num=None, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) +def get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get affiliation participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status details + registrations = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registrations = registrations.filter(tickets__attendance_type=attendance_type) + registrations = registrations.values('affiliation') + + # Count per canonicalized affiliation + organization = dict() + for reg in registrations: + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + + # Sort to have the largest count first (nicer in pie chart) + sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) + labels = [] + data = [] + others_count = 0 + total = 0 + for org, count in sorted_orgs: + total += count + if count > minimum_required: + labels.append(org) + data.append(count) + else: + others_count += count + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def get_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get country participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status counts, aggregated by country_code + registration_counts = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registration_counts = registration_counts.filter(tickets__attendance_type=attendance_type) + registration_counts = registration_counts.values('country_code').annotate(count=Count('country_code')).order_by('-count') + + labels = [] + data = [] + others_count = 0 + total = 0 + for item in registration_counts: + total += item['count'] + if item['count'] > minimum_required: + labels.append(item['country_code']) + data.append(item['count']) + else: + others_count += item['count'] + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def meeting_stats(request, meeting_number=None, stats_type='country'): + """Render statistics for a specific meeting. + + Args: + request: The HTTP request object. + meeting_number: The meeting number (defaults to current). + stats_type: Type of statistics ('country' or 'affiliation'). + + Returns: + Rendered response for the meeting stats template. + """ + current_meeting_number = get_current_ietf_meeting_num() + if meeting_number is None: + meeting_number = current_meeting_number + this_meeting = get_object_or_404( + Meeting.objects.filter(type_id="ietf"), number=meeting_number + ) + + if stats_type == 'affiliation': + minimum_required = 4 + total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + elif stats_type == 'country': + minimum_required = 10 + total_labels, total_data, total_total = get_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + total_chart_data = { + 'labels': total_labels, + 'datasets': [{ + 'label': 'Total Registrations by ' + stats_type, + 'data': total_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': [{ + 'label': 'In Person Registrations by ' + stats_type, + 'data': in_person_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'country'})), + ] + + # Prepare the list of meeting number buttons for the template + possible_meeting_numbers = [('All', urlreverse(meetings_timeline, kwargs={'stats_type': stats_type}))] + if int(meeting_number) > 72: # No registration data before IETF-72 + possible_meeting_numbers.append((int(meeting_number)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)-1, 'stats_type': stats_type}))) + possible_meeting_numbers.append((meeting_number, urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': stats_type}))) + if int(meeting_number) <= int(current_meeting_number): # Allow current meeting +1 + possible_meeting_numbers.append((int(meeting_number)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)+1, 'stats_type': stats_type}))) + + return render(request, "stats/meeting_stats.html", { + "meeting_number": meeting_number, + "meeting_date": this_meeting.date, + "meeting_country": this_meeting.country, + "meeting_city": this_meeting.city, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "minimum_required": minimum_required, + "total_chart_data": total_chart_data, + "total_total": total_total, + "in_person_chart_data": in_person_chart_data, + "in_person_total": in_person_total + }) @login_required def review_stats(request, stats_type=None, acronym=None): + """Render review statistics page with tables and charts for review assignments. + + Shows completion status, results, assignment states, and time series data. + Supports both team-level and reviewer-level views with filtering options. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('completion', 'results', 'states', 'time'). + acronym: Team acronym for reviewer-level view (None for team view). + + Returns: + Rendered response for the review stats template. + """ # This view is a bit complex because we want to show a bunch of # tables with various filtering options, and both a team overview # and a reviewers-within-team overview - and a time series chart. @@ -449,3 +958,46 @@ def time_key_fn(t): "possible_states": possible_states, "selected_state": selected_state, }) + + +@role_required("LLC Staff") +def annual_report_inputs(request, year=None): + if year is None and "year" in request.GET: + return HttpResponseRedirect( + urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}) + ) + year = int(year) if year else datetime.date.today().year - 1 + + from ietf.doc.models import NewRevisionDocEvent + from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people + + download = request.GET.get("download") + if download in ("authors", "submitters"): + addresses = authors_by_year(year) if download == "authors" else submitters_by_year(year) + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="{download}-{year}.csv"' + writer = csv.writer(response) + writer.writerow(sorted(addresses)) + return response + + authors = authors_by_year(year) + submitters = submitters_by_year(year) + author_persons, author_nopersons = unique_people(authors) + submitter_persons, submitter_nopersons = unique_people(submitters) + + draft_count = len(set( + NewRevisionDocEvent.objects.filter( + doc__type_id="draft", time__year=year + ).values_list("doc__name", flat=True) + )) + + return render(request, "stats/annual_report_inputs.html", { + "year": year, + "author_count": len(authors), + "submitter_count": len(submitters), + "author_person_count": author_persons.count(), + "author_noperson_count": len(author_nopersons), + "submitter_person_count": submitter_persons.count(), + "submitter_noperson_count": len(submitter_nopersons), + "draft_count": draft_count, + }) diff --git a/ietf/submit/factories.py b/ietf/submit/factories.py index 1076434b822..69888a055dd 100644 --- a/ietf/submit/factories.py +++ b/ietf/submit/factories.py @@ -21,6 +21,9 @@ class Meta: class SubmissionFactory(factory.django.DjangoModelFactory): state_id = 'uploaded' + submitter_name = factory.Faker("name") + submitter_email = factory.Faker("email") + submitter = factory.LazyAttribute(lambda o: f"{o.submitter_name} <{o.submitter_email}>") @factory.lazy_attribute_sequence def name(self, n): @@ -32,3 +35,4 @@ def auth_key(self): class Meta: model = Submission + exclude = ("submitter_name", "submitter_email") diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ede63d2752c..abe23c1a643 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2011-2023, All Rights Reserved +# Copyright The IETF Trust 2011-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -40,7 +40,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.person.models import Person -from ietf.person.factories import UserFactory, PersonFactory +from ietf.person.factories import UserFactory, PersonFactory, EmailFactory from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource @@ -207,20 +207,24 @@ def test_manualpost_view(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn( - urlreverse( - "ietf.submit.views.submission_status", - kwargs=dict(submission_id=submission.pk) - ), - q("#manual.submissions td a").attr("href") - ) - self.assertIn( - submission.name, - q("#manual.submissions td a").text() + # Validate that the basic submission status URL is on the manual post page + # _without_ an access token, even if logged in as various users. + expected_url = urlreverse( + "ietf.submit.views.submission_status", + kwargs=dict(submission_id=submission.pk) ) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) + for username in ["plain", "secretary"]: + self.client.login(username=username, password=username + "+password") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) - def test_manualpost_cancel(self): - pass class SubmitTests(BaseSubmitTestCase): def setUp(self): @@ -595,7 +599,7 @@ def submit_existing(self, formats, change_authors=True, group_type='wg', stream_ TestBlobstoreManager().emptyTestBlobstores() def _assert_authors_are_action_holders(draft, expect=True): - for author in draft.authors(): + for author in draft.author_persons(): if expect: self.assertIn(author, draft.action_holders.all()) else: @@ -1868,10 +1872,8 @@ def test_submit_bad_author_email(self): name = "draft-authorname-testing-bademail" rev = "00" - author = PersonFactory() - email = author.email_set.first() - email.address = '@bad.email' - email.save() + author = PersonFactory(default_emails=False) + EmailFactory(person=author, primary=True, address="@bad.email") status_url, _ = self.do_submission(name=name, rev=rev, author=author, formats=('xml',)) r = self.client.get(status_url) @@ -2404,7 +2406,7 @@ def test_upload_draft(self): response = r.json() self.assertCountEqual( response.keys(), - ['id', 'name', 'rev', 'status_url'], + ['id', 'name', 'rev', 'status_url', 'submission_url'], ) submission_id = int(response['id']) self.assertEqual(response['name'], 'draft-somebody-test') @@ -2416,6 +2418,13 @@ def test_upload_draft(self): kwargs={'submission_id': submission_id}, ), ) + self.assertEqual( + response['submission_url'], + 'https://datatracker.example.com' + urlreverse( + 'ietf.submit.views.submission_status', + kwargs={'submission_id': submission_id}, + ) + ) self.assertEqual(mock_task.delay.call_count, 1) self.assertEqual(mock_task.delay.call_args.args, (submission_id,)) submission = Submission.objects.get(pk=submission_id) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index a0c7dd8511e..457462e4f2c 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -395,10 +395,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): log.log(f"{submission.name}: updated state and info") - trouble = rebuild_reference_relations(draft, find_submission_filenames(draft)) - if trouble: - log.log('Rebuild_reference_relations trouble: %s'%trouble) - log.log(f"{submission.name}: rebuilt reference relations") + rebuild_reference_relations(draft, find_submission_filenames(draft)) if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00": # automatically set state "WG Document" @@ -1268,7 +1265,7 @@ def process_submission_text(filename, revision): if title: title = _normalize_title(title) - # Translation taable drops \r, \n, <, >. + # Translation table drops \r, \n, <, >. trans_table = str.maketrans("", "", "\r\n<>") authors = [ { @@ -1615,6 +1612,4 @@ def active(dirent): log.log(f"Error processing {item.name}: {e}") ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod/" - if not moddir.endswith("/"): - moddir += "/" - subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir)) + subprocess.call(("/usr/bin/rsync", "-aq", "--delete", f"{moddir}/", str(ftp_moddir))) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 8329a312bb7..2db3f510989 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -182,6 +182,10 @@ def err(code, error, messages=None): settings.IDTRACKER_BASE_URL, urlreverse(api_submission_status, kwargs={'submission_id': submission.pk}), ), + 'submission_url': urljoin( + settings.IDTRACKER_BASE_URL, + urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': submission.pk}), + ), } ) else: diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py new file mode 100644 index 00000000000..bc1a1cbd283 --- /dev/null +++ b/ietf/sync/errata.py @@ -0,0 +1,187 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import datetime +import json +from collections import defaultdict +from typing import DefaultDict + +from django.conf import settings +from django.core.files.storage import storages +from django.db import transaction +from django.db.models import Q + +from ietf.doc.models import Document, DocEvent +from ietf.name.models import DocTagName +from ietf.person.models import Person +from ietf.utils.log import log +from ietf.utils.models import DirtyBits + + +DEFAULT_ERRATA_JSON_BLOB_NAME = "other/errata.json" + +type ErrataJsonEntry = dict[str, str] + + +def get_errata_last_updated() -> datetime.datetime: + """Get timestamp of the last errata.json update + + May raise FileNotFoundError or other storage/S3 exceptions. Be prepared. + """ + red_bucket = storages["red_bucket"] + return red_bucket.get_modified_time( + getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME) + ) + + +def get_errata_data() -> list[ErrataJsonEntry]: + red_bucket = storages["red_bucket"] + with red_bucket.open( + getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME), "r" + ) as f: + errata_data = json.load(f) + return errata_data + + +def errata_map_from_json(errata_data: list[ErrataJsonEntry]): + """Create a dict mapping RFC number to a list of applicable errata records""" + errata = defaultdict(list) + for item in errata_data: + doc_id = item["doc-id"] + if doc_id.upper().startswith("RFC"): + rfc_number = int(doc_id[3:]) + errata[rfc_number].append(item) + return dict(errata) + + +def update_errata_tags(errata_data: list[ErrataJsonEntry]): + tag_has_errata = DocTagName.objects.get(slug="errata") + tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") + system = Person.objects.get(name="(System)") + + errata_map = errata_map_from_json(errata_data) + nums_with_errata = [ + num + for num, errata in errata_map.items() + if any(er["errata_status_code"] != "Rejected" for er in errata) + ] + nums_with_verified_errata = [ + num + for num, errata in errata_map.items() + if any(er["errata_status_code"] == "Verified" for er in errata) + ] + + rfcs_gaining_errata_tag = Document.objects.filter( + type_id="rfc", rfc_number__in=nums_with_errata + ).exclude(tags=tag_has_errata) + + rfcs_gaining_verified_errata_tag = Document.objects.filter( + type_id="rfc", rfc_number__in=nums_with_verified_errata + ).exclude(tags=tag_has_verified_errata) + + rfcs_losing_errata_tag = Document.objects.filter( + type_id="rfc", tags=tag_has_errata + ).exclude(rfc_number__in=nums_with_errata) + + rfcs_losing_verified_errata_tag = Document.objects.filter( + type_id="rfc", tags=tag_has_verified_errata + ).exclude(rfc_number__in=nums_with_verified_errata) + + # map rfc_number to add/remove lists + changes: DefaultDict[Document, dict[str, list[DocTagName]]] = defaultdict( + lambda: {"add": [], "remove": []} + ) + for rfc in rfcs_gaining_errata_tag: + changes[rfc]["add"].append(tag_has_errata) + for rfc in rfcs_gaining_verified_errata_tag: + changes[rfc]["add"].append(tag_has_verified_errata) + for rfc in rfcs_losing_errata_tag: + changes[rfc]["remove"].append(tag_has_errata) + for rfc in rfcs_losing_verified_errata_tag: + changes[rfc]["remove"].append(tag_has_verified_errata) + + for rfc, changeset in changes.items(): + # Update in a transaction per RFC to keep tags and DocEvents consistent. + # With this in place, an interrupted task will be cleanly completed on the + # next run. + with transaction.atomic(): + change_descs = [] + for tag in changeset["add"]: + rfc.tags.add(tag) + change_descs.append(f"added {tag.slug} tag") + for tag in changeset["remove"]: + rfc.tags.remove(tag) + change_descs.append(f"removed {tag.slug} tag") + summary = "Update from RFC Editor: " + ", ".join(change_descs) + if rfc.rfc_number in errata_map and all( + er["errata_status_code"] == "Rejected" + for er in errata_map[rfc.rfc_number] + ): + summary += " (all errata rejected)" + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, # expect no rev + by=system, + type="sync_from_rfc_editor", + desc=summary, + ) + + return {rfc.rfc_number for rfc in changes} + + +def update_errata_from_rfceditor() -> set[int]: + errata_data = get_errata_data() + return update_errata_tags(errata_data) + + +## DirtyBits management for the errata tags + +ERRATA_SLUG = DirtyBits.Slugs.ERRATA + + +def update_errata_dirty_time() -> DirtyBits | None: + try: + last_update = get_errata_last_updated() + except Exception as err: + log(f"Error in get_errata_last_updated: {err}") + return None + else: + dirty_work, created = DirtyBits.objects.update_or_create( + slug=ERRATA_SLUG, defaults={"dirty_time": last_update} + ) + if created: + log(f"Created DirtyBits(slug='{ERRATA_SLUG}')") + return dirty_work + + +def mark_errata_as_processed(when: datetime.datetime): + n_updated = DirtyBits.objects.filter( + Q(processed_time__isnull=True) | Q(processed_time__lt=when), + slug=ERRATA_SLUG, + ).update(processed_time=when) + if n_updated > 0: + log(f"processed_time is now {when.isoformat()}") + else: + log("processed_time not updated, no matching record found") + + +def errata_are_dirty(): + """Does the rfc index need to be updated?""" + dirty_work = update_errata_dirty_time() # creates DirtyBits if needed + if dirty_work is None: + # A None indicates we could not check the timestamp of errata.json. In that + # case, we are not likely to be able to read the blob either, so don't try + # to process it. An error was already logged. + return False + display_processed_time = ( + dirty_work.processed_time.isoformat() + if dirty_work.processed_time is not None + else "never" + ) + log( + f"DirtyBits(slug='{ERRATA_SLUG}'): " + f"dirty_time={dirty_work.dirty_time.isoformat()} " + f"processed_time={display_processed_time}" + ) + return ( + dirty_work.processed_time is None + or dirty_work.dirty_time >= dirty_work.processed_time + ) diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py deleted file mode 100644 index 76357ffbcc3..00000000000 --- a/ietf/sync/rfceditor.py +++ /dev/null @@ -1,863 +0,0 @@ -# Copyright The IETF Trust 2012-2025, All Rights Reserved -# -*- coding: utf-8 -*- - - -import base64 -import datetime -import re -import requests - -from typing import Iterator, Optional, Union -from urllib.parse import urlencode -from xml.dom import pulldom, Node - -from django.conf import settings -from django.db import transaction -from django.db.models import Subquery, OuterRef, F, Q -from django.utils import timezone -from django.utils.encoding import smart_bytes, force_str - -import debug # pyflakes:ignore - -from ietf.doc.models import ( Document, State, StateType, DocEvent, DocRelationshipName, - DocTagName, RelatedDocument, RelatedDocHistory ) -from ietf.doc.expire import move_draft_files_to_archive -from ietf.doc.utils import add_state_change_event, new_state_change_event, prettify_std_name, update_action_holders -from ietf.group.models import Group -from ietf.ipr.models import IprDocRel -from ietf.name.models import StdLevelName, StreamName -from ietf.person.models import Person -from ietf.utils.log import log -from ietf.utils.mail import send_mail_text -from ietf.utils.timezone import datetime_from_date, RPC_TZINFO - -# QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" -# INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" -# POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" - -MIN_ERRATA_RESULTS = 5000 -MIN_INDEX_RESULTS = 8000 -MIN_QUEUE_RESULTS = 10 - -def get_child_text(parent_node, tag_name): - text = [] - for node in parent_node.childNodes: - if node.nodeType == Node.ELEMENT_NODE and node.localName == tag_name: - text.append(node.firstChild.data) - return '\n\n'.join(text) - - -def parse_queue(response): - """Parse RFC Editor queue XML into a bunch of tuples + warnings.""" - - events = pulldom.parse(response) - drafts = [] - warnings = [] - stream = None - - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName == "entry": - events.expandNode(node) - node.normalize() - draft_name = get_child_text(node, "draft").strip() - draft_name = re.sub(r"(-\d\d)?(.txt){1,2}$", "", draft_name) - date_received = get_child_text(node, "date-received") - - state = "" - tags = [] - missref_generation = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "state": - state = child.firstChild.data - # state has some extra annotations encoded, parse - # them out - if '*R' in state: - tags.append("ref") - state = state.replace("*R", "") - if '*A' in state: - tags.append("iana") - state = state.replace("*A", "") - m = re.search(r"\(([0-9]+)G\)", state) - if m: - missref_generation = m.group(1) - state = state.replace("(%sG)" % missref_generation, "") - - # AUTH48 link - auth48 = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "auth48-url": - auth48 = child.firstChild.data - - # cluster link (if it ever gets implemented) - cluster = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "cluster-url": - cluster = child.firstChild.data - - refs = [] - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "normRef": - ref_name = get_child_text(child, "ref-name") - ref_state = get_child_text(child, "ref-state") - in_queue = ref_state.startswith("IN-QUEUE") - refs.append((ref_name, ref_state, in_queue)) - - drafts.append((draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs)) - - elif event == pulldom.START_ELEMENT and node.tagName == "section": - name = node.getAttribute('name') - if name.startswith("IETF"): - stream = "ietf" - elif name.startswith("IAB"): - stream = "iab" - elif name.startswith("IRTF"): - stream = "irtf" - elif name.startswith("INDEPENDENT"): - stream = "ise" - else: - stream = None - warnings.append("unrecognized section " + name) - except Exception as e: - log("Exception when processing an RFC queue entry: %s" % e) - log("node: %s" % node) - raise - - return drafts, warnings - -def update_drafts_from_queue(drafts): - """Given a list of parsed drafts from the RFC Editor queue, update the - documents in the database. Return those that were changed.""" - - tag_mapping = { - 'IANA': DocTagName.objects.get(slug='iana'), - 'REF': DocTagName.objects.get(slug='ref') - } - - slookup = dict((s.slug, s) - for s in State.objects.filter(used=True, type=StateType.objects.get(slug="draft-rfceditor"))) - state_mapping = { - 'AUTH': slookup['auth'], - 'AUTH48': slookup['auth48'], - 'AUTH48-DONE': slookup['auth48-done'], - 'EDIT': slookup['edit'], - 'IANA': slookup['iana'], - 'IESG': slookup['iesg'], - 'ISR': slookup['isr'], - 'ISR-AUTH': slookup['isr-auth'], - 'REF': slookup['ref'], - 'RFC-EDITOR': slookup['rfc-edit'], - 'TI': slookup['tooling-issue'], - 'TO': slookup['timeout'], - 'MISSREF': slookup['missref'], - } - - system = Person.objects.get(name="(System)") - - warnings = [] - - names = [t[0] for t in drafts] - - drafts_in_db = dict((d.name, d) - for d in Document.objects.filter(type="draft", name__in=names)) - - changed = set() - - for name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs in drafts: - if name not in drafts_in_db: - warnings.append("unknown document %s" % name) - continue - - if not state or state not in state_mapping: - warnings.append("unknown state '%s' for %s" % (state, name)) - continue - - d = drafts_in_db[name] - - prev_state = d.get_state("draft-rfceditor") - next_state = state_mapping[state] - events = [] - - # check if we've noted it's been received - if d.get_state_slug("draft-iesg") == "ann" and not prev_state and not d.latest_event(DocEvent, type="rfc_editor_received_announcement"): - e = DocEvent(doc=d, rev=d.rev, by=system, type="rfc_editor_received_announcement") - e.desc = "Announcement was received by RFC Editor" - e.save() - send_mail_text(None, "iesg-secretary@ietf.org", None, - '%s in RFC Editor queue' % d.name, - 'The announcement for %s has been received by the RFC Editor.' % d.name) - # change draft-iesg state to RFC Ed Queue - prev_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="ann") - next_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="rfcqueue") - - d.set_state(next_iesg_state) - e = add_state_change_event(d, system, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - e = update_action_holders(d, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - changed.add(name) - - # check draft-rfceditor state - if prev_state != next_state: - d.set_state(next_state) - - e = new_state_change_event(d, system, prev_state, next_state) # unsaved - if e: - if auth48: - e.desc = re.sub(r"(.*)", "\\1" % auth48, e.desc) - e.save() - events.append(e) - - if auth48: - # Create or update the auth48 URL whether or not this is a state expected to have one. - d.documenturl_set.update_or_create( - tag_id='auth48', # look up existing based on this field - defaults=dict(url=auth48) # create or update with this field - ) - else: - # Remove any existing auth48 URL when an update does not have one. - d.documenturl_set.filter(tag_id='auth48').delete() - - changed.add(name) - - t = DocTagName.objects.filter(slug__in=tags) - if set(t) != set(d.tags.all()): - d.tags.clear() - d.tags.set(t) - changed.add(name) - - if events: - d.save_with_history(events) - - - # remove tags and states for those not in the queue anymore - for d in Document.objects.exclude(name__in=names).filter(states__type="draft-rfceditor").distinct(): - d.tags.remove(*list(tag_mapping.values())) - d.unset_state("draft-rfceditor") - # we do not add a history entry here - most likely we already - # have something that explains what happened - changed.add(name) - - return changed, warnings - - -def parse_index(response): - """Parse RFC Editor index XML into a bunch of tuples.""" - - def normalize_std_name(std_name): - # remove zero padding - prefix = std_name[:3] - if prefix in ("RFC", "FYI", "BCP", "STD"): - try: - return prefix + str(int(std_name[3:])) - except ValueError: - pass - return std_name - - def extract_doc_list(parentNode, tagName): - l = [] - for u in parentNode.getElementsByTagName(tagName): - for d in u.getElementsByTagName("doc-id"): - l.append(normalize_std_name(d.firstChild.data)) - return l - - also_list = {} - data = [] - events = pulldom.parse(response) - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName in ["bcp-entry", "fyi-entry", "std-entry"]: - events.expandNode(node) - node.normalize() - bcpid = normalize_std_name(get_child_text(node, "doc-id")) - doclist = extract_doc_list(node, "is-also") - for docid in doclist: - if docid in also_list: - also_list[docid].append(bcpid) - else: - also_list[docid] = [bcpid] - - elif event == pulldom.START_ELEMENT and node.tagName == "rfc-entry": - events.expandNode(node) - node.normalize() - rfc_number = int(get_child_text(node, "doc-id")[3:]) - title = get_child_text(node, "title") - - authors = [] - for author in node.getElementsByTagName("author"): - authors.append(get_child_text(author, "name")) - - d = node.getElementsByTagName("date")[0] - year = int(get_child_text(d, "year")) - month = get_child_text(d, "month") - month = ["January","February","March","April","May","June","July","August","September","October","November","December"].index(month)+1 - rfc_published_date = datetime.date(year, month, 1) - - current_status = get_child_text(node, "current-status").title() - - updates = extract_doc_list(node, "updates") - updated_by = extract_doc_list(node, "updated-by") - obsoletes = extract_doc_list(node, "obsoletes") - obsoleted_by = extract_doc_list(node, "obsoleted-by") - pages = get_child_text(node, "page-count") - stream = get_child_text(node, "stream") - wg = get_child_text(node, "wg_acronym") - if wg and ((wg == "NON WORKING GROUP") or len(wg) > 15): - wg = None - - l = [] - for fmt in node.getElementsByTagName("format"): - l.append(get_child_text(fmt, "file-format")) - file_formats = (",".join(l)).lower() - - abstract = "" - for abstract in node.getElementsByTagName("abstract"): - abstract = get_child_text(abstract, "p") - - draft = get_child_text(node, "draft") - if draft and re.search(r"-\d\d$", draft): - draft = draft[0:-3] - - if len(node.getElementsByTagName("errata-url")) > 0: - has_errata = 1 - else: - has_errata = 0 - - data.append((rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,[],draft,has_errata,stream,wg,file_formats,pages,abstract)) - except Exception as e: - log("Exception when processing an RFC index entry: %s" % e) - log("node: %s" % node) - raise - for d in data: - k = "RFC%d" % d[0] - if k in also_list: - d[9].extend(also_list[k]) - return data - - -def update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date: Optional[datetime.date] = None -) -> Iterator[tuple[int, list[str], Document, bool]]: - """Given parsed data from the RFC Editor index, update the documents in the database - - Returns an iterator that yields (rfc_number, change_list, doc, rfc_published) for the - RFC document and, if applicable, the I-D that it came from. - - The skip_older_than_date is a bare date, not a datetime. - """ - # Create dict mapping doc-id to list of errata records that apply to it - errata: dict[str, list[dict]] = {} - for item in errata_data: - name = item["doc-id"] - if name.upper().startswith("RFC"): - name = f"RFC{int(name[3:])}" # removes leading 0s on the rfc number - if not name in errata: - errata[name] = [] - errata[name].append(item) - - std_level_mapping = { - "Standard": StdLevelName.objects.get(slug="std"), - "Internet Standard": StdLevelName.objects.get(slug="std"), - "Draft Standard": StdLevelName.objects.get(slug="ds"), - "Proposed Standard": StdLevelName.objects.get(slug="ps"), - "Informational": StdLevelName.objects.get(slug="inf"), - "Experimental": StdLevelName.objects.get(slug="exp"), - "Best Current Practice": StdLevelName.objects.get(slug="bcp"), - "Historic": StdLevelName.objects.get(slug="hist"), - "Unknown": StdLevelName.objects.get(slug="unkn"), - } - - stream_mapping = { - "IETF": StreamName.objects.get(slug="ietf"), - "INDEPENDENT": StreamName.objects.get(slug="ise"), - "IRTF": StreamName.objects.get(slug="irtf"), - "IAB": StreamName.objects.get(slug="iab"), - "Editorial": StreamName.objects.get(slug="editorial"), - "Legacy": StreamName.objects.get(slug="legacy"), - } - - tag_has_errata = DocTagName.objects.get(slug="errata") - tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") - relationship_obsoletes = DocRelationshipName.objects.get(slug="obs") - relationship_updates = DocRelationshipName.objects.get(slug="updates") - rfc_published_state = State.objects.get(type_id="rfc", slug="published") - - system = Person.objects.get(name="(System)") - - first_sync_creating_subseries = not Document.objects.filter(type_id__in=["bcp","std","fyi"]).exists() - - for ( - rfc_number, - title, - authors, - rfc_published_date, - current_status, - updates, - updated_by, - obsoletes, - obsoleted_by, - also, - draft_name, - has_errata, - stream, - wg, - file_formats, - pages, - abstract, - ) in index_data: - if skip_older_than_date and rfc_published_date < skip_older_than_date: - # speed up the process by skipping old entries (n.b., the comparison above is a - # lexical comparison between "YYYY-MM-DD"-formatted dates) - continue - - # we assume two things can happen: we get a new RFC, or an - # attribute has been updated at the RFC Editor (RFC Editor - # attributes take precedence over our local attributes) - rfc_events = [] - rfc_changes = [] - rfc_published = False - - # Find the draft, if any - draft = None - if draft_name: - try: - draft = Document.objects.get(name=draft_name, type_id="draft") - except Document.DoesNotExist: - pass - # Logging below warning turns out to be unhelpful - there are many references - # to such things in the index: - # * all april-1 RFCs have an internal name that looks like a draft name, but there - # was never such a draft. More of these will exist in the future - # * Several documents were created with out-of-band input to the RFC-editor, for a - # variety of reasons. - # - # What this exposes is that the rfc-index needs to stop talking about these things. - # If there is no draft to point to, don't point to one, even if there was an RPC - # internal name in use (and in the RPC database). This will be a requirement on the - # reimplementation of the creation of the rfc-index. - # - # log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}") - - # Find or create the RFC document - creation_args: dict[str, Optional[Union[str, int]]] = {"name": f"rfc{rfc_number}"} - if draft: - creation_args.update( - { - "title": draft.title, - "stream": draft.stream, - "group": draft.group, - "abstract": draft.abstract, - "pages": draft.pages, - "words": draft.words, - "std_level": draft.std_level, - "ad": draft.ad, - "external_url": draft.external_url, - "uploaded_filename": draft.uploaded_filename, - "note": draft.note, - } - ) - doc, created_rfc = Document.objects.get_or_create( - rfc_number=rfc_number, type_id="rfc", defaults=creation_args - ) - if created_rfc: - rfc_changes.append(f"created document {prettify_std_name(doc.name)}") - doc.set_state(rfc_published_state) - if draft: - doc.formal_languages.set(draft.formal_languages.all()) - for author in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = doc - author.save() - - if draft: - draft_events = [] - draft_changes = [] - - # Ensure the draft is in the "rfc" state and move its files to the archive - # if necessary. - if draft.get_state_slug() != "rfc": - draft.set_state( - State.objects.get(used=True, type="draft", slug="rfc") - ) - move_draft_files_to_archive(draft, draft.rev) - draft_changes.append(f"changed state to {draft.get_state()}") - - # Ensure the draft and rfc are linked with a "became_rfc" relationship - r, created_relateddoc = RelatedDocument.objects.get_or_create( - source=draft, target=doc, relationship_id="became_rfc" - ) - if created_relateddoc: - change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( - rel_name=r.relationship.name.lower(), - pretty_draft_name=prettify_std_name(draft_name), - pretty_rfc_name=prettify_std_name(doc.name), - ) - draft_changes.append(change) - rfc_changes.append(change) - - # Always set the "draft-iesg" state. This state should be set for all drafts, so - # log a warning if it is not set. What should happen here is that ietf stream - # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. - # Other stream documents should normally be "idexists" and be left that way. The - # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", - # and changes any other state to "pub". If unset, it changes it to "idexists". - # This reflects historical behavior and should probably be updated, but a migration - # of existing drafts (and validation of the change) is needed before we change the - # handling. - prev_iesg_state = draft.get_state("draft-iesg") - if prev_iesg_state is None: - log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state') - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") - elif prev_iesg_state.slug not in ("pub", "idexists"): - if prev_iesg_state.slug != "rfcqueue": - log( - 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( - doc.name, draft.name, prev_iesg_state.slug - ) - ) - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") - else: - new_iesg_state = prev_iesg_state - - if new_iesg_state != prev_iesg_state: - draft.set_state(new_iesg_state) - draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") - e = update_action_holders(draft, prev_iesg_state, new_iesg_state) - if e: - draft_events.append(e) - - # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. - if draft.stream != doc.stream: - log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( - doc.name, draft.name, draft.stream, doc.stream - )) - elif draft.stream.slug in ["iab", "irtf", "ise"]: - stream_slug = f"draft-stream-{draft.stream.slug}" - prev_state = draft.get_state(stream_slug) - if prev_state is not None and prev_state.slug != "pub": - new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") - draft.set_state(new_state) - draft_changes.append( - f"changed {new_state.type.label} to {new_state}" - ) - e = update_action_holders(draft, prev_state, new_state) - if e: - draft_events.append(e) - if draft_changes: - draft_events.append( - DocEvent.objects.create( - doc=draft, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})", - ) - ) - draft.save_with_history(draft_events) - yield rfc_number, draft_changes, draft, False # yield changes to the draft - - # check attributes - verbed = "set" if created_rfc else "changed" - if title != doc.title: - doc.title = title - rfc_changes.append(f"{verbed} title to '{doc.title}'") - - if abstract and abstract != doc.abstract: - doc.abstract = abstract - rfc_changes.append(f"{verbed} abstract to '{doc.abstract}'") - - if pages and int(pages) != doc.pages: - doc.pages = int(pages) - rfc_changes.append(f"{verbed} pages to {doc.pages}") - - if std_level_mapping[current_status] != doc.std_level: - doc.std_level = std_level_mapping[current_status] - rfc_changes.append(f"{verbed} standardization level to {doc.std_level}") - - if doc.stream != stream_mapping[stream]: - doc.stream = stream_mapping[stream] - rfc_changes.append(f"{verbed} stream to {doc.stream}") - - if doc.get_state() != rfc_published_state: - doc.set_state(rfc_published_state) - rfc_changes.append(f"{verbed} {rfc_published_state.type.label} to {rfc_published_state}") - - # if we have no group assigned, check if RFC Editor has a suggestion - if not doc.group: - if wg: - doc.group = Group.objects.get(acronym=wg) - rfc_changes.append(f"set group to {doc.group}") - else: - doc.group = Group.objects.get( - type="individ" - ) # fallback for newly created doc - - if not doc.latest_event(type="published_rfc"): - e = DocEvent(doc=doc, rev=doc.rev, type="published_rfc") - # unfortunately, rfc_published_date doesn't include the correct day - # at the moment because the data only has month/year, so - # try to deduce it - # - # Note: This is in done PST8PDT to preserve compatibility with events created when - # USE_TZ was False. The published_rfc event was created with a timestamp whose - # server-local datetime (PST8PDT) matched the publication date from the RFC index. - # When switching to USE_TZ=True, the timestamps were migrated so they still - # matched the publication date in PST8PDT. When interpreting the event timestamp - # as a publication date, you must treat it in the PST8PDT time zone. The - # RPC_TZINFO constant in ietf.utils.timezone is defined for this purpose. - d = datetime_from_date(rfc_published_date, RPC_TZINFO) - synthesized = timezone.now().astimezone(RPC_TZINFO) - if abs(d - synthesized) > datetime.timedelta(days=60): - synthesized = d - else: - direction = -1 if (d - synthesized).total_seconds() < 0 else +1 - while synthesized.month != d.month or synthesized.year != d.year: - synthesized += datetime.timedelta(days=direction) - e.time = synthesized - e.by = system - e.desc = "RFC published" - e.save() - rfc_events.append(e) - - rfc_changes.append( - f"added RFC published event at {e.time.strftime('%Y-%m-%d')}" - ) - rfc_published = True - - def parse_relation_list(l): - res = [] - for x in l: - for a in Document.objects.filter(name=x.lower(), type_id="rfc"): - if a not in res: - res.append(a) - return res - - for x in parse_relation_list(obsoletes): - if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_obsoletes - ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_obsoletes - ) - rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( - rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), - ) - ) - - for x in parse_relation_list(updates): - if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_updates - ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_updates - ) - rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( - rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), - ) - ) - - if also: - # recondition also to have proper subseries document names: - conditioned_also = [] - for a in also: - a = a.lower() - subseries_slug = a[:3] - if subseries_slug not in ["bcp", "std", "fyi"]: - log(f"Unexpected 'also' relationship of {a} encountered for {doc}") - continue - maybe_number = a[3:].strip() - if not maybe_number.isdigit(): - log(f"Unexpected 'also' subseries element identifier {a} encountered for {doc}") - continue - else: - subseries_number = int(maybe_number) - conditioned_also.append(f"{subseries_slug}{subseries_number}") # Note the lack of leading zeros - also = conditioned_also - - for a in also: - subseries_doc_name = a - subseries_slug=a[:3] - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=subseries_slug, name=subseries_doc_name) - if created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Imported {subseries_doc_name} into the datatracker via sync to the rfc-index") - else: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Created {subseries_doc_name} via sync to the rfc-index") - _, relationship_created = subseries_doc.relateddocument_set.get_or_create(relationship_id="contains", target=doc) - if relationship_created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index") - rfc_events.append(doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of {subseries_doc.name.upper()} is currently available in the datatracker before this point")) - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index")) - else: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}") - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}")) - - # Delete subseries relations that are no longer current. Use a transaction - # so we are sure we iterate over the same relations that we delete! - with transaction.atomic(): - stale_subseries_relations = doc.relations_that("contains").exclude( - source__name__in=also - ) - for stale_relation in stale_subseries_relations: - stale_subseries_doc = stale_relation.source - rfc_events.append( - doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - ) - stale_subseries_doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - stale_subseries_relations.delete() - - doc_errata = errata.get(f"RFC{rfc_number}", []) - all_rejected = doc_errata and all( - er["errata_status_code"] == "Rejected" for er in doc_errata - ) - if has_errata and not all_rejected: - if not doc.tags.filter(pk=tag_has_errata.pk).exists(): - doc.tags.add(tag_has_errata) - rfc_changes.append("added Errata tag") - has_verified_errata = any( - [er["errata_status_code"] == "Verified" for er in doc_errata] - ) - if ( - has_verified_errata - and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists() - ): - doc.tags.add(tag_has_verified_errata) - rfc_changes.append("added Verified Errata tag") - else: - if doc.tags.filter(pk=tag_has_errata.pk): - doc.tags.remove(tag_has_errata) - if all_rejected: - rfc_changes.append("removed Errata tag (all errata rejected)") - else: - rfc_changes.append("removed Errata tag") - if doc.tags.filter(pk=tag_has_verified_errata.pk): - doc.tags.remove(tag_has_verified_errata) - rfc_changes.append("removed Verified Errata tag") - - if rfc_changes: - rfc_events.append( - DocEvent.objects.create( - doc=doc, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(rfc_changes)})", - ) - ) - doc.save_with_history(rfc_events) - yield rfc_number, rfc_changes, doc, rfc_published # yield changes to the RFC - - if first_sync_creating_subseries: - # First - create the known subseries documents that have ghosted. - # The RFC editor (as of 31 Oct 2023) claims these subseries docs do not exist. - # The datatracker, on the other hand, will say that the series doc currently contains no RFCs. - for name in ["fyi17", "std1", "bcp12", "bcp113", "bcp66"]: - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=name[:3], name=name) - if not created: - log(f"Warning: {name} unexpectedly already exists") - else: - subseries_slug = name[:3] - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - - RelatedDocument.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - RelatedDocHistory.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - IprDocRel.objects.filter( - Q(originaldocumentaliasname__startswith="bcp") | - Q(originaldocumentaliasname__startswith="std") | - Q(originaldocumentaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaldocumentaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(document=F("subseries_target")) - - -def post_approved_draft(url, name): - """Post an approved draft to the RFC Editor so they can retrieve - the data from the Datatracker and start processing it. Returns - response and error (empty string if no error).""" - - if settings.SERVER_MODE != "production": - log(f"In production, would have posted RFC-Editor notification of approved I-D '{name}' to '{url}'") - return "", "" - - # HTTP basic auth - username = "dtracksync" - password = settings.RFC_EDITOR_SYNC_PASSWORD - headers = { - "Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain", - "Authorization": "Basic %s" % force_str(base64.encodebytes(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""), - } - - log("Posting RFC-Editor notification of approved Internet-Draft '%s' to '%s'" % (name, url)) - text = error = "" - - try: - r = requests.post( - url, - headers=headers, - data=smart_bytes(urlencode({ 'draft': name })), - timeout=settings.DEFAULT_REQUESTS_TIMEOUT, - ) - - log("RFC-Editor notification result for Internet-Draft '%s': %s:'%s'" % (name, r.status_code, r.text)) - - if r.status_code != 200: - raise RuntimeError("Status code is not 200 OK (it's %s)." % r.status_code) - - if force_str(r.text) != "OK": - raise RuntimeError('Response is not "OK" (it\'s "%s").' % r.text) - - except Exception as e: - # catch everything so we don't leak exceptions, convert them - # into string instead - msg = "Exception on RFC-Editor notification for Internet-Draft '%s': %s: %s" % (name, type(e), str(e)) - log(msg) - if settings.SERVER_MODE == 'test': - debug.say(msg) - error = str(e) - - return text, error diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py new file mode 100644 index 00000000000..f47974f9002 --- /dev/null +++ b/ietf/sync/rfcindex.py @@ -0,0 +1,826 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import datetime +import json +import shutil +from collections import defaultdict +from collections.abc import Container, Iterable +from dataclasses import dataclass +from itertools import chain +from operator import attrgetter, itemgetter +from pathlib import Path +from textwrap import fill +from urllib.parse import urljoin + +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Q +from lxml import etree + +from django.core.files.storage import storages +from django.db import models +from django.db.models.functions import Substr, Cast +from django.template.loader import render_to_string +from django.utils import timezone + +from ietf.doc.models import Document +from ietf.name.models import StdLevelName +from ietf.utils.log import log +from ietf.utils.models import DirtyBits + +FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] +SS_TXT_MARGIN = 3 +SS_TXT_CUE_COL_WIDTH = 14 + + +def format_rfc_number(n): + """Format an RFC number (or subseries doc number) + + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. + That is for debugging only - tests will fail. + """ + if getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False): + return format(n, "04") + else: + return format(n) + + +def errata_url(rfc: Document): + return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") + + +def red_bucket_input_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_INPUT_PATH) / filename) + + +def red_bucket_output_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_OUTPUT_PATH) / filename) + + +def save_to_red_bucket(filename: str, content: str | bytes): + red_bucket = storages["red_bucket"] + bucket_path = red_bucket_output_path(filename) + if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): + # Django 4.2's FileSystemStorage does not support allow_overwrite. + red_bucket.delete(bucket_path) + red_bucket.save( + bucket_path, + ContentFile(content if isinstance(content, bytes) else content.encode("utf-8")), + ) + log(f"Saved {bucket_path} in red_bucket storage") + + +def save_to_filesystem( + filename: str, content: str | bytes, subdirs: Iterable[str] = () +): + """Save contents to the RFC_PATH in the filesystem + + Always saves directly to settings.RFC_PATH/filename. Additionally saves a copy + to settings.RFC_PATH/subdir/filename for each entry in subdirs. Uses shutil.copy2 + to create the copies, which will preserve mtime and other metadata between copies. + """ + rfc_path = Path(settings.RFC_PATH) + dest_path = rfc_path / filename + dest_path.write_bytes( + content if isinstance(content, bytes) else content.encode("utf-8") + ) + for subdir in subdirs: + (rfc_path / subdir).mkdir(parents=False, exist_ok=True) + shutil.copy2(dest_path, rfc_path / subdir / filename) + + +@dataclass +class UnusableRfcNumber: + rfc_number: int + comment: str + + +def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: + bucket_path = red_bucket_input_path("unusable-rfc-numbers.json") + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record["number"], int) for record in records) + assert all(isinstance(record["comment"], str) for record in records) + return [ + UnusableRfcNumber(rfc_number=record["number"], comment=record["comment"]) + for record in sorted(records, key=itemgetter("number")) + ] + + +def get_april1_rfc_numbers() -> Container[int]: + bucket_path = red_bucket_input_path("april-first-rfc-numbers.json") + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record, int) for record in records) + return records + + +def get_publication_std_levels() -> dict[int, StdLevelName]: + bucket_path = red_bucket_input_path("publication-std-levels.json") + values: dict[int, StdLevelName] = {} + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + # intentionally fall through instead of return here + else: + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE != "development": + raise + else: + assert all(isinstance(record["number"], int) for record in records) + values = { + record["number"]: StdLevelName.objects.get( + slug=record["publication_std_level"] + ) + for record in records + } + # defaultdict to return "unknown" for any missing values + unknown_std_level = StdLevelName.objects.get(slug="unkn") + return defaultdict(lambda: unknown_std_level, values) + + +def format_ordering(rfc_number): + if rfc_number < settings.FIRST_V3_RFC: + ordering = ["txt", "ps", "pdf", "html", "xml"] + else: + ordering = ["html", "txt", "ps", "pdf", "xml"] + return ordering.index # return the method + + +def get_rfc_text_index_entries(): + """Returns RFC entries for rfc-index.txt""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + rfcs = sorted( + chain(published_rfcs, get_unusable_rfc_numbers()), key=attrgetter("rfc_number") + ) + for rfc in rfcs: + if isinstance(rfc, UnusableRfcNumber): + entries.append(f"{format_rfc_number(rfc.rfc_number)} Not Issued.") + else: + assert isinstance(rfc, Document) + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + published_at = rfc.pub_date() + date = ( + published_at.strftime("1 %B %Y") + if rfc.rfc_number in april1_rfc_numbers + else published_at.strftime("%B %Y") + ) + + # formats + formats = ", ".join( + sorted( + [ + format["fmt"] + for format in rfc.formats() + if format["fmt"] in FORMATS_FOR_INDEX + ], + key=format_ordering(rfc.rfc_number), + ) + ).upper() + + # obsoletes + obsoletes = "" + obsoletes_documents = sorted( + rfc.related_that_doc("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoletes_documents) > 0: + obsoletes_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoletes_documents + ) + obsoletes = f" (Obsoletes {obsoletes_names})" + + # obsoleted by + obsoleted_by = "" + obsoleted_by_documents = sorted( + rfc.related_that("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoleted_by_documents) > 0: + obsoleted_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoleted_by_documents + ) + obsoleted_by = f" (Obsoleted by {obsoleted_by_names})" + + # updates + updates = "" + updates_documents = sorted( + rfc.related_that_doc("updates"), + key=attrgetter("rfc_number"), + ) + if len(updates_documents) > 0: + updates_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updates_documents + ) + updates = f" (Updates {updates_names})" + + # updated by + updated_by = "" + updated_by_documents = sorted( + rfc.related_that("updates"), + key=attrgetter("rfc_number"), + ) + if len(updated_by_documents) > 0: + updated_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updated_by_documents + ) + updated_by = f" (Updated by {updated_by_names})" + + doc_relations = f"{obsoletes}{obsoleted_by}{updates}{updated_by} " + + # subseries + subseries = ",".join( + f"{container.type.slug}{format_rfc_number(int(container.name[3:]))}" + for container in rfc.part_of() + ).upper() + if subseries: + subseries = f"(Also {subseries}) " + + entry = fill( + ( + f"{format_rfc_number(rfc.rfc_number)} {rfc.title}. {authors}. {date}. " + f"(Format: {formats}){doc_relations}{subseries}" + f"(Status: {str(rfc.std_level).upper()}) " + f"(DOI: {rfc.doi})" + ), + width=73, + subsequent_indent=" " * 5, + ) + entries.append(entry) + + return entries + + +def subseries_text_line(line, first=False): + """Return subseries text entry line""" + indent = " " * SS_TXT_CUE_COL_WIDTH + if first: + initial_indent = " " * SS_TXT_MARGIN + else: + initial_indent = indent + return fill( + line, + initial_indent=initial_indent, + subsequent_indent=indent, + width=80, + break_on_hyphens=False, + ) + + +def get_bcp_text_index_entries(): + """Returns BCP entries for bcp-index.txt""" + entries = [] + + highest_bcp_number = ( + Document.objects.filter(type_id="bcp") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for bcp_number in range(1, highest_bcp_number + 1): + bcp_name = f"BCP{bcp_number}" + bcp = Document.objects.filter(type_id="bcp", name=f"{bcp_name.lower()}").first() + + if bcp: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{bcp_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this BCP comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(bcp.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", BCP¶{bcp_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + +def get_std_text_index_entries(): + """Returns STD entries for std-index.txt""" + entries = [] + + highest_std_number = ( + Document.objects.filter(type_id="std") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for std_number in range(1, highest_std_number + 1): + std_name = f"STD{std_number}" + std = Document.objects.filter(type_id="std", name=f"{std_name.lower()}").first() + + if std and std.contains(): + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{std_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this STD comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(std.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", STD¶{std_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + +def get_fyi_text_index_entries(): + """Returns FYI entries for fyi-index.txt""" + entries = [] + + highest_fyi_number = ( + Document.objects.filter(type_id="fyi") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for fyi_number in range(1, highest_fyi_number + 1): + fyi_name = f"FYI{fyi_number}" + fyi = Document.objects.filter(type_id="fyi", name=f"{fyi_name.lower()}").first() + + if fyi and fyi.contains(): + entry = subseries_text_line( + ( + f"[{fyi_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(fyi_name) - 2 - SS_TXT_MARGIN)}" + f"For Your Information {fyi_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{fyi_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this FYI comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(fyi.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", FYI¶{fyi_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{fyi_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(fyi_name) - 2 - SS_TXT_MARGIN)}" + f"For Your Information {fyi_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + +def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): + """Add subseries entries for rfc-index.xml""" + # subseries docs annotated with numeric number + ss_docs = list( + Document.objects.filter(type_id=ss_type) + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + ) + if len(ss_docs) == 0: + return # very much not expected + highest_number = ss_docs[0].number + for ss_number in range(1, highest_number + 1): + if ss_docs[-1].number == ss_number: + this_ss_doc = ss_docs.pop() + contained_rfcs = this_ss_doc.contains() + else: + contained_rfcs = [] + if len(contained_rfcs) == 0 and not include_all: + continue + entry = etree.SubElement(rfc_index, f"{ss_type}-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"{ss_type.upper()}{format_rfc_number(ss_number)}" + if len(contained_rfcs) > 0: + is_also = etree.SubElement(entry, "is-also") + for rfc in sorted(contained_rfcs, key=attrgetter("rfc_number")): + etree.SubElement( + is_also, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + + +def add_related_xml_index_entries(root: etree.Element, rfc: Document, tag: str): + relation_getter = { + "obsoletes": lambda doc: doc.related_that_doc("obs"), + "obsoleted-by": lambda doc: doc.related_that("obs"), + "updates": lambda doc: doc.related_that_doc("updates"), + "updated-by": lambda doc: doc.related_that("updates"), + } + related_docs = sorted( + relation_getter[tag](rfc), + key=attrgetter("rfc_number"), + ) + if len(related_docs) > 0: + element = etree.SubElement(root, tag) + for doc in related_docs: + etree.SubElement( + element, "doc-id" + ).text = f"RFC{format_rfc_number(doc.rfc_number)}" + + +def add_rfc_xml_index_entries(rfc_index): + """Add RFC entries for rfc-index.xml""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + publication_statuses = get_publication_std_levels() + + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + + # Iterators for unpublished and published, both sorted by number + unpublished_iter = iter(get_unusable_rfc_numbers()) + published_iter = iter(published_rfcs) + + # Prime the next_* values + next_unpublished = next(unpublished_iter, None) + next_published = next(published_iter, None) + + while next_published is not None or next_unpublished is not None: + if next_unpublished is not None and ( + next_published is None + or next_unpublished.rfc_number < next_published.rfc_number + ): + entry = etree.SubElement(rfc_index, "rfc-not-issued-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(next_unpublished.rfc_number)}" + entries.append(entry) + next_unpublished = next(unpublished_iter, None) + continue + + rfc = next_published # hang on to this + next_published = next(published_iter, None) # prep for next iteration + entry = etree.SubElement(rfc_index, "rfc-entry") + + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + etree.SubElement(entry, "title").text = rfc.title + + for author in rfc.rfcauthor_set.all(): + author_element = etree.SubElement(entry, "author") + etree.SubElement(author_element, "name").text = author.titlepage_name + if author.is_editor: + etree.SubElement(author_element, "title").text = "Editor" + + date = etree.SubElement(entry, "date") + published_at = rfc.pub_date() + etree.SubElement(date, "month").text = published_at.strftime("%B") + if rfc.rfc_number in april1_rfc_numbers: + etree.SubElement(date, "day").text = str(published_at.day) + etree.SubElement(date, "year").text = str(published_at.year) + + format_ = etree.SubElement(entry, "format") + fmts = [ff["fmt"] for ff in rfc.formats() if ff["fmt"] in FORMATS_FOR_INDEX] + for fmt in sorted(fmts, key=format_ordering(rfc.rfc_number)): + match_legacy = getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False) + etree.SubElement(format_, "file-format").text = ( + "ASCII" if match_legacy and fmt == "txt" else fmt.upper() + ) + + etree.SubElement(entry, "page-count").text = str(rfc.pages) + + if len(rfc.keywords) > 0: + keywords = etree.SubElement(entry, "keywords") + for keyword in rfc.keywords: + etree.SubElement(keywords, "kw").text = keyword.strip() + + if rfc.abstract: + abstract = etree.SubElement(entry, "abstract") + for paragraph in rfc.abstract.split("\n\n"): + etree.SubElement(abstract, "p").text = paragraph.strip() + + draft = rfc.came_from_draft() + if draft is not None: + etree.SubElement(entry, "draft").text = f"{draft.name}-{draft.rev}" + + part_of_documents = rfc.part_of() + if len(part_of_documents) > 0: + is_also = etree.SubElement(entry, "is-also") + for doc in part_of_documents: + etree.SubElement(is_also, "doc-id").text = doc.name.upper() + + add_related_xml_index_entries(entry, rfc, "obsoletes") + add_related_xml_index_entries(entry, rfc, "obsoleted-by") + add_related_xml_index_entries(entry, rfc, "updates") + add_related_xml_index_entries(entry, rfc, "updated-by") + + etree.SubElement(entry, "current-status").text = rfc.std_level.name.upper() + etree.SubElement(entry, "publication-status").text = publication_statuses[ + rfc.rfc_number + ].name.upper() + etree.SubElement(entry, "stream").text = ( + "INDEPENDENT" if rfc.stream_id == "ise" else rfc.stream.name + ) + + # Add area / wg_acronym + if rfc.stream_id == "ietf": + if rfc.group.type_id in ["individ", "area"]: + etree.SubElement(entry, "wg_acronym").text = "NON WORKING GROUP" + else: + if rfc.area is not None: + etree.SubElement(entry, "area").text = rfc.area.acronym + if rfc.group: + etree.SubElement(entry, "wg_acronym").text = rfc.group.acronym + + if rfc.tags.filter(slug="errata").exists(): + etree.SubElement(entry, "errata-url").text = errata_url(rfc) + etree.SubElement(entry, "doi").text = rfc.doi + entries.append(entry) + + +def create_rfc_txt_index(): + """Create text index of published documents""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating rfc-index.txt") + index = render_to_string( + "sync/rfc-index.txt", + { + "created_on": created_on, + "rfcs": get_rfc_text_index_entries(), + }, + ) + filename = "rfc-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index) + + +def create_rfc_xml_index(): + """Create XML index of published documents""" + XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" + XSI = "{" + XSI_NAMESPACE + "}" + + log("Creating rfc-index.xml") + rfc_index = etree.Element( + "rfc-index", + nsmap={ + None: "https://www.rfc-editor.org/rfc-index", + "xsi": XSI_NAMESPACE, + }, + attrib={ + XSI + "schemaLocation": ( + "https://www.rfc-editor.org/rfc-index " + "https://www.rfc-editor.org/rfc-index.xsd" + ), + }, + ) + + # add data + add_subseries_xml_index_entries(rfc_index, "bcp", include_all=True) + add_subseries_xml_index_entries(rfc_index, "fyi") + add_rfc_xml_index_entries(rfc_index) + add_subseries_xml_index_entries(rfc_index, "std") + + # make it pretty + pretty_index = etree.tostring( + rfc_index, + encoding="utf-8", + xml_declaration=True, + pretty_print=4, + ) + filename = "rfc-index.xml" + save_to_red_bucket(filename, pretty_index) + save_to_filesystem(filename, pretty_index) + + +def create_bcp_txt_index(): + """Create text index of BCPs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating bcp-index.txt") + index = render_to_string( + "sync/bcp-index.txt", + { + "created_on": created_on, + "bcps": get_bcp_text_index_entries(), + }, + ) + filename = "bcp-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["bcp"]) + + +def create_std_txt_index(): + """Create text index of STDs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating std-index.txt") + index = render_to_string( + "sync/std-index.txt", + { + "created_on": created_on, + "stds": get_std_text_index_entries(), + }, + ) + filename = "std-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["std"]) + + +def create_fyi_txt_index(): + """Create text index of FYIs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating fyi-index.txt") + index = render_to_string( + "sync/fyi-index.txt", + { + "created_on": created_on, + "fyis": get_fyi_text_index_entries(), + }, + ) + filename = "fyi-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["fyi"]) + + +## DirtyBits management for the RFC index + +RFCINDEX_SLUG = DirtyBits.Slugs.RFCINDEX + + +def mark_rfcindex_as_dirty(): + _, created = DirtyBits.objects.update_or_create( + slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()} + ) + if created: + log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')") + + +def mark_rfcindex_as_processed(when: datetime.datetime): + n_updated = DirtyBits.objects.filter( + Q(processed_time__isnull=True) | Q(processed_time__lt=when), + slug=RFCINDEX_SLUG, + ).update(processed_time=when) + if n_updated > 0: + log(f"processed_time is now {when.isoformat()}") + else: + log("processed_time not updated, no matching record found") + + +def rfcindex_is_dirty(): + """Does the rfc index need to be updated?""" + dirty_work, created = DirtyBits.objects.get_or_create( + slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()} + ) + if created: + log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')") + display_processed_time = ( + dirty_work.processed_time.isoformat() + if dirty_work.processed_time is not None + else "never" + ) + log( + f"DirtyBits(slug='{RFCINDEX_SLUG}'): " + f"dirty_time={dirty_work.dirty_time.isoformat()} " + f"processed_time={display_processed_time}" + ) + return ( + dirty_work.processed_time is None + or dirty_work.dirty_time >= dirty_work.processed_time + ) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index e4174d37296..3af5cb8984d 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -1,9 +1,10 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime -import io +from pathlib import Path +from tempfile import NamedTemporaryFile import requests from celery import shared_task @@ -11,92 +12,32 @@ from django.conf import settings from django.utils import timezone -from ietf.doc.models import DocEvent, RelatedDocument +from ietf.doc.models import DocEvent, DocTagName, Document, RelatedDocument, RpcAssignmentDocEvent, State +from ietf.doc.tasks import rebuild_reference_relations_task +from ietf.doc.utils import add_state_change_event, new_state_change_event, update_action_holders +from ietf.person.models import Person +from ietf.utils.mail import send_mail_text from ietf.sync import iana -from ietf.sync import rfceditor -from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue +from ietf.sync.errata import ( + errata_are_dirty, + mark_errata_as_processed, + update_errata_from_rfceditor, +) +from ietf.sync.rfcindex import ( + create_bcp_txt_index, + create_fyi_txt_index, + create_rfc_txt_index, + create_rfc_xml_index, + create_std_txt_index, + rfcindex_is_dirty, mark_rfcindex_as_processed, mark_rfcindex_as_dirty, +) +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils import log -from ietf.utils.timezone import date_today - - -@shared_task -def rfc_editor_index_update_task(full_index=False): - """Update metadata from the RFC index - - Default is to examine only changes in the past 365 days. Call with full_index=True to update - the full RFC index. - - According to comments on the original script, a year's worth took about 20s on production as of - August 2022 - - The original rfc-editor-index-update script had a long-disabled provision for running the - rebuild_reference_relations scripts after the update. That has not been brought over - at all because it should be implemented as its own task if it is needed. - """ - skip_date = None if full_index else date_today() - datetime.timedelta(days=365) - log.log( - "Updating document metadata from RFC index going back to {since}, from {url}".format( - since=skip_date if skip_date is not None else "the beginning", - url=settings.RFC_EDITOR_INDEX_URL, - ) - ) - try: - response = requests.get( - settings.RFC_EDITOR_INDEX_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f'GET request timed out retrieving RFC editor index: {exc}') - return # failed - rfc_index_xml = response.text - index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml)) - try: - response = requests.get( - settings.RFC_EDITOR_ERRATA_JSON_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f'GET request timed out retrieving RFC editor errata: {exc}') - return # failed - errata_data = response.json() - if len(index_data) < rfceditor.MIN_INDEX_RESULTS: - log.log("Not enough index entries, only %s" % len(index_data)) - return # failed - if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: - log.log("Not enough errata entries, only %s" % len(errata_data)) - return # failed - for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date=skip_date - ): - for c in changes: - log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) - - -@shared_task -def rfc_editor_queue_updates_task(): - log.log(f"Updating RFC Editor queue states from {settings.RFC_EDITOR_QUEUE_URL}") - try: - response = requests.get( - settings.RFC_EDITOR_QUEUE_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor queue: {exc}") - return # failed - drafts, warnings = parse_queue(io.StringIO(response.text)) - for w in warnings: - log.log(f"Warning: {w}") - - if len(drafts) < MIN_QUEUE_RESULTS: - log.log("Not enough results, only %s" % len(drafts)) - return # failed - - changed, warnings = update_drafts_from_queue(drafts) - for w in warnings: - log.log(f"Warning: {w}") - - for c in changed: - log.log(f"Updated {c}") @shared_task @@ -110,9 +51,11 @@ def iana_changes_update_task(): MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23) start = ( - timezone.now() - - datetime.timedelta(hours=23) - + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION,) + timezone.now() + - datetime.timedelta(hours=23) + + datetime.timedelta( + seconds=CLOCK_SKEW_COMPENSATION, + ) ) end = start + datetime.timedelta(hours=23) @@ -123,7 +66,9 @@ def iana_changes_update_task(): # requests if necessary text = iana.fetch_changes_json( - settings.IANA_SYNC_CHANGES_URL, t, min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA) + settings.IANA_SYNC_CHANGES_URL, + t, + min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA), ) log.log(f"Retrieved the JSON: {text}") @@ -149,9 +94,9 @@ def iana_protocols_update_task(): # "this needs to be the date where this tool is first deployed" in the original # iana-protocols-updates script)" rfc_must_published_later_than = datetime.datetime( - 2012, - 11, - 26, + 2012, + 11, + 26, tzinfo=datetime.UTC, ) @@ -161,17 +106,17 @@ def iana_protocols_update_task(): timeout=30, ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving IANA protocols page: {exc}') + log.log(f"GET request timed out retrieving IANA protocols page: {exc}") return rfc_numbers = iana.parse_protocol_page(response.text) def batched(l, n): """Split list l up in batches of max size n. - + For Python 3.12 or later, replace this with itertools.batched() """ - return (l[i:i + n] for i in range(0, len(l), n)) + return (l[i : i + n] for i in range(0, len(l), n)) for batch in batched(rfc_numbers, 100): updated = iana.update_rfc_log_from_protocol_page( @@ -182,6 +127,7 @@ def batched(l, n): for d in updated: log.log("Added history entry for %s" % d.display_name()) + @shared_task def fix_subseries_docevents_task(): """Repairs DocEvents related to bugs around removing docs from subseries @@ -222,3 +168,256 @@ def fix_subseries_docevents_task(): DocEvent.objects.filter(type="sync_from_rfc_editor", desc=desc).update( time=obsoleting_time ) + + +@shared_task +def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]): + log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}") + from_file = None + with NamedTemporaryFile(mode="w", delete_on_close=False) as fp: + fp.write(build_from_file_content(rfc_numbers)) + fp.close() + from_file = Path(fp.name) + rsync_helper( + [ + "-a", + "--ignore-existing", + f"--include-from={from_file}", + "--exclude=*", + "rsync.rfc-editor.org::rfcs/", + f"{settings.RFC_PATH}", + ] + ) + load_rfcs_into_blobdb(rfc_numbers) + + rebuild_reference_relations_task.delay([f"rfc{num}" for num in rfc_numbers]) + + +@shared_task +def load_rfcs_into_blobdb_task(start: int, end: int): + """Move file content for rfcs from rfc{start} to rfc{end} inclusive + + As this is expected to be removed once the blobdb is populated, it + will truncate its work to a coded max end. + This will not overwrite any existing blob content, and will only + log a small complaint if asked to load a non-exsiting RFC. + """ + # Protect us from ourselves + if end < start: + return + if start < 1: + start = 1 + if end > 11000: # Arbitrarily chosen + end = 11000 + load_rfcs_into_blobdb(list(range(start, end + 1))) + + +@shared_task +def update_errata_from_rfceditor_task(): + if errata_are_dirty(): + # new_processed_time is the *start* of processing so that any changes after + # this point will trigger another refresh + new_processed_time = timezone.now() + changed_numbers = update_errata_from_rfceditor() + mark_errata_as_processed(new_processed_time) + mark_rfcindex_as_dirty() # ensure any changes are reflected in the indexes + if changed_numbers: + update_rfc_json_task.delay(list(changed_numbers)) + +@shared_task +def update_rfc_json_by_range_list_task(ranges: str) -> None: + """Regenerate RFC JSON for the RFCs described by a range-list string + + See expand_rfc_number_range_list() for the accepted format (e.g. + "[1,100,1000-1004]"). Invalid input is logged and otherwise ignored. + """ + try: + rfc_numbers = expand_rfc_number_range_list(ranges) + except ValueError as e: + log.log( + f"update_rfc_json_by_range_list_task: ignoring invalid input " + f"'{ranges}': {e}" + ) + return + if rfc_numbers: + update_rfc_json_task(rfc_numbers) + +@shared_task +def update_rfc_json_task(rfc_numbers: list[int]) -> None: + from ietf.doc.utils_rfc_json import generate_rfc_json + from ietf.sync.rfcindex import get_publication_std_levels + + try: + pub_levels = get_publication_std_levels() + except Exception as e: + log.log(f"update_rfc_json_task: failed to get publication std levels: {e}") + return + for rfc_number in rfc_numbers: + try: + generate_rfc_json(rfc_number, pub_levels=pub_levels) + except Exception as e: + log.log(f"update_rfc_json_task: failed for RFC {rfc_number}: {e}") + + +@shared_task +def refresh_rfc_index_task(): + if rfcindex_is_dirty(): + # new_processed_time is the *start* of processing so that any changes after + # this point will trigger another refresh + new_processed_time = timezone.now() + + try: + create_rfc_txt_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.txt. {e}") + pass + + try: + create_rfc_xml_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.xml. {e}") + pass + + try: + create_bcp_txt_index() + except Exception as e: + log.log(f"Error: failure in creating bcp-index.txt. {e}") + pass + + try: + create_std_txt_index() + except Exception as e: + log.log(f"Error: failure in creating std-index.txt. {e}") + pass + + try: + create_fyi_txt_index() + except Exception as e: + log.log(f"Error: failure in creating fyi-index.txt. {e}") + pass + + mark_rfcindex_as_processed(new_processed_time) + + +@shared_task +def process_rpc_queue_task(data: list): + in_progress_state = State.objects.get( + used=True, type="draft-rfceditor", slug="in_progress" + ) + blocked_state = State.objects.get(used=True, type="draft-rfceditor", slug="blocked") + system = Person.objects.get(name="(System)") + iana_ref_tags = list(DocTagName.objects.filter(slug__in=["iana", "ref"])) + + names = [obj["name"] for obj in data] + docs_in_db = { + d.name: d for d in Document.objects.filter(type="draft", name__in=names) + } + + for obj in data: + name = obj["name"] + if name not in docs_in_db: + log.log(f"process_rpc_queue_task: unknown document {name}") + continue + + d = docs_in_db[name] + events = [] + prev_state = d.get_state("draft-rfceditor") + + # Same check as ietf.sync.rfceditor.update_drafts_from_queue: + # if this document just arrived at the RFC Editor for the first time, record it. + if ( + d.get_state_slug("draft-iesg") == "ann" + and not prev_state + and not d.latest_event(DocEvent, type="rfc_editor_received_announcement") + ): + e = DocEvent( + doc=d, rev=d.rev, by=system, type="rfc_editor_received_announcement" + ) + e.desc = "Announcement was received by RFC Editor" + e.save() + send_mail_text( + None, + "iesg-secretary@ietf.org", + None, + "%s in RFC Editor queue" % d.name, + "The announcement for %s has been received by the RFC Editor." % d.name, + ) + prev_iesg_state = State.objects.get( + used=True, type="draft-iesg", slug="ann" + ) + next_iesg_state = State.objects.get( + used=True, type="draft-iesg", slug="rfcqueue" + ) + d.set_state(next_iesg_state) + e = add_state_change_event(d, system, prev_iesg_state, next_iesg_state) + if e: + events.append(e) + e = update_action_holders(d, prev_iesg_state, next_iesg_state) + if e: + events.append(e) + + is_blocked = any(a["role"] == "blocked" for a in obj.get("assignment_set", [])) + next_state = blocked_state if is_blocked else in_progress_state + + if prev_state != next_state: + d.set_state(next_state) + e = new_state_change_event(d, system, prev_state, next_state) + if e: + e.save() + events.append(e) + + roles = sorted(a["role"] for a in obj.get("assignment_set", [])) + next_assignments = ", ".join(roles) + blocking_names = sorted( + br["reason"]["name"] for br in obj.get("blocking_reasons", []) + ) + if blocking_names: + next_assignments += ": " + ", ".join(blocking_names) + + if next_assignments == "": + next_assignments = "Awaiting Editor Assignment" + + prev_assignments_event = d.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + prev_assignments = ( + prev_assignments_event.assignments if prev_assignments_event else None + ) + + if next_assignments != prev_assignments: + e = RpcAssignmentDocEvent( + doc=d, + rev=d.rev, + by=system, + type="changed_rpc_assignments", + assignments=next_assignments, + ) + e.desc = f"RPC status changed to {next_assignments}" + if prev_assignments is not None and prev_assignments != "": + e.desc += f" from {prev_assignments}" + e.save() + events.append(e) + + rfc_number = obj.get("rfc_number") + if obj.get("final_approval") and rfc_number: + d.documenturl_set.update_or_create( + tag_id="auth48", + defaults=dict( + url=f"{settings.RFC_EDITOR_QUEUE_SITE_BASE_URL}/final-review/rfc{rfc_number}/" + ), + ) + else: + d.documenturl_set.filter(tag_id="auth48").delete() + + d.tags.remove(*iana_ref_tags) + + if events: + d.save_with_history(events) + + for d in ( + Document.objects.exclude(name__in=names) + .filter(states__type="draft-rfceditor") + .distinct() + ): + d.tags.remove(*iana_ref_tags) + d.unset_state("draft-rfceditor") diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 3432f6214ad..e13dcd19fd7 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,18 +1,13 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2012-2026, All Rights Reserved - -import os -import io import json import datetime from unittest import mock import quopri import requests -from dataclasses import dataclass - -from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import storages from django.urls import reverse as urlreverse from django.utils import timezone from django.test.utils import override_settings @@ -23,20 +18,35 @@ from ietf.doc.factories import ( WgDraftFactory, RfcFactory, - DocumentAuthorFactory, DocEventFactory, - BcpFactory, + WgRfcFactory, +) +from ietf.doc.models import ( + Document, + DocEvent, + DeletedEvent, + DocTagName, + State, + StateDocEvent, ) -from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent from ietf.doc.utils import add_state_change_event -from ietf.group.factories import GroupFactory from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor, tasks +from ietf.sync import iana, tasks +from ietf.sync.errata import ( + update_errata_from_rfceditor, + get_errata_last_updated, + get_errata_data, + errata_map_from_json, + update_errata_dirty_time, + mark_errata_as_processed, + update_errata_tags, +) +from ietf.sync.tasks import update_errata_from_rfceditor_task from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.models import DirtyBits from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase -from ietf.utils.timezone import date_today, RPC_TZINFO class IANASyncTests(TestCase): @@ -286,517 +296,6 @@ def test_notify_page(self): self.assertContains(r, "new changes at") # we don't actually try posting as that would trigger a real run - - -class RFCSyncTests(TestCase): - def write_draft_file(self, name, size): - with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), 'w') as f: - f.write("a" * size) - - def test_rfc_index(self): - area = GroupFactory(type_id='area') - draft_doc = WgDraftFactory( - group__parent=area, - states=[('draft-iesg','rfcqueue')], - ad=Person.objects.get(user__username='ad'), - external_url="http://my-external-url.example.com", - note="this is a note", - ) - DocumentAuthorFactory.create_batch(2, document=draft_doc) - draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared - - RfcFactory(rfc_number=123) - - today = date_today() - - t = ''' - - - BCP0001 - - RFC1234 - RFC2345 - - - - FYI0001 - - RFC1234 - - - - STD0002 - Test - - RFC1234 - - - - RFC1234 - A Testing RFC - - A. Irector - - - %(month)s - %(year)s - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - %(name)s-%(rev)s - - RFC123 - - - BCP0001 - - PROPOSED STANDARD - PROPOSED STANDARD - IETF - %(area)s - %(group)s - http://www.rfc-editor.org/errata_search.php?rfc=1234 -     -    ''' % dict(year=today.strftime("%Y"), - month=today.strftime("%B"), - name=draft_doc.name, - rev=draft_doc.rev, - area=draft_doc.group.parent.acronym, - group=draft_doc.group.acronym) - - errata = [{ - "errata_id":1, - "doc-id":"RFC123", # n.b. this is not the same RFC as in the above index XML! - "errata_status_code":"Verified", - "errata_type_code":"Editorial", - "section": "4.1", - "orig_text":" S: 220-smtp.example.com ESMTP Server", - "correct_text":" S: 220 smtp.example.com ESMTP Server", - "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", - "submit_date":"2007-07-19", - "submitter_name":"Rob Siemborski", - "verifier_id":99, - "verifier_name":None, - "update_date":"2019-09-10 09:09:03"}, - ] - - data = rfceditor.parse_index(io.StringIO(t)) - self.assertEqual(len(data), 1) - rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0] - - # currently, we only check what we actually use - self.assertEqual(rfc_number, 1234) - self.assertEqual(title, "A Testing RFC") - self.assertEqual(rfc_published_date.year, today.year) - self.assertEqual(rfc_published_date.month, today.month) - self.assertEqual(current_status, "Proposed Standard") - self.assertEqual(updates, ["RFC123"]) - self.assertEqual(set(also), set(["BCP1", "FYI1", "STD2"])) - self.assertEqual(draft, draft_doc.name) - self.assertEqual(wg, draft_doc.group.acronym) - self.assertEqual(has_errata, True) - self.assertEqual(stream, "IETF") - self.assertEqual(pages, "42") - self.assertEqual(abstract, "This is some interesting text.") - - draft_filename = "%s-%s.txt" % (draft_doc.name, draft_doc.rev) - self.write_draft_file(draft_filename, 5000) - - event_count_before = draft_doc.docevent_set.count() - draft_title_before = draft_doc.title - draft_abstract_before = draft_doc.abstract - draft_pages_before = draft_doc.pages - - changes = [] - with mock.patch("ietf.sync.rfceditor.log") as mock_log: - for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): - changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list - self.assertEqual(rfc_number, 1234) - if rfc_published: - self.assertEqual(d.type_id, "rfc") - self.assertEqual(d.rfc_number, rfc_number) - else: - self.assertEqual(d.type_id, "draft") - self.assertIsNone(d.rfc_number) - - self.assertFalse(mock_log.called, "No log messages expected") - - draft_doc = Document.objects.get(name=draft_doc.name) - draft_events = draft_doc.docevent_set.all() - self.assertEqual(len(draft_events) - event_count_before, 2) - self.assertEqual(draft_events[0].type, "sync_from_rfc_editor") - self.assertEqual(draft_events[1].type, "changed_action_holders") - self.assertEqual(draft_doc.get_state_slug(), "rfc") - self.assertEqual(draft_doc.get_state_slug("draft-iesg"), "pub") - self.assertCountEqual(draft_doc.action_holders.all(), []) - self.assertEqual(draft_doc.title, draft_title_before) - self.assertEqual(draft_doc.abstract, draft_abstract_before) - self.assertEqual(draft_doc.pages, draft_pages_before) - self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename))) - self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename))) - - rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() - self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.authors(), draft_doc.authors()) - rfc_events = rfc_doc.docevent_set.all() - self.assertEqual(len(rfc_events), 8) - expected_events = [ - ["sync_from_rfc_editor", ""], # Not looking for exact desc match here - see detailed tests below - ["sync_from_rfc_editor", "Imported membership of rfc1234 in std2 via sync to the rfc-index"], - ["std_history_marker", "No history of STD2 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in fyi1 via sync to the rfc-index"], - ["fyi_history_marker", "No history of FYI1 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in bcp1 via sync to the rfc-index"], - ["bcp_history_marker", "No history of BCP1 is currently available in the datatracker before this point"], - ["published_rfc", "RFC published"] - ] - for index, [event_type, desc] in enumerate(expected_events): - self.assertEqual(rfc_events[index].type, event_type) - if index == 0: - self.assertIn("Received changes through RFC Editor sync (created document RFC 1234,", rfc_events[0].desc) - self.assertIn(f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", rfc_events[0].desc) - self.assertIn("set title to 'A Testing RFC'", rfc_events[0].desc) - self.assertIn("set abstract to 'This is some interesting text.'", rfc_events[0].desc) - self.assertIn("set pages to 42", rfc_events[0].desc) - self.assertIn("set standardization level to Proposed Standard", rfc_events[0].desc) - self.assertIn(f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", rfc_events[0].desc) - self.assertIn("created updates relation between RFC 1234 and RFC 123", rfc_events[0].desc) - self.assertIn("added Errata tag", rfc_events[0].desc) - else: - self.assertEqual(rfc_events[index].desc, desc) - self.assertEqual(rfc_events[7].time.astimezone(RPC_TZINFO).date(), today) - for subseries_name in ["bcp1", "fyi1", "std2"]: - sub = Document.objects.filter(type_id=subseries_name[:3],name=subseries_name).first() - self.assertIsNotNone(sub, f"{subseries_name} not created") - self.assertTrue(rfc_doc in sub.contains()) - self.assertTrue(sub in rfc_doc.part_of()) - self.assertEqual(rfc_doc.get_state_slug(), "published") - # Should have an "errata" tag because there is an errata-url in the index XML, but no "verified-errata" tag - # because there is no verified item in the errata JSON with doc-id matching the RFC document. - tag_slugs = rfc_doc.tags.values_list("slug", flat=True) - self.assertTrue("errata" in tag_slugs) - self.assertFalse("verified-errata" in tag_slugs) - # TODO: adjust these when we have subseries document types - # self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc)) - self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists()) - self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists()) - self.assertEqual(rfc_doc.title, "A Testing RFC") - self.assertEqual(rfc_doc.abstract, "This is some interesting text.") - self.assertEqual(rfc_doc.std_level_id, "ps") - self.assertEqual(rfc_doc.pages, 42) - self.assertEqual(rfc_doc.stream, draft_doc.stream) - self.assertEqual(rfc_doc.group, draft_doc.group) - self.assertEqual(rfc_doc.words, draft_doc.words) - self.assertEqual(rfc_doc.ad, draft_doc.ad) - self.assertEqual(rfc_doc.external_url, draft_doc.external_url) - self.assertEqual(rfc_doc.note, draft_doc.note) - - # check that we got the expected changes - self.assertEqual(len(changes), 2) - self.assertEqual(changes[0]["doc_pk"], draft_doc.pk) - self.assertEqual(changes[0]["rfc_published"], False) - self.assertEqual(changes[1]["doc_pk"], rfc_doc.pk) - self.assertEqual(changes[1]["rfc_published"], True) - - # make sure we can apply it again with no changes - changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) - self.assertEqual(len(changed), 0) - - def test_rfc_index_subseries_replacement(self): - today = date_today() - author = PersonFactory(name="Some Bozo") - - # Start with two BCPs, each containing an rfc - rfc1, rfc2, rfc3 = RfcFactory.create_batch(3, authors=[author]) - bcp1 = BcpFactory(contains=[rfc1]) - bcp2 = BcpFactory(contains=[rfc2]) - - def _nameify(doc): - """Convert a name like 'rfc1' to 'RFC0001""" - return f"{doc.name[:3].upper()}{int(doc.name[3:]):04d}" - - # RFC index that replaces rfc2 with rfc3 in bcp2 - index_xml = f""" - - - {_nameify(bcp1)} - - {_nameify(rfc1)} - - - - {_nameify(bcp2)} - - {_nameify(rfc3)} - - - - {_nameify(rfc1)} - {rfc1.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - - {_nameify(bcp1)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     - - {_nameify(rfc2)} - {rfc2.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     - - {_nameify(rfc3)} - {rfc3.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - - {_nameify(bcp2)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     -    """ - data = rfceditor.parse_index(io.StringIO(index_xml)) # parse index - self.assertEqual(len(data), 3) # check that we parsed 3 RFCs - # Process the data by consuming the generator - for _ in rfceditor.update_docs_from_rfc_index(data, []): - pass - # Confirm that the expected changes were made - self.assertCountEqual(rfc1.related_that("contains"), [bcp1]) - self.assertCountEqual(rfc2.related_that("contains"), []) - self.assertCountEqual(rfc3.related_that("contains"), [bcp2]) - - def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): - """Generate an RFC queue xml string for a draft""" - t = ''' -
    - -%(name)s-%(rev)s.txt -2010-09-08 -%(state)s -%(auth48_url)s - -%(ref)s -IN-QUEUE - -A. Author - -%(title)s - -10000000 -%(group)s - -
    -    ''' % dict(name=draft.name, - rev=draft.rev, - title=draft.title, - group=draft.group.name, - ref="draft-ietf-test", - state=state, - auth48_url=(auth48_url or '')) - t = t.replace('\n', '') # strip empty auth48-url tags - return t - - def test_rfc_queue(self): - draft = WgDraftFactory(states=[('draft-iesg','ann')], ad=Person.objects.get(user__username='ad')) - draft.action_holders.add(draft.ad) # add an action holder so we can test that it's removed later - - expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234" - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url=expected_auth48_url) - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - # rfceditor.parse_queue() is tested independently; just sanity check here - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - mailbox_before = len(outbox) - - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - - draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.get_state_slug("draft-rfceditor"), "edit") - self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") - self.assertCountEqual(draft.action_holders.all(), []) - self.assertEqual(set(draft.tags.all()), set(DocTagName.objects.filter(slug__in=("iana", "ref")))) - events = draft.docevent_set.all() - self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state - self.assertEqual(events[1].type, "changed_action_holders") - self.assertEqual(events[2].type, "changed_state") # changed draft-rfceditor state - self.assertEqual(events[3].type, "rfc_editor_received_announcement") - - self.assertEqual(len(outbox), mailbox_before + 1) - self.assertTrue("RFC Editor queue" in outbox[-1]["Subject"]) - - # make sure we can apply it again with no changes - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 0) - self.assertEqual(len(warnings), 0) - - def test_rfceditor_parse_queue(self): - """Test that rfceditor.parse_queue() behaves as expected. - - Currently does a limited test - old comment was - "currently, we only check what we actually use". - """ - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0] - self.assertEqual(draft_name, draft.name) - self.assertEqual(state, "EDIT") - self.assertEqual(set(tags), set(["iana", "ref"])) - self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234") - - def test_rfceditor_parse_queue_TI_state(self): - # Test with TI state introduced 11 Sep 2019 - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='TI', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - __, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(warnings), 0) - - def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None): - """Helper to generate fake output from rfceditor.parse_queue()""" - return [[ - draft.name, # draft_name - '2020-06-03', # date_received - state, - tags or [], - '1', # missref_generation - 'ietf', # stream - auth48_url or '', - '', # cluster - ['draft-ietf-test'], # refs - ]] - - def test_update_draft_auth48_url(self): - """Test that auth48 URLs are handled correctly.""" - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - - # Step 1 setup: update to a state with no auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='EDIT') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) - - # Step 2: update to auth48 state with auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNotNone(auth48_docurl) - self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234') - - # Step 3: update to auth48-done state without auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48-DONE') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) - - def test_post_approved_draft_in_production_only(self): - self.requests_mock.post("https://rfceditor.example.com/", status_code=200, text="OK") - - # be careful playing with SERVER_MODE! - with override_settings(SERVER_MODE="test"): - self.assertEqual( - rfceditor.post_approved_draft("https://rfceditor.example.com/", "some-draft"), - ("", "") - ) - self.assertFalse(self.requests_mock.called) - with override_settings(SERVER_MODE="development"): - self.assertEqual( - rfceditor.post_approved_draft("https://rfceditor.example.com/", "some-draft"), - ("", "") - ) - self.assertFalse(self.requests_mock.called) - with override_settings(SERVER_MODE="production"): - self.assertEqual( - rfceditor.post_approved_draft("https://rfceditor.example.com/", "some-draft"), - ("", "") - ) - self.assertTrue(self.requests_mock.called) - class DiscrepanciesTests(TestCase): def test_discrepancies(self): @@ -881,161 +380,204 @@ def test_rfceditor_undo(self): self.assertTrue(StateDocEvent.objects.filter(desc="First", doc=draft)) -class TaskTests(TestCase): - @override_settings( - RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", - RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", - ) - @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") - @mock.patch("ietf.sync.tasks.rfceditor.parse_index") - @mock.patch("ietf.sync.tasks.requests.get") - def test_rfc_editor_index_update_task( - self, requests_get_mock, parse_index_mock, update_docs_mock - ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked - """rfc_editor_index_update_task calls helpers correctly - - This tests that data flow is as expected. Assumes the individual helpers are - separately tested to function correctly. - """ - @dataclass - class MockIndexData: - """Mock index item that claims to be a specified length""" - length: int - - def __len__(self): - return self.length - - @dataclass - class MockResponse: - """Mock object that contains text and json() that claims to be a specified length""" - text: str - json_length: int = 0 - - def json(self): - return MockIndexData(length=self.json_length) - - # Response objects - index_response = MockResponse(text="this is the index") - errata_response = MockResponse( - text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS +class ErrataTests(TestCase): + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") + def test_get_errata_last_update(self): + red_bucket = storages["red_bucket"] # InMemoryStorage in test + red_bucket.save("myblob.json", ContentFile("file")) + self.assertEqual( + get_errata_last_updated(), red_bucket.get_modified_time("myblob.json") ) - rfc = RfcFactory() - # Test with full_index = False - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, False), + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") + def test_get_errata_data(self): + red_bucket = storages["red_bucket"] # InMemoryStorage in test + red_bucket.save("myblob.json", ContentFile('[{"value": 3}]')) + self.assertEqual( + get_errata_data(), + [{"value": 3}], ) - tasks.rfc_editor_index_update_task(full_index=False) + def test_errata_map_from_json(self): + input_data = [ + { + "doc-id": "not-an-rfc", + "errata_status_code": "Verified", + }, + { + "doc-id": "rfc01234", + "errata_status_code": "Reported", + }, + { + "doc-id": "RFC1001", + "errata_status_code": "Verified" + }, + { + "doc-id": "RfC1234", + "errata_status_code": "Verified", + }, + ] + expected_output = {1001: [input_data[2]], 1234: [input_data[1], input_data[3]]} + self.assertDictEqual(errata_map_from_json(input_data), expected_output) + + @mock.patch("ietf.sync.errata.update_errata_tags") + @mock.patch("ietf.sync.errata.get_errata_data") + def test_update_errata_from_rfceditor(self, mock_get_data, mock_update): + fake_data = object() + fake_changed = {1234, 5678} + mock_get_data.return_value = fake_data + mock_update.return_value = fake_changed + result = update_errata_from_rfceditor() + self.assertTrue(mock_get_data.called) + self.assertTrue(mock_update.called) + self.assertEqual(mock_update.call_args, mock.call(fake_data)) + self.assertEqual(result, fake_changed) + + def test_update_errata_tags(self): + tag_has_errata = DocTagName.objects.get(slug="errata") + tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") + + rfcs = WgRfcFactory.create_batch(10) + rfcs[0].tags.set([tag_has_errata]) + rfcs[1].tags.set([tag_has_errata, tag_has_verified_errata]) + rfcs[2].tags.set([tag_has_errata]) + rfcs[3].tags.set([tag_has_errata, tag_has_verified_errata]) + rfcs[4].tags.set([tag_has_errata]) + rfcs[5].tags.set([tag_has_errata, tag_has_verified_errata]) + + # Only contains the fields we care about, not the full JSON + errata_data = [ + # rfcs[0] had errata and should keep it + {"doc-id": rfcs[0].name, "errata_status_code": "Held for Document Update"}, + {"doc-id": rfcs[0].name, "errata_status_code": "Rejected"}, + # rfcs[1] had errata+verified-errata and should keep both + {"doc-id": rfcs[1].name, "errata_status_code": "Verified"}, + # rfcs[2] had errata and should gain verified-errata + {"doc-id": rfcs[2].name, "errata_status_code": "Verified"}, + # rfcs[3] had errata+verified errata and should lose both + {"doc-id": rfcs[3].name, "errata_status_code": "Rejected"}, + # rfcs[4] had errata and should gain verified-errata + {"doc-id": rfcs[4].name, "errata_status_code": "Verified"}, + {"doc-id": rfcs[4].name, "errata_status_code": "Reported"}, + # rfcs[5] had errata+verified-errata and should lose verified-errata + {"doc-id": rfcs[5].name, "errata_status_code": "Reported"}, + # rfcs[6] had none and should gain errata + {"doc-id": rfcs[6].name, "errata_status_code": "Reported"}, + # rfcs[7] had none and should gain errata+verified-errata + {"doc-id": rfcs[7].name, "errata_status_code": "Verified"}, + # rfcs[8] had none and it should stay that way + {"doc-id": rfcs[8].name, "errata_status_code": "Rejected"}, + # rfcs[9] had none and it should stay that way (no entry at all) + ] + changed = update_errata_tags(errata_data) - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", + self.assertCountEqual(rfcs[0].tags.all(), [tag_has_errata]) + self.assertIsNone(rfcs[0].docevent_set.first()) # no change + + self.assertCountEqual( + rfcs[1].tags.all(), [tag_has_errata, tag_has_verified_errata] ) + self.assertIsNone(rfcs[1].docevent_set.first()) # no change - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) + self.assertCountEqual( + rfcs[2].tags.all(), [tag_has_errata, tag_has_verified_errata] + ) + self.assertEqual(rfcs[2].docevent_set.count(), 1) + self.assertIn(": added verified-errata tag", rfcs[2].docevent_set.first().desc) + + self.assertCountEqual(rfcs[3].tags.all(), []) + self.assertEqual(rfcs[3].docevent_set.count(), 1) + self.assertIn( + ": removed errata tag, removed verified-errata tag (all errata rejected)", + rfcs[3].docevent_set.first().desc, ) - self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) - # Test again with full_index = True - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - tasks.rfc_editor_index_update_task(full_index=True) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", + self.assertCountEqual( + rfcs[4].tags.all(), [tag_has_errata, tag_has_verified_errata] ) + self.assertEqual(rfcs[4].docevent_set.count(), 1) + self.assertIn(": added verified-errata tag", rfcs[4].docevent_set.first().desc) - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) + self.assertCountEqual(rfcs[5].tags.all(), [tag_has_errata]) + self.assertEqual(rfcs[5].docevent_set.count(), 1) + self.assertIn( + ": removed verified-errata tag", rfcs[5].docevent_set.first().desc ) - self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) - # Test error handling - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - requests_get_mock.side_effect = requests.Timeout # timeout on every get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) + self.assertCountEqual(rfcs[6].tags.all(), [tag_has_errata]) + self.assertEqual(rfcs[6].docevent_set.count(), 1) + self.assertIn(": added errata tag", rfcs[6].docevent_set.first().desc) - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - # feed in an index that is too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertTrue(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) + self.assertCountEqual( + rfcs[7].tags.all(), [tag_has_errata, tag_has_verified_errata] + ) + self.assertEqual(rfcs[7].docevent_set.count(), 1) + self.assertIn( + ": added errata tag, added verified-errata tag", + rfcs[7].docevent_set.first().desc, + ) - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - - @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") - @mock.patch("ietf.sync.tasks.update_drafts_from_queue") - @mock.patch("ietf.sync.tasks.parse_queue") - def test_rfc_editor_queue_updates_task(self, mock_parse, mock_update): - # test a request timeout - self.requests_mock.get("https://rfc-editor.example.com/queue/", exc=requests.exceptions.Timeout) - tasks.rfc_editor_queue_updates_task() - self.assertFalse(mock_parse.called) - self.assertFalse(mock_update.called) + self.assertCountEqual(rfcs[8].tags.all(), []) + self.assertIsNone(rfcs[8].docevent_set.first()) # no change + + self.assertCountEqual(rfcs[9].tags.all(), []) + self.assertIsNone(rfcs[9].docevent_set.first()) # no change + + # return value: only RFCs whose tags actually changed + # rfcs[0], rfcs[1], rfcs[8], rfcs[9] had no tag changes + for unchanged_rfc in (rfcs[0], rfcs[1], rfcs[8], rfcs[9]): + self.assertNotIn(unchanged_rfc.rfc_number, changed) + # rfcs[2..7] had tag changes + for changed_rfc in rfcs[2:8]: + self.assertIn(changed_rfc.rfc_number, changed) + + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") + @mock.patch("ietf.sync.errata.get_errata_last_updated") + def test_update_errata_dirty_time(self, mock_last_updated): + ERRATA_SLUG = DirtyBits.Slugs.ERRATA - # now return a value rather than an exception - self.requests_mock.get("https://rfc-editor.example.com/queue/", text="the response") - - # mock returning < MIN_QUEUE_RESULTS values - treated as an error, so no update takes place - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS - 1)], ["a warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertFalse(mock_update.called) - mock_parse.reset_mock() + # No time available + mock_last_updated.side_effect = FileNotFoundError + self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first()) + self.assertIsNone(update_errata_dirty_time()) # no blob yet + self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first()) + + # Now set a time + first_timestamp = timezone.now() - datetime.timedelta(hours=3) + mock_last_updated.return_value = first_timestamp + mock_last_updated.side_effect = None + result = update_errata_dirty_time() + self.assertTrue(isinstance(result, DirtyBits)) + result.refresh_from_db() + self.assertEqual(result.slug, ERRATA_SLUG) + self.assertEqual(result.processed_time, None) + self.assertEqual(result.dirty_time, first_timestamp) + + # Update the time + second_timestamp = timezone.now() + mock_last_updated.return_value = second_timestamp + second_result = update_errata_dirty_time() + self.assertEqual(result.pk, second_result.pk) # should be the same record + result.refresh_from_db() + self.assertEqual(result.slug, ERRATA_SLUG) + self.assertEqual(result.processed_time, None) + self.assertEqual(result.dirty_time, second_timestamp) + + def test_mark_errata_as_processed(self): + ERRATA_SLUG = DirtyBits.Slugs.ERRATA + first_timestamp = timezone.now() + mark_errata_as_processed(first_timestamp) # no DirtyBits is not an error + self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first()) + dbits = DirtyBits.objects.create(slug=ERRATA_SLUG, dirty_time=first_timestamp) + second_timestamp = timezone.now() + mark_errata_as_processed(second_timestamp) + dbits.refresh_from_db() + self.assertEqual(dbits.dirty_time, first_timestamp) + self.assertEqual(dbits.processed_time, second_timestamp) - # mock returning +. MIN_QUEUE_RESULTS - should succeed - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS)], ["a warning"]) - mock_update.return_value = ([1,2,3], ["another warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertEqual(mock_update.call_count, 1) - self.assertEqual(mock_update.call_args, mock.call([n for n in range(rfceditor.MIN_QUEUE_RESULTS)])) +class TaskTests(TestCase): + @override_settings(IANA_SYNC_CHANGES_URL="https://iana.example.com/sync/") @mock.patch("ietf.sync.tasks.iana.update_history_with_changes") @mock.patch("ietf.sync.tasks.iana.parse_changes_json") @@ -1134,3 +676,77 @@ def test_iana_protocols_update_task( self.assertTrue(requests_get_mock.called) self.assertFalse(parse_protocols_mock.called) self.assertFalse(update_rfc_log_mock.called) + + @mock.patch("ietf.sync.tasks.rsync_helper") + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + @mock.patch("ietf.sync.tasks.rebuild_reference_relations_task.delay") + def test_rsync_rfcs_from_rfceditor_task( + self, + rebuild_relations_mock, + load_blobs_mock, + rsync_helper_mock, + ): + tasks.rsync_rfcs_from_rfceditor_task([12345, 54321]) + self.assertTrue(rsync_helper_mock.called) + self.assertTrue(load_blobs_mock.called) + load_blobs_args, load_blobs_kwargs = load_blobs_mock.call_args + self.assertEqual(load_blobs_args, ([12345, 54321],)) + self.assertEqual(load_blobs_kwargs, {}) + self.assertTrue(rebuild_relations_mock.called) + rebuild_args, rebuild_kwargs = rebuild_relations_mock.call_args + self.assertEqual(rebuild_args, (["rfc12345", "rfc54321"],)) + self.assertEqual(rebuild_kwargs, {}) + + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + def test_load_rfcs_into_blobdb_task( + self, + load_blobs_mock, + ): + tasks.load_rfcs_into_blobdb_task(5, 3) + self.assertFalse(load_blobs_mock.called) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(-1, 1) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([1],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(10999, 50000) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([10999, 11000],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(3261, 3263) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([3261, 3262, 3263],)) + self.assertEqual(mock_kwargs, {}) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_update_errata_from_rfceditor_task( + self, + mock_errata_are_dirty, + mock_mark_errata_processed, + mock_mark_rfcindex_dirty, + mock_update, + mock_rfc_json_delay, + ): + mock_errata_are_dirty.return_value = False + update_errata_from_rfceditor_task() + self.assertTrue(mock_errata_are_dirty.called) + self.assertFalse(mock_mark_errata_processed.called) + self.assertFalse(mock_mark_rfcindex_dirty.called) + self.assertFalse(mock_update.called) + + mock_errata_are_dirty.reset_mock() + mock_errata_are_dirty.return_value = True + update_errata_from_rfceditor_task() + self.assertTrue(mock_errata_are_dirty.called) + self.assertTrue(mock_mark_errata_processed.called) + self.assertTrue(mock_mark_rfcindex_dirty.called) + self.assertTrue(mock_update.called) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py new file mode 100644 index 00000000000..226b41af946 --- /dev/null +++ b/ietf/sync/tests_rfcindex.py @@ -0,0 +1,506 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +import re +from pathlib import Path +from unittest import mock + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import storages +from django.test.utils import override_settings +from lxml import etree + +from ietf.doc.factories import ( + BcpFactory, + FyiFactory, + StdFactory, + IndividualRfcFactory, + PublishedRfcDocEventFactory, +) +from ietf.name.models import DocTagName +from ietf.sync.rfcindex import ( + create_bcp_txt_index, + create_fyi_txt_index, + create_rfc_txt_index, + create_rfc_xml_index, + create_std_txt_index, + format_rfc_number, + get_april1_rfc_numbers, + get_publication_std_levels, + get_unusable_rfc_numbers, + red_bucket_input_path, + red_bucket_output_path, + save_to_filesystem, + save_to_red_bucket, + subseries_text_line, +) +from ietf.utils.test_utils import TestCase + + +class RfcIndexTests(TestCase): + """Tests of rfc-index generation + + Tests are limited and should cover more cases. Needs: + * test of subseries docs + * test of related docs (obsoletes/updates + reverse directions) + * more thorough validation of index contents + + Be careful when calling create_rfc_txt_index() or create_rfc_xml_index(). These + will save to a storage by default, which can introduce cross-talk between tests. + Best to patch that method with a mock. + """ + + def setUp(self): + super().setUp() + red_bucket = storages["red_bucket"] + + # Create an unused RFC number + red_bucket.save( + "input/unusable-rfc-numbers.json", + ContentFile(json.dumps([{"number": 123, "comment": ""}])), + ) + + # actual April 1 RFC + self.april_fools_rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=IndividualRfcFactory( + name="rfc4560", + rfc_number=4560, + stream_id="ise", + std_level_id="inf", + ), + ).doc + # Set up a JSON file to flag the April 1 RFC + red_bucket.save( + "input/april-first-rfc-numbers.json", + ContentFile(json.dumps([self.april_fools_rfc.rfc_number])), + ) + + # non-April Fools RFC that happens to have been published on April 1 + self.rfc = PublishedRfcDocEventFactory( + time="2021-04-01T12:00:00Z", + doc__name="rfc10000", + doc__rfc_number=10000, + doc__std_level_id="std", + ).doc + self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + + # Create a BCP with non-April Fools RFC + self.bcp = BcpFactory(contains=[self.rfc], name="bcp11") + + # Create a STD with non-April Fools RFC + self.std = StdFactory(contains=[self.rfc], name="std11") + + # Create a FYI with non-April Fools RFC + self.fyi = FyiFactory(contains=[self.rfc], name="fyi11") + + # Set up a publication-std-levels.json file to indicate the publication + # standard of self.rfc as different from its current value + red_bucket.save( + "input/publication-std-levels.json", + ContentFile( + json.dumps( + [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] + ) + ), + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + red_bucket.delete("input/unusable-rfc-numbers.json") + red_bucket.delete("input/april-first-rfc-numbers.json") + red_bucket.delete("input/publication-std-levels.json") + super().tearDown() + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): + create_rfc_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.txt", contents)) + + self.assertTrue(isinstance(contents, str)) + self.assertIn( + "123 Not Issued.", + contents, + ) + # No zero prefix! + self.assertNotIn( + "0123 Not Issued.", + contents, + ) + + # strip whitespace so line breaks don't interfere with the next few tests + stripped_contents = re.sub(r"\s+", " ", mock_save_blob.call_args[0][1]) + self.assertIn( + f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", + stripped_contents, + ) + # "1 April 2020" may be split across a line wrap (e.g. "1 April\n 2020") + # when the randomly-generated title is long enough to push the date off the line. + # assertRegex handles both wrapped and non-wrapped cases explicitly. + self.assertRegex(contents, r"1\s+April\s+2020") # from the April 1 RFC + self.assertIn( + f"{self.rfc.rfc_number} {self.rfc.title}", + stripped_contents, + ) + self.assertIn("April 2021", stripped_contents) # from the non-April 1 RFC + self.assertNotIn("1 April 2021", stripped_contents) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_xml_index(self, mock_save_blob, mock_save_file): + create_rfc_xml_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.xml") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.xml", contents)) + + self.assertTrue(isinstance(contents, bytes)) + ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string + index = etree.fromstring(contents) + + # We can aspire to validating the schema - currently does not conform because + # XSD expects 4-digit RFC numbers (etc). + # + # xmlschema = etree.XMLSchema(etree.fromstring( + # Path(__file__).with_name("rfc-index.xsd").read_bytes()) + # ) + # xmlschema.assertValid(index) + + children = list(index) # elements as list + # Should be one rfc-not-issued-entry + self.assertEqual(len(children), 16) + self.assertEqual( + [ + c.find(f"{ns}doc-id").text + for c in children + if c.tag == f"{ns}rfc-not-issued-entry" + ], + ["RFC123"], + ) + # Should be two rfc-entries + rfc_entries = { + c.find(f"{ns}doc-id").text: c for c in children if c.tag == f"{ns}rfc-entry" + } + + # Check the April Fool's entry + april_fools_entry = rfc_entries[self.april_fools_rfc.name.upper()] + self.assertEqual( + april_fools_entry.find(f"{ns}title").text, + self.april_fools_rfc.title, + ) + self.assertEqual( + [(c.tag, c.text) for c in april_fools_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}day", "1"), (f"{ns}year", "2020")], + ) + self.assertEqual( + april_fools_entry.find(f"{ns}current-status").text, + "INFORMATIONAL", + ) + self.assertEqual( + april_fools_entry.find(f"{ns}publication-status").text, + "UNKNOWN", + ) + + # Check the Regular entry + rfc_entry = rfc_entries[self.rfc.name.upper()] + self.assertEqual(rfc_entry.find(f"{ns}title").text, self.rfc.title) + self.assertEqual( + rfc_entry.find(f"{ns}current-status").text, "INTERNET STANDARD" + ) + self.assertEqual( + rfc_entry.find(f"{ns}publication-status").text, "PROPOSED STANDARD" + ) + self.assertEqual( + [(c.tag, c.text) for c in rfc_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}year", "2021")], + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_bcp_txt_index(self, mock_save_blob, mock_save_file): + create_bcp_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "bcp-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("bcp-index.txt", contents, ["bcp"]), + ) + + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[BCP1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[BCP10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[BCP12]", + contents, + ) + # Test empty BCPs + self.assertIn( + "Best Current Practice 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[BCP0001]", + contents, + ) + # Has BCP11 with a RFC + self.assertIn( + "Best Current Practice 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "BCP 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_std_txt_index(self, mock_save_blob, mock_save_file): + create_std_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "std-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("std-index.txt", contents, ["std"]), + ) + + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[STD1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[STD10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[STD12]", + contents, + ) + # Test empty STDs + self.assertIn( + "Internet Standard 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[STD0001]", + contents, + ) + # Has STD11 with a RFC + self.assertIn( + "Internet Standard 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "STD 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): + create_fyi_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "fyi-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("fyi-index.txt", contents, ["fyi"]), + ) + + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[FYI1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[FYI10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[FYI12]", + contents, + ) + # Test empty FYIs + self.assertIn( + "For Your Information 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[FYI0001]", + contents, + ) + # Has FYI11 with a RFC + self.assertIn( + "For Your Information 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "FYI 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + +@override_settings(RFCINDEX_INPUT_PATH="input/", RFCINDEX_OUTPUT_PATH="output/") +class HelperTests(TestCase): + INPUT_PATH = "input" + OUTPUT_PATH = "output" + + def test_format_rfc_number(self): + self.assertEqual(format_rfc_number(10), "10") + with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): + self.assertEqual(format_rfc_number(10), "0010") + + def test_red_bucket_input_path(self): + with override_settings(RFCINDEX_INPUT_PATH="bar"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + with override_settings(RFCINDEX_INPUT_PATH="bar/"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + + def test_red_bucket_output_path(self): + self.assertEqual(red_bucket_input_path("foo"), f"{self.INPUT_PATH}/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar/"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + + def test_save_to_red_bucket(self): + red_bucket = storages["red_bucket"] + with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): + save_to_red_bucket("test", "contents \U0001f600") + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") + with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): + save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") + red_bucket.delete(f"{self.OUTPUT_PATH}/test") # clean up like a good child + # check that we can override the path + with override_settings(RFCINDEX_OUTPUT_PATH="fruit"): + save_to_red_bucket("test", "content") + self.assertTrue(red_bucket.exists("fruit/test")) + red_bucket.delete("fruit/test") # clean up like a good child + + def test_save_to_filesystem(self): + rfc_path = Path(settings.RFC_PATH) + self.assertFalse((rfc_path / "test").exists()) + save_to_filesystem("test", "contents \U0001f600") + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertFalse((rfc_path / "subdir" / "test").exists()) + + self.assertFalse((rfc_path / "test2").exists()) + self.assertFalse((rfc_path / "subdir" / "test2").exists()) + save_to_filesystem("test", "contents \U0001f600".encode("utf-8"), ["subdir"]) + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertEqual( + (rfc_path / "subdir" / "test").read_text("utf-8"), "contents \U0001f600" + ) + self.assertEqual( + (rfc_path / "test").stat().st_mtime, + (rfc_path / "subdir" / "test").stat().st_mtime, + ) + + def test_get_unusable_rfc_numbers_raises(self): + """get_unusable_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_unusable_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{self.INPUT_PATH}/unusable-rfc-numbers.json", ContentFile("not json") + ) + with self.assertRaises(json.JSONDecodeError): + get_unusable_rfc_numbers() + red_bucket.delete(f"{self.INPUT_PATH}/unusable-rfc-numbers.json") + + def test_get_april1_rfc_numbers_raises(self): + """get_april1_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_april1_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{self.INPUT_PATH}/april-first-rfc-numbers.json", ContentFile("not json") + ) + with self.assertRaises(json.JSONDecodeError): + get_april1_rfc_numbers() + red_bucket.delete(f"{self.INPUT_PATH}/april-first-rfc-numbers.json") + + def test_get_publication_std_levels_raises(self): + """get_publication_std_levels should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_publication_std_levels() + red_bucket = storages["red_bucket"] + red_bucket.save( + f"{self.INPUT_PATH}/publication-std-levels.json", ContentFile("not json") + ) + with self.assertRaises(json.JSONDecodeError): + get_publication_std_levels() + red_bucket.delete(f"{self.INPUT_PATH}/publication-std-levels.json") + + def test_subseries_text_line(self): + text = "foobar" + self.assertEqual(subseries_text_line(line=text, first=True), f" {text}") + self.assertEqual(subseries_text_line(line=text), f" {text}") diff --git a/ietf/sync/tests_tasks.py b/ietf/sync/tests_tasks.py new file mode 100644 index 00000000000..57284b72980 --- /dev/null +++ b/ietf/sync/tests_tasks.py @@ -0,0 +1,555 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import mock +from django.test.utils import override_settings + +from ietf.doc.factories import WgDraftFactory +from ietf.doc.models import ( + DocEvent, + DocTagName, + Document, + DocumentURL, + RpcAssignmentDocEvent, + State, +) +from ietf.person.models import Person +from ietf.sync import tasks +from ietf.utils.mail import outbox +from ietf.utils.test_utils import TestCase + + +def _make_entry( + doc_name, roles=None, blocking_reasons=None, rfc_number=None, final_approval=None +): + return { + "name": doc_name, + "assignment_set": [{"role": r, "state": "in_progress"} for r in (roles or [])], + "blocking_reasons": blocking_reasons or [], + "rfc_number": rfc_number, + "final_approval": final_approval or [], + } + + +class ProcessRpcQueueTaskTests(TestCase): + def setUp(self): + super().setUp() + self.system = Person.objects.get(name="(System)") + + # --- Unknown document -------------------------------------------------------- + + def test_unknown_document_is_skipped(self): + """Entries with unknown doc names are logged and skipped without raising.""" + tasks.process_rpc_queue_task([_make_entry("draft-does-not-exist")]) + self.assertFalse(Document.objects.filter(name="draft-does-not-exist").exists()) + + # --- First-arrival announcement ---------------------------------------------- + + def test_first_arrival_fires_announcement(self): + """Fires rfc_editor_received_announcement and email on first arrival.""" + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") + self.assertTrue( + draft.docevent_set.filter(type="rfc_editor_received_announcement").exists() + ) + self.assertEqual(len(outbox), mailbox_before + 1) + self.assertIn("RFC Editor queue", outbox[-1]["Subject"]) + self.assertIn("iesg-secretary@ietf.org", outbox[-1]["To"]) + + def test_first_arrival_skipped_if_rfceditor_state_exists(self): + """No announcement when doc already has a draft-rfceditor state.""" + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + draft.set_state( + State.objects.get(used=True, type="draft-rfceditor", slug="in_progress") + ) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse( + draft.docevent_set.filter(type="rfc_editor_received_announcement").exists() + ) + self.assertEqual(len(outbox), mailbox_before) + + def test_first_arrival_skipped_if_announcement_event_exists(self): + """No duplicate announcement when rfc_editor_received_announcement already exists.""" + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="rfc_editor_received_announcement", + desc="Announcement was received by RFC Editor", + ) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name)]) + + self.assertEqual( + draft.docevent_set.filter(type="rfc_editor_received_announcement").count(), + 1, + ) + self.assertEqual(len(outbox), mailbox_before) + + def test_first_arrival_skipped_if_not_ann_iesg_state(self): + """No announcement when IESG state is not 'ann'.""" + draft = WgDraftFactory(states=[("draft-iesg", "rfcqueue")]) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse( + draft.docevent_set.filter(type="rfc_editor_received_announcement").exists() + ) + self.assertEqual(len(outbox), mailbox_before) + + # --- draft-rfceditor state transitions --------------------------------------- + + def test_sets_in_progress_state(self): + """Non-blocked assignment results in in_progress draft-rfceditor state.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-rfceditor"), "in_progress") + + def test_sets_blocked_state(self): + """Assignment with role='blocked' results in blocked draft-rfceditor state.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [{"role": "blocked", "state": "in_progress"}], + "blocking_reasons": [], + "rfc_number": None, + "final_approval": [], + } + ] + ) + + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-rfceditor"), "blocked") + + def test_no_state_change_event_when_state_unchanged(self): + """No state-change DocEvent created when draft-rfceditor state is already correct.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + events_before = draft.docevent_set.filter(type="changed_state").count() + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertEqual( + draft.docevent_set.filter(type="changed_state").count(), events_before + ) + + def test_state_change_event_created_on_transition(self): + """State-change DocEvent is created when draft-rfceditor state changes.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [{"role": "blocked", "state": "in_progress"}], + "blocking_reasons": [], + "rfc_number": None, + "final_approval": [], + } + ] + ) + + self.assertTrue(draft.docevent_set.filter(type="changed_state").exists()) + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-rfceditor"), "blocked") + + # --- RpcAssignmentDocEvent --------------------------------------------------- + + def test_creates_assignment_event_on_first_update(self): + """Creates RpcAssignmentDocEvent when no prior event exists.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [_make_entry(draft.name, roles=["first_editor", "second_editor"])] + ) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIsNotNone(event) + self.assertEqual(event.assignments, "first_editor, second_editor") + + def test_no_assignment_event_when_unchanged(self): + """No new RpcAssignmentDocEvent when assignments match the last recorded ones.""" + draft = WgDraftFactory() + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + events_before = RpcAssignmentDocEvent.objects.filter(doc=draft).count() + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertEqual( + RpcAssignmentDocEvent.objects.filter(doc=draft).count(), events_before + ) + + def test_assignment_desc_includes_previous_assignments(self): + """Assignment event desc includes previous assignments when they exist.""" + draft = WgDraftFactory() + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["second_editor"])]) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIn("from first_editor", event.desc) + + def test_blocking_reasons_appended_to_assignments(self): + """Blocking reason names are appended after ':' in the assignment string, sorted.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [{"role": "blocked", "state": "in_progress"}], + "blocking_reasons": [ + {"reason": {"name": "missing reference"}}, + ], + "rfc_number": None, + "final_approval": [], + } + ] + ) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIsNotNone(event) + self.assertEqual(event.assignments, "blocked: missing reference") + + def test_roles_sorted_in_assignment_string(self): + """Roles are sorted alphabetically in the assignment string.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [_make_entry(draft.name, roles=["second_editor", "first_editor"])] + ) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertEqual(event.assignments, "first_editor, second_editor") + + def test_empty_roles_uses_awaiting_editor_assignment(self): + """Empty assignment_set records 'Awaiting Editor Assignment' rather than an empty string.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task([_make_entry(draft.name)]) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIsNotNone(event) + self.assertEqual(event.assignments, "Awaiting Editor Assignment") + + # --- DocumentURL (auth48) handling ------------------------------------------- + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_created_on_final_approval(self): + """auth48 DocumentURL is created when final_approval is truthy and rfc_number is set.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": 9999, + "final_approval": [{"approved": True}], + } + ] + ) + + url_obj = draft.documenturl_set.filter(tag_id="auth48").first() + self.assertIsNotNone(url_obj) + self.assertEqual(url_obj.url, "https://queue.example.com/final-review/rfc9999/") + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_not_created_without_rfc_number(self): + """No auth48 URL created when rfc_number is None even if final_approval is set.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": None, + "final_approval": [{"approved": True}], + } + ] + ) + + self.assertFalse(draft.documenturl_set.filter(tag_id="auth48").exists()) + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_deleted_when_final_approval_cleared(self): + """Existing auth48 URL is deleted whenever final_approval is empty, regardless of whether assignments changed.""" + draft = WgDraftFactory() + DocumentURL.objects.create( + doc=draft, + tag_id="auth48", + url="https://queue.example.com/final-review/rfc9999/", + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="old_editor", + desc="RPC status changed to old_editor", + ) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse(draft.documenturl_set.filter(tag_id="auth48").exists()) + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_updated_when_rfc_number_changes(self): + """auth48 URL is updated whenever final_approval and rfc_number are set, regardless of whether assignments changed.""" + draft = WgDraftFactory() + DocumentURL.objects.create( + doc=draft, + tag_id="auth48", + url="https://queue.example.com/final-review/rfc8888/", + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="old_editor", + desc="RPC status changed to old_editor", + ) + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": 9999, + "final_approval": [{"approved": True}], + } + ] + ) + + url_obj = draft.documenturl_set.filter(tag_id="auth48").first() + self.assertIsNotNone(url_obj) + self.assertEqual(url_obj.url, "https://queue.example.com/final-review/rfc9999/") + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_created_when_assignments_unchanged(self): + """auth48 URL is created even when assignments have not changed.""" + draft = WgDraftFactory() + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": 9999, + "final_approval": [{"approved": True}], + } + ] + ) + + url_obj = draft.documenturl_set.filter(tag_id="auth48").first() + self.assertIsNotNone(url_obj) + self.assertEqual(url_obj.url, "https://queue.example.com/final-review/rfc9999/") + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_deleted_when_assignments_unchanged(self): + """Existing auth48 URL is deleted even when assignments have not changed.""" + draft = WgDraftFactory() + DocumentURL.objects.create( + doc=draft, + tag_id="auth48", + url="https://queue.example.com/final-review/rfc9999/", + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse(draft.documenturl_set.filter(tag_id="auth48").exists()) + + # --- Tag handling ------------------------------------------------------------ + + def test_removes_iana_and_ref_tags_from_queued_docs(self): + """iana and ref tags are removed from documents in the queue.""" + iana_tag = DocTagName.objects.get(slug="iana") + ref_tag = DocTagName.objects.get(slug="ref") + draft = WgDraftFactory() + draft.tags.add(iana_tag, ref_tag) + + tasks.process_rpc_queue_task([_make_entry(draft.name)]) + + draft = Document.objects.get(pk=draft.pk) + self.assertNotIn(iana_tag, draft.tags.all()) + self.assertNotIn(ref_tag, draft.tags.all()) + + # --- Cleanup of docs no longer in queue -------------------------------------- + + def test_unsets_rfceditor_state_for_docs_not_in_queue(self): + """Documents with draft-rfceditor state but absent from the queue have that state cleared.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + + tasks.process_rpc_queue_task([]) + + draft = Document.objects.get(pk=draft.pk) + self.assertIsNone(draft.get_state("draft-rfceditor")) + + def test_removes_tags_from_docs_not_in_queue(self): + """iana and ref tags are removed from docs with rfceditor state not in the queue.""" + iana_tag = DocTagName.objects.get(slug="iana") + ref_tag = DocTagName.objects.get(slug="ref") + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + draft.tags.add(iana_tag, ref_tag) + + tasks.process_rpc_queue_task([]) + + draft = Document.objects.get(pk=draft.pk) + self.assertNotIn(iana_tag, draft.tags.all()) + self.assertNotIn(ref_tag, draft.tags.all()) + + def test_docs_in_queue_retain_rfceditor_state(self): + """Documents present in the queue keep their draft-rfceditor state.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + draft = Document.objects.get(pk=draft.pk) + self.assertIsNotNone(draft.get_state("draft-rfceditor")) + + +class UpdateErrataFromRfcEditorTaskTests(TestCase): + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_no_update_when_not_dirty( + self, + mock_dirty, + mock_mark_processed, + mock_mark_index, + mock_update, + mock_json_delay, + ): + """When errata are not dirty nothing runs.""" + mock_dirty.return_value = False + tasks.update_errata_from_rfceditor_task() + mock_update.assert_not_called() + mock_json_delay.assert_not_called() + + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_json_task_called_for_changed_rfcs( + self, + mock_dirty, + mock_mark_processed, + mock_mark_index, + mock_update, + mock_json_delay, + ): + """update_rfc_json_task is dispatched with the changed RFC numbers.""" + mock_dirty.return_value = True + mock_update.return_value = {3261, 9000} + tasks.update_errata_from_rfceditor_task() + mock_json_delay.assert_called_once() + called_numbers = mock_json_delay.call_args[0][0] + self.assertCountEqual(called_numbers, [3261, 9000]) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task.delay") + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_json_task_not_called_when_no_changes( + self, + mock_dirty, + mock_mark_processed, + mock_mark_index, + mock_update, + mock_json_delay, + ): + """update_rfc_json_task is not dispatched when no errata tags changed.""" + mock_dirty.return_value = True + mock_update.return_value = set() + tasks.update_errata_from_rfceditor_task() + mock_json_delay.assert_not_called() + + +class UpdateRfcJsonByRangeListTaskTests(TestCase): + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_expands_range_list_and_calls_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[1,100,1000-1004]") + mock_update.assert_called_once_with([1, 100, 1000, 1001, 1002, 1003]) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_invalid_input_is_ignored(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[5-3]") + mock_update.assert_not_called() + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_empty_input_does_not_call_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[]") + mock_update.assert_not_called() diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py new file mode 100644 index 00000000000..ed1dd4a62aa --- /dev/null +++ b/ietf/sync/tests_utils.py @@ -0,0 +1,129 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.test import override_settings +from ietf import settings +from ietf.doc.factories import RfcFactory +from ietf.doc.storage_utils import exists_in_storage, retrieve_str +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) +from ietf.utils.test_utils import TestCase + + +class ExpandRfcNumberRangeListTests(TestCase): + def test_expands_bare_numbers_and_ranges(self): + # ranges follow Python range() conventions: right value is excluded + self.assertEqual( + expand_rfc_number_range_list("[1,100,1000-1004]"), + [1, 100, 1000, 1001, 1002, 1003], + ) + + def test_accepts_input_without_brackets_and_whitespace(self): + self.assertEqual( + expand_rfc_number_range_list(" 1 , 3 - 5 , 9 "), + [1, 3, 4, 9], + ) + + def test_overlapping_ranges_are_sorted_and_deduplicated(self): + self.assertEqual( + expand_rfc_number_range_list("[5,1-4,3-6,2]"), + [1, 2, 3, 4, 5], + ) + + def test_empty_input_yields_empty_list(self): + self.assertEqual(expand_rfc_number_range_list("[]"), []) + self.assertEqual(expand_rfc_number_range_list(""), []) + + def test_rejects_invalid_input(self): + for bad in [ + "[0]", # zero is not positive + "[-5]", # negatives are not accepted + "[abc]", # non-numeric + "[5-3]", # reversed range + "[5-5]", # empty range (start not less than end) + "[1-]", # missing end + "[-1]", # missing start + "[1-2-3]", # malformed range + "[1.5]", # not an integer + ]: + with self.assertRaises(ValueError, msg=f"expected ValueError for {bad!r}"): + expand_rfc_number_range_list(bad) + + +class RsyncHelperTests(TestCase): + def test_rsync_helper(self): + with ( + TemporaryDirectory() as source_dir, + TemporaryDirectory() as dest_dir, + ): + with (Path(source_dir) / "canary.txt").open("w") as canary_source_file: + canary_source_file.write("chirp") + rsync_helper( + [ + "-a", + f"{source_dir}/", + f"{dest_dir}/", + ] + ) + with (Path(dest_dir) / "canary.txt").open("r") as canary_dest_file: + chirp = canary_dest_file.read() + self.assertEqual(chirp, "chirp") + + def test_build_from_file_content(self): + content = build_from_file_content([12345, 54321]) + self.assertEqual( + content, + """prerelease/ +rfc12345.txt +rfc12345.html +rfc12345.xml +rfc12345.pdf +rfc12345.ps +rfc12345.json +prerelease/rfc12345.notprepped.xml +rfc54321.txt +rfc54321.html +rfc54321.xml +rfc54321.pdf +rfc54321.ps +rfc54321.json +prerelease/rfc54321.notprepped.xml +""", + ) + + +class RfcBlobUploadTests(TestCase): + def test_load_rfcs_into_blobdb(self): + with TemporaryDirectory() as faux_rfc_path: + with override_settings(RFC_PATH=faux_rfc_path): + rfc_path = Path(faux_rfc_path) + (rfc_path / "prerelease").mkdir() + for num in [12345, 54321]: + RfcFactory(rfc_number=num) + for ext in settings.RFC_FILE_TYPES + ("json",): + with (rfc_path / f"rfc{num}.{ext}").open("w") as f: + f.write(ext) + with (rfc_path / "rfc{num}.bogon").open("w") as f: + f.write("bogon") + with (rfc_path / "prerelease" / f"rfc{num}.notprepped.xml").open( + "w" + ) as f: + f.write("notprepped") + load_rfcs_into_blobdb([12345, 54321]) + for num in [12345, 54321]: + for ext in settings.RFC_FILE_TYPES + ("json",): + self.assertEqual( + retrieve_str("rfc", f"{ext}/rfc{num}.{ext}"), + ext, + ) + self.assertFalse(exists_in_storage("rfc", f"bogon/rfc{num}.bogon")) + self.assertEqual( + retrieve_str("rfc", f"notprepped/rfc{num}.notprepped.xml"), + "notprepped", + ) diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py new file mode 100644 index 00000000000..b75d80e76bc --- /dev/null +++ b/ietf/sync/utils.py @@ -0,0 +1,128 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import subprocess + +from pathlib import Path + +from django.conf import settings +from ietf.utils import log +from ietf.doc.models import Document +from ietf.doc.storage_utils import AlreadyExistsError, store_bytes + + +def rsync_helper(subprocess_arg_array: list[str]): + subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + + +def _parse_positive_int(value: str) -> int: + """Parse a string as a positive (>= 1) integer, raising ValueError otherwise""" + token = value.strip() + # str.isdigit() rejects signs, whitespace, and non-digits, so anything that + # passes is a non-negative integer literal; guard against zero separately. + if not token.isdigit(): + raise ValueError(f"'{value}' is not a positive integer") + number = int(token) + if number < 1: + raise ValueError(f"'{value}' is not a positive integer") + return number + + +def expand_rfc_number_range_list(ranges: str) -> list[int]: + """Expand a range-list string into a list of RFC numbers + + The string is a comma-separated list of tokens, optionally surrounded by a + pair of square brackets. Each token is either a bare positive integer or a + pair of positive integers separated by a hyphen. A hyphenated pair is + expanded following the convention of Python's range(): the left value is + included and the right value is excluded. For example, "[1,100,1000-1004]" + expands to [1, 100, 1000, 1001, 1002, 1003]. + + The returned list is sorted and deduplicated, so overlapping ranges do not + produce repeated numbers. + + Raises ValueError if the input contains anything other than positive + integers and well-formed (non-reversed) ranges. + """ + numbers: set[int] = set() + stripped = ranges.strip() + if stripped.startswith("[") and stripped.endswith("]"): + stripped = stripped[1:-1] + for raw_token in stripped.split(","): + token = raw_token.strip() + if not token: + continue + if "-" in token: + start_str, _, end_str = token.partition("-") + start = _parse_positive_int(start_str) + end = _parse_positive_int(end_str) + if start >= end: + raise ValueError( + f"'{token}' is not a valid range (start must be less than end)" + ) + numbers.update(range(start, end)) + else: + numbers.add(_parse_positive_int(token)) + return sorted(numbers) + +def build_from_file_content(rfc_numbers: list[int]) -> str: + types_to_sync = settings.RFC_FILE_TYPES + ("json",) + lines = [] + lines.append("prerelease/") + for num in rfc_numbers: + for ext in types_to_sync: + lines.append(f"rfc{num}.{ext}") + lines.append(f"prerelease/rfc{num}.notprepped.xml") + return "\n".join(lines)+"\n" + +def load_rfcs_into_blobdb(numbers: list[int]): + types_to_load = settings.RFC_FILE_TYPES + ("json",) + rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True) + for num in numbers: + if num in rfc_docs: + for ext in types_to_load: + fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}" + if fs_path.is_file(): + with fs_path.open("rb") as f: + bytes = f.read() + mtime = fs_path.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"{ext}/rfc{num}.{ext}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp( + mtime, tz=datetime.UTC + ), + ) + except AlreadyExistsError as e: + log.log(str(e)) + + # store the not-prepped xml + name = f"rfc{num}.notprepped.xml" + source = Path(settings.RFC_PATH) / "prerelease" / name + if source.is_file(): + with open(source, "rb") as f: + bytes = f.read() + mtime = source.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"notprepped/{name}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC), + ) + except AlreadyExistsError as e: + log.log(str(e)) + else: + log.log( + f"Skipping loading rfc{num} into blobdb as no matching Document exists" + ) diff --git a/ietf/sync/views.py b/ietf/sync/views.py index bd539d4e186..58209c2c6ee 100644 --- a/ietf/sync/views.py +++ b/ietf/sync/views.py @@ -35,7 +35,6 @@ def notify(request, org, notification): known_orgs = { "iana": "IANA", - "rfceditor": "RFC Editor", } if org not in known_orgs: @@ -67,29 +66,13 @@ def notify(request, org, notification): known_notifications = { "protocols": "an added reference to an RFC at the IANA protocols page" % settings.IANA_SYNC_PROTOCOLS_URL, "changes": "new changes at the changes JSON dump" % settings.IANA_SYNC_CHANGES_URL, - "queue": "new changes to queue2.xml" % settings.RFC_EDITOR_QUEUE_URL, - "index": "new changes to rfc-index.xml" % settings.RFC_EDITOR_INDEX_URL, } if notification not in known_notifications: raise Http404 if request.method == "POST": - if notification == "index": - log("Queuing RFC Editor index sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_index_update_task.delay() - ) - elif notification == "queue": - log("Queuing RFC Editor queue sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_queue_updates_task.delay() - ) - elif notification == "changes": + if notification == "changes": log("Queuing IANA changes sync from notify view POST") # Wrap in on_commit in case a transaction is open # (As of 2024-11-08, this only runs in a transaction during tests) diff --git a/ietf/templates/base.html b/ietf/templates/base.html index aa449555276..b0df04f30a3 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -15,6 +15,7 @@ {% block title %}No title{% endblock %} + @@ -66,13 +67,17 @@ {% endif %} - + + No, I wish to manage adoption directly, perhaps with non-IETF-stream groups + Back + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/change_stream_state.html b/ietf/templates/doc/draft/change_stream_state.html index 0b13e02fdf6..7f3132b3c8a 100644 --- a/ietf/templates/doc/draft/change_stream_state.html +++ b/ietf/templates/doc/draft/change_stream_state.html @@ -1,7 +1,6 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load django_bootstrap5 %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} {% block title %}Change {{ state_type.label }} for {{ doc }}{% endblock %} {% block content %} {% origin %} @@ -14,12 +13,10 @@

    Help on states

    - Move document to {{ next_states|pluralize:"to one of" }} the recommended next state{{ next_states|pluralize }}: + Move document to {{ next_states|pluralize:"one of" }} the recommended next state{{ next_states|pluralize }}:

    {% for state in next_states %} - {% if state.slug == 'sub-pub' %} - {{ state.name }} - {% else %} + {% if state.slug != 'sub-pub' and state.slug != "wg-lc" %} {% endif %} {% endfor %} @@ -28,7 +25,13 @@

    {% csrf_token %} {% bootstrap_form form %} - Back + Back {% endblock %} {% block js %} diff --git a/ietf/templates/doc/draft/id_expired_email.txt b/ietf/templates/doc/draft/id_expired_email.txt index afbf253ee28..161146a3010 100644 --- a/ietf/templates/doc/draft/id_expired_email.txt +++ b/ietf/templates/doc/draft/id_expired_email.txt @@ -1,4 +1,4 @@ -{% autoescape off %}{{ doc.file_tag|safe }} was just expired. +{% autoescape off %}{{ doc.file_tag|safe }} just expired. This Internet-Draft is in the state "{{ state }}" in the Datatracker. diff --git a/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html new file mode 100644 index 00000000000..61094b053a5 --- /dev/null +++ b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin django_bootstrap5 static %} +={% block title %}Issue Working Group Call for Adoption of {{ doc }}{% endblock %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

    + Issue Working Group Call for Adoption +
    + {{ doc }} +

    + {% if form.errors %} +

    + Please correct the following: +

    + {% endif %} + {% bootstrap_form_errors form %} +
    + {% csrf_token %} + {% bootstrap_form form %} + + + Back +
    +{% endblock %} +{% block js %} + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/issue_working_group_last_call.html b/ietf/templates/doc/draft/issue_working_group_last_call.html new file mode 100644 index 00000000000..d6f35a0e82e --- /dev/null +++ b/ietf/templates/doc/draft/issue_working_group_last_call.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin django_bootstrap5 static %} +={% block title %}Issue Working Group Last Call for {{ doc }}{% endblock %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

    + Issue Working Group Last Call +
    + {{ doc }} +

    + {% if form.errors %} +

    + Please correct the following: +

    + {% endif %} + {% bootstrap_form_errors form %} +
    + {% csrf_token %} + {% bootstrap_form form %} + + + Back +
    +{% endblock %} +{% block js %} + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/rfceditor_post_approved_draft_failed.html b/ietf/templates/doc/draft/rfceditor_post_approved_draft_failed.html deleted file mode 100644 index f976ead9265..00000000000 --- a/ietf/templates/doc/draft/rfceditor_post_approved_draft_failed.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% block title %}Posting approved I-D to RFC Editor failed{% endblock %} -{% block content %} - {% origin %} -

    Posting approved I-D to RFC Editor failed

    -

    - Sorry, when trying to notify the RFC Editor through HTTP, we hit an - error. -

    -

    - We have not changed the Internet-Draft state or sent the announcement - yet so if this is an intermittent error, you can go back and try - again. -

    -

    - The error was: {{ error }} -

    - {% if response %} -

    - The response from the RFC Editor was: - {{ response|linebreaksbr }} -

    - {% endif %} -{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/wg_action_helpers.html b/ietf/templates/doc/draft/wg_action_helpers.html new file mode 100644 index 00000000000..d21f3c0926e --- /dev/null +++ b/ietf/templates/doc/draft/wg_action_helpers.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} +{% block title %}Change IETF WG state for {{ doc }}{% endblock %} +{% block content %} + {% origin %} +

    + Change IETF WG state +
    + {{ doc }} +

    +
    + {% if doc|is_doc_ietf_adoptable %} + Issue WG Call for Adoption + {% endif %} + {% if doc|can_issue_ietf_wg_lc %} + Issue{% if doc|has_had_ietf_wg_lc %} Another{% endif %} Working Group Last Call + {% endif %} + {% if doc|can_submit_to_iesg %} + Submit to IESG for Publication + {% endif %} + Set any WG state directly + Back +
    +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/index_active_drafts.html b/ietf/templates/doc/index_active_drafts.html index 06ea2c4ff5f..607385f56fd 100644 --- a/ietf/templates/doc/index_active_drafts.html +++ b/ietf/templates/doc/index_active_drafts.html @@ -29,7 +29,7 @@

    Active Internet-Drafts

    {% for group in groups %}

    {{ group.name }} ({{ group.acronym }})

    - {% for d in group.active_drafts %} + {% for d in group.active_drafts %}{# n.b., d is a dict, not a Document #}
    {{ d.title }}. diff --git a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt index c4a2401bc29..15ace9495b2 100644 --- a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt +++ b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt @@ -1,15 +1,13 @@ -{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %} -Subject: {{ subject }} +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a {{group.acronym}} WG Call for Adoption of: +{{ doc.name }}-{{ doc.rev }} -This message starts a {{ cfa_duration_weeks }}-week Call for Adoption for this document. + +This Working Group Call for Adoption ends on {{ end_date }} Abstract: {{ doc.abstract }} -File can be retrieved from: -{{ url }} - -Please reply to this message keeping {{ wg_list }} in copy by indicating whether you support or not the adoption of this draft as a WG document. Comments to motivate your preference are highly appreciated. +Please reply to this message and indicate whether or not you support adoption of this Internet-Draft by the {{group.acronym}} WG. Comments to explain your preference are greatly appreciated. Please reply to all recipients of this message and include this message in your response. Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [2]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. @@ -17,5 +15,17 @@ Thank you. [1] https://datatracker.ietf.org/doc/bcp78/ [2] https://datatracker.ietf.org/doc/bcp79/ [3] https://datatracker.ietf.org/doc/rfc6701/ + +The IETF datatracker status page for this Internet-Draft is: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %} +{% if doc.submission.xml_version == "3" %} +There is also an HTML version available at: +{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %} +There is also an HTMLized version available at: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %} +{% if doc.rev != "00" %} +A diff from the previous version is available at: +{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }} +{% endif %} {% endfilter %} {% endautoescape %} diff --git a/ietf/templates/doc/mail/wg_last_call_issued.txt b/ietf/templates/doc/mail/wg_last_call_issued.txt index 35b1e149d7c..114f8bc5e22 100644 --- a/ietf/templates/doc/mail/wg_last_call_issued.txt +++ b/ietf/templates/doc/mail/wg_last_call_issued.txt @@ -1,7 +1,7 @@ -{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %} -Subject: {{ subject }} +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a WG Last Call for: +{{ doc.name }}-{{ doc.rev }} -This message starts a {{ wglc_duration_weeks }}-week WG Last Call for this document. +This Working Group Last Call ends on {{ end_date }} Abstract: {{ doc.abstract }} @@ -9,14 +9,26 @@ Abstract: File can be retrieved from: {{ url }} -Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be motivated and suggestions to resolve them are highly appreciated. +Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be explained and suggestions to resolve them are highly appreciated. -Authors, and WG participants in general, are reminded again of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. +Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. Thank you. [1] https://datatracker.ietf.org/doc/bcp78/ [2] https://datatracker.ietf.org/doc/bcp79/ [3] https://datatracker.ietf.org/doc/rfc6701/ + +The IETF datatracker status page for this Internet-Draft is: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %} +{% if doc.submission.xml_version == "3" %} +There is also an HTML version available at: +{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %} +There is also an HTMLized version available at: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %} +{% if doc.rev != "00" %} +A diff from the previous version is available at: +{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }} +{% endif %} {% endfilter %} {% endautoescape %} diff --git a/ietf/templates/doc/notprepped_wrapper.html b/ietf/templates/doc/notprepped_wrapper.html new file mode 100644 index 00000000000..078db7bc51a --- /dev/null +++ b/ietf/templates/doc/notprepped_wrapper.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% load origin %} +{% block title %}RFC {{ rfc.rfc_number }} — Not-prepped XML{% endblock %} +{% block content %} + {% origin %} +

    RFC {{ rfc.rfc_number }} — Not-prepped XML

    +

    + The not-prepped XML + is the RFC XML v3 source for an RFC at the moment in the publication process + just before the prep tool was used to expand default + values, generate section numbers, resolve cross-references, and embed + boilerplate. +

    + It is useful for authors who want to begin a new draft based on + the RFC's text, such as when creating a bis-draft, and for tools that process + author-facing RFC XML. +

    +

    + + + Download not-prepped XML for RFC {{ rfc.rfc_number }} + +

    +{% endblock %} diff --git a/ietf/templates/doc/opengraph.html b/ietf/templates/doc/opengraph.html index 4fe39b6209f..1c8c5abe912 100644 --- a/ietf/templates/doc/opengraph.html +++ b/ietf/templates/doc/opengraph.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2016-2020, All Rights Reserved #} +{# Copyright The IETF Trust 2016-2025, All Rights Reserved #} {% load origin %} {% load static %} {% load ietf_filters %} @@ -36,7 +36,7 @@ {% else %}{# TODO: We need a card image for individual I-Ds. #} {% endif %} -{% if doc.pk %}{% for author in doc.documentauthor_set.all %} +{% if doc.pk %}{% for author_name in doc.author_names %} {% endfor %}{% endif %} {% if published %}{% endif %} {% if expires %}{% endif %} \ No newline at end of file diff --git a/ietf/templates/doc/review/request_info.html b/ietf/templates/doc/review/request_info.html index 9ad126d59e2..51aea10a023 100644 --- a/ietf/templates/doc/review/request_info.html +++ b/ietf/templates/doc/review/request_info.html @@ -74,13 +74,13 @@ {% person_link review_req.requested_by %} {% endif %} - {% if review_req.doc.authors %} + {% if review_req.doc.author_persons_or_names %} Authors - {% for author in review_req.doc.authors %} - {% person_link author %}{% if not forloop.last %},{% endif %} + {% for person, tp_name in review_req.doc.author_persons_or_names %} + {% if person %}{% person_link person %}{% else %}{{ tp_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %} diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html index 476c81f5986..cb50a1b214e 100644 --- a/ietf/templates/doc/search/search_result_row.html +++ b/ietf/templates/doc/search/search_result_row.html @@ -45,7 +45,7 @@ {% endfor %} - + {% if doc.pages %}{{ doc.pages }} page{{ doc.pages|pluralize }}{% endif %}
    diff --git a/ietf/templates/doc/search/status_columns.html b/ietf/templates/doc/search/status_columns.html index 5ba41bb9c4a..50d70fcb379 100644 --- a/ietf/templates/doc/search/status_columns.html +++ b/ietf/templates/doc/search/status_columns.html @@ -11,7 +11,7 @@ {{ doc.friendly_state|safe }} {% endif %} {% if doc|state:"draft-rfceditor" %} - : {{ doc|state:"draft-rfceditor" }} + : {{ doc|state:"draft-rfceditor" }} {% endif %} {{ doc|auth48_alert_badge }} {{ doc|state_age_colored }} diff --git a/ietf/templates/group/active_teams.html b/ietf/templates/group/active_teams.html index 502d971a206..771dfda290a 100644 --- a/ietf/templates/group/active_teams.html +++ b/ietf/templates/group/active_teams.html @@ -16,21 +16,29 @@

    Active teams

    Chairs - - {% for group in teams %} - - - {{ group.acronym }} - - {{ group.name }} - - {% for chair in group.chairs %} - {% person_link chair.person %}{% if not forloop.last %},{% endif %} - {% endfor %} - - - {% endfor %} - + {% regroup teams by parent as grouped_teams %} + {% for group_entry in grouped_teams %} + + + {% if group_entry.grouper %}{{ group_entry.grouper.name }}{% else %}Other{% endif %} + + + + {% for group in group_entry.list %} + + + {{ group.acronym }} + + {{ group.name }} + + {% for chair in group.chairs %} + {% person_link chair.person %}{% if not forloop.last %},{% endif %} + {% endfor %} + + + {% endfor %} + + {% endfor %} {% endblock %} {% block js %} diff --git a/ietf/templates/group/change_reviewer_settings.html b/ietf/templates/group/change_reviewer_settings.html index 9ecec5633cf..75451fdd750 100644 --- a/ietf/templates/group/change_reviewer_settings.html +++ b/ietf/templates/group/change_reviewer_settings.html @@ -89,7 +89,7 @@

    Unavailable periods

    + value="add_period">Save new period

    History of settings

    diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index cbc2e11536f..0a8b9194f28 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -51,6 +51,13 @@ {{ group.parent.name }} ({{ group.parent.acronym }}) + {% elif group.parent and group.type_id == "team" %} + Parent + + + {{ group.parent.name }} + ({{ group.parent.acronym }}) + {% else %} @@ -444,4 +451,4 @@

    group_stats("{% url 'ietf.group.views.group_stats_data' %}", ".chart"); }); -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/group/manage_review_requests.html b/ietf/templates/group/manage_review_requests.html index 99b23c138af..d240ef24fa0 100644 --- a/ietf/templates/group/manage_review_requests.html +++ b/ietf/templates/group/manage_review_requests.html @@ -66,10 +66,10 @@

    Auto-suggested
    {% endif %} - {% if r.doc.authors %} + {% if r.doc.author_persons_or_names %} Authors: - {% for person in r.doc.authors %} - {% person_link person %}{% if not forloop.last %},{% endif %} + {% for person, tp_name in r.doc.author_persons_or_names %} + {% if person %}{% person_link person %}{% else %}{{ tp_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}
    {% endif %} diff --git a/ietf/templates/group/milestones.html b/ietf/templates/group/milestones.html index df03d370fd0..51579ce80ad 100644 --- a/ietf/templates/group/milestones.html +++ b/ietf/templates/group/milestones.html @@ -20,7 +20,7 @@

    - {% for s in manual %} - {% if user.is_authenticated %} - - {% else %} - - {% endif %} +
    + {% if group.uses_milestone_dates %} Date {% else %} @@ -39,7 +39,7 @@

    {{ milestone.resolved|title }} {% else %} {% if group.uses_milestone_dates %} - {{ milestone.due|date:"M Y" }} + {{ milestone.due|date:"M Y" }} {% else %} {% if forloop.first %}Last{% endif %} {% if forloop.last %}Next{% endif %} diff --git a/ietf/templates/iesg/review_decisions.html b/ietf/templates/iesg/review_decisions.html index 68a9d7fb953..2f69c1809b4 100644 --- a/ietf/templates/iesg/review_decisions.html +++ b/ietf/templates/iesg/review_decisions.html @@ -1,8 +1,8 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} -{% block title %}IESG review decisions in {{ timeframe }}{% endblock %} -{% block content %} +{% block group_subtitle %}IESG review decisions in {{ timeframe }}{% endblock %} +{% block group_content %} {% origin %}

    IESG review decisions in {{ timeframe }}

    diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html index b799636857f..1bde1362c50 100644 --- a/ietf/templates/iesg/working_groups.html +++ b/ietf/templates/iesg/working_groups.html @@ -1,11 +1,11 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2025-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} {% endblock %} -{% block title %}IESG view of working groups{% endblock %} -{% block content %} +{% block group_subtitle %}IESG view of working groups{% endblock %} +{% block group_content %} {% origin %}

    IESG view of working groups

    Area Size and Load

    diff --git a/ietf/templates/liaisons/list_other_sdo.html b/ietf/templates/liaisons/list_other_sdo.html new file mode 100644 index 00000000000..e6a567ae50f --- /dev/null +++ b/ietf/templates/liaisons/list_other_sdo.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin static person_filters %} +{% block pagehead %} + +{% endblock %} +{% block title %} + Other SDO groups +{% endblock %} +{% block content %} + {% origin %} + {% regroup sdos by state.name as sdos_by_state %} + {% for sdos_in_state in sdos_by_state %} +

    {{sdos_in_state.grouper|capfirst}} SDO groups

    + + + + + + + + + + {% for group in sdos_in_state.list%} + + + + + + {% endfor %} + +
    AcronymNameLiaison Managers
    {{ group.acronym }}{{ group.name }} + {% for person in group.liaison_managers %} + {% person_link person %}{% if not forloop.last %}, {% endif %} + {% endfor %} +
    + {% endfor %} +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/approve_proposed_slides.html b/ietf/templates/meeting/approve_proposed_slides.html index 37fb5233949..204473f4551 100644 --- a/ietf/templates/meeting/approve_proposed_slides.html +++ b/ietf/templates/meeting/approve_proposed_slides.html @@ -34,6 +34,6 @@

    + value="disapprove">Decline and Delete -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/meeting/bluesheet.txt b/ietf/templates/meeting/bluesheet.txt index dd3bf36ac7c..5b3960f3aa7 100644 --- a/ietf/templates/meeting/bluesheet.txt +++ b/ietf/templates/meeting/bluesheet.txt @@ -1,7 +1,8 @@ -Bluesheet for {{session}} +{% autoescape off %}Bluesheet for {{session}} ======================================================================== {{ data|length }} attendees. {% for item in data %} {{ item.name }} {{ item.affiliation }}{% endfor %} +{% endautoescape %} diff --git a/ietf/templates/meeting/important_dates.ics b/ietf/templates/meeting/important_dates.ics deleted file mode 100644 index 35079e01eb2..00000000000 --- a/ietf/templates/meeting/important_dates.ics +++ /dev/null @@ -1,5 +0,0 @@ -{% load humanize %}{% autoescape off %}{% load ietf_filters %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical importantdates//EN -{% for meeting in meetings %}{% include "meeting/important_dates_for_meeting.ics" %}{% endfor %}END:VCALENDAR{% endautoescape %} diff --git a/ietf/templates/meeting/important_dates_for_meeting.ics b/ietf/templates/meeting/important_dates_for_meeting.ics deleted file mode 100644 index e6d403da932..00000000000 --- a/ietf/templates/meeting/important_dates_for_meeting.ics +++ /dev/null @@ -1,24 +0,0 @@ -{# Copyright The IETF Trust 2025, All Rights Reserved #} -{% load tz ietf_filters %}{% for d in meeting.important_dates %}BEGIN:VEVENT -UID:ietf-{{ meeting.number }}-{{ d.name_id }}-{{ d.date.isoformat }} -SUMMARY:IETF {{ meeting.number }}: {{ d.name.name }} -CLASS:PUBLIC -DTSTART{% if not d.midnight_cutoff %};VALUE=DATE{% endif %}:{{ d.date|date:"Ymd" }}{% if d.midnight_cutoff %}235900Z{% endif %} -DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %} -TRANSP:TRANSPARENT -DESCRIPTION:{{ d.name.desc }}{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}\n - Register here: https://www.ietf.org/how/meetings/register/{% endif %}{% if d.name.slug == 'opensched' %}\n - To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.meeting.views_session_request.list_view' %}\n - If you are working on a BOF request, it is highly recommended to tell the IESG\n - now by sending an email to iesg@ietf.org to get advance help with the request.{% endif %}{% if d.name.slug == 'cutoffwgreq' %}\n - To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views_session_request.list_view' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n - To request a BOF, please see instructions on Requesting a BOF:\n - https://www.ietf.org/how/bofs/bof-procedures/{% endif %}{% if d.name.slug == 'idcutoff' %}\n - Upload using the I-D Submission Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.submit.views.upload_submission' %}{% endif %}{% if d.name.slug == 'draftwgagenda' or d.name.slug == 'revwgagenda' or d.name.slug == 'procsub' or d.name.slug == 'revslug' %}\n - Upload using the Meeting Materials Management Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views.materials' num=meeting.number %}{% endif %} -END:VEVENT -{% endfor %} diff --git a/ietf/templates/meeting/previously_approved_slides.html b/ietf/templates/meeting/previously_approved_slides.html index 25a4c978633..95975cb63f8 100644 --- a/ietf/templates/meeting/previously_approved_slides.html +++ b/ietf/templates/meeting/previously_approved_slides.html @@ -11,7 +11,7 @@

    {% if submission.status.slug == 'approved' %} approved {% else %} - rejected + declined {% endif %}

    @@ -19,7 +19,7 @@

    {% if submission.status.slug == 'approved' %} approved. {% else %} - rejected. + declined. {% endif %} No further action is needed.

    @@ -33,4 +33,4 @@

    return to this meeting session .

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html index 390ce00cadd..0c59d4ab155 100644 --- a/ietf/templates/meeting/proceedings_attendees.html +++ b/ietf/templates/meeting/proceedings_attendees.html @@ -3,6 +3,7 @@ {% load origin markup_tags static %} {% block pagehead %} + {% if chart_data %}{% endif %} {% endblock %} {% block title %}IETF {{ meeting.number }} proceedings{% endblock %} {% block content %} @@ -14,8 +15,52 @@

    Attendee list of IETF {{ meeting.number }} meeting

    - + + {% if chart_data %} +
    +
    +
    Onsite: {{ stats.onsite }}
    +
    Remote: {{ stats.remote }}
    +
    Total: {{ stats.total }}
    +
    + +
    + + + + {{ chart_data|json_script:"attendees-chart-data" }} + {% endif %}{# chart_data #} + {% if template %} + {{template|safe}} {% else %} @@ -44,4 +89,8 @@

    Attendee list of IETF {{ meeting.number }} meeting

    {% endblock %} {% block js %} -{% endblock %} \ No newline at end of file + {% if chart_data %} + + + {% endif %} +{% endblock %} diff --git a/ietf/templates/meeting/session_details.html b/ietf/templates/meeting/session_details.html index 55fa3d38571..a4d9ba10904 100644 --- a/ietf/templates/meeting/session_details.html +++ b/ietf/templates/meeting/session_details.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin ietf_filters static %} {% block title %}{{ meeting }} : {{ group.acronym }}{% endblock %} {% block morecss %} @@ -53,69 +53,36 @@

    Unscheduled Sessions

    {% endif %} {% if forloop.last %}{% endif %} {% endfor %} + {% if user|has_role:"Secretariat" %} +
    +
    + Secretariat Only +
    +
    +
    + {% csrf_token %} + + +
    +
    + {% endif %} + {% comment %} + The existence of an element with id canManageMaterialsFlag is checked in + session_details.js to determine whether it should init the sortable tables. + Not the most elegant approach, but it works. + {% endcomment %} + {% if can_manage_materials %}
    {% endif %} {% endblock %} {% block js %} - {% if can_manage_materials %} - {% endif %} + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/upcoming.ics b/ietf/templates/meeting/upcoming.ics deleted file mode 100644 index 5eca7ec81df..00000000000 --- a/ietf/templates/meeting/upcoming.ics +++ /dev/null @@ -1,32 +0,0 @@ -{% load humanize tz %}{% autoescape off %}{% load ietf_filters textfilters %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical upcoming//EN -{{vtimezones}}{% for item in assignments %}BEGIN:VEVENT -UID:ietf-{{item.session.meeting.number}}-{{item.timeslot.pk}} -SUMMARY:{% if item.session.name %}{{item.session.group.acronym|lower}} - {{item.session.name|ics_esc}}{% else %}{{item.session.group.acronym|lower}} - {{item.session.group.name}}{%endif%} -{% if item.schedule.meeting.city %}LOCATION:{{item.schedule.meeting.city}},{{item.schedule.meeting.country}} -{% endif %}STATUS:{{item.session.ical_status}} -CLASS:PUBLIC -DTSTART{% ics_date_time item.timeslot.local_start_time item.schedule.meeting.time_zone %} -DTEND{% ics_date_time item.timeslot.local_end_time item.schedule.meeting.time_zone %} -DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %} -URL:{{item.session.agenda.get_href}}{% endif %} -DESCRIPTION:{% if item.timeslot.name %}{{item.timeslot.name|ics_esc}}\n{% endif %}{% if item.session.agenda_note %} - Note: {{item.session.agenda_note|ics_esc}}\n{% endif %}{% for material in item.session.materials.all %} - \n{{material.type}}{% if material.type.name != "Agenda" %} - ({{material.title|ics_esc}}){% endif %}: - {{material.get_href}}\n{% endfor %}{% if item.session.remote_instructions %} - Remote instructions: {{ item.session.remote_instructions }}\n{% endif %} -END:VEVENT -{% endfor %}{% for meeting in ietfs %}BEGIN:VEVENT -UID:ietf-{{ meeting.number }} -SUMMARY:IETF {{ meeting.number }}{% if meeting.city %} -LOCATION:{{ meeting.city }},{{ meeting.country }}{% endif %} -CLASS:PUBLIC -DTSTART;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.date|date:"Ymd" }} -DTEND;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.end_date|next_day|date:"Ymd" }} -DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %} -URL:{{ request.scheme }}://{{ request.get_host }}{% url 'agenda' num=meeting.number %} -END:VEVENT -{% endfor %}END:VCALENDAR{% endautoescape %} diff --git a/ietf/templates/nomcom/view_feedback.html b/ietf/templates/nomcom/view_feedback.html index 93d61b7f420..0758329d606 100644 --- a/ietf/templates/nomcom/view_feedback.html +++ b/ietf/templates/nomcom/view_feedback.html @@ -43,7 +43,7 @@

    Declined each nominated position

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - @@ -82,7 +82,7 @@

    Feedback related to topics

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - diff --git a/ietf/templates/person/merge.html b/ietf/templates/person/merge.html index 36499ecdbce..5c3e6b09380 100644 --- a/ietf/templates/person/merge.html +++ b/ietf/templates/person/merge.html @@ -1,5 +1,5 @@ +{# Copyright The IETF Trust 2018-2025, All Rights Reserved #} {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} {% load static %} {% load django_bootstrap5 %} {% block title %}Merge Persons{% endblock %} @@ -8,45 +8,17 @@

    Merge Person Records

    This tool will merge two Person records into one. If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records.

    - - {% if method == 'post' %} - {% csrf_token %} - {% endif %} +
    {% bootstrap_field form.source %} - {% if source %} - {% with person=source %} - {% include "person/person_info.html" %} - {% endwith %} - {% endif %}
    {% bootstrap_field form.target %} - {% if target %} - {% with person=target %} - {% include "person/person_info.html" %} - {% endwith %} - {% endif %}
    - {% if change_details %}{% endif %} - {% if warn_messages %} - {% for message in warn_messages %}{% endfor %} - {% endif %} - {% if method == 'post' %} - - Swap - - {% endif %} - {% endblock %} \ No newline at end of file diff --git a/ietf/templates/person/merge_request_email.txt b/ietf/templates/person/merge_request_email.txt new file mode 100644 index 00000000000..0a695f036c5 --- /dev/null +++ b/ietf/templates/person/merge_request_email.txt @@ -0,0 +1,23 @@ +Hello, + +We have identified multiple IETF Datatracker accounts that may represent a single person: + +https://datatracker.ietf.org/person/{{ source_account }} + +and + +https://datatracker.ietf.org/person/{{ target_account }} + +If this is so then it is important that we merge the accounts. + +This email is being sent to the primary emails associated with each Datatracker account. + +Please respond to this message individually from the email account(s) you control so we can take the appropriate action. + +If these should be merged, please identify which account you would like to keep the login credentials from. + +If you are associated with but no longer have access to one of the email accounts, then please let us know and we will follow up to determine how to proceed. + + +{{ sender_name }} +IETF Support \ No newline at end of file diff --git a/ietf/templates/person/merge_submit.html b/ietf/templates/person/merge_submit.html new file mode 100644 index 00000000000..30e1999f815 --- /dev/null +++ b/ietf/templates/person/merge_submit.html @@ -0,0 +1,57 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load django_bootstrap5 %} +{% block title %}Merge Persons{% endblock %} +{% block content %} +

    Merge Person Records

    +

    + This tool will merge two Person records into one. If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records. +

    + + {% csrf_token %} +
    +
    + {% bootstrap_field form.source %} + {% if source %} + {% with person=source %} + {% include "person/person_info.html" %} + {% endwith %} + {% endif %} +
    +
    + {% bootstrap_field form.target %} + {% if target %} + {% with person=target %} + {% include "person/person_info.html" %} + {% endwith %} + {% endif %} +
    +
    + {% if change_details %}{% endif %} + {% if warn_messages %} + {% for message in warn_messages %}{% endfor %} + {% endif %} + + + Swap + + + + + + Send Email + + + Back + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/person/person_link.html b/ietf/templates/person/person_link.html index f3f7e1a5b7a..b77fe8d6dfd 100644 --- a/ietf/templates/person/person_link.html +++ b/ietf/templates/person/person_link.html @@ -1,7 +1,7 @@ {% if email and email == "system@datatracker.ietf.org" or name and name == "(System)" %}(System){% else %}{% if email or name %}{{ name }}{% if email and with_email %} {% if titlepage_name %}{{ titlepage_name }}{% else %}{{ name }}{% endif %}{% if email and with_email %} diff --git a/ietf/templates/person/send_merge_request.html b/ietf/templates/person/send_merge_request.html new file mode 100644 index 00000000000..f0c6272dca2 --- /dev/null +++ b/ietf/templates/person/send_merge_request.html @@ -0,0 +1,20 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load django_bootstrap5 %} +{% block title %}Send Merge Notice{% endblock %} +{% block content %} +

    Send Merge Notice

    + {% if form.non_field_errors %}
    {{ form.non_field_errors }}
    {% endif %} +
    + {% csrf_token %} + {% bootstrap_field form.to layout='horizontal' %} + {% bootstrap_field form.frm layout='horizontal' %} + {% bootstrap_field form.reply_to layout='horizontal' %} + {% bootstrap_field form.subject layout='horizontal' %} + {% bootstrap_field form.body layout='horizontal' %} + +     Cancel + +{% endblock %} diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 00000000000..15add7ece3c --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    + +

    Summary

    +
    + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }}
    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 2000168525d..38c8069507f 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -14,8 +14,14 @@

    Reviews of Internet-Drafts in review teams (requires login) +
  • + Per country/affiliation registration for a specific meeting, by default IETF-{{ current_meeting }}. +
    • +
  • + Per country registration timeline for last meetings +

    • - Statistics on meetings and authorship are not currently available. + Statistics on authorship are not currently available.

    {% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html new file mode 100644 index 00000000000..fc41949a2ef --- /dev/null +++ b/ietf/templates/stats/meeting_stats.html @@ -0,0 +1,58 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

    + {% block title %} + Statistics for IETF-{{ meeting_number }} ({{ meeting_date }}, {{ meeting_city }}, {{ meeting_country }}) Registrations + {% endblock %} +

    +
    + +
    + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
    + +
    + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
    +
    +

    + This page provides a visual representation of the total registrations for IETF-{{ meeting_number }} by {{ stats_type }}. + Only categories having more than {{ minimum_required }} registrations are displayed separately, + else they are grouped under "Other". +

    +
    +
    +

    Total Registrations by {{ stats_type|title }} ({{ total_total}} in total)

    +
    + +
    +
    +
    +

    In Person Registrations by {{ stats_type|title }} ({{ in_person_total}} in total)

    +
    + +
    +
    +
    +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meetings_timeline.html b/ietf/templates/stats/meetings_timeline.html new file mode 100644 index 00000000000..40f46880ccf --- /dev/null +++ b/ietf/templates/stats/meetings_timeline.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + {{ stats_type|json_script:"stats-type-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

    + {% block title %} + Statistics for IETF Meeting Registrations + {% endblock %} +

    + +
    + +
    + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
    + +
    + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
    +
    +

    + {% if stats_type == 'total' %} + This page provides a timeline of meeting registrations. + {% else %} + This page provides a timeline of meeting registrations by {{ stats_type }} with a limit of {{ top_n }} categories. + {% endif %} + Panning can be done via the mouse or with a finger. Zooming is done via the mouse wheel or via a pinch gesture. Press ESC + or click to reset panning/zooming. +

    +
    +
    + {% if stats_type == 'total' %} +

    Total Registrations

    + {% else %} +

    Total Registrations by {{ stats_type|title }}

    + {% endif %} +
    + +
    +
    + {% if stats_type != 'total' %} +
    + {% if stats_type == 'total' %} +

    Total In Person Registrations

    + {% else %} +

    In Person Registrations by {{ stats_type|title }}

    + {% endif %} +
    + +
    +
    + {% endif %} +
    +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/submit/manual_post.html b/ietf/templates/submit/manual_post.html index 6e4a2ba42ac..0da83e750fc 100644 --- a/ietf/templates/submit/manual_post.html +++ b/ietf/templates/submit/manual_post.html @@ -1,5 +1,5 @@ {% extends "submit/submit_base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} @@ -27,17 +27,9 @@

    Submissions needing manual posting

    - - {{ s.name }}-{{ s.rev }} - - - {{ s.name }}-{{ s.rev }} - + {{ s.name }}-{{ s.rev }} + {{ s.submission_date }} {% if s.passes_checks %} diff --git a/ietf/templates/submit/upload_submission.html b/ietf/templates/submit/upload_submission.html index 7313d8f000b..b8b1aca29c9 100644 --- a/ietf/templates/submit/upload_submission.html +++ b/ietf/templates/submit/upload_submission.html @@ -73,6 +73,11 @@ $(document).ready(function() { if ($("#checkbox").is(':checked')) $("#other-formats").collapse('show') + + $("form").one('submit', function() { + $("button").attr('disabled', 'disabled'); + return true; + }) }); {% endblock %} diff --git a/ietf/templates/sync/bcp-index.txt b/ietf/templates/sync/bcp-index.txt new file mode 100644 index 00000000000..dd19920eba8 --- /dev/null +++ b/ietf/templates/sync/bcp-index.txt @@ -0,0 +1,52 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all BCPs in numeric order. The BCPs +form a sub-series of the RFC document series, specifically those RFCs +with the status BEST CURRENT PRACTICE. + +BCP citations appear in this format: + + [BCP#] Best Current Practice #, + . + At the time of writing, this BCP comprises the following: + + Author 1, Author 2, "Title of the RFC", BCP #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [BCP3] Best Current Practice 3, + . + At the time of writing, this BCP comprises the following: + + F. Kastenholz, "Variance for The PPP Compression Control Protocol + and The PPP Encryption Control Protocol", BCP 3, RFC 1915, + DOI 10.17487/RFC1915, February 1996, + . + +Key to fields: + +# is the BCP number. + +№ is the RFC number. + +BCPs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + --------- + + + +{% for bcp in bcps %}{{bcp|safe}} + +{% endfor %} diff --git a/ietf/templates/sync/fyi-index.txt b/ietf/templates/sync/fyi-index.txt new file mode 100644 index 00000000000..cf9d57d5706 --- /dev/null +++ b/ietf/templates/sync/fyi-index.txt @@ -0,0 +1,52 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + FYI INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all FYIs in numeric order. The FYIs +(For Your Information) documents form a sub-series of the RFC series, +specifically those documents that may be of particular interest +to Internet users. The corresponding RFCs have status INFORMATIONAL. + +FYI citations appear in this format: + + [FYI#] For Your Information #, + . + At the time of writing, this FYI comprises the following: + + Author 1, Author 2, "Title of the RFC", FYI #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [FYI8] For Your Information 8, + . + At the time of writing, this FYI comprises the following: + + B. Fraser, "Site Security Handbook", FYI 8, RFC 2196, + DOI 10.17487/RFC2196, September 1997, + . + +Key to fields: + +# is the FYI number. + +№ is the RFC number. + +FYIs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + FYI INDEX + --------- + + + +{% for fyi in fyis %}{{fyi|safe}} + +{% endfor %} diff --git a/ietf/templates/sync/rfc-index.txt b/ietf/templates/sync/rfc-index.txt new file mode 100644 index 00000000000..0f01ddfa905 --- /dev/null +++ b/ietf/templates/sync/rfc-index.txt @@ -0,0 +1,69 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all RFCs in numeric order. + +RFC citations appear in this format: + + #### Title of RFC. Author 1, Author 2, Author 3. Issue date. + (Format: ASCII) (Obsoletes xxx) (Obsoleted by xxx) (Updates xxx) + (Updated by xxx) (Also FYI ####) (Status: ssssss) (DOI: ddd) + +or + + #### Not Issued. + +For example: + + 1129 Internet Time Synchronization: The Network Time Protocol. D.L. + Mills. October 1989. (Format: TXT, PS, PDF, HTML) (Also RFC1119) + (Status: INFORMATIONAL) (DOI: 10.17487/RFC1129) + +Key to citations: + +#### is the RFC number. + +Following the RFC number are the title, the author(s), and the +publication date of the RFC. Each of these is terminated by a period. + +Following the number are the title (terminated with a period), the +author, or list of authors (terminated with a period), and the date +(terminated with a period). + +The format follows in parentheses. One or more of the following formats +are listed: text (TXT), PostScript (PS), Portable Document Format +(PDF), HTML, XML. + +Obsoletes xxxx refers to other RFCs that this one replaces; +Obsoleted by xxxx refers to RFCs that have replaced this one. +Updates xxxx refers to other RFCs that this one merely updates (but +does not replace); Updated by xxxx refers to RFCs that have updated +(but not replaced) this one. Generally, only immediately succeeding +and/or preceding RFCs are indicated, not the entire history of each +related earlier or later RFC in a related series. + +The (Also FYI ##) or (Also STD ##) or (Also BCP ##) phrase gives the +equivalent FYI, STD, or BCP number if the RFC is also in those +document sub-series. The Status field gives the document's +current status (see RFC 2026). The (DOI ddd) field gives the +Digital Object Identifier. + +RFCs may be obtained in a number of ways, using HTTP, FTP, or email. +See the RFC Editor Web page http://www.rfc-editor.org + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + --------- + + + +{% for rfc in rfcs %}{{rfc|safe}} + +{% endfor %} diff --git a/ietf/templates/sync/std-index.txt b/ietf/templates/sync/std-index.txt new file mode 100644 index 00000000000..a4a5fba9461 --- /dev/null +++ b/ietf/templates/sync/std-index.txt @@ -0,0 +1,51 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all STDs in numeric order. Each +STD represents a single Internet Standard technical specification, +composed of one or more RFCs with Internet Standard status. + +STD citations appear in this format: + + [STD#] Internet Standard #, + . + At the time of writing, this STD comprises the following: + + Author 1, Author 2, "Title of the RFC", STD #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [STD6] Internet Standard 6, + . + At the time of writing, this STD comprises the following: + + J. Postel, "User Datagram Protocol", STD 6, RFC 768, + DOI 10.17487/RFC0768, August 1980, + . + +Key to fields: + +# is the STD number. + +№ is the RFC number. + +STDs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + --------- + + + +{% for std in stds %}{{std|safe}} + +{% endfor %} diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index 6c1c8726e10..cb8841cdc67 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -1,58 +1,30 @@ -# Copyright The IETF Trust 2011-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2011-2026, All Rights Reserved from django.contrib import admin -from django.utils.encoding import force_str - -def name(obj): - if hasattr(obj, 'abbrev'): - return obj.abbrev() - elif hasattr(obj, 'name'): - if callable(obj.name): - name = obj.name() - else: - name = force_str(obj.name) - if name: - return name - return str(obj) - -def admin_link(field, label=None, ordering="", display=name, suffix=""): - if not label: - label = field.capitalize().replace("_", " ").strip() - if ordering == "": - ordering = field - def _link(self): - obj = self - for attr in field.split("__"): - obj = getattr(obj, attr) - if callable(obj): - obj = obj() - if hasattr(obj, "all"): - objects = obj.all() - elif callable(obj): - objects = obj() - if not hasattr(objects, "__iter__"): - objects = [ objects ] - elif hasattr(obj, "__iter__"): - objects = obj - else: - objects = [ obj ] - chunks = [] - for obj in objects: - app = obj._meta.app_label - model = obj.__class__.__name__.lower() - id = obj.pk - chunks += [ '%(display)s' % - {'app':app, "model": model, "id":id, "display": display(obj), "suffix":suffix, } ] - return ", ".join(chunks) - _link.allow_tags = True - _link.short_description = label - _link.admin_order_field = ordering - return _link - -from .models import DumpInfo +from .models import DumpInfo, DirtyBits + + +class SaferStackedInline(admin.StackedInline): + """StackedInline without delete by default""" + + can_delete = False # no delete button + show_change_link = True # show a link to the resource (where it can be deleted) + + +class SaferTabularInline(admin.TabularInline): + """TabularInline without delete by default""" + + can_delete = False # no delete button + show_change_link = True # show a link to the resource (where it can be deleted) + + +@admin.register(DumpInfo) class DumpInfoAdmin(admin.ModelAdmin): - list_display = ['date', 'host', 'tz'] - list_filter = ['date'] -admin.site.register(DumpInfo, DumpInfoAdmin) + list_display = ["date", "host", "tz"] + list_filter = ["date"] + + +@admin.register(DirtyBits) +class DirtyBitsAdmin(admin.ModelAdmin): + list_display = ["slug", "dirty_time", "processed_time"] diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py index ba3fecebc6f..6e8765612f8 100644 --- a/ietf/utils/fields.py +++ b/ietf/utils/fields.py @@ -1,10 +1,11 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import json import re +from email.utils import parseaddr import debug # pyflakes:ignore @@ -16,6 +17,7 @@ from django.core.exceptions import ValidationError from django.utils.dateparse import parse_duration + class MultiEmailField(forms.Field): def to_python(self, value): "Normalize data to a list of strings." @@ -38,6 +40,25 @@ def validate(self, value): for email in value: validate_email(email) + +def validate_name_addr_email(value): + "Validate name-addr style email address" + name, addr = parseaddr(value) + if not addr: + raise ValidationError("Invalid email format.") + try: + validate_email(addr) # validate the actual address part + except ValidationError: + raise ValidationError("Invalid email address.") + + +class NameAddrEmailField(forms.CharField): + def validate(self, value): + "Check if value consists only of valid emails." + super().validate(value) + validate_name_addr_email(value) + + def yyyymmdd_to_strftime_format(fmt): translation_table = sorted([ ("yyyy", "%Y"), diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 2d34f8361c1..c878ba49f10 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -83,34 +83,6 @@ def create_default_tasks(self): ), ) - PeriodicTask.objects.get_or_create( - name="Partial sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=False)), - defaults=dict( - enabled=False, - crontab=self.crontabs["every_15m_except_midnight"], # don't collide with full sync - description=( - "Reparse the last _year_ of RFC index entries until " - "https://github.com/ietf-tools/datatracker/issues/3734 is addressed. " - "This takes about 20s on production as of 2022-08-11." - ) - ), - ) - - PeriodicTask.objects.get_or_create( - name="Full sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=True)), - defaults=dict( - enabled=False, - crontab=self.crontabs["daily"], - description=( - "Run an extended version of the rfc editor update to catch changes with backdated timestamps" - ), - ), - ) - PeriodicTask.objects.get_or_create( name="Fetch meeting attendance", task="ietf.stats.tasks.fetch_meeting_attendance_task", diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py index 7654f67cd15..943f3789efa 100644 --- a/ietf/utils/meetecho.py +++ b/ietf/utils/meetecho.py @@ -508,8 +508,13 @@ def _should_send_update(self, session): return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time) def add(self, session: "Session", slides: "Document", order: int): + """Add a slide deck to the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False # Would like to confirm that session.presentations includes the slides Document, but we can't # (same problem regarding unsaved Documents discussed in the docstring) @@ -524,11 +529,16 @@ def add(self, session: "Session", slides: "Document", order: int): "order": order, } ) + return True def delete(self, session: "Session", slides: "Document"): - """Delete a slide deck from the session""" + """Delete a slide deck from the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False if session.presentations.filter(document=slides).exists(): # "order" problems are very likely to result if we delete slides that are actually still @@ -543,12 +553,17 @@ def delete(self, session: "Session", slides: "Document"): id=slides.pk, ) if session.presentations.filter(document__type_id="slides").exists(): - self.send_update(session) # adjust order to fill in the hole + self._send_update(session) # adjust order to fill in the hole + return True def revise(self, session: "Session", slides: "Document"): - """Replace existing deck with its current state""" + """Replace existing deck with its current state + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False sp = session.presentations.filter(document=slides).first() if sp is None: @@ -561,11 +576,13 @@ def revise(self, session: "Session", slides: "Document"): id=slides.pk, ) self.add(session, slides, order) # fill in the hole + return True - def send_update(self, session: "Session"): - if not self._should_send_update(session): - return - + def _send_update(self, session: "Session"): + """Notify of the current state of the session's slides (no time window check) + + This is a private helper - use send_update() (no leading underscore) instead. + """ self.api.update_slide_decks( wg_token=self.wg_token(session.group), session=str(session.pk), @@ -580,3 +597,14 @@ def send_update(self, session: "Session"): for deck in session.presentations.filter(document__type="slides") ] ) + + def send_update(self, session: "Session"): + """Notify of the current state of the session's slides + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ + if not self._should_send_update(session): + return False + self._send_update(session) + return True diff --git a/ietf/utils/migrations/0003_dirtybits.py b/ietf/utils/migrations/0003_dirtybits.py new file mode 100644 index 00000000000..11f6ed09f6f --- /dev/null +++ b/ietf/utils/migrations/0003_dirtybits.py @@ -0,0 +1,37 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("utils", "0002_delete_versioninfo"), + ] + + operations = [ + migrations.CreateModel( + name="DirtyBits", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "slug", + models.CharField( + choices=[("rfcindex", "RFC Index")], max_length=40, unique=True + ), + ), + ("dirty_time", models.DateTimeField(blank=True, null=True)), + ("processed_time", models.DateTimeField(blank=True, null=True)), + ], + options={ + "verbose_name_plural": "dirty bits", + }, + ), + ] diff --git a/ietf/utils/migrations/0004_alter_dirtybits_slug.py b/ietf/utils/migrations/0004_alter_dirtybits_slug.py new file mode 100644 index 00000000000..e17ea6cadd3 --- /dev/null +++ b/ietf/utils/migrations/0004_alter_dirtybits_slug.py @@ -0,0 +1,21 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("utils", "0003_dirtybits"), + ] + + operations = [ + migrations.AlterField( + model_name="dirtybits", + name="slug", + field=models.CharField( + choices=[("rfcindex", "RFC Index"), ("errata", "Errata Tags")], + max_length=40, + unique=True, + ), + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 21af5766e99..64f7f253f29 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -1,14 +1,36 @@ -# Copyright The IETF Trust 2015-2020, All Rights Reserved +# Copyright The IETF Trust 2015-2026, All Rights Reserved import itertools from django.db import models + +class DirtyBits(models.Model): + """A weak semaphore mechanism for coordination with celery beat tasks + + Web workers will set the "dirty_time" value for a given dirtybit slug. + Celery workers will do work if "processed_time" < "dirty_time" and update + "processed_time". + """ + + class Slugs(models.TextChoices): + RFCINDEX = "rfcindex", "RFC Index" + ERRATA = "errata", "Errata Tags" + + # next line can become `...choices=Slugs)` when we get to Django 5.x + slug = models.CharField(max_length=40, blank=False, choices=Slugs.choices, unique=True) + dirty_time = models.DateTimeField(null=True, blank=True) + processed_time = models.DateTimeField(null=True, blank=True) + + class Meta: + verbose_name_plural = "dirty bits" + + class DumpInfo(models.Model): date = models.DateTimeField() host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') - + class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 00000000000..9a969a52178 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved + +from typing import List, Set, Tuple +from django.db.models import QuerySet + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" + addresses = set() + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): + addresses.update([a["email"] for a in submission.authors]) + return addresses + + +def submitters_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year, state="posted" + ).values_list("submitter", flat=True) + ] + ) + + +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lengths of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (persons, set(addresses) - set(known_email)) + + diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 1252cfef146..63206eb33a7 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -1,6 +1,4 @@ -# Copyright The IETF Trust 2014-2019, All Rights Reserved -# -*- coding: utf-8 -*- -# Autogenerated by the mkresources management command 2014-11-13 05:39 +# Copyright The IETF Trust 2014-2026, All Rights Reserved from ietf.api import ModelResource @@ -12,7 +10,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo +from ietf.utils.models import DirtyBits, DumpInfo class UserResource(ModelResource): @@ -43,3 +41,9 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) + + +class DirtyBitsResource(ModelResource): + class Meta: + queryset = DirtyBits.objects.none() +api.utils.register(DirtyBitsResource()) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py new file mode 100644 index 00000000000..3c6e1b8f26f --- /dev/null +++ b/ietf/utils/searchindex.py @@ -0,0 +1,425 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +"""Search indexing utilities""" + +import re +from itertools import batched +from math import floor +from typing import Iterable +from urllib.parse import urljoin + +import httpx # just for exceptions +import requests +import typesense +import typesense.exceptions +from django.conf import settings +from typesense.types.document import DocumentSchema + +from ietf.doc.models import Document, StoredObject +from ietf.doc.storage_utils import retrieve_str +from ietf.utils.log import log + +# Error classes that might succeed just by retrying a failed attempt. +# Must be a tuple for use with isinstance() +RETRYABLE_ERROR_CLASSES = ( + httpx.ConnectError, + httpx.ConnectTimeout, + typesense.exceptions.Timeout, + typesense.exceptions.ServerError, + typesense.exceptions.ServiceUnavailable, +) + +DEFAULT_SETTINGS = { + "TYPESENSE_API_URL": "", + "TYPESENSE_API_KEY": "", + "TYPESENSE_COLLECTION_NAME": "docs", + "TASK_RETRY_DELAY": 10, + "TASK_MAX_RETRIES": 12, +} + + +def get_settings(): + return DEFAULT_SETTINGS | getattr(settings, "SEARCHINDEX_CONFIG", {}) + + +def enabled(): + _settings = get_settings() + return _settings["TYPESENSE_API_URL"] != "" + + +def get_typesense_client() -> typesense.Client: + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + return client + + +def get_collection_name() -> str: + _settings = get_settings() + collection_name = _settings["TYPESENSE_COLLECTION_NAME"] + assert isinstance(collection_name, str) + return collection_name + + +def _sanitize_text(content: str): + """Sanitize content text for search + + Aggressively simplifies whitespace, removes most punctuation + """ + # REs (with approximate names) + RE_DOT_OR_BANG_SPACE = r"\. |! " # -> " " (space) + RE_COMMENT_OR_TOC_CRUD = r"<--|-->|--+|\+|\.\.+" # -> "" + RE_BRACKETED_REF = r"\[[a-zA-Z0-9 -]+\]" # -> "" + RE_DOTTED_NUMBERS = r"[0-9]+\.[0-9]+(\.[0-9]+)?" # -> "" + RE_MULTIPLE_WHITESPACE = r"\s+" # -> " " (space) + # Replacement values (for clarity of intent) + SPACE = " " + EMPTY = "" + # Sanitizing begins here, order is significant! + content = re.sub(RE_DOT_OR_BANG_SPACE, SPACE, content.strip()) + content = re.sub(RE_COMMENT_OR_TOC_CRUD, EMPTY, content) + content = re.sub(RE_BRACKETED_REF, EMPTY, content) + content = re.sub(RE_DOTTED_NUMBERS, EMPTY, content) + content = re.sub(RE_MULTIPLE_WHITESPACE, SPACE, content) + return content.strip() + + +def _sanitize_abstract(abstract: str): + """Sanitize abstract text for search + + Simplifies whitespace but mostly leaves text intact. Abstract text will be + displayed in search results, so a light touch is needed. + """ + abstract = abstract.strip() + abstract = re.sub("\r\n|\n\r|\r", "\n", abstract) # normalize on \n + abstract = "\n".join(line.strip() for line in abstract.split("\n")) # strip by line + return abstract + + +def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: + assert rfc.type_id == "rfc" + assert rfc.rfc_number is not None + assert rfc.pages is not None + + keywords: list[str] = rfc.keywords # help type checking + + subseries = rfc.part_of() + if len(subseries) > 1: + log( + f"RFC {rfc.rfc_number} is in multiple subseries. " + f"Indexing as {subseries[0].name}" + ) + subseries = subseries[0] if len(subseries) > 0 else None + obsoleted_by = rfc.related_that("obs") + is_obsoleted = len(obsoleted_by) > 0 + updated_by = rfc.related_that("updates") + is_updated = len(updated_by) > 0 + is_historic = rfc.std_level.slug == "hist" + + stored_txt = ( + StoredObject.objects.exclude_deleted() + .filter(store="rfc", doc_name=rfc.name, name__startswith="txt/") + .first() + ) + content = "" + if stored_txt is not None: + # Should be available in the blobdb, but be cautious... + try: + content = retrieve_str(kind=stored_txt.store, name=stored_txt.name) + except Exception as err: + log(f"Unable to retrieve {stored_txt} from storage: {err}") + + ts_document = { + "id": f"doc-{rfc.pk}", + "rfcNumber": rfc.rfc_number, + "rfc": str(rfc.rfc_number), + "filename": rfc.name, + "title": rfc.title, + "abstract": _sanitize_abstract(rfc.abstract), + "pages": rfc.pages, + "keywords": keywords, + "type": "rfc", + "state": [state.name for state in rfc.states.all()], + "status": {"slug": rfc.std_level.slug, "name": rfc.std_level.name}, + "date": floor(rfc.time.timestamp()), + "publicationDate": floor(rfc.pub_datetime().timestamp()), + "stream": {"slug": rfc.stream.slug, "name": rfc.stream.name}, + "authors": [ + {"name": rfc_author.titlepage_name, "affiliation": rfc_author.affiliation} + for rfc_author in rfc.rfcauthor_set.all() + ], + "flags": { + "hiddenDefault": is_obsoleted or is_historic, + "obsoleted": is_obsoleted, + "updated": is_updated, + }, + "obsoletedBy": [str(doc.rfc_number) for doc in obsoleted_by], + "updatedBy": [str(doc.rfc_number) for doc in updated_by], + "ranking": rfc.rfc_number, + } + if subseries is not None: + ts_document["subseries"] = { + "acronym": subseries.type.slug, + "number": int(subseries.name[len(subseries.type.slug):]), + "total": len(subseries.contains()), + } + if rfc.group is not None: + ts_document["group"] = { + "acronym": rfc.group.acronym, + "name": rfc.group.name, + "full": f"{rfc.group.acronym} - {rfc.group.name}", + "type": rfc.group.type.slug, + } + if ( + rfc.group.parent is not None + and rfc.stream_id not in ["ise", "irtf", "iab"] # exclude editorial? + ): + ts_document["area"] = { + "acronym": rfc.group.parent.acronym, + "name": rfc.group.parent.name, + "full": f"{rfc.group.parent.acronym} - {rfc.group.parent.name}", + } + if rfc.ad is not None: + ts_document["adName"] = rfc.ad.name + if content != "": + ts_document["content"] = _sanitize_text(content) + return ts_document + + +def update_or_create_rfc_entry(rfc: Document): + """Update/create index entries for one RFC""" + ts_document = typesense_doc_from_rfc(rfc) + client = get_typesense_client() + client.collections[get_collection_name()].documents.upsert(ts_document) + + +def update_or_create_rfc_entries( + rfcs: Iterable[Document], batchsize: int | None = None +): + """Update/create index entries for RFCs in bulk + + If batchsize is set, computes index data in batches of batchsize and adds to the + index. Will make a total of (len(rfcs) // batchsize) + 1 API calls. + + N.b. that typesense has a server-side batch size that defaults to 40, which should + "almost never be changed from the default." This does not change that. Further, + the python client library's import_ method has a batch_size parameter that does + client-side batching. We don't use that, either. + """ + success_count = 0 + fail_count = 0 + client = get_typesense_client() + batches = [rfcs] if batchsize is None else batched(rfcs, batchsize) + for batch in batches: + tdoc_batch = [typesense_doc_from_rfc(rfc) for rfc in batch] + results = client.collections[get_collection_name()].documents.import_( + tdoc_batch, {"action": "upsert"} + ) + for tdoc, result in zip(tdoc_batch, results): + if result["success"]: + success_count += 1 + else: + fail_count += 1 + log(f"Failed to index RFC {tdoc['rfcNumber']}: {result['error']}") + log(f"Added {success_count} RFCs to the index, failed to add {fail_count}") + + +DOCS_SCHEMA = { + "enable_nested_fields": True, + "default_sorting_field": "ranking", + "fields": [ + # RFC number in integer form, for sorting asc/desc in search results + # Omit field for drafts + { + "name": "rfcNumber", + "type": "int32", + "facet": False, + "optional": True, + "sort": True, + }, + # RFC number in string form, for direct matching with ranking + # Omit field for drafts + {"name": "rfc", "type": "string", "facet": False, "optional": True}, + # For drafts that correspond to an RFC, insert the RFC number + # Omit field for rfcs or if not relevant + {"name": "ref", "type": "string", "facet": False, "optional": True}, + # Filename of the document (without the extension, e.g. "rfc1234" + # or "draft-ietf-abc-def-02") + {"name": "filename", "type": "string", "facet": False, "infix": True}, + # Title of the draft / rfc + {"name": "title", "type": "string", "facet": False}, + # Abstract of the draft / rfc + {"name": "abstract", "type": "string", "facet": False}, + # Number of pages + {"name": "pages", "type": "int32", "facet": False}, + # A list of search keywords if relevant, set to empty array otherwise + {"name": "keywords", "type": "string[]", "facet": True}, + # Type of the document + # Accepted values: "draft" or "rfc" + {"name": "type", "type": "string", "facet": True}, + # State(s) of the document (e.g. "Published", "Adopted by a WG", etc.) + # Use the full name, not the slug + {"name": "state", "type": "string[]", "facet": True, "optional": True}, + # Status (Standard Level Name) + # Object with properties "slug" and "name" + # e.g.: { slug: "std", "name": "Internet Standard" } + {"name": "status", "type": "object", "facet": True, "optional": True}, + # The subseries it is part of. (e.g. "BCP") + # Omit otherwise. + { + "name": "subseries.acronym", + "type": "string", + "facet": True, + "optional": True, + }, + # The subseries number it is part of. (e.g. 123) + # Omit otherwise. + { + "name": "subseries.number", + "type": "int32", + "facet": True, + "sort": True, + "optional": True, + }, + # The total of RFCs in the subseries + # Omit if not part of a subseries + { + "name": "subseries.total", + "type": "int32", + "facet": False, + "sort": False, + "optional": True, + }, + # Date of the document, in unix epoch seconds (can be negative for < 1970) + {"name": "date", "type": "int64", "facet": False}, + # Expiration date of the document, in unix epoch seconds (can be negative + # for < 1970). Omit field for RFCs + {"name": "expires", "type": "int64", "facet": False, "optional": True}, + # Publication date of the RFC, in unix epoch seconds (can be negative + # for < 1970). Omit field for drafts + { + "name": "publicationDate", + "type": "int64", + "facet": True, + "optional": True, + }, + # Working Group + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "ntp", + # "name": "Network Time Protocols", + # "full": "ntp - Network Time Protocols", + # } + {"name": "group", "type": "object", "facet": True, "optional": True}, + # Area + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "mpls", + # "name": "Multiprotocol Label Switching", + # "full": "mpls - Multiprotocol Label Switching", + # } + {"name": "area", "type": "object", "facet": True, "optional": True}, + # Stream + # Object with properties "slug" and "name" + # e.g.: { slug: "ietf", "name": "IETF" } + {"name": "stream", "type": "object", "facet": True, "optional": True}, + # List of authors + # Array of objects with properties "name" and "affiliation" + # e.g.: + # [ + # {"name": "John Doe", "affiliation": "ACME Inc."}, + # {"name": "Ada Lovelace", "affiliation": "Babbage Corps."}, + # ] + {"name": "authors", "type": "object[]", "facet": True, "optional": True}, + # Area Director Name (e.g. "Leonardo DaVinci") + {"name": "adName", "type": "string", "facet": True, "optional": True}, + # Whether the document should be hidden by default in search results or not. + {"name": "flags.hiddenDefault", "type": "bool", "facet": True}, + # Whether the document is obsoleted by another document or not. + {"name": "flags.obsoleted", "type": "bool", "facet": True}, + # Whether the document is updated by another document or not. + {"name": "flags.updated", "type": "bool", "facet": True}, + # List of documents that obsolete this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.obsoleted" is set to True. + { + "name": "obsoletedBy", + "type": "string[]", + "facet": False, + "optional": True, + }, + # List of documents that update this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.updated" is set to True. + {"name": "updatedBy", "type": "string[]", "facet": False, "optional": True}, + # Sanitized content of the document. + # Make sure to remove newlines, double whitespaces, symbols and tags. + { + "name": "content", + "type": "string", + "facet": False, + "optional": True, + "store": False, + }, + # Ranking value to use when no explicit sorting is used during search + # Set to the RFC number for RFCs and the revision number for drafts + # This ensures newer RFCs get listed first in the default search results + # (without a query) + {"name": "ranking", "type": "int32", "facet": False}, + ], +} + +SEARCH_PRESETS = { + "red": { + "collection": "docs", + "infix": "off,always,off,off,off,off,off,off", + "query_by": "rfc,filename,title,abstract,keywords,authors,group,area", + "query_by_weights": "127,50,50,20,20,5,2,1" + }, + "red-content": { + "collection": "docs", + "infix": "off,always,off,off,off", + "query_by": "rfc,filename,keywords,authors,content", + "query_by_weights": "127,50,20,5,1" + }, +} + + +def create_collection(): + collection_name = get_collection_name() + log(f"Creating '{collection_name}' collection") + client = get_typesense_client() + client.collections.create({"name": get_collection_name()} | DOCS_SCHEMA) + + +def delete_collection(): + collection_name = get_collection_name() + log(f"Deleting '{collection_name}' collection") + client = get_typesense_client() + try: + client.collections[collection_name].delete() + except typesense.exceptions.ObjectNotFound: + pass + + +def upsert_presets(): + # typesense-python does not support presets, so use requests + _settings = get_settings() + api_base = _settings["TYPESENSE_API_URL"] + api_key = _settings["TYPESENSE_API_KEY"] + for preset_name, payload in SEARCH_PRESETS.items(): + log(f"Upserting '{preset_name}' preset") + response = requests.put( + urljoin(api_base, f"/presets/{preset_name}"), + json={"value": payload}, + headers={ + "X-TYPESENSE-API-KEY": api_key, + }, + timeout=3, + ) + response.raise_for_status() diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 1a3d4e5c3d1..a23416e87f2 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -70,7 +70,7 @@ from django.template.loaders.filesystem import Loader as BaseLoader from django.test.runner import DiscoverRunner from django.core.management import call_command -from django.urls import URLResolver # type: ignore +from django.urls import URLResolver, resolve, Resolver404 # type: ignore from django.template.backends.django import DjangoTemplates from django.template.backends.django import Template # type: ignore[attr-defined] from django.utils import timezone @@ -88,6 +88,30 @@ from mypy_boto3_s3.service_resource import Bucket +class UrlCoverageWarning(UserWarning): + """Warning category for URL coverage-related warnings""" + # URLs for which we don't expect patterns to match + IGNORE_URLS = ( + "/_doesnotexist/", + "/sitemap.xml.", + ) + + +class UninterestingPatternWarning(UrlCoverageWarning): + """Warning category for unexpected URL match patterns + + These are common, caused by tests that hit a URL that is not selected for + coverage checking. The warning is in place to help with a putative future + review of whether we're selecting the right patterns to check for coverage. + """ + pass + + +# Configure warnings for reasonable output quantity +warnings.simplefilter("once", UrlCoverageWarning) +warnings.simplefilter("ignore", UninterestingPatternWarning) + + loaded_templates: set[str] = set() visited_urls: set[str] = set() test_database_name: Optional[str] = None @@ -550,21 +574,38 @@ def ignore_pattern(regex, pattern): ) or pattern.callback == django.views.static.serve) - patterns = [(regex, re.compile(regex, re.U), obj) for regex, obj in url_patterns - if not ignore_pattern(regex, obj)] + patterns ={ + regex: obj + for regex, obj in url_patterns + if not ignore_pattern(regex, obj) + } covered = set() for url in visited_urls: - for regex, compiled, obj in patterns: - if regex not in covered and compiled.match(url[1:]): # strip leading / - covered.add(regex) - break + try: + resolved = resolve(url) # let Django resolve the URL for us + except Resolver404: + if url not in UrlCoverageWarning.IGNORE_URLS: + warnings.warn( + f"Unable to resolve visited URL {url}", UrlCoverageWarning + ) + continue + if resolved.route not in patterns: + warnings.warn( + f"WARNING: url resolved to an unexpected pattern (url='{url}', " + f"resolved to r'{resolved.route}'", + UninterestingPatternWarning, + ) + continue + covered.add(resolved.route) self.runner.coverage_data["url"] = { - "coverage": 1.0*len(covered)/len(patterns), - "covered": dict( (k, (o.lookup_str, k in covered)) for k,p,o in patterns ), + "coverage": 1.0 * len(covered) / len(patterns), + "covered": dict( + (k, (o.lookup_str, k in covered)) for k, o in patterns.items() + ), "format": 4, - } + } self.report_test_result("url") else: diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 86c5a0c1c31..5faf83d93ff 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -38,6 +38,7 @@ import re import email import html5lib +import rest_framework.test import requests_mock import shutil import sys @@ -312,3 +313,11 @@ def tearDown(self): shutil.rmtree(dir) self.requests_mock.stop() super().tearDown() + + +class APITestCase(TestCase): + """Test case that uses rest_framework's APIClient + + This is equivalent to rest_framework.test.APITestCase, but picks up our + """ + client_class = rest_framework.test.APIClient diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 3288309095c..99c33f34b33 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -60,7 +60,6 @@ set_url_coverage, ) from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.text import parse_unicode from ietf.utils.timezone import timezone_not_near_midnight from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output @@ -864,24 +863,6 @@ def test_assertion(self): assertion('False') settings.SERVER_MODE = 'test' -class TestRFC2047Strings(TestCase): - def test_parse_unicode(self): - names = ( - ('=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=', 'ዳዊት በቀለ'), - ('=?utf-8?b?5Li9IOmDnA==?=', '丽 郜'), - ('=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=', 'कम्बोज डार'), - ('=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=', 'Ηράκλεια Λιόντη'), - ('=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=', 'ישראל רוזנפלד'), - ('=?utf-8?b?5Li95Y2OIOeahw==?=', '丽华 皇'), - ('=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=', 'ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ'), - ('=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=', 'ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ'), - ('=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=', 'Iñigo Sanç Ibáñez de la Peña'), - ('Mart van Oostendorp', 'Mart van Oostendorp'), - ('', ''), - ) - for encoded_str, unicode in names: - self.assertEqual(unicode, parse_unicode(encoded_str)) - class TestAndroidSiteManifest(TestCase): def test_manifest(self): r = self.client.get(urlreverse('site.webmanifest')) diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py index 502e936483a..c076a3df745 100644 --- a/ietf/utils/tests_meetecho.py +++ b/ietf/utils/tests_meetecho.py @@ -547,7 +547,8 @@ def test_add(self, mock_add, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) session = SessionFactory() slides_doc = DocumentFactory(type_id="slides") - sm.add(session, slides_doc, 13) + retval = sm.add(session, slides_doc, 13) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_add.called) self.assertEqual( @@ -565,6 +566,14 @@ def test_add(self, mock_add, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.add(session, slides_doc, 14) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") def test_delete(self, mock_delete, mock_update, mock_wg_token): @@ -580,7 +589,8 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): sm.delete(session, slides_doc) # can't remove slides still attached to the session self.assertFalse(any([mock_wg_token.called, mock_delete.called, mock_update.called])) - sm.delete(session, removed_slides_doc) + retval = sm.delete(session, removed_slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -609,9 +619,18 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): # Delete the other session and check that we don't make the update call slides.delete() - sm.delete(session, slides_doc) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_delete.called) self.assertFalse(mock_update.called) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, False) @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") @patch("ietf.utils.meetecho.MeetechoAPI.add_slide_deck") @@ -619,7 +638,8 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides", order=23) slides_doc = slides.document - sm.revise(slides.session, slides.document) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -642,13 +662,22 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") def test_send_update(self, mock_send_update, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides") SessionPresentationFactory(session=slides.session, document__type_id="agenda") - sm.send_update(slides.session) + retval = sm.send_update(slides.session) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_send_update.called) self.assertEqual( @@ -667,3 +696,11 @@ def test_send_update(self, mock_send_update, mock_wg_token): ] ) ) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.send_update(slides.session) + self.assertIs(retval, False) diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 00000000000..83daa15cc14 --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,189 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + authors2020, + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + authors2021, + set( + [ + "author0@example.net", + "author0@example.com", + "author2@example.net", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + sub2020, + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py new file mode 100644 index 00000000000..b97bc2e266c --- /dev/null +++ b/ietf/utils/tests_searchindex.py @@ -0,0 +1,325 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from unittest import mock + +import requests.exceptions +import typesense.exceptions +from django.conf import settings +from django.test.utils import override_settings + +from . import searchindex +from .test_utils import TestCase +from ..blobdb.models import Blob +from ..doc.factories import ( + WgDraftFactory, + WgRfcFactory, + PublishedRfcDocEventFactory, + BcpFactory, + StdFactory, +) +from ..doc.models import Document, RelatedDocument +from ..doc.storage_utils import store_str +from ..person.factories import PersonFactory + + +class SearchindexTests(TestCase): + def test_enabled(self): + with override_settings(): + try: + del settings.SEARCHINDEX_CONFIG + except AttributeError: + pass + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_KEY": "this-is-not-a-key"} + ): + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_URL": "http://example.com"} + ): + self.assertTrue(searchindex.enabled()) + + def test_sanitize_text(self): + dirty_text = """ + + This is text. It + is <---- full of \tprobl.....ems! Fix it. + """ + sanitized = "This is text It is full of problems Fix it." + self.assertEqual(searchindex._sanitize_text(dirty_text), sanitized) + + def test_sanitize_abstract(self): + dirty_abstract = ( + "Mixed\n" + "Newlines\r" + "And\r\n" + "Things\n\r" + " Sometimes\n" + "\n" + "With\r\n" + "\r\n" + "Double \n\r" + "\n\r" + " Newlines\r" + "\r" + "Whee!" + ) + sanitized = ( + "Mixed\n" + "Newlines\n" + "And\n" + "Things\n" + "Sometimes\n" + "\n" + "With\n" + "\n" + "Double\n" + "\n" + "Newlines\n" + "\n" + "Whee!" + ) + self.assertEqual(searchindex._sanitize_abstract(dirty_abstract), sanitized) + + def test_typesense_doc_from_rfc(self): + not_rfc = WgDraftFactory() + assert isinstance(not_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.typesense_doc_from_rfc(not_rfc) + + invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) + assert isinstance(invalid_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.typesense_doc_from_rfc(invalid_rfc) + + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + result = searchindex.typesense_doc_from_rfc(rfc) + # Check a few values, not exhaustive + self.assertEqual(result["id"], f"doc-{rfc.pk}") + self.assertEqual(result["rfcNumber"], rfc.rfc_number) + self.assertEqual(result["abstract"], searchindex._sanitize_abstract(rfc.abstract)) + self.assertEqual(result["pages"], rfc.pages) + self.assertNotIn("adName", result) + self.assertNotIn("content", result) # no blob + self.assertNotIn("subseries", result) + + # repeat, this time with contents, an AD, and subseries docs + store_str( + kind="rfc", + name=f"txt/{rfc.name}.txt", + content="The contents of this RFC", + doc_name=rfc.name, + doc_rev=rfc.rev, # expected to be None + ) + rfc.ad = PersonFactory(name="Alfred D. Rector") + # Put it in two Subseries docs to be sure this does not break things + # (the typesense schema does not support this for real at the moment) + BcpFactory(contains=[rfc], name="bcp1234") + StdFactory(contains=[rfc], name="std1234") + result = searchindex.typesense_doc_from_rfc(rfc) + # Check a few values, not exhaustive + self.assertEqual( + result["content"], + searchindex._sanitize_text("The contents of this RFC"), + ) + self.assertEqual(result["adName"], "Alfred D. Rector") + self.assertIn("subseries", result) + ss_dict = result["subseries"] + # We should get one of the two subseries docs, but neither is more correct + # than the other... + self.assertTrue( + any( + ss_dict == {"acronym": ss_type, "number": 1234, "total": 1} + for ss_type in ["bcp", "std"] + ) + ) + + # Finally, delete the contents blob and make sure things don't blow up + Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertNotIn("content", result) + + def test_typesense_doc_from_rfc_flags_obsoleted(self): + """typesense docs should set correct flags for obsoleted RFC""" + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + self.assertEqual(len(rfc.related_that("obs")), 0) + self.assertEqual(len(rfc.related_that("updates")), 0) + self.assertNotEqual(rfc.std_level.slug, "hist") + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertFalse(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + RelatedDocument.objects.create( + source=(PublishedRfcDocEventFactory().doc), + target=rfc, + relationship_id="obs", + ) + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertTrue(result["flags"]["hiddenDefault"]) + self.assertTrue(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + def test_typesense_doc_from_rfc_flags_updated(self): + """typesense docs should set flags correctly for updated RFC""" + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + self.assertEqual(len(rfc.related_that("obs")), 0) + self.assertEqual(len(rfc.related_that("updates")), 0) + self.assertNotEqual(rfc.std_level.slug, "hist") + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertFalse(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + RelatedDocument.objects.create( + source=(PublishedRfcDocEventFactory().doc), + target=rfc, + relationship_id="updates", + ) + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertFalse(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertTrue(result["flags"]["updated"]) + + def test_typesense_doc_from_rfc_flags_historic(self): + """typesense docs should set flags correctly for historic RFC""" + rfc = PublishedRfcDocEventFactory(doc__std_level_id="hist").doc + assert isinstance(rfc, Document) + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertTrue(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.upsert + self.assertTrue(mock_upsert.called) + self.assertEqual(mock_upsert.call_args, mock.call(fake_tdoc)) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entries( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entries([rfc] * 50) # list of docs... + self.assertEqual(mock_ts_client_constructor.call_count, 1) + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 1) + self.assertEqual( + mock_import_.call_args, mock.call([fake_tdoc] * 50, {"action": "upsert"}) + ) + + mock_import_.reset_mock() + searchindex.update_or_create_rfc_entries([rfc] * 50, batchsize=20) + self.assertEqual(mock_ts_client_constructor.call_count, 2) # one more + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 3) + self.assertEqual( + mock_import_.call_args_list, + [ + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 10, {"action": "upsert"}), + ], + ) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_create_collection(self, mock_ts_client_constructor): + searchindex.create_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections.create.called) + self.assertEqual(mock_collections.create.call_args[0][0]["name"], "frogs") + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_delete_collection(self, mock_ts_client_constructor): + searchindex.delete_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections["frogs"].delete.called) + + mock_collections["frogs"].side_effect = typesense.exceptions.ObjectNotFound + searchindex.delete_collection() # should ignore the exception + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + def test_upsert_presets(self): + self.requests_mock.put( + "http://ts.example.com/presets/red", text="ok", status_code=201 + ) + self.requests_mock.put( + "http://ts.example.com/presets/red-content", text="ok", status_code=202 + ) + searchindex.upsert_presets() + + self.requests_mock.put( + "http://ts.example.com/presets/red", text="not ok", status_code=400 + ) + with self.assertRaises(requests.exceptions.HTTPError): + searchindex.upsert_presets() + + self.requests_mock.put( + "http://ts.example.com/presets/red", text="ok", status_code=200 + ) + self.requests_mock.put( + "http://ts.example.com/presets/red-content", text="not ok", status_code=400 + ) + with self.assertRaises(requests.exceptions.HTTPError): + searchindex.upsert_presets() diff --git a/ietf/utils/tests_text.py b/ietf/utils/tests_text.py new file mode 100644 index 00000000000..51aa2eff132 --- /dev/null +++ b/ietf/utils/tests_text.py @@ -0,0 +1,71 @@ +# Copyright The IETF Trust 2021-2026, All Rights Reserved +from ietf.utils.test_utils import TestCase +from ietf.utils.text import parse_unicode, decode_document_content + + +class TestDecoders(TestCase): + def test_parse_unicode(self): + names = ( + ("=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=", "ዳዊት በቀለ"), + ("=?utf-8?b?5Li9IOmDnA==?=", "丽 郜"), + ("=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=", "कम्बोज डार"), + ("=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=", "Ηράκλεια Λιόντη"), + ("=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=", "ישראל רוזנפלד"), + ("=?utf-8?b?5Li95Y2OIOeahw==?=", "丽华 皇"), + ("=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=", "ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ"), + ( + "=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=", + "ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ", + ), + ( + "=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=", + "Iñigo Sanç Ibáñez de la Peña", + ), + ("Mart van Oostendorp", "Mart van Oostendorp"), + ("", ""), + ) + for encoded_str, unicode in names: + self.assertEqual(unicode, parse_unicode(encoded_str)) + + def test_decode_document_content(self): + utf8_bytes = "𒀭𒊩𒌆𒄈𒋢".encode("utf-8") # ends with 4-byte character + latin1_bytes = "àéîøü".encode("latin-1") + other_bytes = "àéîøü".encode("macintosh") # different from its latin-1 encoding + assert other_bytes.decode("macintosh") != other_bytes.decode("latin-1"),\ + "test broken: other_bytes must decode differently as latin-1" + + # simplest case + self.assertEqual( + decode_document_content(utf8_bytes), + utf8_bytes.decode(), + ) + # losing 1-4 bytes from the end leave the last character incomplete; the + # decoder should decode all but that last character + self.assertEqual( + decode_document_content(utf8_bytes[:-1]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-2]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-3]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-4]), + utf8_bytes.decode()[:-1], + ) + + # latin-1 is also simple + self.assertEqual( + decode_document_content(latin1_bytes), + latin1_bytes.decode("latin-1"), + ) + + # other character sets are just treated as latin1 (bug? feature? you decide) + self.assertEqual( + decode_document_content(other_bytes), + other_bytes.decode("latin-1"), + ) diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 590ec3fd30a..2763056e1a9 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -263,3 +263,21 @@ def parse_unicode(text): else: text = decoded_string return text + + +def decode_document_content(content: bytes) -> str: + """Decode document contents as utf-8 or latin1 + + Method was developed in DocumentInfo.text() where it gave acceptable results + for existing documents / RFCs. + """ + try: + return content.decode("utf-8") + except UnicodeDecodeError: + pass + for back in range(1, 4): + try: + return content[:-back].decode("utf-8") + except UnicodeDecodeError: + pass + return content.decode("latin-1") # everything is legal in latin-1 diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py index e08dfa02f2c..5e82c225dc6 100644 --- a/ietf/utils/timezone.py +++ b/ietf/utils/timezone.py @@ -15,7 +15,7 @@ DEADLINE_TZINFO = ZoneInfo('PST8PDT') # Time zone for dates from the RPC. This value is baked into the timestamps on DocEvents -# of type="published_rfc" - see Document.pub_date() and ietf.sync.refceditor.update_docs_from_rfc_index() +# of type="published_rfc" - see Document.pub_date() and the rfc publication workflow # for more information about how that works. RPC_TZINFO = ZoneInfo('PST8PDT') diff --git a/ietf/utils/unicodenormalize.py b/ietf/utils/unicodenormalize.py new file mode 100644 index 00000000000..8644dbdb79f --- /dev/null +++ b/ietf/utils/unicodenormalize.py @@ -0,0 +1,9 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import unicodedata + +def normalize_for_sorting(text): + """Normalize text for proper accent-aware sorting.""" + # Normalize the text to NFD (decomposed form) + decomposed = unicodedata.normalize('NFD', text) + # Filter out combining diacritical marks + return ''.join(char for char in decomposed if not unicodedata.combining(char)) diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 92a20f5a266..a99de72724e 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -33,8 +33,9 @@ # Note that this is an instantiation of the regex validator, _not_ the # regex-string validator defined right below validate_no_control_chars = RegexValidator( - regex="^[^\x00-\x1f]*$", - message="Please enter a string without control characters." ) + regex="^[^\x01-\x1f]*$", + message="Please enter a string without control characters.", +) @deconstructible diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py index f555a0a16a0..325b8499a96 100644 --- a/ietf/utils/xmldraft.py +++ b/ietf/utils/xmldraft.py @@ -102,6 +102,17 @@ def _document_name(self, ref): number = int(maybe_number) return f"{label}{number}" + target = ref.get("target") + if isinstance(target, str): + target = target.lower() + if target.startswith("https://datatracker.ietf.org/doc/"): + # len("https://datatracker.ietf.org/doc/")==33 + m = re.match(r"^(draft-[a-z0-9-]*[a-z0-9])([/-]\d{2})?/?$",target[33:]) + if m: + name = m.group(1) + return name + + # if we couldn't find a match so far, try the seriesInfo series_query = " or ".join(f"@name='{x.upper()}'" for x in series) for info in ref.xpath( diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 392e306b548..ef8c259933b 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -8,23 +8,11 @@ spec: selector: matchLabels: app: auth - strategy: - type: Recreate template: metadata: labels: app: auth spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -71,11 +59,15 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 250m + memory: 4Gi # ----------------------------------------------------- # Nginx Container # ----------------------------------------------------- - name: nginx - image: "ghcr.io/nginxinc/nginx-unprivileged:1.27" + image: "ghcr.io/nginx/nginx-unprivileged:1.30" imagePullPolicy: IfNotPresent ports: - containerPort: 8080 @@ -96,6 +88,10 @@ spec: - name: dt-cfg mountPath: /etc/nginx/conf.d/default.conf subPath: nginx-auth.conf + resources: + requests: + cpu: 10m + memory: 10Mi # ----------------------------------------------------- # ScoutAPM Container # ----------------------------------------------------- @@ -121,6 +117,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # "nobody" user by default runAsGroup: 65534 # "nogroup" group by default + resources: + requests: + cpu: 10m + memory: 50Mi volumes: # To be overriden with the actual shared volume - name: dt-vol @@ -139,8 +139,6 @@ spec: - name: nginx-tmp emptyDir: sizeLimit: "500Mi" - dnsPolicy: ClusterFirst - restartPolicy: Always terminationGracePeriodSeconds: 60 --- apiVersion: v1 diff --git a/k8s/beat.yaml b/k8s/beat.yaml index cc98beecf62..cc171fb7d1c 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -17,16 +17,6 @@ spec: labels: app: beat spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -58,6 +48,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 100m + memory: 250Mi volumes: # To be overridden with the actual shared volume - name: dt-vol @@ -69,4 +63,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 10 diff --git a/k8s/celery.yaml b/k8s/celery.yaml index a2799f2a6d8..f6cea2acc7a 100644 --- a/k8s/celery.yaml +++ b/k8s/celery.yaml @@ -17,16 +17,6 @@ spec: labels: app: celery spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -62,6 +52,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 100m + memory: 1Gi # ----------------------------------------------------- # ScoutAPM Container # ----------------------------------------------------- @@ -87,6 +81,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # "nobody" user by default runAsGroup: 65534 # "nogroup" group by default + resources: + requests: + cpu: 10m + memory: 50Mi volumes: # To be overridden with the actual shared volume - name: dt-vol diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 50a2c696878..5183893bc8c 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: datatracker - strategy: - type: Recreate template: metadata: labels: @@ -61,11 +59,15 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 250m + memory: 4Gi # ----------------------------------------------------- # Nginx Container # ----------------------------------------------------- - name: nginx - image: "ghcr.io/nginxinc/nginx-unprivileged:1.27" + image: "ghcr.io/nginx/nginx-unprivileged:1.30" imagePullPolicy: IfNotPresent ports: - containerPort: 8080 @@ -87,6 +89,10 @@ spec: # Replaces the original default.conf mountPath: /etc/nginx/conf.d/default.conf subPath: nginx-datatracker.conf + resources: + requests: + cpu: 10m + memory: 10Mi # ----------------------------------------------------- # ScoutAPM Container # ----------------------------------------------------- @@ -112,6 +118,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # "nobody" user by default runAsGroup: 65534 # "nogroup" group by default + resources: + requests: + cpu: 10m + memory: 50Mi initContainers: - name: migration image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" @@ -160,8 +170,6 @@ spec: - name: nginx-tmp emptyDir: sizeLimit: "500Mi" - dnsPolicy: ClusterFirst - restartPolicy: Always terminationGracePeriodSeconds: 60 --- apiVersion: v1 diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 8f73f3d0d50..68b732d7459 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -13,16 +13,6 @@ spec: labels: app: memcached spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -46,6 +36,10 @@ spec: # memcached image sets up uid/gid 11211 runAsUser: 11211 runAsGroup: 11211 + resources: + requests: + cpu: 100m + memory: 100Mi # ----------------------------------------------------- # Memcached Exporter for Prometheus # ----------------------------------------------------- @@ -64,6 +58,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # nobody runAsGroup: 65534 # nobody + resources: + requests: + cpu: 10m + memory: 20Mi dnsPolicy: ClusterFirst restartPolicy: Always terminationGracePeriodSeconds: 30 diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index 95aa838064b..cdb5c665e22 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -1,3 +1,8 @@ +upstream datatracker_backend { + server 127.0.0.1:8000; + keepalive 0; # default = 32 since nginx 1.29.7 +} + server { listen 8080 default_server; server_name _; @@ -33,7 +38,7 @@ server { proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world - proxy_pass http://localhost:8000; + proxy_pass http://datatracker_backend; # Set timeouts longer than Cloudflare proxy limits proxy_connect_timeout 60; # nginx default (Cf = 15) proxy_read_timeout 120; # nginx default = 60 (Cf = 100) diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index 882d7563c25..77ce902b17a 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -1,3 +1,8 @@ +upstream datatracker_backend { + server 127.0.0.1:8000; + keepalive 0; # default = 32 since nginx 1.29.7 +} + server { listen 8080 default_server; server_name _; @@ -22,7 +27,7 @@ server { proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world - proxy_pass http://localhost:8000; + proxy_pass http://datatracker_backend; # Set timeouts longer than Cloudflare proxy limits proxy_connect_timeout 60; # nginx default (Cf = 15) proxy_read_timeout 120; # nginx default = 60 (Cf = 100) diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index 780a3992390..e69aa7a1aa1 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -13,16 +13,6 @@ spec: labels: app: rabbitmq spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -72,6 +62,10 @@ spec: # rabbitmq image sets up uid/gid 100/101 runAsUser: 100 runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 150Mi initContainers: # ----------------------------------------------------- # Init RabbitMQ data diff --git a/k8s/replicator.yaml b/k8s/replicator.yaml index 9c462bd96ba..0b06fe4fdc5 100644 --- a/k8s/replicator.yaml +++ b/k8s/replicator.yaml @@ -17,16 +17,6 @@ spec: labels: app: replicator spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -62,6 +52,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 100m + memory: 500Mi volumes: # To be overridden with the actual shared volume - name: dt-vol diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml index ba90af9c2a5..4d326521467 100644 --- a/k8s/secrets.yaml +++ b/k8s/secrets.yaml @@ -39,7 +39,6 @@ stringData: DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret @@ -80,4 +79,4 @@ stringData: # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file + DATATRACKER_SCOUT_NAME: "StagingDatatracker" diff --git a/k8s/settings_local.py b/k8s/settings_local.py index f8ffacc83fc..5dc31bac0e5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- from base64 import b64decode @@ -18,7 +18,7 @@ def _multiline_to_list(s): - """Helper to split at newlines and conver to list""" + """Helper to split at newlines and convert to list""" return [item.strip() for item in s.split("\n")] @@ -44,12 +44,6 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") -_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) -if _RFC_EDITOR_SYNC_PASSWORD is not None: - RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") -else: - raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") - _YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) if _YOUTUBE_API_KEY is not None: YOUTUBE_API_KEY = _YOUTUBE_API_KEY @@ -80,6 +74,22 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None +) +if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None +) +if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: + RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get( + "DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None +) +if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL + # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" @@ -139,14 +149,18 @@ def _multiline_to_list(s): EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) +_broker_url = os.environ.get("DATATRACKER_BROKER_URL", None) _celery_password = os.environ.get("CELERY_PASSWORD", None) -if _celery_password is None: - raise RuntimeError("CELERY_PASSWORD must be set") -CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), - password=_celery_password, - queue=os.environ.get("RABBITMQ_QUEUE", "dt"), -) +if _broker_url is not None: + CELERY_BROKER_URL = _broker_url +elif _celery_password is not None: + CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), + password=_celery_password, + queue=os.environ.get("RABBITMQ_QUEUE", "dt"), + ) +else: + raise RuntimeError("DATATRACKER_BROKER_URL or CELERY_PASSWORD must be set") # mailarchive API key _mailing_list_archive_api_key = os.environ.get( @@ -234,8 +248,19 @@ def _multiline_to_list(s): EMAIL_COPY_TO = "" -# Until we teach the datatracker to look beyond cloudflare for this check -IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 +# I-D Submission settings + +# Until we teach the datatracker to look beyond cloudflare for this check, it needs +# to be very large. 5000 has been working without complaint. +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = int( + os.environ.get("DATATRACKER_IDSUBMIT_MAX_DAILY_SAME_SUBMITTER", "5000") +) + +# Default is 20 minutes. Allow override via environment. +if "DATATRACKER_IDSUBMIT_MAX_VALIDATION_TIME" in os.environ: + IDSUBMIT_MAX_VALIDATION_TIME = datetime.timedelta( + minutes=int(os.environ.get("DATATRACKER_IDSUBMIT_MAX_VALIDATION_TIME")) + ) # Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting if "DATATRACKER_MATOMO_SITE_ID" in os.environ: @@ -306,6 +331,16 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", @@ -367,6 +402,7 @@ def _multiline_to_list(s): "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set" ) _blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "") +_blob_store_bucket_suffix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_SUFFIX", "") _blob_store_enable_profiling = ( os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" ) @@ -386,6 +422,9 @@ def _multiline_to_list(s): if storagename in ["staging"]: continue replica_storagename = f"r2-{storagename}" + adjusted_bucket_name = ( + _blob_store_bucket_prefix + storagename + _blob_store_bucket_suffix + ).strip() STORAGES[replica_storagename] = { "BACKEND": "ietf.doc.storage.MetadataS3Storage", "OPTIONS": dict( @@ -402,11 +441,42 @@ def _multiline_to_list(s): retries={"total_max_attempts": _blob_store_max_attempts}, ), verify=False, - bucket_name=f"{_blob_store_bucket_prefix}{storagename}".strip(), + bucket_name=adjusted_bucket_name, ietf_log_blob_timing=_blob_store_enable_profiling, ), } +# Configure storage for the red bucket - assume it uses the same credentials as +# other blobs +_red_bucket_name = os.environ.get("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME", "").strip() +if _red_bucket_name == "": + raise RuntimeError("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME must be set") + +STORAGES["red_bucket"] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, + security_token=None, + client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, + ), + verify=False, + bucket_name=_red_bucket_name, + ), +} +RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default +if "DATATRACKER_RFCINDEX_OUTPUT_PATH" in os.environ: + RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH") +if "DATATRACKER_RFCINDEX_INPUT_PATH" in os.environ: + RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH") + # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( os.environ.get("DATATRACKER_BLOBDB_REPLICATION_ENABLED", "true").lower() == "true" @@ -428,3 +498,34 @@ def _multiline_to_list(s): PASSWORD_POLICY_ENFORCE_AT_LOGIN = ( os.environ.get("DATATRACKER_ENFORCE_PW_POLICY", "true").lower() != "false" ) + +# Typesense search indexing +SEARCHINDEX_CONFIG = { + "TYPESENSE_API_URL": os.environ.get("DATATRACKER_TYPESENSE_API_URL", ""), + "TYPESENSE_API_KEY": os.environ.get("DATATRACKER_TYPESENSE_API_KEY", ""), + "TASK_RETRY_DELAY": int( + os.environ.get("DATATRACKER_SEARCHINDEX_TASK_RETRY_DELAY", "10") + ), + "TASK_MAX_RETRIES": int( + os.environ.get("DATATRACKER_SEARCHINDEX_TASK_MAX_RETRIES", "12") + ), +} + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_API_KEY = os.environ.get( + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY", None +) +if ERRATA_METADATA_NOTIFICATION_API_KEY is not None: + ERRATA_METADATA_NOTIFICATION_URL = os.environ.get( + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL", None + ) + if ERRATA_METADATA_NOTIFICATION_URL is None: + raise RuntimeError( + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL must be set if " + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY is provided" + ) + +# name (with path) of errata.json in the red bucket +ERRATA_JSON_BLOB_NAME = os.environ.get( + "DATATRACKER_ERRATA_JSON_BLOB_NAME", "other/errata.json" +) diff --git a/mypy.ini b/mypy.ini index 19df7ec9b00..4acaf98c950 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,6 +2,9 @@ ignore_missing_imports = True +# allow PEP 695 type aliases (flag needed until mypy >= 1.13) +enable_incomplete_feature = NewGenericSyntax + plugins = mypy_django_plugin.main diff --git a/package.json b/package.json index e2e6fd7dab7..29ead19d239 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,16 @@ "@fullcalendar/luxon3": "6.1.11", "@fullcalendar/timegrid": "6.1.11", "@fullcalendar/vue3": "6.1.11", + "@kurkle/color": "0.3.1", "@popperjs/core": "2.11.8", "@twuni/emojify": "1.0.2", "bootstrap": "5.3.3", "bootstrap-icons": "1.11.3", "browser-fs-access": "0.35.0", "caniuse-lite": "1.0.30001603", + "chart.js": "^4.5.1", + "chartjs-plugin-autocolors": "0.3.1", + "chartjs-plugin-zoom": "2.2.0", "d3": "7.9.0", "file-saver": "2.0.5", "highcharts": "11.4.0", @@ -112,6 +116,7 @@ "ietf/static/images/irtf-logo-white.svg", "ietf/static/images/irtf-logo.svg", "ietf/static/js/add_session_recordings.js", + "ietf/static/js/attendees-chart.js", "ietf/static/js/agenda_filter.js", "ietf/static/js/agenda_materials.js", "ietf/static/js/announcement.js", @@ -145,9 +150,13 @@ "ietf/static/js/manage-community-list.js", "ietf/static/js/manage-review-requests.js", "ietf/static/js/meeting-interim-request.js", + "ietf/static/js/meeting_stats.js", + "ietf/static/js/meeting_timeline.js", "ietf/static/js/moment.js", + "ietf/static/js/navbar-doc-search.js", "ietf/static/js/password_strength.js", "ietf/static/js/select2.js", + "ietf/static/js/session_details.js", "ietf/static/js/session_details_form.js", "ietf/static/js/session_form.js", "ietf/static/js/session_request.js", diff --git a/patch/django-cookie-delete-with-all-settings.patch b/patch/django-cookie-delete-settings-and-CVE-2026-35192.patch similarity index 63% rename from patch/django-cookie-delete-with-all-settings.patch rename to patch/django-cookie-delete-settings-and-CVE-2026-35192.patch index fb8bbbe4fed..3f625c33bb9 100644 --- a/patch/django-cookie-delete-with-all-settings.patch +++ b/patch/django-cookie-delete-settings-and-CVE-2026-35192.patch @@ -9,9 +9,9 @@ samesite=settings.SESSION_COOKIE_SAMESITE, ) ---- django/http/response.py.orig 2020-08-13 11:16:04.060627793 +0200 -+++ django/http/response.py 2020-08-13 11:54:03.482476973 +0200 -@@ -282,20 +282,28 @@ +--- django/http/response.py.orig 2025-12-02 22:12:05.197283001 +0000 ++++ django/http/response.py 2025-12-02 22:26:01.396576013 +0000 +@@ -286,20 +286,28 @@ value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) @@ -44,9 +44,9 @@ expires="Thu, 01 Jan 1970 00:00:00 GMT", samesite=samesite, ) ---- django/contrib/sessions/middleware.py.orig 2020-08-13 12:12:12.401898114 +0200 -+++ django/contrib/sessions/middleware.py 2020-08-13 12:14:52.690520659 +0200 -@@ -38,6 +38,8 @@ +--- django/contrib/sessions/middleware.py.old 2026-05-12 15:18:07.673997003 +0000 ++++ django/contrib/sessions/middleware.py 2026-05-12 15:18:15.770997007 +0000 +@@ -38,12 +38,15 @@ settings.SESSION_COOKIE_NAME, path=settings.SESSION_COOKIE_PATH, domain=settings.SESSION_COOKIE_DOMAIN, @@ -54,4 +54,23 @@ + httponly=settings.SESSION_COOKIE_HTTPONLY or None, samesite=settings.SESSION_COOKIE_SAMESITE, ) - patch_vary_headers(response, ("Cookie",)) +- patch_vary_headers(response, ("Cookie",)) ++ need_vary_cookie = True + else: +- if accessed: +- patch_vary_headers(response, ("Cookie",)) ++ # If the session was accessed, it must be varied on, regardless of ++ # whether it was modified or will be saved. ++ need_vary_cookie = accessed + if (modified or settings.SESSION_SAVE_EVERY_REQUEST) and not empty: + if request.session.get_expire_at_browser_close(): + max_age = None +@@ -74,4 +77,8 @@ + httponly=settings.SESSION_COOKIE_HTTPONLY or None, + samesite=settings.SESSION_COOKIE_SAMESITE, + ) ++ # With a session cookie set, it must be varied on. ++ need_vary_cookie = True ++ if need_vary_cookie: ++ patch_vary_headers(response, ("Cookie",)) + return response diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js index 412a3fe9b8c..2248027a38a 100644 --- a/playwright/tests/meeting/agenda.spec.js +++ b/playwright/tests/meeting/agenda.spec.js @@ -1219,7 +1219,12 @@ test.describe('future - desktop', () => { await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible() } } else { - await expect(eventButtons).toHaveCount(0) + if (event.links.calendar) { + await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar`)).toHaveAttribute('href', event.links.calendar) + await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible() + } else { + await expect(eventButtons).toHaveCount(0) + } } } } diff --git a/requirements.txt b/requirements.txt index cf7c920fa3f..485e4eb4a3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ setuptools>=80.9.0 # Require this first, to prevent later errors aiosmtpd>=1.4.6 argon2-cffi>=25.1.0 # For the Argon2 password hasher option beautifulsoup4>=4.13.4 # Only used in tests -bibtexparser>=1.4.3 # Only used in tests +bibtexparser>=1.4.4 # Only used in tests bleach>=6.2.0 # project is deprecated but supported types-bleach>=6.2.0 boto3>=1.39.15 @@ -13,14 +13,16 @@ botocore>=1.39.15 celery>=5.5.3 coverage>=7.9.2 defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency -Django>4.2,<5 +Django>=4.2.30,<5 django-admin-rangefilter>=0.13.3 django-analytical>=3.2.0 django-bootstrap5>=25.1 -django-celery-beat>=2.7.0,<2.8.0 # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit +django-celery-beat>=2.9.0 django-celery-results>=2.6.0 +django-csp>=3.7 django-cors-headers>=4.7.0 django-debug-toolbar>=6.0.0 +django-filter>=24.3 django-markup>=1.10 # Limited use - need to reconcile against direct use of markdown django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return django-simple-history>=3.10.1 @@ -39,23 +41,30 @@ gunicorn>=23.0.0 hashids>=1.3.1 html2text>=2025.4.15 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests +httpx>=0.28.1 # Indirect req of typesense, but we import and refer to exceptions icalendar>=5.0.0 inflect>= 7.5.0 jsonfield>=3.2.0 # deprecated - need to replace with Django's JSONField jsonschema[format]>=4.25.0 jwcrypto>=1.5.6 # for signed notifications - this is aspirational, and is not really used. logging_tree>=1.10 # Used only by the showloggers management command -lxml>=6.0.0 +lxml>=6.1.0 markdown>=3.8.0 types-markdown>=3.8.0 mock>=5.2.0 # should replace with unittest.mock and remove dependency types-mock>=5.2.0 -mypy~=1.7.0 # Version requirements determined by django-stubs. +mypy~=1.11.2 # Version requirements loosely determined by django-stubs. oic>=1.7.0 # Used only by tests +opentelemetry-sdk>=1.38.0 +opentelemetry-instrumentation-django>=0.59b0 +opentelemetry-instrumentation-psycopg2>=0.59b0 +opentelemetry-instrumentation-pymemcache>=0.59b0 +opentelemetry-instrumentation-requests>=0.59b0 +opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 pyang>=2.6.1 -pydyf>=0.11.0 +pydyf>=0.12.1 pyflakes>=3.4.0 pyopenssl>=25.1.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency pyquery>=2.0.1 @@ -66,7 +75,8 @@ python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2025.2.0.20250809 # match pytz version +types-pytz==2025.2.0.20251108 # match pytz version +typesense>=2.0.0 requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 @@ -80,6 +90,6 @@ unidecode>=1.4.0 urllib3>=2.5.0 weasyprint>=66.0 xml2rfc>=3.30.0 -xym>=0.6,<1.0 +xym>=0.6,<0.10.0 zxcvbn>=4.5.0 -types-zxcvbn~=4.5.0.20250223 # match zxcvbn version +types-zxcvbn~=4.5.0.20260518 # match zxcvbn version diff --git a/yarn.lock b/yarn.lock index 54768ac3913..47d675d6b9f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -532,6 +532,20 @@ __metadata: languageName: node linkType: hard +"@kurkle/color@npm:0.3.1": + version: 0.3.1 + resolution: "@kurkle/color@npm:0.3.1" + checksum: e6be5c081bf5acfd4a1803dcd5a0733caf450e73148d5f02dc536b1ff0c60c959c23472a26c9c3c6c78ada04fb6a53c9202db9b2de8ea56f6eeec381f9cc3a1a + languageName: node + linkType: hard + +"@kurkle/color@npm:^0.3.0": + version: 0.3.4 + resolution: "@kurkle/color@npm:0.3.4" + checksum: b95c6abe0241ba1745b3c84de3b464296b95ce577110b54f46e6c6dcc9a0966491533df43812bd6c66f92cf818e385d1390b280cd5851d4afb52fc37f8a6c0b9 + languageName: node + linkType: hard + "@lezer/common@npm:^0.15.0, @lezer/common@npm:^0.15.7": version: 0.15.12 resolution: "@lezer/common@npm:0.15.12" @@ -1944,6 +1958,13 @@ __metadata: languageName: node linkType: hard +"@types/hammerjs@npm:^2.0.45": + version: 2.0.46 + resolution: "@types/hammerjs@npm:2.0.46" + checksum: caba6ec788d19905c71092670b58514b3d1f5eee5382bf9205e8df688d51e7857b7994e2dd7aed57fac8977bdf0e456d67fbaf23440a4385b8ce25fe2af1ec39 + languageName: node + linkType: hard + "@types/istanbul-lib-coverage@npm:^2.0.1": version: 2.0.4 resolution: "@types/istanbul-lib-coverage@npm:2.0.4" @@ -2728,6 +2749,37 @@ browserlist@latest: languageName: node linkType: hard +"chart.js@npm:^4.5.1": + version: 4.5.1 + resolution: "chart.js@npm:4.5.1" + dependencies: + "@kurkle/color": ^0.3.0 + checksum: 34b35b373642994b2adac197e91363625930530e29fc1baa6dbb411b5e1295f9f6572922003a0224a21a3019aec916567c1ed00c33b1373081f189fc188e5a7b + languageName: node + linkType: hard + +"chartjs-plugin-autocolors@npm:0.3.1": + version: 0.3.1 + resolution: "chartjs-plugin-autocolors@npm:0.3.1" + peerDependencies: + "@kurkle/color": ^0.3.1 + chart.js: ">=2" + checksum: de4f87b5bb3e042aa1d3de3886425bbd2340a55ca455b645569d0def602079833182ef214e205ff4466fb5ab1e708761cf37eb51ab3cd622284242c05ed94128 + languageName: node + linkType: hard + +"chartjs-plugin-zoom@npm:2.2.0": + version: 2.2.0 + resolution: "chartjs-plugin-zoom@npm:2.2.0" + dependencies: + "@types/hammerjs": ^2.0.45 + hammerjs: ^2.0.8 + peerDependencies: + chart.js: ">=3.2.0" + checksum: a540e3834082eeb4dedb5ec6ca381f94d7e101075c19a7b65f2a4cd2d12685b3a416e718c9cf7145799802874fb397f69b71a955dfc56b035946cde4d1eb6c8e + languageName: node + linkType: hard + "chokidar@npm:>=3.0.0 <4.0.0": version: 3.5.3 resolution: "chokidar@npm:3.5.3" @@ -4616,6 +4668,13 @@ browserlist@latest: languageName: node linkType: hard +"hammerjs@npm:^2.0.8": + version: 2.0.8 + resolution: "hammerjs@npm:2.0.8" + checksum: b092da7d1565a165d7edb53ef0ce212837a8b11f897aa3cf81a7818b66686b0ab3f4747fbce8fc8a41d1376594639ce3a054b0fd4889ca8b5b136a29ca500e27 + languageName: node + linkType: hard + "has-bigints@npm:^1.0.1, has-bigints@npm:^1.0.2": version: 1.0.2 resolution: "has-bigints@npm:1.0.2" @@ -7030,6 +7089,7 @@ browserlist@latest: "@fullcalendar/luxon3": 6.1.11 "@fullcalendar/timegrid": 6.1.11 "@fullcalendar/vue3": 6.1.11 + "@kurkle/color": 0.3.1 "@parcel/optimizer-data-url": 2.12.0 "@parcel/transformer-inline-string": 2.12.0 "@parcel/transformer-sass": 2.12.0 @@ -7044,6 +7104,9 @@ browserlist@latest: browserlist: latest c8: 9.1.0 caniuse-lite: 1.0.30001603 + chart.js: ^4.5.1 + chartjs-plugin-autocolors: 0.3.1 + chartjs-plugin-zoom: 2.2.0 d3: 7.9.0 eslint: 8.57.0 eslint-config-standard: 17.1.0